aboutsummaryrefslogtreecommitdiffstats
path: root/include/linux
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2013-11-14 19:32:31 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2013-11-14 19:32:31 -0500
commitd8fe4acc88da8fbbe360b6592c9d0abbb85117dc (patch)
treee3d5edc0ad3541a1daf37b60c5fe67b6fe347d22 /include/linux
parent3aeb58ab6216d864821e8dafb248e8d77403f3e9 (diff)
parent8d3ef556aba2b5b7d8b7144f7be1814d75ea3cc6 (diff)
Merge branch 'akpm' (patch-bomb from Andrew Morton)
Merge patches from Andrew Morton: - memstick fixes - the rest of MM - various misc bits that were awaiting merges from linux-next into mainline: seq_file, printk, rtc, completions, w1, softirqs, llist, kfifo, hfsplus * emailed patches from Andrew Morton <akpm@linux-foundation.org>: (72 commits) cmdline-parser: fix build hfsplus: Fix undefined __divdi3 in hfsplus_init_header_node() kfifo API type safety kfifo: kfifo_copy_{to,from}_user: fix copied bytes calculation sound/core/memalloc.c: use gen_pool_dma_alloc() to allocate iram buffer llists-move-llist_reverse_order-from-raid5-to-llistc-fix llists: move llist_reverse_order from raid5 to llist.c kernel: fix generic_exec_single indentation kernel-provide-a-__smp_call_function_single-stub-for-config_smp-fix kernel: provide a __smp_call_function_single stub for !CONFIG_SMP kernel: remove CONFIG_USE_GENERIC_SMP_HELPERS revert "softirq: Add support for triggering softirq work on softirqs" drivers/w1/masters/w1-gpio.c: use dev_get_platdata() sched: remove INIT_COMPLETION tree-wide: use reinit_completion instead of INIT_COMPLETION sched: replace INIT_COMPLETION with reinit_completion drivers/rtc/rtc-hid-sensor-time.c: enable HID input processing early drivers/rtc/rtc-hid-sensor-time.c: use dev_get_platdata() vsprintf: ignore %n again seq_file: remove "%n" usage from seq_file users ...
Diffstat (limited to 'include/linux')
-rw-r--r--include/linux/cmdline-parser.h2
-rw-r--r--include/linux/completion.h28
-rw-r--r--include/linux/huge_mm.h17
-rw-r--r--include/linux/hugetlb.h26
-rw-r--r--include/linux/interrupt.h22
-rw-r--r--include/linux/kfifo.h47
-rw-r--r--include/linux/llist.h2
-rw-r--r--include/linux/lockref.h7
-rw-r--r--include/linux/mm.h139
-rw-r--r--include/linux/mm_types.h21
-rw-r--r--include/linux/seq_file.h15
-rw-r--r--include/linux/smp.h16
-rw-r--r--include/linux/swapops.h7
13 files changed, 230 insertions, 119 deletions
diff --git a/include/linux/cmdline-parser.h b/include/linux/cmdline-parser.h
index 98e892ef6d5a..a0f9280421ec 100644
--- a/include/linux/cmdline-parser.h
+++ b/include/linux/cmdline-parser.h
@@ -8,6 +8,8 @@
8#define CMDLINEPARSEH 8#define CMDLINEPARSEH
9 9
10#include <linux/blkdev.h> 10#include <linux/blkdev.h>
11#include <linux/fs.h>
12#include <linux/slab.h>
11 13
12/* partition flags */ 14/* partition flags */
13#define PF_RDONLY 0x01 /* Device is read only */ 15#define PF_RDONLY 0x01 /* Device is read only */
diff --git a/include/linux/completion.h b/include/linux/completion.h
index 22c33e35bcb2..5d5aaae3af43 100644
--- a/include/linux/completion.h
+++ b/include/linux/completion.h
@@ -19,8 +19,8 @@
19 * 19 *
20 * See also: complete(), wait_for_completion() (and friends _timeout, 20 * See also: complete(), wait_for_completion() (and friends _timeout,
21 * _interruptible, _interruptible_timeout, and _killable), init_completion(), 21 * _interruptible, _interruptible_timeout, and _killable), init_completion(),
22 * and macros DECLARE_COMPLETION(), DECLARE_COMPLETION_ONSTACK(), and 22 * reinit_completion(), and macros DECLARE_COMPLETION(),
23 * INIT_COMPLETION(). 23 * DECLARE_COMPLETION_ONSTACK().
24 */ 24 */
25struct completion { 25struct completion {
26 unsigned int done; 26 unsigned int done;
@@ -65,7 +65,7 @@ struct completion {
65 65
66/** 66/**
67 * init_completion - Initialize a dynamically allocated completion 67 * init_completion - Initialize a dynamically allocated completion
68 * @x: completion structure that is to be initialized 68 * @x: pointer to completion structure that is to be initialized
69 * 69 *
70 * This inline function will initialize a dynamically created completion 70 * This inline function will initialize a dynamically created completion
71 * structure. 71 * structure.
@@ -76,6 +76,18 @@ static inline void init_completion(struct completion *x)
76 init_waitqueue_head(&x->wait); 76 init_waitqueue_head(&x->wait);
77} 77}
78 78
79/**
80 * reinit_completion - reinitialize a completion structure
81 * @x: pointer to completion structure that is to be reinitialized
82 *
83 * This inline function should be used to reinitialize a completion structure so it can
84 * be reused. This is especially important after complete_all() is used.
85 */
86static inline void reinit_completion(struct completion *x)
87{
88 x->done = 0;
89}
90
79extern void wait_for_completion(struct completion *); 91extern void wait_for_completion(struct completion *);
80extern void wait_for_completion_io(struct completion *); 92extern void wait_for_completion_io(struct completion *);
81extern int wait_for_completion_interruptible(struct completion *x); 93extern int wait_for_completion_interruptible(struct completion *x);
@@ -94,14 +106,4 @@ extern bool completion_done(struct completion *x);
94extern void complete(struct completion *); 106extern void complete(struct completion *);
95extern void complete_all(struct completion *); 107extern void complete_all(struct completion *);
96 108
97/**
98 * INIT_COMPLETION - reinitialize a completion structure
99 * @x: completion structure to be reinitialized
100 *
101 * This macro should be used to reinitialize a completion structure so it can
102 * be reused. This is especially important after complete_all() is used.
103 */
104#define INIT_COMPLETION(x) ((x).done = 0)
105
106
107#endif 109#endif
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 3935428c57cf..91672e2deec3 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -54,7 +54,8 @@ enum page_check_address_pmd_flag {
54extern pmd_t *page_check_address_pmd(struct page *page, 54extern pmd_t *page_check_address_pmd(struct page *page,
55 struct mm_struct *mm, 55 struct mm_struct *mm,
56 unsigned long address, 56 unsigned long address,
57 enum page_check_address_pmd_flag flag); 57 enum page_check_address_pmd_flag flag,
58 spinlock_t **ptl);
58 59
59#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT) 60#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
60#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER) 61#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)
@@ -129,15 +130,15 @@ extern void __vma_adjust_trans_huge(struct vm_area_struct *vma,
129 unsigned long start, 130 unsigned long start,
130 unsigned long end, 131 unsigned long end,
131 long adjust_next); 132 long adjust_next);
132extern int __pmd_trans_huge_lock(pmd_t *pmd, 133extern int __pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma,
133 struct vm_area_struct *vma); 134 spinlock_t **ptl);
134/* mmap_sem must be held on entry */ 135/* mmap_sem must be held on entry */
135static inline int pmd_trans_huge_lock(pmd_t *pmd, 136static inline int pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma,
136 struct vm_area_struct *vma) 137 spinlock_t **ptl)
137{ 138{
138 VM_BUG_ON(!rwsem_is_locked(&vma->vm_mm->mmap_sem)); 139 VM_BUG_ON(!rwsem_is_locked(&vma->vm_mm->mmap_sem));
139 if (pmd_trans_huge(*pmd)) 140 if (pmd_trans_huge(*pmd))
140 return __pmd_trans_huge_lock(pmd, vma); 141 return __pmd_trans_huge_lock(pmd, vma, ptl);
141 else 142 else
142 return 0; 143 return 0;
143} 144}
@@ -215,8 +216,8 @@ static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
215 long adjust_next) 216 long adjust_next)
216{ 217{
217} 218}
218static inline int pmd_trans_huge_lock(pmd_t *pmd, 219static inline int pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma,
219 struct vm_area_struct *vma) 220 spinlock_t **ptl)
220{ 221{
221 return 0; 222 return 0;
222} 223}
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 0393270466c3..acd2010328f3 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -392,6 +392,15 @@ static inline int hugepage_migration_support(struct hstate *h)
392 return pmd_huge_support() && (huge_page_shift(h) == PMD_SHIFT); 392 return pmd_huge_support() && (huge_page_shift(h) == PMD_SHIFT);
393} 393}
394 394
395static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
396 struct mm_struct *mm, pte_t *pte)
397{
398 if (huge_page_size(h) == PMD_SIZE)
399 return pmd_lockptr(mm, (pmd_t *) pte);
400 VM_BUG_ON(huge_page_size(h) == PAGE_SIZE);
401 return &mm->page_table_lock;
402}
403
395#else /* CONFIG_HUGETLB_PAGE */ 404#else /* CONFIG_HUGETLB_PAGE */
396struct hstate {}; 405struct hstate {};
397#define alloc_huge_page_node(h, nid) NULL 406#define alloc_huge_page_node(h, nid) NULL
@@ -401,6 +410,7 @@ struct hstate {};
401#define hstate_sizelog(s) NULL 410#define hstate_sizelog(s) NULL
402#define hstate_vma(v) NULL 411#define hstate_vma(v) NULL
403#define hstate_inode(i) NULL 412#define hstate_inode(i) NULL
413#define page_hstate(page) NULL
404#define huge_page_size(h) PAGE_SIZE 414#define huge_page_size(h) PAGE_SIZE
405#define huge_page_mask(h) PAGE_MASK 415#define huge_page_mask(h) PAGE_MASK
406#define vma_kernel_pagesize(v) PAGE_SIZE 416#define vma_kernel_pagesize(v) PAGE_SIZE
@@ -421,6 +431,22 @@ static inline pgoff_t basepage_index(struct page *page)
421#define dissolve_free_huge_pages(s, e) do {} while (0) 431#define dissolve_free_huge_pages(s, e) do {} while (0)
422#define pmd_huge_support() 0 432#define pmd_huge_support() 0
423#define hugepage_migration_support(h) 0 433#define hugepage_migration_support(h) 0
434
435static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
436 struct mm_struct *mm, pte_t *pte)
437{
438 return &mm->page_table_lock;
439}
424#endif /* CONFIG_HUGETLB_PAGE */ 440#endif /* CONFIG_HUGETLB_PAGE */
425 441
442static inline spinlock_t *huge_pte_lock(struct hstate *h,
443 struct mm_struct *mm, pte_t *pte)
444{
445 spinlock_t *ptl;
446
447 ptl = huge_pte_lockptr(h, mm, pte);
448 spin_lock(ptl);
449 return ptl;
450}
451
426#endif /* _LINUX_HUGETLB_H */ 452#endif /* _LINUX_HUGETLB_H */
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index c9e831dc80bc..db43b58a3355 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -11,8 +11,6 @@
11#include <linux/irqnr.h> 11#include <linux/irqnr.h>
12#include <linux/hardirq.h> 12#include <linux/hardirq.h>
13#include <linux/irqflags.h> 13#include <linux/irqflags.h>
14#include <linux/smp.h>
15#include <linux/percpu.h>
16#include <linux/hrtimer.h> 14#include <linux/hrtimer.h>
17#include <linux/kref.h> 15#include <linux/kref.h>
18#include <linux/workqueue.h> 16#include <linux/workqueue.h>
@@ -392,15 +390,6 @@ extern void __raise_softirq_irqoff(unsigned int nr);
392extern void raise_softirq_irqoff(unsigned int nr); 390extern void raise_softirq_irqoff(unsigned int nr);
393extern void raise_softirq(unsigned int nr); 391extern void raise_softirq(unsigned int nr);
394 392
395/* This is the worklist that queues up per-cpu softirq work.
396 *
397 * send_remote_sendirq() adds work to these lists, and
398 * the softirq handler itself dequeues from them. The queues
399 * are protected by disabling local cpu interrupts and they must
400 * only be accessed by the local cpu that they are for.
401 */
402DECLARE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
403
404DECLARE_PER_CPU(struct task_struct *, ksoftirqd); 393DECLARE_PER_CPU(struct task_struct *, ksoftirqd);
405 394
406static inline struct task_struct *this_cpu_ksoftirqd(void) 395static inline struct task_struct *this_cpu_ksoftirqd(void)
@@ -408,17 +397,6 @@ static inline struct task_struct *this_cpu_ksoftirqd(void)
408 return this_cpu_read(ksoftirqd); 397 return this_cpu_read(ksoftirqd);
409} 398}
410 399
411/* Try to send a softirq to a remote cpu. If this cannot be done, the
412 * work will be queued to the local cpu.
413 */
414extern void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq);
415
416/* Like send_remote_softirq(), but the caller must disable local cpu interrupts
417 * and compute the current cpu, passed in as 'this_cpu'.
418 */
419extern void __send_remote_softirq(struct call_single_data *cp, int cpu,
420 int this_cpu, int softirq);
421
422/* Tasklets --- multithreaded analogue of BHs. 400/* Tasklets --- multithreaded analogue of BHs.
423 401
424 Main feature differing them of generic softirqs: tasklet 402 Main feature differing them of generic softirqs: tasklet
diff --git a/include/linux/kfifo.h b/include/linux/kfifo.h
index 10308c6a3d1c..552d51efb429 100644
--- a/include/linux/kfifo.h
+++ b/include/linux/kfifo.h
@@ -1,7 +1,7 @@
1/* 1/*
2 * A generic kernel FIFO implementation 2 * A generic kernel FIFO implementation
3 * 3 *
4 * Copyright (C) 2009/2010 Stefani Seibold <stefani@seibold.net> 4 * Copyright (C) 2013 Stefani Seibold <stefani@seibold.net>
5 * 5 *
6 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by 7 * it under the terms of the GNU General Public License as published by
@@ -67,9 +67,10 @@ struct __kfifo {
67 union { \ 67 union { \
68 struct __kfifo kfifo; \ 68 struct __kfifo kfifo; \
69 datatype *type; \ 69 datatype *type; \
70 const datatype *const_type; \
70 char (*rectype)[recsize]; \ 71 char (*rectype)[recsize]; \
71 ptrtype *ptr; \ 72 ptrtype *ptr; \
72 const ptrtype *ptr_const; \ 73 ptrtype const *ptr_const; \
73 } 74 }
74 75
75#define __STRUCT_KFIFO(type, size, recsize, ptrtype) \ 76#define __STRUCT_KFIFO(type, size, recsize, ptrtype) \
@@ -386,16 +387,12 @@ __kfifo_int_must_check_helper( \
386#define kfifo_put(fifo, val) \ 387#define kfifo_put(fifo, val) \
387({ \ 388({ \
388 typeof((fifo) + 1) __tmp = (fifo); \ 389 typeof((fifo) + 1) __tmp = (fifo); \
389 typeof((val) + 1) __val = (val); \ 390 typeof(*__tmp->const_type) __val = (val); \
390 unsigned int __ret; \ 391 unsigned int __ret; \
391 const size_t __recsize = sizeof(*__tmp->rectype); \ 392 size_t __recsize = sizeof(*__tmp->rectype); \
392 struct __kfifo *__kfifo = &__tmp->kfifo; \ 393 struct __kfifo *__kfifo = &__tmp->kfifo; \
393 if (0) { \
394 typeof(__tmp->ptr_const) __dummy __attribute__ ((unused)); \
395 __dummy = (typeof(__val))NULL; \
396 } \
397 if (__recsize) \ 394 if (__recsize) \
398 __ret = __kfifo_in_r(__kfifo, __val, sizeof(*__val), \ 395 __ret = __kfifo_in_r(__kfifo, &__val, sizeof(__val), \
399 __recsize); \ 396 __recsize); \
400 else { \ 397 else { \
401 __ret = !kfifo_is_full(__tmp); \ 398 __ret = !kfifo_is_full(__tmp); \
@@ -404,7 +401,7 @@ __kfifo_int_must_check_helper( \
404 ((typeof(__tmp->type))__kfifo->data) : \ 401 ((typeof(__tmp->type))__kfifo->data) : \
405 (__tmp->buf) \ 402 (__tmp->buf) \
406 )[__kfifo->in & __tmp->kfifo.mask] = \ 403 )[__kfifo->in & __tmp->kfifo.mask] = \
407 *(typeof(__tmp->type))__val; \ 404 (typeof(*__tmp->type))__val; \
408 smp_wmb(); \ 405 smp_wmb(); \
409 __kfifo->in++; \ 406 __kfifo->in++; \
410 } \ 407 } \
@@ -415,7 +412,7 @@ __kfifo_int_must_check_helper( \
415/** 412/**
416 * kfifo_get - get data from the fifo 413 * kfifo_get - get data from the fifo
417 * @fifo: address of the fifo to be used 414 * @fifo: address of the fifo to be used
418 * @val: the var where to store the data to be added 415 * @val: address where to store the data
419 * 416 *
420 * This macro reads the data from the fifo. 417 * This macro reads the data from the fifo.
421 * It returns 0 if the fifo was empty. Otherwise it returns the number 418 * It returns 0 if the fifo was empty. Otherwise it returns the number
@@ -428,12 +425,10 @@ __kfifo_int_must_check_helper( \
428__kfifo_uint_must_check_helper( \ 425__kfifo_uint_must_check_helper( \
429({ \ 426({ \
430 typeof((fifo) + 1) __tmp = (fifo); \ 427 typeof((fifo) + 1) __tmp = (fifo); \
431 typeof((val) + 1) __val = (val); \ 428 typeof(__tmp->ptr) __val = (val); \
432 unsigned int __ret; \ 429 unsigned int __ret; \
433 const size_t __recsize = sizeof(*__tmp->rectype); \ 430 const size_t __recsize = sizeof(*__tmp->rectype); \
434 struct __kfifo *__kfifo = &__tmp->kfifo; \ 431 struct __kfifo *__kfifo = &__tmp->kfifo; \
435 if (0) \
436 __val = (typeof(__tmp->ptr))0; \
437 if (__recsize) \ 432 if (__recsize) \
438 __ret = __kfifo_out_r(__kfifo, __val, sizeof(*__val), \ 433 __ret = __kfifo_out_r(__kfifo, __val, sizeof(*__val), \
439 __recsize); \ 434 __recsize); \
@@ -456,7 +451,7 @@ __kfifo_uint_must_check_helper( \
456/** 451/**
457 * kfifo_peek - get data from the fifo without removing 452 * kfifo_peek - get data from the fifo without removing
458 * @fifo: address of the fifo to be used 453 * @fifo: address of the fifo to be used
459 * @val: the var where to store the data to be added 454 * @val: address where to store the data
460 * 455 *
461 * This reads the data from the fifo without removing it from the fifo. 456 * This reads the data from the fifo without removing it from the fifo.
462 * It returns 0 if the fifo was empty. Otherwise it returns the number 457 * It returns 0 if the fifo was empty. Otherwise it returns the number
@@ -469,12 +464,10 @@ __kfifo_uint_must_check_helper( \
469__kfifo_uint_must_check_helper( \ 464__kfifo_uint_must_check_helper( \
470({ \ 465({ \
471 typeof((fifo) + 1) __tmp = (fifo); \ 466 typeof((fifo) + 1) __tmp = (fifo); \
472 typeof((val) + 1) __val = (val); \ 467 typeof(__tmp->ptr) __val = (val); \
473 unsigned int __ret; \ 468 unsigned int __ret; \
474 const size_t __recsize = sizeof(*__tmp->rectype); \ 469 const size_t __recsize = sizeof(*__tmp->rectype); \
475 struct __kfifo *__kfifo = &__tmp->kfifo; \ 470 struct __kfifo *__kfifo = &__tmp->kfifo; \
476 if (0) \
477 __val = (typeof(__tmp->ptr))NULL; \
478 if (__recsize) \ 471 if (__recsize) \
479 __ret = __kfifo_out_peek_r(__kfifo, __val, sizeof(*__val), \ 472 __ret = __kfifo_out_peek_r(__kfifo, __val, sizeof(*__val), \
480 __recsize); \ 473 __recsize); \
@@ -508,14 +501,10 @@ __kfifo_uint_must_check_helper( \
508#define kfifo_in(fifo, buf, n) \ 501#define kfifo_in(fifo, buf, n) \
509({ \ 502({ \
510 typeof((fifo) + 1) __tmp = (fifo); \ 503 typeof((fifo) + 1) __tmp = (fifo); \
511 typeof((buf) + 1) __buf = (buf); \ 504 typeof(__tmp->ptr_const) __buf = (buf); \
512 unsigned long __n = (n); \ 505 unsigned long __n = (n); \
513 const size_t __recsize = sizeof(*__tmp->rectype); \ 506 const size_t __recsize = sizeof(*__tmp->rectype); \
514 struct __kfifo *__kfifo = &__tmp->kfifo; \ 507 struct __kfifo *__kfifo = &__tmp->kfifo; \
515 if (0) { \
516 typeof(__tmp->ptr_const) __dummy __attribute__ ((unused)); \
517 __dummy = (typeof(__buf))NULL; \
518 } \
519 (__recsize) ?\ 508 (__recsize) ?\
520 __kfifo_in_r(__kfifo, __buf, __n, __recsize) : \ 509 __kfifo_in_r(__kfifo, __buf, __n, __recsize) : \
521 __kfifo_in(__kfifo, __buf, __n); \ 510 __kfifo_in(__kfifo, __buf, __n); \
@@ -561,14 +550,10 @@ __kfifo_uint_must_check_helper( \
561__kfifo_uint_must_check_helper( \ 550__kfifo_uint_must_check_helper( \
562({ \ 551({ \
563 typeof((fifo) + 1) __tmp = (fifo); \ 552 typeof((fifo) + 1) __tmp = (fifo); \
564 typeof((buf) + 1) __buf = (buf); \ 553 typeof(__tmp->ptr) __buf = (buf); \
565 unsigned long __n = (n); \ 554 unsigned long __n = (n); \
566 const size_t __recsize = sizeof(*__tmp->rectype); \ 555 const size_t __recsize = sizeof(*__tmp->rectype); \
567 struct __kfifo *__kfifo = &__tmp->kfifo; \ 556 struct __kfifo *__kfifo = &__tmp->kfifo; \
568 if (0) { \
569 typeof(__tmp->ptr) __dummy = NULL; \
570 __buf = __dummy; \
571 } \
572 (__recsize) ?\ 557 (__recsize) ?\
573 __kfifo_out_r(__kfifo, __buf, __n, __recsize) : \ 558 __kfifo_out_r(__kfifo, __buf, __n, __recsize) : \
574 __kfifo_out(__kfifo, __buf, __n); \ 559 __kfifo_out(__kfifo, __buf, __n); \
@@ -773,14 +758,10 @@ __kfifo_uint_must_check_helper( \
773__kfifo_uint_must_check_helper( \ 758__kfifo_uint_must_check_helper( \
774({ \ 759({ \
775 typeof((fifo) + 1) __tmp = (fifo); \ 760 typeof((fifo) + 1) __tmp = (fifo); \
776 typeof((buf) + 1) __buf = (buf); \ 761 typeof(__tmp->ptr) __buf = (buf); \
777 unsigned long __n = (n); \ 762 unsigned long __n = (n); \
778 const size_t __recsize = sizeof(*__tmp->rectype); \ 763 const size_t __recsize = sizeof(*__tmp->rectype); \
779 struct __kfifo *__kfifo = &__tmp->kfifo; \ 764 struct __kfifo *__kfifo = &__tmp->kfifo; \
780 if (0) { \
781 typeof(__tmp->ptr) __dummy __attribute__ ((unused)) = NULL; \
782 __buf = __dummy; \
783 } \
784 (__recsize) ? \ 765 (__recsize) ? \
785 __kfifo_out_peek_r(__kfifo, __buf, __n, __recsize) : \ 766 __kfifo_out_peek_r(__kfifo, __buf, __n, __recsize) : \
786 __kfifo_out_peek(__kfifo, __buf, __n); \ 767 __kfifo_out_peek(__kfifo, __buf, __n); \
diff --git a/include/linux/llist.h b/include/linux/llist.h
index 8828a78dec9a..fbf10a0bc095 100644
--- a/include/linux/llist.h
+++ b/include/linux/llist.h
@@ -195,4 +195,6 @@ static inline struct llist_node *llist_del_all(struct llist_head *head)
195 195
196extern struct llist_node *llist_del_first(struct llist_head *head); 196extern struct llist_node *llist_del_first(struct llist_head *head);
197 197
198struct llist_node *llist_reverse_order(struct llist_node *head);
199
198#endif /* LLIST_H */ 200#endif /* LLIST_H */
diff --git a/include/linux/lockref.h b/include/linux/lockref.h
index 13dfd36a3294..c8929c3832db 100644
--- a/include/linux/lockref.h
+++ b/include/linux/lockref.h
@@ -15,10 +15,15 @@
15 */ 15 */
16 16
17#include <linux/spinlock.h> 17#include <linux/spinlock.h>
18#include <generated/bounds.h>
19
20#define USE_CMPXCHG_LOCKREF \
21 (IS_ENABLED(CONFIG_ARCH_USE_CMPXCHG_LOCKREF) && \
22 IS_ENABLED(CONFIG_SMP) && !BLOATED_SPINLOCKS)
18 23
19struct lockref { 24struct lockref {
20 union { 25 union {
21#ifdef CONFIG_CMPXCHG_LOCKREF 26#if USE_CMPXCHG_LOCKREF
22 aligned_u64 lock_count; 27 aligned_u64 lock_count;
23#endif 28#endif
24 struct { 29 struct {
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 42a35d94b82c..0548eb201e05 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1316,32 +1316,85 @@ static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long a
1316} 1316}
1317#endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */ 1317#endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */
1318 1318
1319#if USE_SPLIT_PTLOCKS 1319#if USE_SPLIT_PTE_PTLOCKS
1320/* 1320#if BLOATED_SPINLOCKS
1321 * We tuck a spinlock to guard each pagetable page into its struct page, 1321void __init ptlock_cache_init(void);
1322 * at page->private, with BUILD_BUG_ON to make sure that this will not 1322extern bool ptlock_alloc(struct page *page);
1323 * overflow into the next struct page (as it might with DEBUG_SPINLOCK). 1323extern void ptlock_free(struct page *page);
1324 * When freeing, reset page->mapping so free_pages_check won't complain. 1324
1325 */ 1325static inline spinlock_t *ptlock_ptr(struct page *page)
1326#define __pte_lockptr(page) &((page)->ptl) 1326{
1327#define pte_lock_init(_page) do { \ 1327 return page->ptl;
1328 spin_lock_init(__pte_lockptr(_page)); \ 1328}
1329} while (0) 1329#else /* BLOATED_SPINLOCKS */
1330#define pte_lock_deinit(page) ((page)->mapping = NULL) 1330static inline void ptlock_cache_init(void) {}
1331#define pte_lockptr(mm, pmd) ({(void)(mm); __pte_lockptr(pmd_page(*(pmd)));}) 1331static inline bool ptlock_alloc(struct page *page)
1332#else /* !USE_SPLIT_PTLOCKS */ 1332{
1333 return true;
1334}
1335
1336static inline void ptlock_free(struct page *page)
1337{
1338}
1339
1340static inline spinlock_t *ptlock_ptr(struct page *page)
1341{
1342 return &page->ptl;
1343}
1344#endif /* BLOATED_SPINLOCKS */
1345
1346static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
1347{
1348 return ptlock_ptr(pmd_page(*pmd));
1349}
1350
1351static inline bool ptlock_init(struct page *page)
1352{
1353 /*
1354 * prep_new_page() initialize page->private (and therefore page->ptl)
1355 * with 0. Make sure nobody took it in use in between.
1356 *
1357 * It can happen if arch try to use slab for page table allocation:
1358 * slab code uses page->slab_cache and page->first_page (for tail
1359 * pages), which share storage with page->ptl.
1360 */
1361 VM_BUG_ON(*(unsigned long *)&page->ptl);
1362 if (!ptlock_alloc(page))
1363 return false;
1364 spin_lock_init(ptlock_ptr(page));
1365 return true;
1366}
1367
1368/* Reset page->mapping so free_pages_check won't complain. */
1369static inline void pte_lock_deinit(struct page *page)
1370{
1371 page->mapping = NULL;
1372 ptlock_free(page);
1373}
1374
1375#else /* !USE_SPLIT_PTE_PTLOCKS */
1333/* 1376/*
1334 * We use mm->page_table_lock to guard all pagetable pages of the mm. 1377 * We use mm->page_table_lock to guard all pagetable pages of the mm.
1335 */ 1378 */
1336#define pte_lock_init(page) do {} while (0) 1379static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
1337#define pte_lock_deinit(page) do {} while (0) 1380{
1338#define pte_lockptr(mm, pmd) ({(void)(pmd); &(mm)->page_table_lock;}) 1381 return &mm->page_table_lock;
1339#endif /* USE_SPLIT_PTLOCKS */ 1382}
1383static inline void ptlock_cache_init(void) {}
1384static inline bool ptlock_init(struct page *page) { return true; }
1385static inline void pte_lock_deinit(struct page *page) {}
1386#endif /* USE_SPLIT_PTE_PTLOCKS */
1387
1388static inline void pgtable_init(void)
1389{
1390 ptlock_cache_init();
1391 pgtable_cache_init();
1392}
1340 1393
1341static inline void pgtable_page_ctor(struct page *page) 1394static inline bool pgtable_page_ctor(struct page *page)
1342{ 1395{
1343 pte_lock_init(page);
1344 inc_zone_page_state(page, NR_PAGETABLE); 1396 inc_zone_page_state(page, NR_PAGETABLE);
1397 return ptlock_init(page);
1345} 1398}
1346 1399
1347static inline void pgtable_page_dtor(struct page *page) 1400static inline void pgtable_page_dtor(struct page *page)
@@ -1378,6 +1431,52 @@ static inline void pgtable_page_dtor(struct page *page)
1378 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \ 1431 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
1379 NULL: pte_offset_kernel(pmd, address)) 1432 NULL: pte_offset_kernel(pmd, address))
1380 1433
1434#if USE_SPLIT_PMD_PTLOCKS
1435
1436static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
1437{
1438 return ptlock_ptr(virt_to_page(pmd));
1439}
1440
1441static inline bool pgtable_pmd_page_ctor(struct page *page)
1442{
1443#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1444 page->pmd_huge_pte = NULL;
1445#endif
1446 return ptlock_init(page);
1447}
1448
1449static inline void pgtable_pmd_page_dtor(struct page *page)
1450{
1451#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1452 VM_BUG_ON(page->pmd_huge_pte);
1453#endif
1454 ptlock_free(page);
1455}
1456
1457#define pmd_huge_pte(mm, pmd) (virt_to_page(pmd)->pmd_huge_pte)
1458
1459#else
1460
1461static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
1462{
1463 return &mm->page_table_lock;
1464}
1465
1466static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }
1467static inline void pgtable_pmd_page_dtor(struct page *page) {}
1468
1469#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
1470
1471#endif
1472
1473static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
1474{
1475 spinlock_t *ptl = pmd_lockptr(mm, pmd);
1476 spin_lock(ptl);
1477 return ptl;
1478}
1479
1381extern void free_area_init(unsigned long * zones_size); 1480extern void free_area_init(unsigned long * zones_size);
1382extern void free_area_init_node(int nid, unsigned long * zones_size, 1481extern void free_area_init_node(int nid, unsigned long * zones_size,
1383 unsigned long zone_start_pfn, unsigned long *zholes_size); 1482 unsigned long zone_start_pfn, unsigned long *zholes_size);
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index a3198e5aaf4e..10f5a7272b80 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -23,7 +23,9 @@
23 23
24struct address_space; 24struct address_space;
25 25
26#define USE_SPLIT_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS) 26#define USE_SPLIT_PTE_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS)
27#define USE_SPLIT_PMD_PTLOCKS (USE_SPLIT_PTE_PTLOCKS && \
28 IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK))
27 29
28/* 30/*
29 * Each physical page in the system has a struct page associated with 31 * Each physical page in the system has a struct page associated with
@@ -63,6 +65,9 @@ struct page {
63 * this page is only used to 65 * this page is only used to
64 * free other pages. 66 * free other pages.
65 */ 67 */
68#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS
69 pgtable_t pmd_huge_pte; /* protected by page->ptl */
70#endif
66 }; 71 };
67 72
68 union { 73 union {
@@ -141,9 +146,13 @@ struct page {
141 * indicates order in the buddy 146 * indicates order in the buddy
142 * system if PG_buddy is set. 147 * system if PG_buddy is set.
143 */ 148 */
144#if USE_SPLIT_PTLOCKS 149#if USE_SPLIT_PTE_PTLOCKS
150#if BLOATED_SPINLOCKS
151 spinlock_t *ptl;
152#else
145 spinlock_t ptl; 153 spinlock_t ptl;
146#endif 154#endif
155#endif
147 struct kmem_cache *slab_cache; /* SL[AU]B: Pointer to slab */ 156 struct kmem_cache *slab_cache; /* SL[AU]B: Pointer to slab */
148 struct page *first_page; /* Compound tail pages */ 157 struct page *first_page; /* Compound tail pages */
149 }; 158 };
@@ -309,14 +318,14 @@ enum {
309 NR_MM_COUNTERS 318 NR_MM_COUNTERS
310}; 319};
311 320
312#if USE_SPLIT_PTLOCKS && defined(CONFIG_MMU) 321#if USE_SPLIT_PTE_PTLOCKS && defined(CONFIG_MMU)
313#define SPLIT_RSS_COUNTING 322#define SPLIT_RSS_COUNTING
314/* per-thread cached information, */ 323/* per-thread cached information, */
315struct task_rss_stat { 324struct task_rss_stat {
316 int events; /* for synchronization threshold */ 325 int events; /* for synchronization threshold */
317 int count[NR_MM_COUNTERS]; 326 int count[NR_MM_COUNTERS];
318}; 327};
319#endif /* USE_SPLIT_PTLOCKS */ 328#endif /* USE_SPLIT_PTE_PTLOCKS */
320 329
321struct mm_rss_stat { 330struct mm_rss_stat {
322 atomic_long_t count[NR_MM_COUNTERS]; 331 atomic_long_t count[NR_MM_COUNTERS];
@@ -339,6 +348,7 @@ struct mm_struct {
339 pgd_t * pgd; 348 pgd_t * pgd;
340 atomic_t mm_users; /* How many users with user space? */ 349 atomic_t mm_users; /* How many users with user space? */
341 atomic_t mm_count; /* How many references to "struct mm_struct" (users count as 1) */ 350 atomic_t mm_count; /* How many references to "struct mm_struct" (users count as 1) */
351 atomic_long_t nr_ptes; /* Page table pages */
342 int map_count; /* number of VMAs */ 352 int map_count; /* number of VMAs */
343 353
344 spinlock_t page_table_lock; /* Protects page tables and some counters */ 354 spinlock_t page_table_lock; /* Protects page tables and some counters */
@@ -360,7 +370,6 @@ struct mm_struct {
360 unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE */ 370 unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE */
361 unsigned long stack_vm; /* VM_GROWSUP/DOWN */ 371 unsigned long stack_vm; /* VM_GROWSUP/DOWN */
362 unsigned long def_flags; 372 unsigned long def_flags;
363 unsigned long nr_ptes; /* Page table pages */
364 unsigned long start_code, end_code, start_data, end_data; 373 unsigned long start_code, end_code, start_data, end_data;
365 unsigned long start_brk, brk, start_stack; 374 unsigned long start_brk, brk, start_stack;
366 unsigned long arg_start, arg_end, env_start, env_end; 375 unsigned long arg_start, arg_end, env_start, env_end;
@@ -406,7 +415,7 @@ struct mm_struct {
406#ifdef CONFIG_MMU_NOTIFIER 415#ifdef CONFIG_MMU_NOTIFIER
407 struct mmu_notifier_mm *mmu_notifier_mm; 416 struct mmu_notifier_mm *mmu_notifier_mm;
408#endif 417#endif
409#ifdef CONFIG_TRANSPARENT_HUGEPAGE 418#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
410 pgtable_t pmd_huge_pte; /* protected by page_table_lock */ 419 pgtable_t pmd_huge_pte; /* protected by page_table_lock */
411#endif 420#endif
412#ifdef CONFIG_CPUMASK_OFFSTACK 421#ifdef CONFIG_CPUMASK_OFFSTACK
diff --git a/include/linux/seq_file.h b/include/linux/seq_file.h
index 4e32edc8f506..52e0097f61f0 100644
--- a/include/linux/seq_file.h
+++ b/include/linux/seq_file.h
@@ -20,6 +20,7 @@ struct seq_file {
20 size_t size; 20 size_t size;
21 size_t from; 21 size_t from;
22 size_t count; 22 size_t count;
23 size_t pad_until;
23 loff_t index; 24 loff_t index;
24 loff_t read_pos; 25 loff_t read_pos;
25 u64 version; 26 u64 version;
@@ -79,6 +80,20 @@ static inline void seq_commit(struct seq_file *m, int num)
79 } 80 }
80} 81}
81 82
83/**
84 * seq_setwidth - set padding width
85 * @m: the seq_file handle
86 * @size: the max number of bytes to pad.
87 *
88 * Call seq_setwidth() for setting max width, then call seq_printf() etc. and
89 * finally call seq_pad() to pad the remaining bytes.
90 */
91static inline void seq_setwidth(struct seq_file *m, size_t size)
92{
93 m->pad_until = m->count + size;
94}
95void seq_pad(struct seq_file *m, char c);
96
82char *mangle_path(char *s, const char *p, const char *esc); 97char *mangle_path(char *s, const char *p, const char *esc);
83int seq_open(struct file *, const struct seq_operations *); 98int seq_open(struct file *, const struct seq_operations *);
84ssize_t seq_read(struct file *, char __user *, size_t, loff_t *); 99ssize_t seq_read(struct file *, char __user *, size_t, loff_t *);
diff --git a/include/linux/smp.h b/include/linux/smp.h
index 731f5237d5f4..5da22ee42e16 100644
--- a/include/linux/smp.h
+++ b/include/linux/smp.h
@@ -49,6 +49,9 @@ void on_each_cpu_cond(bool (*cond_func)(int cpu, void *info),
49 smp_call_func_t func, void *info, bool wait, 49 smp_call_func_t func, void *info, bool wait,
50 gfp_t gfp_flags); 50 gfp_t gfp_flags);
51 51
52void __smp_call_function_single(int cpuid, struct call_single_data *data,
53 int wait);
54
52#ifdef CONFIG_SMP 55#ifdef CONFIG_SMP
53 56
54#include <linux/preempt.h> 57#include <linux/preempt.h>
@@ -95,9 +98,6 @@ int smp_call_function(smp_call_func_t func, void *info, int wait);
95void smp_call_function_many(const struct cpumask *mask, 98void smp_call_function_many(const struct cpumask *mask,
96 smp_call_func_t func, void *info, bool wait); 99 smp_call_func_t func, void *info, bool wait);
97 100
98void __smp_call_function_single(int cpuid, struct call_single_data *data,
99 int wait);
100
101int smp_call_function_any(const struct cpumask *mask, 101int smp_call_function_any(const struct cpumask *mask,
102 smp_call_func_t func, void *info, int wait); 102 smp_call_func_t func, void *info, int wait);
103 103
@@ -106,14 +106,10 @@ void kick_all_cpus_sync(void);
106/* 106/*
107 * Generic and arch helpers 107 * Generic and arch helpers
108 */ 108 */
109#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
110void __init call_function_init(void); 109void __init call_function_init(void);
111void generic_smp_call_function_single_interrupt(void); 110void generic_smp_call_function_single_interrupt(void);
112#define generic_smp_call_function_interrupt \ 111#define generic_smp_call_function_interrupt \
113 generic_smp_call_function_single_interrupt 112 generic_smp_call_function_single_interrupt
114#else
115static inline void call_function_init(void) { }
116#endif
117 113
118/* 114/*
119 * Mark the boot cpu "online" so that it can call console drivers in 115 * Mark the boot cpu "online" so that it can call console drivers in
@@ -155,12 +151,6 @@ smp_call_function_any(const struct cpumask *mask, smp_call_func_t func,
155 151
156static inline void kick_all_cpus_sync(void) { } 152static inline void kick_all_cpus_sync(void) { }
157 153
158static inline void __smp_call_function_single(int cpuid,
159 struct call_single_data *data, int wait)
160{
161 on_each_cpu(data->func, data->info, wait);
162}
163
164#endif /* !SMP */ 154#endif /* !SMP */
165 155
166/* 156/*
diff --git a/include/linux/swapops.h b/include/linux/swapops.h
index 8d4fa82bfb91..c0f75261a728 100644
--- a/include/linux/swapops.h
+++ b/include/linux/swapops.h
@@ -139,7 +139,8 @@ static inline void make_migration_entry_read(swp_entry_t *entry)
139 139
140extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 140extern void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
141 unsigned long address); 141 unsigned long address);
142extern void migration_entry_wait_huge(struct mm_struct *mm, pte_t *pte); 142extern void migration_entry_wait_huge(struct vm_area_struct *vma,
143 struct mm_struct *mm, pte_t *pte);
143#else 144#else
144 145
145#define make_migration_entry(page, write) swp_entry(0, 0) 146#define make_migration_entry(page, write) swp_entry(0, 0)
@@ -151,8 +152,8 @@ static inline int is_migration_entry(swp_entry_t swp)
151static inline void make_migration_entry_read(swp_entry_t *entryp) { } 152static inline void make_migration_entry_read(swp_entry_t *entryp) { }
152static inline void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, 153static inline void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
153 unsigned long address) { } 154 unsigned long address) { }
154static inline void migration_entry_wait_huge(struct mm_struct *mm, 155static inline void migration_entry_wait_huge(struct vm_area_struct *vma,
155 pte_t *pte) { } 156 struct mm_struct *mm, pte_t *pte) { }
156static inline int is_write_migration_entry(swp_entry_t entry) 157static inline int is_write_migration_entry(swp_entry_t entry)
157{ 158{
158 return 0; 159 return 0;