aboutsummaryrefslogtreecommitdiffstats
path: root/include/linux
diff options
context:
space:
mode:
Diffstat (limited to 'include/linux')
-rw-r--r--include/linux/backing-dev-defs.h3
-rw-r--r--include/linux/backing-dev.h69
-rw-r--r--include/linux/cma.h2
-rw-r--r--include/linux/compiler-gcc.h13
-rw-r--r--include/linux/compiler.h66
-rw-r--r--include/linux/dma-contiguous.h4
-rw-r--r--include/linux/memcontrol.h8
7 files changed, 82 insertions, 83 deletions
diff --git a/include/linux/backing-dev-defs.h b/include/linux/backing-dev-defs.h
index a23209b43842..1b4d69f68c33 100644
--- a/include/linux/backing-dev-defs.h
+++ b/include/linux/backing-dev-defs.h
@@ -116,6 +116,8 @@ struct bdi_writeback {
116 struct list_head work_list; 116 struct list_head work_list;
117 struct delayed_work dwork; /* work item used for writeback */ 117 struct delayed_work dwork; /* work item used for writeback */
118 118
119 struct list_head bdi_node; /* anchored at bdi->wb_list */
120
119#ifdef CONFIG_CGROUP_WRITEBACK 121#ifdef CONFIG_CGROUP_WRITEBACK
120 struct percpu_ref refcnt; /* used only for !root wb's */ 122 struct percpu_ref refcnt; /* used only for !root wb's */
121 struct fprop_local_percpu memcg_completions; 123 struct fprop_local_percpu memcg_completions;
@@ -150,6 +152,7 @@ struct backing_dev_info {
150 atomic_long_t tot_write_bandwidth; 152 atomic_long_t tot_write_bandwidth;
151 153
152 struct bdi_writeback wb; /* the root writeback info for this bdi */ 154 struct bdi_writeback wb; /* the root writeback info for this bdi */
155 struct list_head wb_list; /* list of all wbs */
153#ifdef CONFIG_CGROUP_WRITEBACK 156#ifdef CONFIG_CGROUP_WRITEBACK
154 struct radix_tree_root cgwb_tree; /* radix tree of active cgroup wbs */ 157 struct radix_tree_root cgwb_tree; /* radix tree of active cgroup wbs */
155 struct rb_root cgwb_congested_tree; /* their congested states */ 158 struct rb_root cgwb_congested_tree; /* their congested states */
diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h
index d5eb4ad1c534..c85f74946a8b 100644
--- a/include/linux/backing-dev.h
+++ b/include/linux/backing-dev.h
@@ -19,13 +19,17 @@
19#include <linux/slab.h> 19#include <linux/slab.h>
20 20
21int __must_check bdi_init(struct backing_dev_info *bdi); 21int __must_check bdi_init(struct backing_dev_info *bdi);
22void bdi_destroy(struct backing_dev_info *bdi); 22void bdi_exit(struct backing_dev_info *bdi);
23 23
24__printf(3, 4) 24__printf(3, 4)
25int bdi_register(struct backing_dev_info *bdi, struct device *parent, 25int bdi_register(struct backing_dev_info *bdi, struct device *parent,
26 const char *fmt, ...); 26 const char *fmt, ...);
27int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev); 27int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev);
28void bdi_unregister(struct backing_dev_info *bdi);
29
28int __must_check bdi_setup_and_register(struct backing_dev_info *, char *); 30int __must_check bdi_setup_and_register(struct backing_dev_info *, char *);
31void bdi_destroy(struct backing_dev_info *bdi);
32
29void wb_start_writeback(struct bdi_writeback *wb, long nr_pages, 33void wb_start_writeback(struct bdi_writeback *wb, long nr_pages,
30 bool range_cyclic, enum wb_reason reason); 34 bool range_cyclic, enum wb_reason reason);
31void wb_start_background_writeback(struct bdi_writeback *wb); 35void wb_start_background_writeback(struct bdi_writeback *wb);
@@ -408,61 +412,6 @@ static inline void unlocked_inode_to_wb_end(struct inode *inode, bool locked)
408 rcu_read_unlock(); 412 rcu_read_unlock();
409} 413}
410 414
411struct wb_iter {
412 int start_memcg_id;
413 struct radix_tree_iter tree_iter;
414 void **slot;
415};
416
417static inline struct bdi_writeback *__wb_iter_next(struct wb_iter *iter,
418 struct backing_dev_info *bdi)
419{
420 struct radix_tree_iter *titer = &iter->tree_iter;
421
422 WARN_ON_ONCE(!rcu_read_lock_held());
423
424 if (iter->start_memcg_id >= 0) {
425 iter->slot = radix_tree_iter_init(titer, iter->start_memcg_id);
426 iter->start_memcg_id = -1;
427 } else {
428 iter->slot = radix_tree_next_slot(iter->slot, titer, 0);
429 }
430
431 if (!iter->slot)
432 iter->slot = radix_tree_next_chunk(&bdi->cgwb_tree, titer, 0);
433 if (iter->slot)
434 return *iter->slot;
435 return NULL;
436}
437
438static inline struct bdi_writeback *__wb_iter_init(struct wb_iter *iter,
439 struct backing_dev_info *bdi,
440 int start_memcg_id)
441{
442 iter->start_memcg_id = start_memcg_id;
443
444 if (start_memcg_id)
445 return __wb_iter_next(iter, bdi);
446 else
447 return &bdi->wb;
448}
449
450/**
451 * bdi_for_each_wb - walk all wb's of a bdi in ascending memcg ID order
452 * @wb_cur: cursor struct bdi_writeback pointer
453 * @bdi: bdi to walk wb's of
454 * @iter: pointer to struct wb_iter to be used as iteration buffer
455 * @start_memcg_id: memcg ID to start iteration from
456 *
457 * Iterate @wb_cur through the wb's (bdi_writeback's) of @bdi in ascending
458 * memcg ID order starting from @start_memcg_id. @iter is struct wb_iter
459 * to be used as temp storage during iteration. rcu_read_lock() must be
460 * held throughout iteration.
461 */
462#define bdi_for_each_wb(wb_cur, bdi, iter, start_memcg_id) \
463 for ((wb_cur) = __wb_iter_init(iter, bdi, start_memcg_id); \
464 (wb_cur); (wb_cur) = __wb_iter_next(iter, bdi))
465
466#else /* CONFIG_CGROUP_WRITEBACK */ 415#else /* CONFIG_CGROUP_WRITEBACK */
467 416
468static inline bool inode_cgwb_enabled(struct inode *inode) 417static inline bool inode_cgwb_enabled(struct inode *inode)
@@ -522,14 +471,6 @@ static inline void wb_blkcg_offline(struct blkcg *blkcg)
522{ 471{
523} 472}
524 473
525struct wb_iter {
526 int next_id;
527};
528
529#define bdi_for_each_wb(wb_cur, bdi, iter, start_blkcg_id) \
530 for ((iter)->next_id = (start_blkcg_id); \
531 ({ (wb_cur) = !(iter)->next_id++ ? &(bdi)->wb : NULL; }); )
532
533static inline int inode_congested(struct inode *inode, int cong_bits) 474static inline int inode_congested(struct inode *inode, int cong_bits)
534{ 475{
535 return wb_congested(&inode_to_bdi(inode)->wb, cong_bits); 476 return wb_congested(&inode_to_bdi(inode)->wb, cong_bits);
diff --git a/include/linux/cma.h b/include/linux/cma.h
index f7ef093ec49a..29f9e774ab76 100644
--- a/include/linux/cma.h
+++ b/include/linux/cma.h
@@ -26,6 +26,6 @@ extern int __init cma_declare_contiguous(phys_addr_t base,
26extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size, 26extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size,
27 unsigned int order_per_bit, 27 unsigned int order_per_bit,
28 struct cma **res_cma); 28 struct cma **res_cma);
29extern struct page *cma_alloc(struct cma *cma, unsigned int count, unsigned int align); 29extern struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align);
30extern bool cma_release(struct cma *cma, const struct page *pages, unsigned int count); 30extern bool cma_release(struct cma *cma, const struct page *pages, unsigned int count);
31#endif 31#endif
diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h
index dfaa7b3e9ae9..8efb40e61d6e 100644
--- a/include/linux/compiler-gcc.h
+++ b/include/linux/compiler-gcc.h
@@ -237,12 +237,25 @@
237#define KASAN_ABI_VERSION 3 237#define KASAN_ABI_VERSION 3
238#endif 238#endif
239 239
240#if GCC_VERSION >= 40902
241/*
242 * Tell the compiler that address safety instrumentation (KASAN)
243 * should not be applied to that function.
244 * Conflicts with inlining: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67368
245 */
246#define __no_sanitize_address __attribute__((no_sanitize_address))
247#endif
248
240#endif /* gcc version >= 40000 specific checks */ 249#endif /* gcc version >= 40000 specific checks */
241 250
242#if !defined(__noclone) 251#if !defined(__noclone)
243#define __noclone /* not needed */ 252#define __noclone /* not needed */
244#endif 253#endif
245 254
255#if !defined(__no_sanitize_address)
256#define __no_sanitize_address
257#endif
258
246/* 259/*
247 * A trick to suppress uninitialized variable warning without generating any 260 * A trick to suppress uninitialized variable warning without generating any
248 * code 261 * code
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index c836eb2dc44d..3d7810341b57 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -198,19 +198,45 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect);
198 198
199#include <uapi/linux/types.h> 199#include <uapi/linux/types.h>
200 200
201static __always_inline void __read_once_size(const volatile void *p, void *res, int size) 201#define __READ_ONCE_SIZE \
202({ \
203 switch (size) { \
204 case 1: *(__u8 *)res = *(volatile __u8 *)p; break; \
205 case 2: *(__u16 *)res = *(volatile __u16 *)p; break; \
206 case 4: *(__u32 *)res = *(volatile __u32 *)p; break; \
207 case 8: *(__u64 *)res = *(volatile __u64 *)p; break; \
208 default: \
209 barrier(); \
210 __builtin_memcpy((void *)res, (const void *)p, size); \
211 barrier(); \
212 } \
213})
214
215static __always_inline
216void __read_once_size(const volatile void *p, void *res, int size)
202{ 217{
203 switch (size) { 218 __READ_ONCE_SIZE;
204 case 1: *(__u8 *)res = *(volatile __u8 *)p; break; 219}
205 case 2: *(__u16 *)res = *(volatile __u16 *)p; break; 220
206 case 4: *(__u32 *)res = *(volatile __u32 *)p; break; 221#ifdef CONFIG_KASAN
207 case 8: *(__u64 *)res = *(volatile __u64 *)p; break; 222/*
208 default: 223 * This function is not 'inline' because __no_sanitize_address confilcts
209 barrier(); 224 * with inlining. Attempt to inline it may cause a build failure.
210 __builtin_memcpy((void *)res, (const void *)p, size); 225 * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67368
211 barrier(); 226 * '__maybe_unused' allows us to avoid defined-but-not-used warnings.
212 } 227 */
228static __no_sanitize_address __maybe_unused
229void __read_once_size_nocheck(const volatile void *p, void *res, int size)
230{
231 __READ_ONCE_SIZE;
232}
233#else
234static __always_inline
235void __read_once_size_nocheck(const volatile void *p, void *res, int size)
236{
237 __READ_ONCE_SIZE;
213} 238}
239#endif
214 240
215static __always_inline void __write_once_size(volatile void *p, void *res, int size) 241static __always_inline void __write_once_size(volatile void *p, void *res, int size)
216{ 242{
@@ -248,8 +274,22 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s
248 * required ordering. 274 * required ordering.
249 */ 275 */
250 276
251#define READ_ONCE(x) \ 277#define __READ_ONCE(x, check) \
252 ({ union { typeof(x) __val; char __c[1]; } __u; __read_once_size(&(x), __u.__c, sizeof(x)); __u.__val; }) 278({ \
279 union { typeof(x) __val; char __c[1]; } __u; \
280 if (check) \
281 __read_once_size(&(x), __u.__c, sizeof(x)); \
282 else \
283 __read_once_size_nocheck(&(x), __u.__c, sizeof(x)); \
284 __u.__val; \
285})
286#define READ_ONCE(x) __READ_ONCE(x, 1)
287
288/*
289 * Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need
290 * to hide memory access from KASAN.
291 */
292#define READ_ONCE_NOCHECK(x) __READ_ONCE(x, 0)
253 293
254#define WRITE_ONCE(x, val) \ 294#define WRITE_ONCE(x, val) \
255({ \ 295({ \
diff --git a/include/linux/dma-contiguous.h b/include/linux/dma-contiguous.h
index 569bbd039896..fec734df1524 100644
--- a/include/linux/dma-contiguous.h
+++ b/include/linux/dma-contiguous.h
@@ -111,7 +111,7 @@ static inline int dma_declare_contiguous(struct device *dev, phys_addr_t size,
111 return ret; 111 return ret;
112} 112}
113 113
114struct page *dma_alloc_from_contiguous(struct device *dev, int count, 114struct page *dma_alloc_from_contiguous(struct device *dev, size_t count,
115 unsigned int order); 115 unsigned int order);
116bool dma_release_from_contiguous(struct device *dev, struct page *pages, 116bool dma_release_from_contiguous(struct device *dev, struct page *pages,
117 int count); 117 int count);
@@ -144,7 +144,7 @@ int dma_declare_contiguous(struct device *dev, phys_addr_t size,
144} 144}
145 145
146static inline 146static inline
147struct page *dma_alloc_from_contiguous(struct device *dev, int count, 147struct page *dma_alloc_from_contiguous(struct device *dev, size_t count,
148 unsigned int order) 148 unsigned int order)
149{ 149{
150 return NULL; 150 return NULL;
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 6452ff4c463f..3e3318ddfc0e 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -676,8 +676,9 @@ enum {
676 676
677struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg); 677struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg);
678struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb); 678struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb);
679void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pavail, 679void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
680 unsigned long *pdirty, unsigned long *pwriteback); 680 unsigned long *pheadroom, unsigned long *pdirty,
681 unsigned long *pwriteback);
681 682
682#else /* CONFIG_CGROUP_WRITEBACK */ 683#else /* CONFIG_CGROUP_WRITEBACK */
683 684
@@ -687,7 +688,8 @@ static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
687} 688}
688 689
689static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb, 690static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb,
690 unsigned long *pavail, 691 unsigned long *pfilepages,
692 unsigned long *pheadroom,
691 unsigned long *pdirty, 693 unsigned long *pdirty,
692 unsigned long *pwriteback) 694 unsigned long *pwriteback)
693{ 695{