diff options
Diffstat (limited to 'include/linux')
| -rw-r--r-- | include/linux/backing-dev-defs.h | 3 | ||||
| -rw-r--r-- | include/linux/backing-dev.h | 69 | ||||
| -rw-r--r-- | include/linux/cma.h | 2 | ||||
| -rw-r--r-- | include/linux/compiler-gcc.h | 13 | ||||
| -rw-r--r-- | include/linux/compiler.h | 66 | ||||
| -rw-r--r-- | include/linux/dma-contiguous.h | 4 | ||||
| -rw-r--r-- | include/linux/memcontrol.h | 8 |
7 files changed, 82 insertions, 83 deletions
diff --git a/include/linux/backing-dev-defs.h b/include/linux/backing-dev-defs.h index a23209b43842..1b4d69f68c33 100644 --- a/include/linux/backing-dev-defs.h +++ b/include/linux/backing-dev-defs.h | |||
| @@ -116,6 +116,8 @@ struct bdi_writeback { | |||
| 116 | struct list_head work_list; | 116 | struct list_head work_list; |
| 117 | struct delayed_work dwork; /* work item used for writeback */ | 117 | struct delayed_work dwork; /* work item used for writeback */ |
| 118 | 118 | ||
| 119 | struct list_head bdi_node; /* anchored at bdi->wb_list */ | ||
| 120 | |||
| 119 | #ifdef CONFIG_CGROUP_WRITEBACK | 121 | #ifdef CONFIG_CGROUP_WRITEBACK |
| 120 | struct percpu_ref refcnt; /* used only for !root wb's */ | 122 | struct percpu_ref refcnt; /* used only for !root wb's */ |
| 121 | struct fprop_local_percpu memcg_completions; | 123 | struct fprop_local_percpu memcg_completions; |
| @@ -150,6 +152,7 @@ struct backing_dev_info { | |||
| 150 | atomic_long_t tot_write_bandwidth; | 152 | atomic_long_t tot_write_bandwidth; |
| 151 | 153 | ||
| 152 | struct bdi_writeback wb; /* the root writeback info for this bdi */ | 154 | struct bdi_writeback wb; /* the root writeback info for this bdi */ |
| 155 | struct list_head wb_list; /* list of all wbs */ | ||
| 153 | #ifdef CONFIG_CGROUP_WRITEBACK | 156 | #ifdef CONFIG_CGROUP_WRITEBACK |
| 154 | struct radix_tree_root cgwb_tree; /* radix tree of active cgroup wbs */ | 157 | struct radix_tree_root cgwb_tree; /* radix tree of active cgroup wbs */ |
| 155 | struct rb_root cgwb_congested_tree; /* their congested states */ | 158 | struct rb_root cgwb_congested_tree; /* their congested states */ |
diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h index d5eb4ad1c534..c85f74946a8b 100644 --- a/include/linux/backing-dev.h +++ b/include/linux/backing-dev.h | |||
| @@ -19,13 +19,17 @@ | |||
| 19 | #include <linux/slab.h> | 19 | #include <linux/slab.h> |
| 20 | 20 | ||
| 21 | int __must_check bdi_init(struct backing_dev_info *bdi); | 21 | int __must_check bdi_init(struct backing_dev_info *bdi); |
| 22 | void bdi_destroy(struct backing_dev_info *bdi); | 22 | void bdi_exit(struct backing_dev_info *bdi); |
| 23 | 23 | ||
| 24 | __printf(3, 4) | 24 | __printf(3, 4) |
| 25 | int bdi_register(struct backing_dev_info *bdi, struct device *parent, | 25 | int bdi_register(struct backing_dev_info *bdi, struct device *parent, |
| 26 | const char *fmt, ...); | 26 | const char *fmt, ...); |
| 27 | int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev); | 27 | int bdi_register_dev(struct backing_dev_info *bdi, dev_t dev); |
| 28 | void bdi_unregister(struct backing_dev_info *bdi); | ||
| 29 | |||
| 28 | int __must_check bdi_setup_and_register(struct backing_dev_info *, char *); | 30 | int __must_check bdi_setup_and_register(struct backing_dev_info *, char *); |
| 31 | void bdi_destroy(struct backing_dev_info *bdi); | ||
| 32 | |||
| 29 | void wb_start_writeback(struct bdi_writeback *wb, long nr_pages, | 33 | void wb_start_writeback(struct bdi_writeback *wb, long nr_pages, |
| 30 | bool range_cyclic, enum wb_reason reason); | 34 | bool range_cyclic, enum wb_reason reason); |
| 31 | void wb_start_background_writeback(struct bdi_writeback *wb); | 35 | void wb_start_background_writeback(struct bdi_writeback *wb); |
| @@ -408,61 +412,6 @@ static inline void unlocked_inode_to_wb_end(struct inode *inode, bool locked) | |||
| 408 | rcu_read_unlock(); | 412 | rcu_read_unlock(); |
| 409 | } | 413 | } |
| 410 | 414 | ||
| 411 | struct wb_iter { | ||
| 412 | int start_memcg_id; | ||
| 413 | struct radix_tree_iter tree_iter; | ||
| 414 | void **slot; | ||
| 415 | }; | ||
| 416 | |||
| 417 | static inline struct bdi_writeback *__wb_iter_next(struct wb_iter *iter, | ||
| 418 | struct backing_dev_info *bdi) | ||
| 419 | { | ||
| 420 | struct radix_tree_iter *titer = &iter->tree_iter; | ||
| 421 | |||
| 422 | WARN_ON_ONCE(!rcu_read_lock_held()); | ||
| 423 | |||
| 424 | if (iter->start_memcg_id >= 0) { | ||
| 425 | iter->slot = radix_tree_iter_init(titer, iter->start_memcg_id); | ||
| 426 | iter->start_memcg_id = -1; | ||
| 427 | } else { | ||
| 428 | iter->slot = radix_tree_next_slot(iter->slot, titer, 0); | ||
| 429 | } | ||
| 430 | |||
| 431 | if (!iter->slot) | ||
| 432 | iter->slot = radix_tree_next_chunk(&bdi->cgwb_tree, titer, 0); | ||
| 433 | if (iter->slot) | ||
| 434 | return *iter->slot; | ||
| 435 | return NULL; | ||
| 436 | } | ||
| 437 | |||
| 438 | static inline struct bdi_writeback *__wb_iter_init(struct wb_iter *iter, | ||
| 439 | struct backing_dev_info *bdi, | ||
| 440 | int start_memcg_id) | ||
| 441 | { | ||
| 442 | iter->start_memcg_id = start_memcg_id; | ||
| 443 | |||
| 444 | if (start_memcg_id) | ||
| 445 | return __wb_iter_next(iter, bdi); | ||
| 446 | else | ||
| 447 | return &bdi->wb; | ||
| 448 | } | ||
| 449 | |||
| 450 | /** | ||
| 451 | * bdi_for_each_wb - walk all wb's of a bdi in ascending memcg ID order | ||
| 452 | * @wb_cur: cursor struct bdi_writeback pointer | ||
| 453 | * @bdi: bdi to walk wb's of | ||
| 454 | * @iter: pointer to struct wb_iter to be used as iteration buffer | ||
| 455 | * @start_memcg_id: memcg ID to start iteration from | ||
| 456 | * | ||
| 457 | * Iterate @wb_cur through the wb's (bdi_writeback's) of @bdi in ascending | ||
| 458 | * memcg ID order starting from @start_memcg_id. @iter is struct wb_iter | ||
| 459 | * to be used as temp storage during iteration. rcu_read_lock() must be | ||
| 460 | * held throughout iteration. | ||
| 461 | */ | ||
| 462 | #define bdi_for_each_wb(wb_cur, bdi, iter, start_memcg_id) \ | ||
| 463 | for ((wb_cur) = __wb_iter_init(iter, bdi, start_memcg_id); \ | ||
| 464 | (wb_cur); (wb_cur) = __wb_iter_next(iter, bdi)) | ||
| 465 | |||
| 466 | #else /* CONFIG_CGROUP_WRITEBACK */ | 415 | #else /* CONFIG_CGROUP_WRITEBACK */ |
| 467 | 416 | ||
| 468 | static inline bool inode_cgwb_enabled(struct inode *inode) | 417 | static inline bool inode_cgwb_enabled(struct inode *inode) |
| @@ -522,14 +471,6 @@ static inline void wb_blkcg_offline(struct blkcg *blkcg) | |||
| 522 | { | 471 | { |
| 523 | } | 472 | } |
| 524 | 473 | ||
| 525 | struct wb_iter { | ||
| 526 | int next_id; | ||
| 527 | }; | ||
| 528 | |||
| 529 | #define bdi_for_each_wb(wb_cur, bdi, iter, start_blkcg_id) \ | ||
| 530 | for ((iter)->next_id = (start_blkcg_id); \ | ||
| 531 | ({ (wb_cur) = !(iter)->next_id++ ? &(bdi)->wb : NULL; }); ) | ||
| 532 | |||
| 533 | static inline int inode_congested(struct inode *inode, int cong_bits) | 474 | static inline int inode_congested(struct inode *inode, int cong_bits) |
| 534 | { | 475 | { |
| 535 | return wb_congested(&inode_to_bdi(inode)->wb, cong_bits); | 476 | return wb_congested(&inode_to_bdi(inode)->wb, cong_bits); |
diff --git a/include/linux/cma.h b/include/linux/cma.h index f7ef093ec49a..29f9e774ab76 100644 --- a/include/linux/cma.h +++ b/include/linux/cma.h | |||
| @@ -26,6 +26,6 @@ extern int __init cma_declare_contiguous(phys_addr_t base, | |||
| 26 | extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size, | 26 | extern int cma_init_reserved_mem(phys_addr_t base, phys_addr_t size, |
| 27 | unsigned int order_per_bit, | 27 | unsigned int order_per_bit, |
| 28 | struct cma **res_cma); | 28 | struct cma **res_cma); |
| 29 | extern struct page *cma_alloc(struct cma *cma, unsigned int count, unsigned int align); | 29 | extern struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align); |
| 30 | extern bool cma_release(struct cma *cma, const struct page *pages, unsigned int count); | 30 | extern bool cma_release(struct cma *cma, const struct page *pages, unsigned int count); |
| 31 | #endif | 31 | #endif |
diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h index dfaa7b3e9ae9..8efb40e61d6e 100644 --- a/include/linux/compiler-gcc.h +++ b/include/linux/compiler-gcc.h | |||
| @@ -237,12 +237,25 @@ | |||
| 237 | #define KASAN_ABI_VERSION 3 | 237 | #define KASAN_ABI_VERSION 3 |
| 238 | #endif | 238 | #endif |
| 239 | 239 | ||
| 240 | #if GCC_VERSION >= 40902 | ||
| 241 | /* | ||
| 242 | * Tell the compiler that address safety instrumentation (KASAN) | ||
| 243 | * should not be applied to that function. | ||
| 244 | * Conflicts with inlining: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67368 | ||
| 245 | */ | ||
| 246 | #define __no_sanitize_address __attribute__((no_sanitize_address)) | ||
| 247 | #endif | ||
| 248 | |||
| 240 | #endif /* gcc version >= 40000 specific checks */ | 249 | #endif /* gcc version >= 40000 specific checks */ |
| 241 | 250 | ||
| 242 | #if !defined(__noclone) | 251 | #if !defined(__noclone) |
| 243 | #define __noclone /* not needed */ | 252 | #define __noclone /* not needed */ |
| 244 | #endif | 253 | #endif |
| 245 | 254 | ||
| 255 | #if !defined(__no_sanitize_address) | ||
| 256 | #define __no_sanitize_address | ||
| 257 | #endif | ||
| 258 | |||
| 246 | /* | 259 | /* |
| 247 | * A trick to suppress uninitialized variable warning without generating any | 260 | * A trick to suppress uninitialized variable warning without generating any |
| 248 | * code | 261 | * code |
diff --git a/include/linux/compiler.h b/include/linux/compiler.h index c836eb2dc44d..3d7810341b57 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h | |||
| @@ -198,19 +198,45 @@ void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect); | |||
| 198 | 198 | ||
| 199 | #include <uapi/linux/types.h> | 199 | #include <uapi/linux/types.h> |
| 200 | 200 | ||
| 201 | static __always_inline void __read_once_size(const volatile void *p, void *res, int size) | 201 | #define __READ_ONCE_SIZE \ |
| 202 | ({ \ | ||
| 203 | switch (size) { \ | ||
| 204 | case 1: *(__u8 *)res = *(volatile __u8 *)p; break; \ | ||
| 205 | case 2: *(__u16 *)res = *(volatile __u16 *)p; break; \ | ||
| 206 | case 4: *(__u32 *)res = *(volatile __u32 *)p; break; \ | ||
| 207 | case 8: *(__u64 *)res = *(volatile __u64 *)p; break; \ | ||
| 208 | default: \ | ||
| 209 | barrier(); \ | ||
| 210 | __builtin_memcpy((void *)res, (const void *)p, size); \ | ||
| 211 | barrier(); \ | ||
| 212 | } \ | ||
| 213 | }) | ||
| 214 | |||
| 215 | static __always_inline | ||
| 216 | void __read_once_size(const volatile void *p, void *res, int size) | ||
| 202 | { | 217 | { |
| 203 | switch (size) { | 218 | __READ_ONCE_SIZE; |
| 204 | case 1: *(__u8 *)res = *(volatile __u8 *)p; break; | 219 | } |
| 205 | case 2: *(__u16 *)res = *(volatile __u16 *)p; break; | 220 | |
| 206 | case 4: *(__u32 *)res = *(volatile __u32 *)p; break; | 221 | #ifdef CONFIG_KASAN |
| 207 | case 8: *(__u64 *)res = *(volatile __u64 *)p; break; | 222 | /* |
| 208 | default: | 223 | * This function is not 'inline' because __no_sanitize_address confilcts |
| 209 | barrier(); | 224 | * with inlining. Attempt to inline it may cause a build failure. |
| 210 | __builtin_memcpy((void *)res, (const void *)p, size); | 225 | * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67368 |
| 211 | barrier(); | 226 | * '__maybe_unused' allows us to avoid defined-but-not-used warnings. |
| 212 | } | 227 | */ |
| 228 | static __no_sanitize_address __maybe_unused | ||
| 229 | void __read_once_size_nocheck(const volatile void *p, void *res, int size) | ||
| 230 | { | ||
| 231 | __READ_ONCE_SIZE; | ||
| 232 | } | ||
| 233 | #else | ||
| 234 | static __always_inline | ||
| 235 | void __read_once_size_nocheck(const volatile void *p, void *res, int size) | ||
| 236 | { | ||
| 237 | __READ_ONCE_SIZE; | ||
| 213 | } | 238 | } |
| 239 | #endif | ||
| 214 | 240 | ||
| 215 | static __always_inline void __write_once_size(volatile void *p, void *res, int size) | 241 | static __always_inline void __write_once_size(volatile void *p, void *res, int size) |
| 216 | { | 242 | { |
| @@ -248,8 +274,22 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s | |||
| 248 | * required ordering. | 274 | * required ordering. |
| 249 | */ | 275 | */ |
| 250 | 276 | ||
| 251 | #define READ_ONCE(x) \ | 277 | #define __READ_ONCE(x, check) \ |
| 252 | ({ union { typeof(x) __val; char __c[1]; } __u; __read_once_size(&(x), __u.__c, sizeof(x)); __u.__val; }) | 278 | ({ \ |
| 279 | union { typeof(x) __val; char __c[1]; } __u; \ | ||
| 280 | if (check) \ | ||
| 281 | __read_once_size(&(x), __u.__c, sizeof(x)); \ | ||
| 282 | else \ | ||
| 283 | __read_once_size_nocheck(&(x), __u.__c, sizeof(x)); \ | ||
| 284 | __u.__val; \ | ||
| 285 | }) | ||
| 286 | #define READ_ONCE(x) __READ_ONCE(x, 1) | ||
| 287 | |||
| 288 | /* | ||
| 289 | * Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need | ||
| 290 | * to hide memory access from KASAN. | ||
| 291 | */ | ||
| 292 | #define READ_ONCE_NOCHECK(x) __READ_ONCE(x, 0) | ||
| 253 | 293 | ||
| 254 | #define WRITE_ONCE(x, val) \ | 294 | #define WRITE_ONCE(x, val) \ |
| 255 | ({ \ | 295 | ({ \ |
diff --git a/include/linux/dma-contiguous.h b/include/linux/dma-contiguous.h index 569bbd039896..fec734df1524 100644 --- a/include/linux/dma-contiguous.h +++ b/include/linux/dma-contiguous.h | |||
| @@ -111,7 +111,7 @@ static inline int dma_declare_contiguous(struct device *dev, phys_addr_t size, | |||
| 111 | return ret; | 111 | return ret; |
| 112 | } | 112 | } |
| 113 | 113 | ||
| 114 | struct page *dma_alloc_from_contiguous(struct device *dev, int count, | 114 | struct page *dma_alloc_from_contiguous(struct device *dev, size_t count, |
| 115 | unsigned int order); | 115 | unsigned int order); |
| 116 | bool dma_release_from_contiguous(struct device *dev, struct page *pages, | 116 | bool dma_release_from_contiguous(struct device *dev, struct page *pages, |
| 117 | int count); | 117 | int count); |
| @@ -144,7 +144,7 @@ int dma_declare_contiguous(struct device *dev, phys_addr_t size, | |||
| 144 | } | 144 | } |
| 145 | 145 | ||
| 146 | static inline | 146 | static inline |
| 147 | struct page *dma_alloc_from_contiguous(struct device *dev, int count, | 147 | struct page *dma_alloc_from_contiguous(struct device *dev, size_t count, |
| 148 | unsigned int order) | 148 | unsigned int order) |
| 149 | { | 149 | { |
| 150 | return NULL; | 150 | return NULL; |
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 6452ff4c463f..3e3318ddfc0e 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h | |||
| @@ -676,8 +676,9 @@ enum { | |||
| 676 | 676 | ||
| 677 | struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg); | 677 | struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg); |
| 678 | struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb); | 678 | struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb); |
| 679 | void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pavail, | 679 | void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages, |
| 680 | unsigned long *pdirty, unsigned long *pwriteback); | 680 | unsigned long *pheadroom, unsigned long *pdirty, |
| 681 | unsigned long *pwriteback); | ||
| 681 | 682 | ||
| 682 | #else /* CONFIG_CGROUP_WRITEBACK */ | 683 | #else /* CONFIG_CGROUP_WRITEBACK */ |
| 683 | 684 | ||
| @@ -687,7 +688,8 @@ static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb) | |||
| 687 | } | 688 | } |
| 688 | 689 | ||
| 689 | static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb, | 690 | static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb, |
| 690 | unsigned long *pavail, | 691 | unsigned long *pfilepages, |
| 692 | unsigned long *pheadroom, | ||
| 691 | unsigned long *pdirty, | 693 | unsigned long *pdirty, |
| 692 | unsigned long *pwriteback) | 694 | unsigned long *pwriteback) |
| 693 | { | 695 | { |
