From e26b53d0b287056646a0dffce8bc6b0f053f3823 Mon Sep 17 00:00:00 2001 From: Shaohua Li Date: Tue, 15 Oct 2013 09:05:01 +0800 Subject: percpu_ida: make percpu_ida percpu size/batch configurable Make percpu_ida percpu size/batch configurable. The block-mq-tag will use it. After block-mq uses percpu_ida to manage tags, performance is improved. My test is done in a 2 sockets machine, 12 process cross the 2 sockets. So if there is lock contention or ipi, should be stressed heavily. Testing is done for null-blk. hw_queue_depth nopatch iops patch iops 64 ~800k/s ~1470k/s 2048 ~4470k/s ~4340k/s Cc: Andrew Morton Signed-off-by: Shaohua Li Signed-off-by: Jens Axboe --- include/linux/percpu_ida.h | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) (limited to 'include/linux/percpu_ida.h') diff --git a/include/linux/percpu_ida.h b/include/linux/percpu_ida.h index 0b23edbee309..56c14033e7e7 100644 --- a/include/linux/percpu_ida.h +++ b/include/linux/percpu_ida.h @@ -16,6 +16,8 @@ struct percpu_ida { * percpu_ida_init() */ unsigned nr_tags; + unsigned percpu_max_size; + unsigned percpu_batch_size; struct percpu_ida_cpu __percpu *tag_cpu; @@ -51,10 +53,24 @@ struct percpu_ida { } ____cacheline_aligned_in_smp; }; +/* + * Number of tags we move between the percpu freelist and the global freelist at + * a time + */ +#define IDA_DEFAULT_PCPU_BATCH_MOVE 32U +/* Max size of percpu freelist, */ +#define IDA_DEFAULT_PCPU_SIZE ((IDA_DEFAULT_PCPU_BATCH_MOVE * 3) / 2) + int percpu_ida_alloc(struct percpu_ida *pool, gfp_t gfp); void percpu_ida_free(struct percpu_ida *pool, unsigned tag); void percpu_ida_destroy(struct percpu_ida *pool); -int percpu_ida_init(struct percpu_ida *pool, unsigned long nr_tags); +int __percpu_ida_init(struct percpu_ida *pool, unsigned long nr_tags, + unsigned long max_size, unsigned long batch_size); +static inline int percpu_ida_init(struct percpu_ida *pool, unsigned long nr_tags) +{ + return __percpu_ida_init(pool, nr_tags, IDA_DEFAULT_PCPU_SIZE, + IDA_DEFAULT_PCPU_BATCH_MOVE); +} #endif /* __PERCPU_IDA_H__ */ -- cgit v1.2.2 From 7fc2ba17e8bf9f218cac10cc2a3de613d9d9086d Mon Sep 17 00:00:00 2001 From: Shaohua Li Date: Tue, 15 Oct 2013 09:05:02 +0800 Subject: percpu_ida: add percpu_ida_for_each_free Add a new API to iterate free ids. blk-mq-tag will use it. Note, this doesn't guarantee to iterate all free ids restrictly. Caller should be aware of this. blk-mq uses it to do sanity check for request timedout, so can tolerate the limitation. Cc: Andrew Morton Signed-off-by: Shaohua Li Signed-off-by: Jens Axboe --- include/linux/percpu_ida.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include/linux/percpu_ida.h') diff --git a/include/linux/percpu_ida.h b/include/linux/percpu_ida.h index 56c14033e7e7..63510ae6f933 100644 --- a/include/linux/percpu_ida.h +++ b/include/linux/percpu_ida.h @@ -73,4 +73,8 @@ static inline int percpu_ida_init(struct percpu_ida *pool, unsigned long nr_tags IDA_DEFAULT_PCPU_BATCH_MOVE); } +typedef int (*percpu_ida_cb)(unsigned, void *); +int percpu_ida_for_each_free(struct percpu_ida *pool, percpu_ida_cb fn, + void *data); + #endif /* __PERCPU_IDA_H__ */ -- cgit v1.2.2 From 1dddc01af0d42b21058e0cb9c1ca9e8d5204d9b0 Mon Sep 17 00:00:00 2001 From: Shaohua Li Date: Tue, 15 Oct 2013 09:05:03 +0800 Subject: percpu_ida: add an API to return free tags Add an API to return free tags, blk-mq-tag will use it. Note, this just returns a snapshot of free tags number. blk-mq-tag has two usages of it. One is for info output for diagnosis. The other is to quickly check if there are free tags for request dispatch checking. Neither requires very precise. Cc: Andrew Morton Signed-off-by: Shaohua Li Signed-off-by: Jens Axboe --- include/linux/percpu_ida.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux/percpu_ida.h') diff --git a/include/linux/percpu_ida.h b/include/linux/percpu_ida.h index 63510ae6f933..1900bd0fa639 100644 --- a/include/linux/percpu_ida.h +++ b/include/linux/percpu_ida.h @@ -77,4 +77,5 @@ typedef int (*percpu_ida_cb)(unsigned, void *); int percpu_ida_for_each_free(struct percpu_ida *pool, percpu_ida_cb fn, void *data); +unsigned percpu_ida_free_tags(struct percpu_ida *pool, int cpu); #endif /* __PERCPU_IDA_H__ */ -- cgit v1.2.2