From 212079cf4ee99e492a57b817e796825d423a30bb Mon Sep 17 00:00:00 2001 From: Ken Chen Date: Sun, 1 May 2005 08:59:15 -0700 Subject: [PATCH] aio: remove superfluous kiocb member initialization This patch removes superfluous kiocb member initialization in the AIO allocation and deallocation path. For example, in really_put_req(), right before kiocb is returned to slab, 5 variables are reset to NULL. The same variables will be initialized at the kiocb allocation time, so why bother reset them knowing that they will be set to valid data at alloc time? Another example: ki_retry is initialized in __aio_get_req, but is initialized again in io_submit_one. Signed-off-by: Ken Chen Cc: Benjamin LaHaise Cc: Suparna Bhattacharya Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/aio.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) (limited to 'fs/aio.c') diff --git a/fs/aio.c b/fs/aio.c index a82214d2e46d..9f807a541fbe 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -405,7 +405,6 @@ static struct kiocb fastcall *__aio_get_req(struct kioctx *ctx) req->ki_ctx = ctx; req->ki_cancel = NULL; req->ki_retry = NULL; - req->ki_obj.user = NULL; req->ki_dtor = NULL; req->private = NULL; INIT_LIST_HEAD(&req->ki_run_list); @@ -451,11 +450,6 @@ static inline void really_put_req(struct kioctx *ctx, struct kiocb *req) { if (req->ki_dtor) req->ki_dtor(req); - req->ki_ctx = NULL; - req->ki_filp = NULL; - req->ki_obj.user = NULL; - req->ki_dtor = NULL; - req->private = NULL; kmem_cache_free(kiocb_cachep, req); ctx->reqs_active--; @@ -1515,8 +1509,7 @@ int fastcall io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, } req->ki_filp = file; - iocb->aio_key = req->ki_key; - ret = put_user(iocb->aio_key, &user_iocb->aio_key); + ret = put_user(req->ki_key, &user_iocb->aio_key); if (unlikely(ret)) { dprintk("EFAULT: aio_key\n"); goto out_put_req; @@ -1531,8 +1524,6 @@ int fastcall io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, req->ki_opcode = iocb->aio_lio_opcode; init_waitqueue_func_entry(&req->ki_wait, aio_wake_function); INIT_LIST_HEAD(&req->ki_wait.task_list); - req->ki_run_list.next = req->ki_run_list.prev = NULL; - req->ki_retry = NULL; req->ki_retried = 0; req->ki_kicked = 0; req->ki_queued = 0; -- cgit v1.2.2 From 4bf69b2a06090c01c27f25ea5cd1440f7bf9256f Mon Sep 17 00:00:00 2001 From: Ken Chen Date: Sun, 1 May 2005 08:59:15 -0700 Subject: [PATCH] aio: ring wrapping simplification Since the tail pointer in aio_ring structure never wrap ring size more than once, so a simple compare is sufficient to wrap the index around. This avoid a more expensive mod operation. Signed-off-by: Ken Chen Cc: Benjamin LaHaise Cc: Suparna Bhattacharya Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/aio.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs/aio.c') diff --git a/fs/aio.c b/fs/aio.c index 9f807a541fbe..40517f35daae 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -978,7 +978,8 @@ int fastcall aio_complete(struct kiocb *iocb, long res, long res2) tail = info->tail; event = aio_ring_event(info, tail, KM_IRQ0); - tail = (tail + 1) % info->nr; + if (++tail >= info->nr) + tail = 0; event->obj = (u64)(unsigned long)iocb->ki_obj.user; event->data = iocb->ki_user_data; -- cgit v1.2.2 From 644d3a088a3b862ed0b57c286cf58a6bd338ce08 Mon Sep 17 00:00:00 2001 From: Ken Chen Date: Sun, 1 May 2005 08:59:15 -0700 Subject: [PATCH] aio: clean up debug code Clean up code that was previously used for debug purpose. Remove aio_run, aio_wakeups, iocb->ki_queued and iocb->ki_kicked. Also clean up unused variable count in __aio_run_iocbs() and debug code in read_events(). Signed-off-by: Ken Chen Cc: Benjamin LaHaise Cc: Suparna Bhattacharya Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/aio.c | 33 +++++---------------------------- 1 file changed, 5 insertions(+), 28 deletions(-) (limited to 'fs/aio.c') diff --git a/fs/aio.c b/fs/aio.c index 40517f35daae..674bb47fed29 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -40,9 +40,6 @@ #define dprintk(x...) do { ; } while (0) #endif -static long aio_run = 0; /* for testing only */ -static long aio_wakeups = 0; /* for testing only */ - /*------ sysctl variables----*/ atomic_t aio_nr = ATOMIC_INIT(0); /* current system wide number of aio requests */ unsigned aio_max_nr = 0x10000; /* system wide maximum number of aio requests */ @@ -617,7 +614,6 @@ static inline int __queue_kicked_iocb(struct kiocb *iocb) if (list_empty(&iocb->ki_run_list)) { list_add_tail(&iocb->ki_run_list, &ctx->run_list); - iocb->ki_queued++; return 1; } return 0; @@ -658,10 +654,8 @@ static ssize_t aio_run_iocb(struct kiocb *iocb) } if (!(iocb->ki_retried & 0xff)) { - pr_debug("%ld retry: %d of %d (kick %ld, Q %ld run %ld, wake %ld)\n", - iocb->ki_retried, - iocb->ki_nbytes - iocb->ki_left, iocb->ki_nbytes, - iocb->ki_kicked, iocb->ki_queued, aio_run, aio_wakeups); + pr_debug("%ld retry: %d of %d\n", iocb->ki_retried, + iocb->ki_nbytes - iocb->ki_left, iocb->ki_nbytes); } if (!(retry = iocb->ki_retry)) { @@ -768,7 +762,6 @@ out: static int __aio_run_iocbs(struct kioctx *ctx) { struct kiocb *iocb; - int count = 0; LIST_HEAD(run_list); list_splice_init(&ctx->run_list, &run_list); @@ -783,9 +776,7 @@ static int __aio_run_iocbs(struct kioctx *ctx) aio_run_iocb(iocb); if (__aio_put_req(ctx, iocb)) /* drop extra ref */ put_ioctx(ctx); - count++; } - aio_run++; if (!list_empty(&ctx->run_list)) return 1; return 0; @@ -884,10 +875,8 @@ static void queue_kicked_iocb(struct kiocb *iocb) spin_lock_irqsave(&ctx->ctx_lock, flags); run = __queue_kicked_iocb(iocb); spin_unlock_irqrestore(&ctx->ctx_lock, flags); - if (run) { + if (run) aio_queue_work(ctx); - aio_wakeups++; - } } /* @@ -907,7 +896,6 @@ void fastcall kick_iocb(struct kiocb *iocb) return; } - iocb->ki_kicked++; /* If its already kicked we shouldn't queue it again */ if (!kiocbTryKick(iocb)) { queue_kicked_iocb(iocb); @@ -1003,10 +991,8 @@ int fastcall aio_complete(struct kiocb *iocb, long res, long res2) pr_debug("added to ring %p at [%lu]\n", iocb, tail); - pr_debug("%ld retries: %d of %d (kicked %ld, Q %ld run %ld wake %ld)\n", - iocb->ki_retried, - iocb->ki_nbytes - iocb->ki_left, iocb->ki_nbytes, - iocb->ki_kicked, iocb->ki_queued, aio_run, aio_wakeups); + pr_debug("%ld retries: %d of %d\n", iocb->ki_retried, + iocb->ki_nbytes - iocb->ki_left, iocb->ki_nbytes); put_rq: /* everything turned out well, dispose of the aiocb. */ ret = __aio_put_req(ctx, iocb); @@ -1114,7 +1100,6 @@ static int read_events(struct kioctx *ctx, int i = 0; struct io_event ent; struct aio_timeout to; - int event_loop = 0; /* testing only */ int retry = 0; /* needed to zero any padding within an entry (there shouldn't be @@ -1181,7 +1166,6 @@ retry: if (to.timed_out) /* Only check after read evt */ break; schedule(); - event_loop++; if (signal_pending(tsk)) { ret = -EINTR; break; @@ -1209,9 +1193,6 @@ retry: if (timeout) clear_timeout(&to); out: - pr_debug("event loop executed %d times\n", event_loop); - pr_debug("aio_run %ld\n", aio_run); - pr_debug("aio_wakeups %ld\n", aio_wakeups); return i ? i : ret; } @@ -1526,10 +1507,6 @@ int fastcall io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, init_waitqueue_func_entry(&req->ki_wait, aio_wake_function); INIT_LIST_HEAD(&req->ki_wait.task_list); req->ki_retried = 0; - req->ki_kicked = 0; - req->ki_queued = 0; - aio_run = 0; - aio_wakeups = 0; ret = aio_setup_iocb(req); -- cgit v1.2.2 From 954d3e95369cf73b4bc1e570729f68264a0e6fe0 Mon Sep 17 00:00:00 2001 From: Ken Chen Date: Sun, 1 May 2005 08:59:16 -0700 Subject: [PATCH] aio: optimize io_submit_one() This patch optimizes io_submit_one to call aio_run_iocb() directly if ctx->run_list is empty. When the list is empty, the operation of adding to the list, then call to __aio_run_iocbs() is unnecessary because these operations are done in one atomic step. ctx->run_list always has only one element in this case. This optimization speeds up industry standard db transaction processing benchmark by 0.2%. Signed-off-by: Ken Chen Cc: Benjamin LaHaise Cc: Suparna Bhattacharya Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/aio.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'fs/aio.c') diff --git a/fs/aio.c b/fs/aio.c index 674bb47fed29..7afa222f6802 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -1514,10 +1514,14 @@ int fastcall io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, goto out_put_req; spin_lock_irq(&ctx->ctx_lock); - list_add_tail(&req->ki_run_list, &ctx->run_list); - /* drain the run list */ - while (__aio_run_iocbs(ctx)) - ; + if (likely(list_empty(&ctx->run_list))) { + aio_run_iocb(req); + } else { + list_add_tail(&req->ki_run_list, &ctx->run_list); + /* drain the run list */ + while (__aio_run_iocbs(ctx)) + ; + } spin_unlock_irq(&ctx->ctx_lock); aio_put_req(req); /* drop extra ref to req */ return 0; -- cgit v1.2.2