aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--fs/aio.c47
-rw-r--r--include/linux/aio.h5
2 files changed, 12 insertions, 40 deletions
diff --git a/fs/aio.c b/fs/aio.c
index 09fe1f334631..69a608a43760 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -361,7 +361,6 @@ EXPORT_SYMBOL(kiocb_set_cancel_fn);
361static int kiocb_cancel(struct kioctx *ctx, struct kiocb *kiocb) 361static int kiocb_cancel(struct kioctx *ctx, struct kiocb *kiocb)
362{ 362{
363 kiocb_cancel_fn *old, *cancel; 363 kiocb_cancel_fn *old, *cancel;
364 int ret = -EINVAL;
365 364
366 /* 365 /*
367 * Don't want to set kiocb->ki_cancel = KIOCB_CANCELLED unless it 366 * Don't want to set kiocb->ki_cancel = KIOCB_CANCELLED unless it
@@ -371,21 +370,13 @@ static int kiocb_cancel(struct kioctx *ctx, struct kiocb *kiocb)
371 cancel = ACCESS_ONCE(kiocb->ki_cancel); 370 cancel = ACCESS_ONCE(kiocb->ki_cancel);
372 do { 371 do {
373 if (!cancel || cancel == KIOCB_CANCELLED) 372 if (!cancel || cancel == KIOCB_CANCELLED)
374 return ret; 373 return -EINVAL;
375 374
376 old = cancel; 375 old = cancel;
377 cancel = cmpxchg(&kiocb->ki_cancel, old, KIOCB_CANCELLED); 376 cancel = cmpxchg(&kiocb->ki_cancel, old, KIOCB_CANCELLED);
378 } while (cancel != old); 377 } while (cancel != old);
379 378
380 atomic_inc(&kiocb->ki_users); 379 return cancel(kiocb);
381 spin_unlock_irq(&ctx->ctx_lock);
382
383 ret = cancel(kiocb);
384
385 spin_lock_irq(&ctx->ctx_lock);
386 aio_put_req(kiocb);
387
388 return ret;
389} 380}
390 381
391static void free_ioctx_rcu(struct rcu_head *head) 382static void free_ioctx_rcu(struct rcu_head *head)
@@ -599,16 +590,16 @@ static void kill_ioctx(struct kioctx *ctx)
599/* wait_on_sync_kiocb: 590/* wait_on_sync_kiocb:
600 * Waits on the given sync kiocb to complete. 591 * Waits on the given sync kiocb to complete.
601 */ 592 */
602ssize_t wait_on_sync_kiocb(struct kiocb *iocb) 593ssize_t wait_on_sync_kiocb(struct kiocb *req)
603{ 594{
604 while (atomic_read(&iocb->ki_users)) { 595 while (!req->ki_ctx) {
605 set_current_state(TASK_UNINTERRUPTIBLE); 596 set_current_state(TASK_UNINTERRUPTIBLE);
606 if (!atomic_read(&iocb->ki_users)) 597 if (req->ki_ctx)
607 break; 598 break;
608 io_schedule(); 599 io_schedule();
609 } 600 }
610 __set_current_state(TASK_RUNNING); 601 __set_current_state(TASK_RUNNING);
611 return iocb->ki_user_data; 602 return req->ki_user_data;
612} 603}
613EXPORT_SYMBOL(wait_on_sync_kiocb); 604EXPORT_SYMBOL(wait_on_sync_kiocb);
614 605
@@ -687,14 +678,8 @@ out:
687} 678}
688 679
689/* aio_get_req 680/* aio_get_req
690 * Allocate a slot for an aio request. Increments the ki_users count 681 * Allocate a slot for an aio request.
691 * of the kioctx so that the kioctx stays around until all requests are 682 * Returns NULL if no requests are free.
692 * complete. Returns NULL if no requests are free.
693 *
694 * Returns with kiocb->ki_users set to 2. The io submit code path holds
695 * an extra reference while submitting the i/o.
696 * This prevents races between the aio code path referencing the
697 * req (after submitting it) and aio_complete() freeing the req.
698 */ 683 */
699static inline struct kiocb *aio_get_req(struct kioctx *ctx) 684static inline struct kiocb *aio_get_req(struct kioctx *ctx)
700{ 685{
@@ -707,7 +692,6 @@ static inline struct kiocb *aio_get_req(struct kioctx *ctx)
707 if (unlikely(!req)) 692 if (unlikely(!req))
708 goto out_put; 693 goto out_put;
709 694
710 atomic_set(&req->ki_users, 1);
711 req->ki_ctx = ctx; 695 req->ki_ctx = ctx;
712 return req; 696 return req;
713out_put: 697out_put:
@@ -726,13 +710,6 @@ static void kiocb_free(struct kiocb *req)
726 kmem_cache_free(kiocb_cachep, req); 710 kmem_cache_free(kiocb_cachep, req);
727} 711}
728 712
729void aio_put_req(struct kiocb *req)
730{
731 if (atomic_dec_and_test(&req->ki_users))
732 kiocb_free(req);
733}
734EXPORT_SYMBOL(aio_put_req);
735
736static struct kioctx *lookup_ioctx(unsigned long ctx_id) 713static struct kioctx *lookup_ioctx(unsigned long ctx_id)
737{ 714{
738 struct mm_struct *mm = current->mm; 715 struct mm_struct *mm = current->mm;
@@ -771,9 +748,9 @@ void aio_complete(struct kiocb *iocb, long res, long res2)
771 * - the sync task helpfully left a reference to itself in the iocb 748 * - the sync task helpfully left a reference to itself in the iocb
772 */ 749 */
773 if (is_sync_kiocb(iocb)) { 750 if (is_sync_kiocb(iocb)) {
774 BUG_ON(atomic_read(&iocb->ki_users) != 1);
775 iocb->ki_user_data = res; 751 iocb->ki_user_data = res;
776 atomic_set(&iocb->ki_users, 0); 752 smp_wmb();
753 iocb->ki_ctx = ERR_PTR(-EXDEV);
777 wake_up_process(iocb->ki_obj.tsk); 754 wake_up_process(iocb->ki_obj.tsk);
778 return; 755 return;
779 } 756 }
@@ -845,7 +822,7 @@ void aio_complete(struct kiocb *iocb, long res, long res2)
845 eventfd_signal(iocb->ki_eventfd, 1); 822 eventfd_signal(iocb->ki_eventfd, 1);
846 823
847 /* everything turned out well, dispose of the aiocb. */ 824 /* everything turned out well, dispose of the aiocb. */
848 aio_put_req(iocb); 825 kiocb_free(iocb);
849 826
850 /* 827 /*
851 * We have to order our ring_info tail store above and test 828 * We have to order our ring_info tail store above and test
@@ -1269,7 +1246,7 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
1269 return 0; 1246 return 0;
1270out_put_req: 1247out_put_req:
1271 put_reqs_available(ctx, 1); 1248 put_reqs_available(ctx, 1);
1272 aio_put_req(req); 1249 kiocb_free(req);
1273 return ret; 1250 return ret;
1274} 1251}
1275 1252
diff --git a/include/linux/aio.h b/include/linux/aio.h
index b570472355d1..c4f07ffa1cbb 100644
--- a/include/linux/aio.h
+++ b/include/linux/aio.h
@@ -30,8 +30,6 @@ struct kiocb;
30typedef int (kiocb_cancel_fn)(struct kiocb *); 30typedef int (kiocb_cancel_fn)(struct kiocb *);
31 31
32struct kiocb { 32struct kiocb {
33 atomic_t ki_users;
34
35 struct file *ki_filp; 33 struct file *ki_filp;
36 struct kioctx *ki_ctx; /* NULL for sync ops */ 34 struct kioctx *ki_ctx; /* NULL for sync ops */
37 kiocb_cancel_fn *ki_cancel; 35 kiocb_cancel_fn *ki_cancel;
@@ -65,7 +63,6 @@ static inline bool is_sync_kiocb(struct kiocb *kiocb)
65static inline void init_sync_kiocb(struct kiocb *kiocb, struct file *filp) 63static inline void init_sync_kiocb(struct kiocb *kiocb, struct file *filp)
66{ 64{
67 *kiocb = (struct kiocb) { 65 *kiocb = (struct kiocb) {
68 .ki_users = ATOMIC_INIT(1),
69 .ki_ctx = NULL, 66 .ki_ctx = NULL,
70 .ki_filp = filp, 67 .ki_filp = filp,
71 .ki_obj.tsk = current, 68 .ki_obj.tsk = current,
@@ -75,7 +72,6 @@ static inline void init_sync_kiocb(struct kiocb *kiocb, struct file *filp)
75/* prototypes */ 72/* prototypes */
76#ifdef CONFIG_AIO 73#ifdef CONFIG_AIO
77extern ssize_t wait_on_sync_kiocb(struct kiocb *iocb); 74extern ssize_t wait_on_sync_kiocb(struct kiocb *iocb);
78extern void aio_put_req(struct kiocb *iocb);
79extern void aio_complete(struct kiocb *iocb, long res, long res2); 75extern void aio_complete(struct kiocb *iocb, long res, long res2);
80struct mm_struct; 76struct mm_struct;
81extern void exit_aio(struct mm_struct *mm); 77extern void exit_aio(struct mm_struct *mm);
@@ -84,7 +80,6 @@ extern long do_io_submit(aio_context_t ctx_id, long nr,
84void kiocb_set_cancel_fn(struct kiocb *req, kiocb_cancel_fn *cancel); 80void kiocb_set_cancel_fn(struct kiocb *req, kiocb_cancel_fn *cancel);
85#else 81#else
86static inline ssize_t wait_on_sync_kiocb(struct kiocb *iocb) { return 0; } 82static inline ssize_t wait_on_sync_kiocb(struct kiocb *iocb) { return 0; }
87static inline void aio_put_req(struct kiocb *iocb) { }
88static inline void aio_complete(struct kiocb *iocb, long res, long res2) { } 83static inline void aio_complete(struct kiocb *iocb, long res, long res2) { }
89struct mm_struct; 84struct mm_struct;
90static inline void exit_aio(struct mm_struct *mm) { } 85static inline void exit_aio(struct mm_struct *mm) { }