Merge branch 'akpm' (incoming from Andrew)

Merge more incoming from Andrew Morton:

 - Various fixes which were stalled or which I picked up recently

 - A large rotorooting of the AIO code.  Allegedly to improve
   performance but I don't really have good performance numbers (I might
   have lost the email) and I can't raise Kent today.  I held this out
   of 3.9 and we could give it another cycle if it's all too late/scary.

I ended up taking only the first two thirds of the AIO rotorooting.  I
left the percpu parts and the batch completion for later.  - Linus

* emailed patches from Andrew Morton <akpm@linux-foundation.org>: (33 commits)
  aio: don't include aio.h in sched.h
  aio: kill ki_retry
  aio: kill ki_key
  aio: give shared kioctx fields their own cachelines
  aio: kill struct aio_ring_info
  aio: kill batch allocation
  aio: change reqs_active to include unreaped completions
  aio: use cancellation list lazily
  aio: use flush_dcache_page()
  aio: make aio_read_evt() more efficient, convert to hrtimers
  wait: add wait_event_hrtimeout()
  aio: refcounting cleanup
  aio: make aio_put_req() lockless
  aio: do fget() after aio_get_req()
  aio: dprintk() -> pr_debug()
  aio: move private stuff out of aio.h
  aio: add kiocb_cancel()
  aio: kill return value of aio_complete()
  char: add aio_{read,write} to /dev/{null,zero}
  aio: remove retry-based AIO
  ...

10 files changed, 141 insertions, 175 deletions

diff --git a/include/linux/aio.h b/include/linux/aio.h
index 31ff6dba4872..1bdf965339f9 100644
--- a/include/linux/aio.h
+++ b/include/linux/aio.h
@@ -9,91 +9,32 @@
 
 #include <linux/atomic.h>
 
-#define AIO_MAXSEGS             4
-#define AIO_KIOGRP_NR_ATOMIC    8
-
 struct kioctx;
+struct kiocb;
 
-/* Notes on cancelling a kiocb:
- *      If a kiocb is cancelled, aio_complete may return 0 to indicate 
- *      that cancel has not yet disposed of the kiocb.  All cancel 
- *      operations *must* call aio_put_req to dispose of the kiocb 
- *      to guard against races with the completion code.
- */
-#define KIOCB_C_CANCELLED       0x01
-#define KIOCB_C_COMPLETE        0x02
-
-#define KIOCB_SYNC_KEY          (~0U)
+#define KIOCB_KEY               0
 
-/* ki_flags bits */
 /*
- * This may be used for cancel/retry serialization in the future, but
- * for now it's unused and we probably don't want modules to even
- * think they can use it.
+ * We use ki_cancel == KIOCB_CANCELLED to indicate that a kiocb has been either
+ * cancelled or completed (this makes a certain amount of sense because
+ * successful cancellation - io_cancel() - does deliver the completion to
+ * userspace).
+ *
+ * And since most things don't implement kiocb cancellation and we'd really like
+ * kiocb completion to be lockless when possible, we use ki_cancel to
+ * synchronize cancellation and completion - we only set it to KIOCB_CANCELLED
+ * with xchg() or cmpxchg(), see batch_complete_aio() and kiocb_cancel().
  */
-/* #define KIF_LOCKED           0 */
-#define KIF_KICKED              1
-#define KIF_CANCELLED           2
-
-#define kiocbTryLock(iocb)      test_and_set_bit(KIF_LOCKED, &(iocb)->ki_flags)
-#define kiocbTryKick(iocb)      test_and_set_bit(KIF_KICKED, &(iocb)->ki_flags)
+#define KIOCB_CANCELLED         ((void *) (~0ULL))
 
-#define kiocbSetLocked(iocb)    set_bit(KIF_LOCKED, &(iocb)->ki_flags)
-#define kiocbSetKicked(iocb)    set_bit(KIF_KICKED, &(iocb)->ki_flags)
-#define kiocbSetCancelled(iocb) set_bit(KIF_CANCELLED, &(iocb)->ki_flags)
+typedef int (kiocb_cancel_fn)(struct kiocb *, struct io_event *);
 
-#define kiocbClearLocked(iocb)  clear_bit(KIF_LOCKED, &(iocb)->ki_flags)
-#define kiocbClearKicked(iocb)  clear_bit(KIF_KICKED, &(iocb)->ki_flags)
-#define kiocbClearCancelled(iocb)       clear_bit(KIF_CANCELLED, &(iocb)->ki_flags)
-
-#define kiocbIsLocked(iocb)     test_bit(KIF_LOCKED, &(iocb)->ki_flags)
-#define kiocbIsKicked(iocb)     test_bit(KIF_KICKED, &(iocb)->ki_flags)
-#define kiocbIsCancelled(iocb)  test_bit(KIF_CANCELLED, &(iocb)->ki_flags)
-
-/* is there a better place to document function pointer methods? */
-/**
- * ki_retry     -       iocb forward progress callback
- * @kiocb:      The kiocb struct to advance by performing an operation.
- *
- * This callback is called when the AIO core wants a given AIO operation
- * to make forward progress.  The kiocb argument describes the operation
- * that is to be performed.  As the operation proceeds, perhaps partially,
- * ki_retry is expected to update the kiocb with progress made.  Typically
- * ki_retry is set in the AIO core and it itself calls file_operations
- * helpers.
- *
- * ki_retry's return value determines when the AIO operation is completed
- * and an event is generated in the AIO event ring.  Except the special
- * return values described below, the value that is returned from ki_retry
- * is transferred directly into the completion ring as the operation's
- * resulting status.  Once this has happened ki_retry *MUST NOT* reference
- * the kiocb pointer again.
- *
- * If ki_retry returns -EIOCBQUEUED it has made a promise that aio_complete()
- * will be called on the kiocb pointer in the future.  The AIO core will
- * not ask the method again -- ki_retry must ensure forward progress.
- * aio_complete() must be called once and only once in the future, multiple
- * calls may result in undefined behaviour.
- *
- * If ki_retry returns -EIOCBRETRY it has made a promise that kick_iocb()
- * will be called on the kiocb pointer in the future.  This may happen
- * through generic helpers that associate kiocb->ki_wait with a wait
- * queue head that ki_retry uses via current->io_wait.  It can also happen
- * with custom tracking and manual calls to kick_iocb(), though that is
- * discouraged.  In either case, kick_iocb() must be called once and only
- * once.  ki_retry must ensure forward progress, the AIO core will wait
- * indefinitely for kick_iocb() to be called.
- */
 struct kiocb {
-        struct list_head        ki_run_list;
-        unsigned long           ki_flags;
-        int                     ki_users;
-        unsigned                ki_key;         /* id of this request */
+        atomic_t                ki_users;
 
         struct file             *ki_filp;
-        struct kioctx           *ki_ctx;        /* may be NULL for sync ops */
-        int                     (*ki_cancel)(struct kiocb *, struct io_event *);
-        ssize_t                 (*ki_retry)(struct kiocb *);
+        struct kioctx           *ki_ctx;        /* NULL for sync ops */
+        kiocb_cancel_fn         *ki_cancel;
         void                    (*ki_dtor)(struct kiocb *);
 
         union {
@@ -117,7 +58,6 @@ struct kiocb {
 
         struct list_head        ki_list;        /* the aio core uses this
                                                  * for cancellation */
-        struct list_head        ki_batch;       /* batch allocation */
 
         /*
          * If the aio_resfd field of the userspace iocb is not zero,
@@ -128,106 +68,40 @@ struct kiocb {
 
 static inline bool is_sync_kiocb(struct kiocb *kiocb)
 {
-        return kiocb->ki_key == KIOCB_SYNC_KEY;
+        return kiocb->ki_ctx == NULL;
 }
 
 static inline void init_sync_kiocb(struct kiocb *kiocb, struct file *filp)
 {
         *kiocb = (struct kiocb) {
-                        .ki_users = 1,
-                        .ki_key = KIOCB_SYNC_KEY,
+                        .ki_users = ATOMIC_INIT(1),
+                        .ki_ctx = NULL,
                         .ki_filp = filp,
                         .ki_obj.tsk = current,
                 };
 }
 
-#define AIO_RING_MAGIC                  0xa10a10a1
-#define AIO_RING_COMPAT_FEATURES        1
-#define AIO_RING_INCOMPAT_FEATURES      0
-struct aio_ring {
-        unsigned        id;     /* kernel internal index number */
-        unsigned        nr;     /* number of io_events */
-        unsigned        head;
-        unsigned        tail;
-
-        unsigned        magic;
-        unsigned        compat_features;
-        unsigned        incompat_features;
-        unsigned        header_length;  /* size of aio_ring */
-
-
-        struct io_event         io_events[0];
-}; /* 128 bytes + ring size */
-
-#define AIO_RING_PAGES  8
-struct aio_ring_info {
-        unsigned long           mmap_base;
-        unsigned long           mmap_size;
-
-        struct page             **ring_pages;
-        spinlock_t              ring_lock;
-        long                    nr_pages;
-
-        unsigned                nr, tail;
-
-        struct page             *internal_pages[AIO_RING_PAGES];
-};
-
-static inline unsigned aio_ring_avail(struct aio_ring_info *info,
-                                        struct aio_ring *ring)
-{
-        return (ring->head + info->nr - 1 - ring->tail) % info->nr;
-}
-
-struct kioctx {
-        atomic_t                users;
-        int                     dead;
-        struct mm_struct        *mm;
-
-        /* This needs improving */
-        unsigned long           user_id;
-        struct hlist_node       list;
-
-        wait_queue_head_t       wait;
-
-        spinlock_t              ctx_lock;
-
-        int                     reqs_active;
-        struct list_head        active_reqs;    /* used for cancellation */
-        struct list_head        run_list;       /* used for kicked reqs */
-
-        /* sys_io_setup currently limits this to an unsigned int */
-        unsigned                max_reqs;
-
-        struct aio_ring_info    ring_info;
-
-        struct delayed_work     wq;
-
-        struct rcu_head         rcu_head;
-};
-
 /* prototypes */
-extern unsigned aio_max_size;
-
 #ifdef CONFIG_AIO
 extern ssize_t wait_on_sync_kiocb(struct kiocb *iocb);
-extern int aio_put_req(struct kiocb *iocb);
-extern void kick_iocb(struct kiocb *iocb);
-extern int aio_complete(struct kiocb *iocb, long res, long res2);
+extern void aio_put_req(struct kiocb *iocb);
+extern void aio_complete(struct kiocb *iocb, long res, long res2);
 struct mm_struct;
 extern void exit_aio(struct mm_struct *mm);
 extern long do_io_submit(aio_context_t ctx_id, long nr,
                          struct iocb __user *__user *iocbpp, bool compat);
+void kiocb_set_cancel_fn(struct kiocb *req, kiocb_cancel_fn *cancel);
 #else
 static inline ssize_t wait_on_sync_kiocb(struct kiocb *iocb) { return 0; }
-static inline int aio_put_req(struct kiocb *iocb) { return 0; }
-static inline void kick_iocb(struct kiocb *iocb) { }
-static inline int aio_complete(struct kiocb *iocb, long res, long res2) { return 0; }
+static inline void aio_put_req(struct kiocb *iocb) { }
+static inline void aio_complete(struct kiocb *iocb, long res, long res2) { }
 struct mm_struct;
 static inline void exit_aio(struct mm_struct *mm) { }
 static inline long do_io_submit(aio_context_t ctx_id, long nr,
                                 struct iocb __user * __user *iocbpp,
                                 bool compat) { return 0; }
+static inline void kiocb_set_cancel_fn(struct kiocb *req,
+                                       kiocb_cancel_fn *cancel) { }
 #endif /* CONFIG_AIO */
 
 static inline struct kiocb *list_kiocb(struct list_head *h)
diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
index 3bff9ce09cf7..5047355b9a0f 100644
--- a/include/linux/cgroup.h
+++ b/include/linux/cgroup.h
@@ -28,6 +28,7 @@ struct cgroup_subsys;
 struct inode;
 struct cgroup;
 struct css_id;
+struct eventfd_ctx;
 
 extern int cgroup_init_early(void);
 extern int cgroup_init(void);
diff --git a/include/linux/errno.h b/include/linux/errno.h
index f6bf082d4d4f..89627b9187f9 100644
--- a/include/linux/errno.h
+++ b/include/linux/errno.h
@@ -28,6 +28,5 @@
 #define EBADTYPE        527     /* Type not supported by server */
 #define EJUKEBOX        528     /* Request initiated, but will not complete before timeout */
 #define EIOCBQUEUED     529     /* iocb queued, will get completion event */
-#define EIOCBRETRY      530     /* iocb queued, will trigger a retry */
 
 #endif
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 3a62df310f2e..6b4890fa57e7 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -189,8 +189,7 @@ static inline struct hugetlbfs_sb_info *HUGETLBFS_SB(struct super_block *sb)
 
 extern const struct file_operations hugetlbfs_file_operations;
 extern const struct vm_operations_struct hugetlb_vm_ops;
-struct file *hugetlb_file_setup(const char *name, unsigned long addr,
-                                size_t size, vm_flags_t acct,
+struct file *hugetlb_file_setup(const char *name, size_t size, vm_flags_t acct,
                                 struct user_struct **user, int creat_flags,
                                 int page_size_log);
 
@@ -209,8 +208,8 @@ static inline int is_file_hugepages(struct file *file)
 
 #define is_file_hugepages(file)                 0
 static inline struct file *
-hugetlb_file_setup(const char *name, unsigned long addr, size_t size,
-                vm_flags_t acctflag, struct user_struct **user, int creat_flags,
+hugetlb_file_setup(const char *name, size_t size, vm_flags_t acctflag,
+                struct user_struct **user, int creat_flags,
                 int page_size_log)
 {
         return ERR_PTR(-ENOSYS);
@@ -288,6 +287,13 @@ static inline struct hstate *hstate_file(struct file *f)
         return hstate_inode(file_inode(f));
 }
 
+static inline struct hstate *hstate_sizelog(int page_size_log)
+{
+        if (!page_size_log)
+                return &default_hstate;
+        return size_to_hstate(1 << page_size_log);
+}
+
 static inline struct hstate *hstate_vma(struct vm_area_struct *vma)
 {
         return hstate_file(vma->vm_file);
@@ -352,11 +358,12 @@ static inline int hstate_index(struct hstate *h)
         return h - hstates;
 }
 
-#else
+#else   /* CONFIG_HUGETLB_PAGE */
 struct hstate {};
 #define alloc_huge_page_node(h, nid) NULL
 #define alloc_bootmem_huge_page(h) NULL
 #define hstate_file(f) NULL
+#define hstate_sizelog(s) NULL
 #define hstate_vma(v) NULL
 #define hstate_inode(i) NULL
 #define huge_page_size(h) PAGE_SIZE
@@ -371,6 +378,6 @@ static inline unsigned int pages_per_huge_page(struct hstate *h)
 }
 #define hstate_index_to_shift(index) 0
 #define hstate_index(h) 0
-#endif
+#endif  /* CONFIG_HUGETLB_PAGE */
 
 #endif /* _LINUX_HUGETLB_H */
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 1a7f19e7f1a0..e0c8528a41a4 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -951,13 +951,19 @@ void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
  * (see walk_page_range for more details)
  */
 struct mm_walk {
-        int (*pgd_entry)(pgd_t *, unsigned long, unsigned long, struct mm_walk *);
-        int (*pud_entry)(pud_t *, unsigned long, unsigned long, struct mm_walk *);
-        int (*pmd_entry)(pmd_t *, unsigned long, unsigned long, struct mm_walk *);
-        int (*pte_entry)(pte_t *, unsigned long, unsigned long, struct mm_walk *);
-        int (*pte_hole)(unsigned long, unsigned long, struct mm_walk *);
-        int (*hugetlb_entry)(pte_t *, unsigned long,
-                             unsigned long, unsigned long, struct mm_walk *);
+        int (*pgd_entry)(pgd_t *pgd, unsigned long addr,
+                         unsigned long next, struct mm_walk *walk);
+        int (*pud_entry)(pud_t *pud, unsigned long addr,
+                         unsigned long next, struct mm_walk *walk);
+        int (*pmd_entry)(pmd_t *pmd, unsigned long addr,
+                         unsigned long next, struct mm_walk *walk);
+        int (*pte_entry)(pte_t *pte, unsigned long addr,
+                         unsigned long next, struct mm_walk *walk);
+        int (*pte_hole)(unsigned long addr, unsigned long next,
+                        struct mm_walk *walk);
+        int (*hugetlb_entry)(pte_t *pte, unsigned long hmask,
+                             unsigned long addr, unsigned long next,
+                             struct mm_walk *walk);
         struct mm_struct *mm;
         void *private;
 };
diff --git a/include/linux/pid_namespace.h b/include/linux/pid_namespace.h
index 731e4ecee3bd..e2772666f004 100644
--- a/include/linux/pid_namespace.h
+++ b/include/linux/pid_namespace.h
@@ -4,6 +4,7 @@
 #include <linux/sched.h>
 #include <linux/bug.h>
 #include <linux/mm.h>
+#include <linux/workqueue.h>
 #include <linux/threads.h>
 #include <linux/nsproxy.h>
 #include <linux/kref.h>
diff --git a/include/linux/random.h b/include/linux/random.h
index 347ce553a306..3b9377d6b7a5 100644
--- a/include/linux/random.h
+++ b/include/linux/random.h
@@ -29,13 +29,6 @@ u32 prandom_u32(void);
 void prandom_bytes(void *buf, int nbytes);
 void prandom_seed(u32 seed);
 
-/*
- * These macros are preserved for backward compatibility and should be
- * removed as soon as a transition is finished.
- */
-#define random32() prandom_u32()
-#define srandom32(seed) prandom_seed(seed)
-
 u32 prandom_u32_state(struct rnd_state *);
 void prandom_bytes_state(struct rnd_state *state, void *buf, int nbytes);
 
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 4800e9d1864c..022c085ac3c5 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -313,8 +313,6 @@ extern void schedule_preempt_disabled(void);
 struct nsproxy;
 struct user_namespace;
 
-#include <linux/aio.h>
-
 #ifdef CONFIG_MMU
 extern void arch_pick_mmap_layout(struct mm_struct *mm);
 extern unsigned long
diff --git a/include/linux/wait.h b/include/linux/wait.h
index 7cb64d4b499d..ac38be2692d8 100644
--- a/include/linux/wait.h
+++ b/include/linux/wait.h
@@ -330,6 +330,92 @@ do {									\
         __ret;                                                          \
 })
 
+#define __wait_event_hrtimeout(wq, condition, timeout, state)           \
+({                                                                      \
+        int __ret = 0;                                                  \
+        DEFINE_WAIT(__wait);                                            \
+        struct hrtimer_sleeper __t;                                     \
+                                                                        \
+        hrtimer_init_on_stack(&__t.timer, CLOCK_MONOTONIC,              \
+                              HRTIMER_MODE_REL);                        \
+        hrtimer_init_sleeper(&__t, current);                            \
+        if ((timeout).tv64 != KTIME_MAX)                                \
+                hrtimer_start_range_ns(&__t.timer, timeout,             \
+                                       current->timer_slack_ns,         \
+                                       HRTIMER_MODE_REL);               \
+                                                                        \
+        for (;;) {                                                      \
+                prepare_to_wait(&wq, &__wait, state);                   \
+                if (condition)                                          \
+                        break;                                          \
+                if (state == TASK_INTERRUPTIBLE &&                      \
+                    signal_pending(current)) {                          \
+                        __ret = -ERESTARTSYS;                           \
+                        break;                                          \
+                }                                                       \
+                if (!__t.task) {                                        \
+                        __ret = -ETIME;                                 \
+                        break;                                          \
+                }                                                       \
+                schedule();                                             \
+        }                                                               \
+                                                                        \
+        hrtimer_cancel(&__t.timer);                                     \
+        destroy_hrtimer_on_stack(&__t.timer);                           \
+        finish_wait(&wq, &__wait);                                      \
+        __ret;                                                          \
+})
+
+/**
+ * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
+ * @wq: the waitqueue to wait on
+ * @condition: a C expression for the event to wait for
+ * @timeout: timeout, as a ktime_t
+ *
+ * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
+ * @condition evaluates to true or a signal is received.
+ * The @condition is checked each time the waitqueue @wq is woken up.
+ *
+ * wake_up() has to be called after changing any variable that could
+ * change the result of the wait condition.
+ *
+ * The function returns 0 if @condition became true, or -ETIME if the timeout
+ * elapsed.
+ */
+#define wait_event_hrtimeout(wq, condition, timeout)                    \
+({                                                                      \
+        int __ret = 0;                                                  \
+        if (!(condition))                                               \
+                __ret = __wait_event_hrtimeout(wq, condition, timeout,  \
+                                               TASK_UNINTERRUPTIBLE);   \
+        __ret;                                                          \
+})
+
+/**
+ * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
+ * @wq: the waitqueue to wait on
+ * @condition: a C expression for the event to wait for
+ * @timeout: timeout, as a ktime_t
+ *
+ * The process is put to sleep (TASK_INTERRUPTIBLE) until the
+ * @condition evaluates to true or a signal is received.
+ * The @condition is checked each time the waitqueue @wq is woken up.
+ *
+ * wake_up() has to be called after changing any variable that could
+ * change the result of the wait condition.
+ *
+ * The function returns 0 if @condition became true, -ERESTARTSYS if it was
+ * interrupted by a signal, or -ETIME if the timeout elapsed.
+ */
+#define wait_event_interruptible_hrtimeout(wq, condition, timeout)      \
+({                                                                      \
+        long __ret = 0;                                                 \
+        if (!(condition))                                               \
+                __ret = __wait_event_hrtimeout(wq, condition, timeout,  \
+                                               TASK_INTERRUPTIBLE);     \
+        __ret;                                                          \
+})
+
 #define __wait_event_interruptible_exclusive(wq, condition, ret)        \
 do {                                                                    \
         DEFINE_WAIT(__wait);                                            \
diff --git a/include/linux/writeback.h b/include/linux/writeback.h
index 9a9367c0c076..579a5007c696 100644
--- a/include/linux/writeback.h
+++ b/include/linux/writeback.h
@@ -5,6 +5,7 @@
 #define WRITEBACK_H
 
 #include <linux/sched.h>
+#include <linux/workqueue.h>
 #include <linux/fs.h>
 
 DECLARE_PER_CPU(int, dirty_throttle_leaks);