1 files changed, 25 insertions, 13 deletions
diff --git a/fs/direct-io.c b/fs/direct-io.c
index b912270942fa..9f34bb9b1ecb 100644
--- a/fs/direct-io.c
+++ b/fs/direct-io.c
@@ -104,6 +104,18 @@ struct dio {
        unsigned cur_page_len;          /* Nr of bytes at cur_page_offset */
        sector_t cur_page_block;        /* Where it starts */
+        /* BIO completion state */
+        spinlock_t bio_lock;            /* protects BIO fields below */
+        unsigned long refcount;         /* direct_io_worker() and bios */
+        struct bio *bio_list;           /* singly linked via bi_private */
+        struct task_struct *waiter;     /* waiting task (NULL if none) */
+        /* AIO related stuff */
+        struct kiocb *iocb;             /* kiocb */
+        int is_async;                   /* is IO async ? */
+        int io_error;                   /* IO error in completion path */
+        ssize_t result;                 /* IO result */
        /*
         * Page fetching state. These variables belong to dio_refill_pages().
         */
@@ -115,22 +127,16 @@ struct dio {
         * Page queue.  These variables belong to dio_refill_pages() and
         * dio_get_page().
         */
-        struct page *pages[DIO_PAGES];  /* page buffer */
        unsigned head;                  /* next page to process */
        unsigned tail;                  /* last valid page + 1 */
        int page_errors;                /* errno from get_user_pages() */
-        /* BIO completion state */
+        /*
-        spinlock_t bio_lock;            /* protects BIO fields below */
+         * pages[] (and any fields placed after it) are not zeroed out at
-        unsigned long refcount;         /* direct_io_worker() and bios */
+         * allocation time.  Don't add new fields after pages[] unless you
-        struct bio *bio_list;           /* singly linked via bi_private */
+         * wish that they not be zeroed.
-        struct task_struct *waiter;     /* waiting task (NULL if none) */
+         */
+        struct page *pages[DIO_PAGES];  /* page buffer */
-        /* AIO related stuff */
-        struct kiocb *iocb;             /* kiocb */
-        int is_async;                   /* is IO async ? */
-        int io_error;                   /* IO error in completion path */
-        ssize_t result;                 /* IO result */
 };
 /*
@@ -1151,10 +1157,16 @@ __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode,
                }
        }
-        dio = kzalloc(sizeof(*dio), GFP_KERNEL);
+        dio = kmalloc(sizeof(*dio), GFP_KERNEL);
        retval = -ENOMEM;
        if (!dio)
                goto out;
+        /*
+         * Believe it or not, zeroing out the page array caused a .5%
+         * performance regression in a database benchmark.  So, we take
+         * care to only zero out what's needed.
+         */
+        memset(dio, 0, offsetof(struct dio, pages));
        /*
         * For block device access DIO_NO_LOCKING is used,

diff --git a/fs/direct-io.c b/fs/direct-io.c index b912270942fa..9f34bb9b1ecb 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c
@@ -104,6 +104,18 @@ struct dio {
104	unsigned cur_page_len; /* Nr of bytes at cur_page_offset */	104	unsigned cur_page_len; /* Nr of bytes at cur_page_offset */
105	sector_t cur_page_block; /* Where it starts */	105	sector_t cur_page_block; /* Where it starts */
106		106
		107	/* BIO completion state */
		108	spinlock_t bio_lock; /* protects BIO fields below */
		109	unsigned long refcount; /* direct_io_worker() and bios */
		110	struct bio bio_list; / singly linked via bi_private */
		111	struct task_struct waiter; / waiting task (NULL if none) */
		112
		113	/* AIO related stuff */
		114	struct kiocb iocb; / kiocb */
		115	int is_async; /* is IO async ? */
		116	int io_error; /* IO error in completion path */
		117	ssize_t result; /* IO result */
		118
107	/*	119	/*
108	* Page fetching state. These variables belong to dio_refill_pages().	120	* Page fetching state. These variables belong to dio_refill_pages().
109	*/	121	*/
@@ -115,22 +127,16 @@ struct dio {
115	* Page queue. These variables belong to dio_refill_pages() and	127	* Page queue. These variables belong to dio_refill_pages() and
116	* dio_get_page().	128	* dio_get_page().
117	*/	129	*/
118	struct page pages[DIO_PAGES]; / page buffer */
119	unsigned head; /* next page to process */	130	unsigned head; /* next page to process */
120	unsigned tail; /* last valid page + 1 */	131	unsigned tail; /* last valid page + 1 */
121	int page_errors; /* errno from get_user_pages() */	132	int page_errors; /* errno from get_user_pages() */
122		133
123	/* BIO completion state */	134	/*
124	spinlock_t bio_lock; /* protects BIO fields below */	135	* pages[] (and any fields placed after it) are not zeroed out at
125	unsigned long refcount; /* direct_io_worker() and bios */	136	* allocation time. Don't add new fields after pages[] unless you
126	struct bio bio_list; / singly linked via bi_private */	137	* wish that they not be zeroed.
127	struct task_struct waiter; / waiting task (NULL if none) */	138	*/
128		139	struct page pages[DIO_PAGES]; / page buffer */
129	/* AIO related stuff */
130	struct kiocb iocb; / kiocb */
131	int is_async; /* is IO async ? */
132	int io_error; /* IO error in completion path */
133	ssize_t result; /* IO result */
134	};	140	};
135		141
136	/*	142	/*
@@ -1151,10 +1157,16 @@ __blockdev_direct_IO(int rw, struct kiocb iocb, struct inode inode,
1151	}	1157	}
1152	}	1158	}
1153		1159
1154	dio = kzalloc(sizeof(*dio), GFP_KERNEL);	1160	dio = kmalloc(sizeof(*dio), GFP_KERNEL);
1155	retval = -ENOMEM;	1161	retval = -ENOMEM;
1156	if (!dio)	1162	if (!dio)
1157	goto out;	1163	goto out;
		1164	/*
		1165	* Believe it or not, zeroing out the page array caused a .5%
		1166	* performance regression in a database benchmark. So, we take
		1167	* care to only zero out what's needed.
		1168	*/
		1169	memset(dio, 0, offsetof(struct dio, pages));
1158		1170
1159	/*	1171	/*
1160	* For block device access DIO_NO_LOCKING is used,	1172	* For block device access DIO_NO_LOCKING is used,