Linux-2.6.12-rc2v2.6.12-rc2

Initial git repository build. I'm not bothering with the full history,
even though we have it. We can create a separate "historical" git
archive of that later if we want to, and in the meantime it's about
3.2GB when imported into git - space that would just make the early
git days unnecessarily complicated, when we don't have a lot of good
infrastructure for it.

Let it rip!

54 files changed, 28483 insertions, 0 deletions

diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
new file mode 100644
index 000000000000..ac43f98062fd
--- /dev/null
+++ b/drivers/md/Kconfig
@@ -0,0 +1,240 @@
+#
+# Block device driver configuration
+#
+
+menu "Multi-device support (RAID and LVM)"
+
+config MD
+        bool "Multiple devices driver support (RAID and LVM)"
+        help
+          Support multiple physical spindles through a single logical device.
+          Required for RAID and logical volume management.
+
+config BLK_DEV_MD
+        tristate "RAID support"
+        depends on MD
+        ---help---
+          This driver lets you combine several hard disk partitions into one
+          logical block device. This can be used to simply append one
+          partition to another one or to combine several redundant hard disks
+          into a RAID1/4/5 device so as to provide protection against hard
+          disk failures. This is called "Software RAID" since the combining of
+          the partitions is done by the kernel. "Hardware RAID" means that the
+          combining is done by a dedicated controller; if you have such a
+          controller, you do not need to say Y here.
+
+          More information about Software RAID on Linux is contained in the
+          Software RAID mini-HOWTO, available from
+          <http://www.tldp.org/docs.html#howto>. There you will also learn
+          where to get the supporting user space utilities raidtools.
+
+          If unsure, say N.
+
+config MD_LINEAR
+        tristate "Linear (append) mode"
+        depends on BLK_DEV_MD
+        ---help---
+          If you say Y here, then your multiple devices driver will be able to
+          use the so-called linear mode, i.e. it will combine the hard disk
+          partitions by simply appending one to the other.
+
+          To compile this as a module, choose M here: the module
+          will be called linear.
+
+          If unsure, say Y.
+
+config MD_RAID0
+        tristate "RAID-0 (striping) mode"
+        depends on BLK_DEV_MD
+        ---help---
+          If you say Y here, then your multiple devices driver will be able to
+          use the so-called raid0 mode, i.e. it will combine the hard disk
+          partitions into one logical device in such a fashion as to fill them
+          up evenly, one chunk here and one chunk there. This will increase
+          the throughput rate if the partitions reside on distinct disks.
+
+          Information about Software RAID on Linux is contained in the
+          Software-RAID mini-HOWTO, available from
+          <http://www.tldp.org/docs.html#howto>. There you will also
+          learn where to get the supporting user space utilities raidtools.
+
+          To compile this as a module, choose M here: the module
+          will be called raid0.
+
+          If unsure, say Y.
+
+config MD_RAID1
+        tristate "RAID-1 (mirroring) mode"
+        depends on BLK_DEV_MD
+        ---help---
+          A RAID-1 set consists of several disk drives which are exact copies
+          of each other.  In the event of a mirror failure, the RAID driver
+          will continue to use the operational mirrors in the set, providing
+          an error free MD (multiple device) to the higher levels of the
+          kernel.  In a set with N drives, the available space is the capacity
+          of a single drive, and the set protects against a failure of (N - 1)
+          drives.
+
+          Information about Software RAID on Linux is contained in the
+          Software-RAID mini-HOWTO, available from
+          <http://www.tldp.org/docs.html#howto>.  There you will also
+          learn where to get the supporting user space utilities raidtools.
+
+          If you want to use such a RAID-1 set, say Y.  To compile this code
+          as a module, choose M here: the module will be called raid1.
+
+          If unsure, say Y.
+
+config MD_RAID10
+        tristate "RAID-10 (mirrored striping) mode (EXPERIMENTAL)"
+        depends on BLK_DEV_MD && EXPERIMENTAL
+        ---help---
+          RAID-10 provides a combination of striping (RAID-0) and
+          mirroring (RAID-1) with easier configuration and more flexable
+          layout.
+          Unlike RAID-0, but like RAID-1, RAID-10 requires all devices to
+          be the same size (or at least, only as much as the smallest device
+          will be used).
+          RAID-10 provides a variety of layouts that provide different levels
+          of redundancy and performance.
+
+          RAID-10 requires mdadm-1.7.0 or later, available at:
+
+          ftp://ftp.kernel.org/pub/linux/utils/raid/mdadm/
+
+          If unsure, say Y.
+
+config MD_RAID5
+        tristate "RAID-4/RAID-5 mode"
+        depends on BLK_DEV_MD
+        ---help---
+          A RAID-5 set of N drives with a capacity of C MB per drive provides
+          the capacity of C * (N - 1) MB, and protects against a failure
+          of a single drive. For a given sector (row) number, (N - 1) drives
+          contain data sectors, and one drive contains the parity protection.
+          For a RAID-4 set, the parity blocks are present on a single drive,
+          while a RAID-5 set distributes the parity across the drives in one
+          of the available parity distribution methods.
+
+          Information about Software RAID on Linux is contained in the
+          Software-RAID mini-HOWTO, available from
+          <http://www.tldp.org/docs.html#howto>. There you will also
+          learn where to get the supporting user space utilities raidtools.
+
+          If you want to use such a RAID-4/RAID-5 set, say Y.  To
+          compile this code as a module, choose M here: the module
+          will be called raid5.
+
+          If unsure, say Y.
+
+config MD_RAID6
+        tristate "RAID-6 mode"
+        depends on BLK_DEV_MD
+        ---help---
+          A RAID-6 set of N drives with a capacity of C MB per drive
+          provides the capacity of C * (N - 2) MB, and protects
+          against a failure of any two drives. For a given sector
+          (row) number, (N - 2) drives contain data sectors, and two
+          drives contains two independent redundancy syndromes.  Like
+          RAID-5, RAID-6 distributes the syndromes across the drives
+          in one of the available parity distribution methods.
+
+          RAID-6 requires mdadm-1.5.0 or later, available at:
+
+          ftp://ftp.kernel.org/pub/linux/utils/raid/mdadm/
+
+          If you want to use such a RAID-6 set, say Y.  To compile
+          this code as a module, choose M here: the module will be
+          called raid6.
+
+          If unsure, say Y.
+
+config MD_MULTIPATH
+        tristate "Multipath I/O support"
+        depends on BLK_DEV_MD
+        help
+          Multipath-IO is the ability of certain devices to address the same
+          physical disk over multiple 'IO paths'. The code ensures that such
+          paths can be defined and handled at runtime, and ensures that a
+          transparent failover to the backup path(s) happens if a IO errors
+          arrives on the primary path.
+
+          If unsure, say N.
+
+config MD_FAULTY
+        tristate "Faulty test module for MD"
+        depends on BLK_DEV_MD
+        help
+          The "faulty" module allows for a block device that occasionally returns
+          read or write errors.  It is useful for testing.
+
+          In unsure, say N.
+
+config BLK_DEV_DM
+        tristate "Device mapper support"
+        depends on MD
+        ---help---
+          Device-mapper is a low level volume manager.  It works by allowing
+          people to specify mappings for ranges of logical sectors.  Various
+          mapping types are available, in addition people may write their own
+          modules containing custom mappings if they wish.
+
+          Higher level volume managers such as LVM2 use this driver.
+
+          To compile this as a module, choose M here: the module will be
+          called dm-mod.
+
+          If unsure, say N.
+
+config DM_CRYPT
+        tristate "Crypt target support"
+        depends on BLK_DEV_DM && EXPERIMENTAL
+        select CRYPTO
+        ---help---
+          This device-mapper target allows you to create a device that
+          transparently encrypts the data on it. You'll need to activate
+          the ciphers you're going to use in the cryptoapi configuration.
+
+          Information on how to use dm-crypt can be found on
+
+          <http://www.saout.de/misc/dm-crypt/>
+
+          To compile this code as a module, choose M here: the module will
+          be called dm-crypt.
+
+          If unsure, say N.
+
+config DM_SNAPSHOT
+       tristate "Snapshot target (EXPERIMENTAL)"
+       depends on BLK_DEV_DM && EXPERIMENTAL
+       ---help---
+         Allow volume managers to take writeable snapshots of a device.
+
+config DM_MIRROR
+       tristate "Mirror target (EXPERIMENTAL)"
+       depends on BLK_DEV_DM && EXPERIMENTAL
+       ---help---
+         Allow volume managers to mirror logical volumes, also
+         needed for live data migration tools such as 'pvmove'.
+
+config DM_ZERO
+        tristate "Zero target (EXPERIMENTAL)"
+        depends on BLK_DEV_DM && EXPERIMENTAL
+        ---help---
+          A target that discards writes, and returns all zeroes for
+          reads.  Useful in some recovery situations.
+
+config DM_MULTIPATH
+        tristate "Multipath target (EXPERIMENTAL)"
+        depends on BLK_DEV_DM && EXPERIMENTAL
+        ---help---
+          Allow volume managers to support multipath hardware.
+
+config DM_MULTIPATH_EMC
+        tristate "EMC CX/AX multipath support (EXPERIMENTAL)"
+        depends on DM_MULTIPATH && BLK_DEV_DM && EXPERIMENTAL
+        ---help---
+          Multipath support for EMC CX/AX series hardware.
+
+endmenu
+
diff --git a/drivers/md/Makefile b/drivers/md/Makefile
new file mode 100644
index 000000000000..90de9c146a5f
--- /dev/null
+++ b/drivers/md/Makefile
@@ -0,0 +1,107 @@
+#
+# Makefile for the kernel software RAID and LVM drivers.
+#
+
+dm-mod-objs     := dm.o dm-table.o dm-target.o dm-linear.o dm-stripe.o \
+                   dm-ioctl.o dm-io.o kcopyd.o
+dm-multipath-objs := dm-hw-handler.o dm-path-selector.o dm-mpath.o
+dm-snapshot-objs := dm-snap.o dm-exception-store.o
+dm-mirror-objs  := dm-log.o dm-raid1.o
+raid6-objs      := raid6main.o raid6algos.o raid6recov.o raid6tables.o \
+                   raid6int1.o raid6int2.o raid6int4.o \
+                   raid6int8.o raid6int16.o raid6int32.o \
+                   raid6altivec1.o raid6altivec2.o raid6altivec4.o \
+                   raid6altivec8.o \
+                   raid6mmx.o raid6sse1.o raid6sse2.o
+hostprogs-y     := mktables
+
+# Note: link order is important.  All raid personalities
+# and xor.o must come before md.o, as they each initialise 
+# themselves, and md.o may use the personalities when it 
+# auto-initialised.
+
+obj-$(CONFIG_MD_LINEAR)         += linear.o
+obj-$(CONFIG_MD_RAID0)          += raid0.o
+obj-$(CONFIG_MD_RAID1)          += raid1.o
+obj-$(CONFIG_MD_RAID10)         += raid10.o
+obj-$(CONFIG_MD_RAID5)          += raid5.o xor.o
+obj-$(CONFIG_MD_RAID6)          += raid6.o xor.o
+obj-$(CONFIG_MD_MULTIPATH)      += multipath.o
+obj-$(CONFIG_MD_FAULTY)         += faulty.o
+obj-$(CONFIG_BLK_DEV_MD)        += md.o
+obj-$(CONFIG_BLK_DEV_DM)        += dm-mod.o
+obj-$(CONFIG_DM_CRYPT)          += dm-crypt.o
+obj-$(CONFIG_DM_MULTIPATH)      += dm-multipath.o dm-round-robin.o
+obj-$(CONFIG_DM_MULTIPATH_EMC)  += dm-emc.o
+obj-$(CONFIG_DM_SNAPSHOT)       += dm-snapshot.o
+obj-$(CONFIG_DM_MIRROR)         += dm-mirror.o
+obj-$(CONFIG_DM_ZERO)           += dm-zero.o
+
+quiet_cmd_unroll = UNROLL  $@
+      cmd_unroll = $(PERL) $(srctree)/$(src)/unroll.pl $(UNROLL) \
+                   < $< > $@ || ( rm -f $@ && exit 1 )
+
+ifeq ($(CONFIG_ALTIVEC),y)
+altivec_flags := -maltivec -mabi=altivec
+endif
+
+targets += raid6int1.c
+$(obj)/raid6int1.c:   UNROLL := 1
+$(obj)/raid6int1.c:   $(src)/raid6int.uc $(src)/unroll.pl FORCE
+        $(call if_changed,unroll)
+
+targets += raid6int2.c
+$(obj)/raid6int2.c:   UNROLL := 2
+$(obj)/raid6int2.c:   $(src)/raid6int.uc $(src)/unroll.pl FORCE
+        $(call if_changed,unroll)
+
+targets += raid6int4.c
+$(obj)/raid6int4.c:   UNROLL := 4
+$(obj)/raid6int4.c:   $(src)/raid6int.uc $(src)/unroll.pl FORCE
+        $(call if_changed,unroll)
+
+targets += raid6int8.c
+$(obj)/raid6int8.c:   UNROLL := 8
+$(obj)/raid6int8.c:   $(src)/raid6int.uc $(src)/unroll.pl FORCE
+        $(call if_changed,unroll)
+
+targets += raid6int16.c
+$(obj)/raid6int16.c:  UNROLL := 16
+$(obj)/raid6int16.c:  $(src)/raid6int.uc $(src)/unroll.pl FORCE
+        $(call if_changed,unroll)
+
+targets += raid6int32.c
+$(obj)/raid6int32.c:  UNROLL := 32
+$(obj)/raid6int32.c:  $(src)/raid6int.uc $(src)/unroll.pl FORCE
+        $(call if_changed,unroll)
+
+CFLAGS_raid6altivec1.o += $(altivec_flags)
+targets += raid6altivec1.c
+$(obj)/raid6altivec1.c:   UNROLL := 1
+$(obj)/raid6altivec1.c:   $(src)/raid6altivec.uc $(src)/unroll.pl FORCE
+        $(call if_changed,unroll)
+
+CFLAGS_raid6altivec2.o += $(altivec_flags)
+targets += raid6altivec2.c
+$(obj)/raid6altivec2.c:   UNROLL := 2
+$(obj)/raid6altivec2.c:   $(src)/raid6altivec.uc $(src)/unroll.pl FORCE
+        $(call if_changed,unroll)
+
+CFLAGS_raid6altivec4.o += $(altivec_flags)
+targets += raid6altivec4.c
+$(obj)/raid6altivec4.c:   UNROLL := 4
+$(obj)/raid6altivec4.c:   $(src)/raid6altivec.uc $(src)/unroll.pl FORCE
+        $(call if_changed,unroll)
+
+CFLAGS_raid6altivec8.o += $(altivec_flags)
+targets += raid6altivec8.c
+$(obj)/raid6altivec8.c:   UNROLL := 8
+$(obj)/raid6altivec8.c:   $(src)/raid6altivec.uc $(src)/unroll.pl FORCE
+        $(call if_changed,unroll)
+
+quiet_cmd_mktable = TABLE   $@
+      cmd_mktable = $(obj)/mktables > $@ || ( rm -f $@ && exit 1 )
+
+targets += raid6tables.c
+$(obj)/raid6tables.c: $(obj)/mktables FORCE
+        $(call if_changed,mktable)
diff --git a/drivers/md/dm-bio-list.h b/drivers/md/dm-bio-list.h
new file mode 100644
index 000000000000..bc021e1fd4d1
--- /dev/null
+++ b/drivers/md/dm-bio-list.h
@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2004 Red Hat UK Ltd.
+ *
+ * This file is released under the GPL.
+ */
+
+#ifndef DM_BIO_LIST_H
+#define DM_BIO_LIST_H
+
+#include <linux/bio.h>
+
+struct bio_list {
+        struct bio *head;
+        struct bio *tail;
+};
+
+static inline void bio_list_init(struct bio_list *bl)
+{
+        bl->head = bl->tail = NULL;
+}
+
+static inline void bio_list_add(struct bio_list *bl, struct bio *bio)
+{
+        bio->bi_next = NULL;
+
+        if (bl->tail)
+                bl->tail->bi_next = bio;
+        else
+                bl->head = bio;
+
+        bl->tail = bio;
+}
+
+static inline void bio_list_merge(struct bio_list *bl, struct bio_list *bl2)
+{
+        if (bl->tail)
+                bl->tail->bi_next = bl2->head;
+        else
+                bl->head = bl2->head;
+
+        bl->tail = bl2->tail;
+}
+
+static inline struct bio *bio_list_pop(struct bio_list *bl)
+{
+        struct bio *bio = bl->head;
+
+        if (bio) {
+                bl->head = bl->head->bi_next;
+                if (!bl->head)
+                        bl->tail = NULL;
+
+                bio->bi_next = NULL;
+        }
+
+        return bio;
+}
+
+static inline struct bio *bio_list_get(struct bio_list *bl)
+{
+        struct bio *bio = bl->head;
+
+        bl->head = bl->tail = NULL;
+
+        return bio;
+}
+
+#endif
diff --git a/drivers/md/dm-bio-record.h b/drivers/md/dm-bio-record.h
new file mode 100644
index 000000000000..d3ec217847d6
--- /dev/null
+++ b/drivers/md/dm-bio-record.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ */
+
+#ifndef DM_BIO_RECORD_H
+#define DM_BIO_RECORD_H
+
+#include <linux/bio.h>
+
+/*
+ * There are lots of mutable fields in the bio struct that get
+ * changed by the lower levels of the block layer.  Some targets,
+ * such as multipath, may wish to resubmit a bio on error.  The
+ * functions in this file help the target record and restore the
+ * original bio state.
+ */
+struct dm_bio_details {
+        sector_t bi_sector;
+        struct block_device *bi_bdev;
+        unsigned int bi_size;
+        unsigned short bi_idx;
+        unsigned long bi_flags;
+};
+
+static inline void dm_bio_record(struct dm_bio_details *bd, struct bio *bio)
+{
+        bd->bi_sector = bio->bi_sector;
+        bd->bi_bdev = bio->bi_bdev;
+        bd->bi_size = bio->bi_size;
+        bd->bi_idx = bio->bi_idx;
+        bd->bi_flags = bio->bi_flags;
+}
+
+static inline void dm_bio_restore(struct dm_bio_details *bd, struct bio *bio)
+{
+        bio->bi_sector = bd->bi_sector;
+        bio->bi_bdev = bd->bi_bdev;
+        bio->bi_size = bd->bi_size;
+        bio->bi_idx = bd->bi_idx;
+        bio->bi_flags = bd->bi_flags;
+}
+
+#endif
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
new file mode 100644
index 000000000000..77619a56e2bf
--- /dev/null
+++ b/drivers/md/dm-crypt.c
@@ -0,0 +1,977 @@
+/*
+ * Copyright (C) 2003 Christophe Saout <christophe@saout.de>
+ * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
+ *
+ * This file is released under the GPL.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/bio.h>
+#include <linux/blkdev.h>
+#include <linux/mempool.h>
+#include <linux/slab.h>
+#include <linux/crypto.h>
+#include <linux/workqueue.h>
+#include <asm/atomic.h>
+#include <asm/scatterlist.h>
+#include <asm/page.h>
+
+#include "dm.h"
+
+#define PFX     "crypt: "
+
+/*
+ * per bio private data
+ */
+struct crypt_io {
+        struct dm_target *target;
+        struct bio *bio;
+        struct bio *first_clone;
+        struct work_struct work;
+        atomic_t pending;
+        int error;
+};
+
+/*
+ * context holding the current state of a multi-part conversion
+ */
+struct convert_context {
+        struct bio *bio_in;
+        struct bio *bio_out;
+        unsigned int offset_in;
+        unsigned int offset_out;
+        unsigned int idx_in;
+        unsigned int idx_out;
+        sector_t sector;
+        int write;
+};
+
+struct crypt_config;
+
+struct crypt_iv_operations {
+        int (*ctr)(struct crypt_config *cc, struct dm_target *ti,
+                   const char *opts);
+        void (*dtr)(struct crypt_config *cc);
+        const char *(*status)(struct crypt_config *cc);
+        int (*generator)(struct crypt_config *cc, u8 *iv, sector_t sector);
+};
+
+/*
+ * Crypt: maps a linear range of a block device
+ * and encrypts / decrypts at the same time.
+ */
+struct crypt_config {
+        struct dm_dev *dev;
+        sector_t start;
+
+        /*
+         * pool for per bio private data and
+         * for encryption buffer pages
+         */
+        mempool_t *io_pool;
+        mempool_t *page_pool;
+
+        /*
+         * crypto related data
+         */
+        struct crypt_iv_operations *iv_gen_ops;
+        char *iv_mode;
+        void *iv_gen_private;
+        sector_t iv_offset;
+        unsigned int iv_size;
+
+        struct crypto_tfm *tfm;
+        unsigned int key_size;
+        u8 key[0];
+};
+
+#define MIN_IOS        256
+#define MIN_POOL_PAGES 32
+#define MIN_BIO_PAGES  8
+
+static kmem_cache_t *_crypt_io_pool;
+
+/*
+ * Mempool alloc and free functions for the page
+ */
+static void *mempool_alloc_page(unsigned int __nocast gfp_mask, void *data)
+{
+        return alloc_page(gfp_mask);
+}
+
+static void mempool_free_page(void *page, void *data)
+{
+        __free_page(page);
+}
+
+
+/*
+ * Different IV generation algorithms:
+ *
+ * plain: the initial vector is the 32-bit low-endian version of the sector
+ *        number, padded with zeros if neccessary.
+ *
+ * ess_iv: "encrypted sector|salt initial vector", the sector number is
+ *         encrypted with the bulk cipher using a salt as key. The salt
+ *         should be derived from the bulk cipher's key via hashing.
+ *
+ * plumb: unimplemented, see:
+ * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454
+ */
+
+static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
+{
+        memset(iv, 0, cc->iv_size);
+        *(u32 *)iv = cpu_to_le32(sector & 0xffffffff);
+
+        return 0;
+}
+
+static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
+                              const char *opts)
+{
+        struct crypto_tfm *essiv_tfm;
+        struct crypto_tfm *hash_tfm;
+        struct scatterlist sg;
+        unsigned int saltsize;
+        u8 *salt;
+
+        if (opts == NULL) {
+                ti->error = PFX "Digest algorithm missing for ESSIV mode";
+                return -EINVAL;
+        }
+
+        /* Hash the cipher key with the given hash algorithm */
+        hash_tfm = crypto_alloc_tfm(opts, 0);
+        if (hash_tfm == NULL) {
+                ti->error = PFX "Error initializing ESSIV hash";
+                return -EINVAL;
+        }
+
+        if (crypto_tfm_alg_type(hash_tfm) != CRYPTO_ALG_TYPE_DIGEST) {
+                ti->error = PFX "Expected digest algorithm for ESSIV hash";
+                crypto_free_tfm(hash_tfm);
+                return -EINVAL;
+        }
+
+        saltsize = crypto_tfm_alg_digestsize(hash_tfm);
+        salt = kmalloc(saltsize, GFP_KERNEL);
+        if (salt == NULL) {
+                ti->error = PFX "Error kmallocing salt storage in ESSIV";
+                crypto_free_tfm(hash_tfm);
+                return -ENOMEM;
+        }
+
+        sg.page = virt_to_page(cc->key);
+        sg.offset = offset_in_page(cc->key);
+        sg.length = cc->key_size;
+        crypto_digest_digest(hash_tfm, &sg, 1, salt);
+        crypto_free_tfm(hash_tfm);
+
+        /* Setup the essiv_tfm with the given salt */
+        essiv_tfm = crypto_alloc_tfm(crypto_tfm_alg_name(cc->tfm),
+                                     CRYPTO_TFM_MODE_ECB);
+        if (essiv_tfm == NULL) {
+                ti->error = PFX "Error allocating crypto tfm for ESSIV";
+                kfree(salt);
+                return -EINVAL;
+        }
+        if (crypto_tfm_alg_blocksize(essiv_tfm)
+            != crypto_tfm_alg_ivsize(cc->tfm)) {
+                ti->error = PFX "Block size of ESSIV cipher does "
+                                "not match IV size of block cipher";
+                crypto_free_tfm(essiv_tfm);
+                kfree(salt);
+                return -EINVAL;
+        }
+        if (crypto_cipher_setkey(essiv_tfm, salt, saltsize) < 0) {
+                ti->error = PFX "Failed to set key for ESSIV cipher";
+                crypto_free_tfm(essiv_tfm);
+                kfree(salt);
+                return -EINVAL;
+        }
+        kfree(salt);
+
+        cc->iv_gen_private = (void *)essiv_tfm;
+        return 0;
+}
+
+static void crypt_iv_essiv_dtr(struct crypt_config *cc)
+{
+        crypto_free_tfm((struct crypto_tfm *)cc->iv_gen_private);
+        cc->iv_gen_private = NULL;
+}
+
+static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
+{
+        struct scatterlist sg = { NULL, };
+
+        memset(iv, 0, cc->iv_size);
+        *(u64 *)iv = cpu_to_le64(sector);
+
+        sg.page = virt_to_page(iv);
+        sg.offset = offset_in_page(iv);
+        sg.length = cc->iv_size;
+        crypto_cipher_encrypt((struct crypto_tfm *)cc->iv_gen_private,
+                              &sg, &sg, cc->iv_size);
+
+        return 0;
+}
+
+static struct crypt_iv_operations crypt_iv_plain_ops = {
+        .generator = crypt_iv_plain_gen
+};
+
+static struct crypt_iv_operations crypt_iv_essiv_ops = {
+        .ctr       = crypt_iv_essiv_ctr,
+        .dtr       = crypt_iv_essiv_dtr,
+        .generator = crypt_iv_essiv_gen
+};
+
+
+static inline int
+crypt_convert_scatterlist(struct crypt_config *cc, struct scatterlist *out,
+                          struct scatterlist *in, unsigned int length,
+                          int write, sector_t sector)
+{
+        u8 iv[cc->iv_size];
+        int r;
+
+        if (cc->iv_gen_ops) {
+                r = cc->iv_gen_ops->generator(cc, iv, sector);
+                if (r < 0)
+                        return r;
+
+                if (write)
+                        r = crypto_cipher_encrypt_iv(cc->tfm, out, in, length, iv);
+                else
+                        r = crypto_cipher_decrypt_iv(cc->tfm, out, in, length, iv);
+        } else {
+                if (write)
+                        r = crypto_cipher_encrypt(cc->tfm, out, in, length);
+                else
+                        r = crypto_cipher_decrypt(cc->tfm, out, in, length);
+        }
+
+        return r;
+}
+
+static void
+crypt_convert_init(struct crypt_config *cc, struct convert_context *ctx,
+                   struct bio *bio_out, struct bio *bio_in,
+                   sector_t sector, int write)
+{
+        ctx->bio_in = bio_in;
+        ctx->bio_out = bio_out;
+        ctx->offset_in = 0;
+        ctx->offset_out = 0;
+        ctx->idx_in = bio_in ? bio_in->bi_idx : 0;
+        ctx->idx_out = bio_out ? bio_out->bi_idx : 0;
+        ctx->sector = sector + cc->iv_offset;
+        ctx->write = write;
+}
+
+/*
+ * Encrypt / decrypt data from one bio to another one (can be the same one)
+ */
+static int crypt_convert(struct crypt_config *cc,
+                         struct convert_context *ctx)
+{
+        int r = 0;
+
+        while(ctx->idx_in < ctx->bio_in->bi_vcnt &&
+              ctx->idx_out < ctx->bio_out->bi_vcnt) {
+                struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, ctx->idx_in);
+                struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, ctx->idx_out);
+                struct scatterlist sg_in = {
+                        .page = bv_in->bv_page,
+                        .offset = bv_in->bv_offset + ctx->offset_in,
+                        .length = 1 << SECTOR_SHIFT
+                };
+                struct scatterlist sg_out = {
+                        .page = bv_out->bv_page,
+                        .offset = bv_out->bv_offset + ctx->offset_out,
+                        .length = 1 << SECTOR_SHIFT
+                };
+
+                ctx->offset_in += sg_in.length;
+                if (ctx->offset_in >= bv_in->bv_len) {
+                        ctx->offset_in = 0;
+                        ctx->idx_in++;
+                }
+
+                ctx->offset_out += sg_out.length;
+                if (ctx->offset_out >= bv_out->bv_len) {
+                        ctx->offset_out = 0;
+                        ctx->idx_out++;
+                }
+
+                r = crypt_convert_scatterlist(cc, &sg_out, &sg_in, sg_in.length,
+                                              ctx->write, ctx->sector);
+                if (r < 0)
+                        break;
+
+                ctx->sector++;
+        }
+
+        return r;
+}
+
+/*
+ * Generate a new unfragmented bio with the given size
+ * This should never violate the device limitations
+ * May return a smaller bio when running out of pages
+ */
+static struct bio *
+crypt_alloc_buffer(struct crypt_config *cc, unsigned int size,
+                   struct bio *base_bio, unsigned int *bio_vec_idx)
+{
+        struct bio *bio;
+        unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
+        int gfp_mask = GFP_NOIO | __GFP_HIGHMEM;
+        unsigned long flags = current->flags;
+        unsigned int i;
+
+        /*
+         * Tell VM to act less aggressively and fail earlier.
+         * This is not necessary but increases throughput.
+         * FIXME: Is this really intelligent?
+         */
+        current->flags &= ~PF_MEMALLOC;
+
+        if (base_bio)
+                bio = bio_clone(base_bio, GFP_NOIO);
+        else
+                bio = bio_alloc(GFP_NOIO, nr_iovecs);
+        if (!bio) {
+                if (flags & PF_MEMALLOC)
+                        current->flags |= PF_MEMALLOC;
+                return NULL;
+        }
+
+        /* if the last bio was not complete, continue where that one ended */
+        bio->bi_idx = *bio_vec_idx;
+        bio->bi_vcnt = *bio_vec_idx;
+        bio->bi_size = 0;
+        bio->bi_flags &= ~(1 << BIO_SEG_VALID);
+
+        /* bio->bi_idx pages have already been allocated */
+        size -= bio->bi_idx * PAGE_SIZE;
+
+        for(i = bio->bi_idx; i < nr_iovecs; i++) {
+                struct bio_vec *bv = bio_iovec_idx(bio, i);
+
+                bv->bv_page = mempool_alloc(cc->page_pool, gfp_mask);
+                if (!bv->bv_page)
+                        break;
+
+                /*
+                 * if additional pages cannot be allocated without waiting,
+                 * return a partially allocated bio, the caller will then try
+                 * to allocate additional bios while submitting this partial bio
+                 */
+                if ((i - bio->bi_idx) == (MIN_BIO_PAGES - 1))
+                        gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT;
+
+                bv->bv_offset = 0;
+                if (size > PAGE_SIZE)
+                        bv->bv_len = PAGE_SIZE;
+                else
+                        bv->bv_len = size;
+
+                bio->bi_size += bv->bv_len;
+                bio->bi_vcnt++;
+                size -= bv->bv_len;
+        }
+
+        if (flags & PF_MEMALLOC)
+                current->flags |= PF_MEMALLOC;
+
+        if (!bio->bi_size) {
+                bio_put(bio);
+                return NULL;
+        }
+
+        /*
+         * Remember the last bio_vec allocated to be able
+         * to correctly continue after the splitting.
+         */
+        *bio_vec_idx = bio->bi_vcnt;
+
+        return bio;
+}
+
+static void crypt_free_buffer_pages(struct crypt_config *cc,
+                                    struct bio *bio, unsigned int bytes)
+{
+        unsigned int i, start, end;
+        struct bio_vec *bv;
+
+        /*
+         * This is ugly, but Jens Axboe thinks that using bi_idx in the
+         * endio function is too dangerous at the moment, so I calculate the
+         * correct position using bi_vcnt and bi_size.
+         * The bv_offset and bv_len fields might already be modified but we
+         * know that we always allocated whole pages.
+         * A fix to the bi_idx issue in the kernel is in the works, so
+         * we will hopefully be able to revert to the cleaner solution soon.
+         */
+        i = bio->bi_vcnt - 1;
+        bv = bio_iovec_idx(bio, i);
+        end = (i << PAGE_SHIFT) + (bv->bv_offset + bv->bv_len) - bio->bi_size;
+        start = end - bytes;
+
+        start >>= PAGE_SHIFT;
+        if (!bio->bi_size)
+                end = bio->bi_vcnt;
+        else
+                end >>= PAGE_SHIFT;
+
+        for(i = start; i < end; i++) {
+                bv = bio_iovec_idx(bio, i);
+                BUG_ON(!bv->bv_page);
+                mempool_free(bv->bv_page, cc->page_pool);
+                bv->bv_page = NULL;
+        }
+}
+
+/*
+ * One of the bios was finished. Check for completion of
+ * the whole request and correctly clean up the buffer.
+ */
+static void dec_pending(struct crypt_io *io, int error)
+{
+        struct crypt_config *cc = (struct crypt_config *) io->target->private;
+
+        if (error < 0)
+                io->error = error;
+
+        if (!atomic_dec_and_test(&io->pending))
+                return;
+
+        if (io->first_clone)
+                bio_put(io->first_clone);
+
+        bio_endio(io->bio, io->bio->bi_size, io->error);
+
+        mempool_free(io, cc->io_pool);
+}
+
+/*
+ * kcryptd:
+ *
+ * Needed because it would be very unwise to do decryption in an
+ * interrupt context, so bios returning from read requests get
+ * queued here.
+ */
+static struct workqueue_struct *_kcryptd_workqueue;
+
+static void kcryptd_do_work(void *data)
+{
+        struct crypt_io *io = (struct crypt_io *) data;
+        struct crypt_config *cc = (struct crypt_config *) io->target->private;
+        struct convert_context ctx;
+        int r;
+
+        crypt_convert_init(cc, &ctx, io->bio, io->bio,
+                           io->bio->bi_sector - io->target->begin, 0);
+        r = crypt_convert(cc, &ctx);
+
+        dec_pending(io, r);
+}
+
+static void kcryptd_queue_io(struct crypt_io *io)
+{
+        INIT_WORK(&io->work, kcryptd_do_work, io);
+        queue_work(_kcryptd_workqueue, &io->work);
+}
+
+/*
+ * Decode key from its hex representation
+ */
+static int crypt_decode_key(u8 *key, char *hex, unsigned int size)
+{
+        char buffer[3];
+        char *endp;
+        unsigned int i;
+
+        buffer[2] = '\0';
+
+        for(i = 0; i < size; i++) {
+                buffer[0] = *hex++;
+                buffer[1] = *hex++;
+
+                key[i] = (u8)simple_strtoul(buffer, &endp, 16);
+
+                if (endp != &buffer[2])
+                        return -EINVAL;
+        }
+
+        if (*hex != '\0')
+                return -EINVAL;
+
+        return 0;
+}
+
+/*
+ * Encode key into its hex representation
+ */
+static void crypt_encode_key(char *hex, u8 *key, unsigned int size)
+{
+        unsigned int i;
+
+        for(i = 0; i < size; i++) {
+                sprintf(hex, "%02x", *key);
+                hex += 2;
+                key++;
+        }
+}
+
+/*
+ * Construct an encryption mapping:
+ * <cipher> <key> <iv_offset> <dev_path> <start>
+ */
+static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+        struct crypt_config *cc;
+        struct crypto_tfm *tfm;
+        char *tmp;
+        char *cipher;
+        char *chainmode;
+        char *ivmode;
+        char *ivopts;
+        unsigned int crypto_flags;
+        unsigned int key_size;
+
+        if (argc != 5) {
+                ti->error = PFX "Not enough arguments";
+                return -EINVAL;
+        }
+
+        tmp = argv[0];
+        cipher = strsep(&tmp, "-");
+        chainmode = strsep(&tmp, "-");
+        ivopts = strsep(&tmp, "-");
+        ivmode = strsep(&ivopts, ":");
+
+        if (tmp)
+                DMWARN(PFX "Unexpected additional cipher options");
+
+        key_size = strlen(argv[1]) >> 1;
+
+        cc = kmalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL);
+        if (cc == NULL) {
+                ti->error =
+                        PFX "Cannot allocate transparent encryption context";
+                return -ENOMEM;
+        }
+
+        cc->key_size = key_size;
+        if ((!key_size && strcmp(argv[1], "-") != 0) ||
+            (key_size && crypt_decode_key(cc->key, argv[1], key_size) < 0)) {
+                ti->error = PFX "Error decoding key";
+                goto bad1;
+        }
+
+        /* Compatiblity mode for old dm-crypt cipher strings */
+        if (!chainmode || (strcmp(chainmode, "plain") == 0 && !ivmode)) {
+                chainmode = "cbc";
+                ivmode = "plain";
+        }
+
+        /* Choose crypto_flags according to chainmode */
+        if (strcmp(chainmode, "cbc") == 0)
+                crypto_flags = CRYPTO_TFM_MODE_CBC;
+        else if (strcmp(chainmode, "ecb") == 0)
+                crypto_flags = CRYPTO_TFM_MODE_ECB;
+        else {
+                ti->error = PFX "Unknown chaining mode";
+                goto bad1;
+        }
+
+        if (crypto_flags != CRYPTO_TFM_MODE_ECB && !ivmode) {
+                ti->error = PFX "This chaining mode requires an IV mechanism";
+                goto bad1;
+        }
+
+        tfm = crypto_alloc_tfm(cipher, crypto_flags);
+        if (!tfm) {
+                ti->error = PFX "Error allocating crypto tfm";
+                goto bad1;
+        }
+        if (crypto_tfm_alg_type(tfm) != CRYPTO_ALG_TYPE_CIPHER) {
+                ti->error = PFX "Expected cipher algorithm";
+                goto bad2;
+        }
+
+        cc->tfm = tfm;
+
+        /*
+         * Choose ivmode. Valid modes: "plain", "essiv:<esshash>".
+         * See comments at iv code
+         */
+
+        if (ivmode == NULL)
+                cc->iv_gen_ops = NULL;
+        else if (strcmp(ivmode, "plain") == 0)
+                cc->iv_gen_ops = &crypt_iv_plain_ops;
+        else if (strcmp(ivmode, "essiv") == 0)
+                cc->iv_gen_ops = &crypt_iv_essiv_ops;
+        else {
+                ti->error = PFX "Invalid IV mode";
+                goto bad2;
+        }
+
+        if (cc->iv_gen_ops && cc->iv_gen_ops->ctr &&
+            cc->iv_gen_ops->ctr(cc, ti, ivopts) < 0)
+                goto bad2;
+
+        if (tfm->crt_cipher.cit_decrypt_iv && tfm->crt_cipher.cit_encrypt_iv)
+                /* at least a 64 bit sector number should fit in our buffer */
+                cc->iv_size = max(crypto_tfm_alg_ivsize(tfm),
+                                  (unsigned int)(sizeof(u64) / sizeof(u8)));
+        else {
+                cc->iv_size = 0;
+                if (cc->iv_gen_ops) {
+                        DMWARN(PFX "Selected cipher does not support IVs");
+                        if (cc->iv_gen_ops->dtr)
+                                cc->iv_gen_ops->dtr(cc);
+                        cc->iv_gen_ops = NULL;
+                }
+        }
+
+        cc->io_pool = mempool_create(MIN_IOS, mempool_alloc_slab,
+                                     mempool_free_slab, _crypt_io_pool);
+        if (!cc->io_pool) {
+                ti->error = PFX "Cannot allocate crypt io mempool";
+                goto bad3;
+        }
+
+        cc->page_pool = mempool_create(MIN_POOL_PAGES, mempool_alloc_page,
+                                       mempool_free_page, NULL);
+        if (!cc->page_pool) {
+                ti->error = PFX "Cannot allocate page mempool";
+                goto bad4;
+        }
+
+        if (tfm->crt_cipher.cit_setkey(tfm, cc->key, key_size) < 0) {
+                ti->error = PFX "Error setting key";
+                goto bad5;
+        }
+
+        if (sscanf(argv[2], SECTOR_FORMAT, &cc->iv_offset) != 1) {
+                ti->error = PFX "Invalid iv_offset sector";
+                goto bad5;
+        }
+
+        if (sscanf(argv[4], SECTOR_FORMAT, &cc->start) != 1) {
+                ti->error = PFX "Invalid device sector";
+                goto bad5;
+        }
+
+        if (dm_get_device(ti, argv[3], cc->start, ti->len,
+                          dm_table_get_mode(ti->table), &cc->dev)) {
+                ti->error = PFX "Device lookup failed";
+                goto bad5;
+        }
+
+        if (ivmode && cc->iv_gen_ops) {
+                if (ivopts)
+                        *(ivopts - 1) = ':';
+                cc->iv_mode = kmalloc(strlen(ivmode) + 1, GFP_KERNEL);
+                if (!cc->iv_mode) {
+                        ti->error = PFX "Error kmallocing iv_mode string";
+                        goto bad5;
+                }
+                strcpy(cc->iv_mode, ivmode);
+        } else
+                cc->iv_mode = NULL;
+
+        ti->private = cc;
+        return 0;
+
+bad5:
+        mempool_destroy(cc->page_pool);
+bad4:
+        mempool_destroy(cc->io_pool);
+bad3:
+        if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
+                cc->iv_gen_ops->dtr(cc);
+bad2:
+        crypto_free_tfm(tfm);
+bad1:
+        kfree(cc);
+        return -EINVAL;
+}
+
+static void crypt_dtr(struct dm_target *ti)
+{
+        struct crypt_config *cc = (struct crypt_config *) ti->private;
+
+        mempool_destroy(cc->page_pool);
+        mempool_destroy(cc->io_pool);
+
+        if (cc->iv_mode)
+                kfree(cc->iv_mode);
+        if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
+                cc->iv_gen_ops->dtr(cc);
+        crypto_free_tfm(cc->tfm);
+        dm_put_device(ti, cc->dev);
+        kfree(cc);
+}
+
+static int crypt_endio(struct bio *bio, unsigned int done, int error)
+{
+        struct crypt_io *io = (struct crypt_io *) bio->bi_private;
+        struct crypt_config *cc = (struct crypt_config *) io->target->private;
+
+        if (bio_data_dir(bio) == WRITE) {
+                /*
+                 * free the processed pages, even if
+                 * it's only a partially completed write
+                 */
+                crypt_free_buffer_pages(cc, bio, done);
+        }
+
+        if (bio->bi_size)
+                return 1;
+
+        bio_put(bio);
+
+        /*
+         * successful reads are decrypted by the worker thread
+         */
+        if ((bio_data_dir(bio) == READ)
+            && bio_flagged(bio, BIO_UPTODATE)) {
+                kcryptd_queue_io(io);
+                return 0;
+        }
+
+        dec_pending(io, error);
+        return error;
+}
+
+static inline struct bio *
+crypt_clone(struct crypt_config *cc, struct crypt_io *io, struct bio *bio,
+            sector_t sector, unsigned int *bvec_idx,
+            struct convert_context *ctx)
+{
+        struct bio *clone;
+
+        if (bio_data_dir(bio) == WRITE) {
+                clone = crypt_alloc_buffer(cc, bio->bi_size,
+                                 io->first_clone, bvec_idx);
+                if (clone) {
+                        ctx->bio_out = clone;
+                        if (crypt_convert(cc, ctx) < 0) {
+                                crypt_free_buffer_pages(cc, clone,
+                                                        clone->bi_size);
+                                bio_put(clone);
+                                return NULL;
+                        }
+                }
+        } else {
+                /*
+                 * The block layer might modify the bvec array, so always
+                 * copy the required bvecs because we need the original
+                 * one in order to decrypt the whole bio data *afterwards*.
+                 */
+                clone = bio_alloc(GFP_NOIO, bio_segments(bio));
+                if (clone) {
+                        clone->bi_idx = 0;
+                        clone->bi_vcnt = bio_segments(bio);
+                        clone->bi_size = bio->bi_size;
+                        memcpy(clone->bi_io_vec, bio_iovec(bio),
+                               sizeof(struct bio_vec) * clone->bi_vcnt);
+                }
+        }
+
+        if (!clone)
+                return NULL;
+
+        clone->bi_private = io;
+        clone->bi_end_io = crypt_endio;
+        clone->bi_bdev = cc->dev->bdev;
+        clone->bi_sector = cc->start + sector;
+        clone->bi_rw = bio->bi_rw;
+
+        return clone;
+}
+
+static int crypt_map(struct dm_target *ti, struct bio *bio,
+                     union map_info *map_context)
+{
+        struct crypt_config *cc = (struct crypt_config *) ti->private;
+        struct crypt_io *io = mempool_alloc(cc->io_pool, GFP_NOIO);
+        struct convert_context ctx;
+        struct bio *clone;
+        unsigned int remaining = bio->bi_size;
+        sector_t sector = bio->bi_sector - ti->begin;
+        unsigned int bvec_idx = 0;
+
+        io->target = ti;
+        io->bio = bio;
+        io->first_clone = NULL;
+        io->error = 0;
+        atomic_set(&io->pending, 1); /* hold a reference */
+
+        if (bio_data_dir(bio) == WRITE)
+                crypt_convert_init(cc, &ctx, NULL, bio, sector, 1);
+
+        /*
+         * The allocated buffers can be smaller than the whole bio,
+         * so repeat the whole process until all the data can be handled.
+         */
+        while (remaining) {
+                clone = crypt_clone(cc, io, bio, sector, &bvec_idx, &ctx);
+                if (!clone)
+                        goto cleanup;
+
+                if (!io->first_clone) {
+                        /*
+                         * hold a reference to the first clone, because it
+                         * holds the bio_vec array and that can't be freed
+                         * before all other clones are released
+                         */
+                        bio_get(clone);
+                        io->first_clone = clone;
+                }
+                atomic_inc(&io->pending);
+
+                remaining -= clone->bi_size;
+                sector += bio_sectors(clone);
+
+                generic_make_request(clone);
+
+                /* out of memory -> run queues */
+                if (remaining)
+                        blk_congestion_wait(bio_data_dir(clone), HZ/100);
+        }
+
+        /* drop reference, clones could have returned before we reach this */
+        dec_pending(io, 0);
+        return 0;
+
+cleanup:
+        if (io->first_clone) {
+                dec_pending(io, -ENOMEM);
+                return 0;
+        }
+
+        /* if no bio has been dispatched yet, we can directly return the error */
+        mempool_free(io, cc->io_pool);
+        return -ENOMEM;
+}
+
+static int crypt_status(struct dm_target *ti, status_type_t type,
+                        char *result, unsigned int maxlen)
+{
+        struct crypt_config *cc = (struct crypt_config *) ti->private;
+        const char *cipher;
+        const char *chainmode = NULL;
+        unsigned int sz = 0;
+
+        switch (type) {
+        case STATUSTYPE_INFO:
+                result[0] = '\0';
+                break;
+
+        case STATUSTYPE_TABLE:
+                cipher = crypto_tfm_alg_name(cc->tfm);
+
+                switch(cc->tfm->crt_cipher.cit_mode) {
+                case CRYPTO_TFM_MODE_CBC:
+                        chainmode = "cbc";
+                        break;
+                case CRYPTO_TFM_MODE_ECB:
+                        chainmode = "ecb";
+                        break;
+                default:
+                        BUG();
+                }
+
+                if (cc->iv_mode)
+                        DMEMIT("%s-%s-%s ", cipher, chainmode, cc->iv_mode);
+                else
+                        DMEMIT("%s-%s ", cipher, chainmode);
+
+                if (cc->key_size > 0) {
+                        if ((maxlen - sz) < ((cc->key_size << 1) + 1))
+                                return -ENOMEM;
+
+                        crypt_encode_key(result + sz, cc->key, cc->key_size);
+                        sz += cc->key_size << 1;
+                } else {
+                        if (sz >= maxlen)
+                                return -ENOMEM;
+                        result[sz++] = '-';
+                }
+
+                DMEMIT(" " SECTOR_FORMAT " %s " SECTOR_FORMAT,
+                       cc->iv_offset, cc->dev->name, cc->start);
+                break;
+        }
+        return 0;
+}
+
+static struct target_type crypt_target = {
+        .name   = "crypt",
+        .version= {1, 1, 0},
+        .module = THIS_MODULE,
+        .ctr    = crypt_ctr,
+        .dtr    = crypt_dtr,
+        .map    = crypt_map,
+        .status = crypt_status,
+};
+
+static int __init dm_crypt_init(void)
+{
+        int r;
+
+        _crypt_io_pool = kmem_cache_create("dm-crypt_io",
+                                           sizeof(struct crypt_io),
+                                           0, 0, NULL, NULL);
+        if (!_crypt_io_pool)
+                return -ENOMEM;
+
+        _kcryptd_workqueue = create_workqueue("kcryptd");
+        if (!_kcryptd_workqueue) {
+                r = -ENOMEM;
+                DMERR(PFX "couldn't create kcryptd");
+                goto bad1;
+        }
+
+        r = dm_register_target(&crypt_target);
+        if (r < 0) {
+                DMERR(PFX "register failed %d", r);
+                goto bad2;
+        }
+
+        return 0;
+
+bad2:
+        destroy_workqueue(_kcryptd_workqueue);
+bad1:
+        kmem_cache_destroy(_crypt_io_pool);
+        return r;
+}
+
+static void __exit dm_crypt_exit(void)
+{
+        int r = dm_unregister_target(&crypt_target);
+
+        if (r < 0)
+                DMERR(PFX "unregister failed %d", r);
+
+        destroy_workqueue(_kcryptd_workqueue);
+        kmem_cache_destroy(_crypt_io_pool);
+}
+
+module_init(dm_crypt_init);
+module_exit(dm_crypt_exit);
+
+MODULE_AUTHOR("Christophe Saout <christophe@saout.de>");
+MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-emc.c b/drivers/md/dm-emc.c
new file mode 100644
index 000000000000..700658664594
--- /dev/null
+++ b/drivers/md/dm-emc.c
@@ -0,0 +1,359 @@
+/*
+ * Copyright (C) 2004 SUSE LINUX Products GmbH. All rights reserved.
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ *
+ * Multipath support for EMC CLARiiON AX/CX-series hardware.
+ */
+
+#include "dm.h"
+#include "dm-hw-handler.h"
+#include <scsi/scsi.h>
+#include <scsi/scsi_cmnd.h>
+
+struct emc_handler {
+        spinlock_t lock;
+
+        /* Whether we should send the short trespass command (FC-series)
+         * or the long version (default for AX/CX CLARiiON arrays). */
+        unsigned short_trespass;
+        /* Whether or not to honor SCSI reservations when initiating a
+         * switch-over. Default: Don't. */
+        unsigned hr;
+
+        unsigned char sense[SCSI_SENSE_BUFFERSIZE];
+};
+
+#define TRESPASS_PAGE 0x22
+#define EMC_FAILOVER_TIMEOUT (60 * HZ)
+
+/* Code borrowed from dm-lsi-rdac by Mike Christie */
+
+static inline void free_bio(struct bio *bio)
+{
+        __free_page(bio->bi_io_vec[0].bv_page);
+        bio_put(bio);
+}
+
+static int emc_endio(struct bio *bio, unsigned int bytes_done, int error)
+{
+        struct path *path = bio->bi_private;
+
+        if (bio->bi_size)
+                return 1;
+
+        /* We also need to look at the sense keys here whether or not to
+         * switch to the next PG etc.
+         *
+         * For now simple logic: either it works or it doesn't.
+         */
+        if (error)
+                dm_pg_init_complete(path, MP_FAIL_PATH);
+        else
+                dm_pg_init_complete(path, 0);
+
+        /* request is freed in block layer */
+        free_bio(bio);
+
+        return 0;
+}
+
+static struct bio *get_failover_bio(struct path *path, unsigned data_size)
+{
+        struct bio *bio;
+        struct page *page;
+
+        bio = bio_alloc(GFP_ATOMIC, 1);
+        if (!bio) {
+                DMERR("dm-emc: get_failover_bio: bio_alloc() failed.");
+                return NULL;
+        }
+
+        bio->bi_rw |= (1 << BIO_RW);
+        bio->bi_bdev = path->dev->bdev;
+        bio->bi_sector = 0;
+        bio->bi_private = path;
+        bio->bi_end_io = emc_endio;
+
+        page = alloc_page(GFP_ATOMIC);
+        if (!page) {
+                DMERR("dm-emc: get_failover_bio: alloc_page() failed.");
+                bio_put(bio);
+                return NULL;
+        }
+
+        if (bio_add_page(bio, page, data_size, 0) != data_size) {
+                DMERR("dm-emc: get_failover_bio: alloc_page() failed.");
+                __free_page(page);
+                bio_put(bio);
+                return NULL;
+        }
+
+        return bio;
+}
+
+static struct request *get_failover_req(struct emc_handler *h,
+                                        struct bio *bio, struct path *path)
+{
+        struct request *rq;
+        struct block_device *bdev = bio->bi_bdev;
+        struct request_queue *q = bdev_get_queue(bdev);
+
+        /* FIXME: Figure out why it fails with GFP_ATOMIC. */
+        rq = blk_get_request(q, WRITE, __GFP_WAIT);
+        if (!rq) {
+                DMERR("dm-emc: get_failover_req: blk_get_request failed");
+                return NULL;
+        }
+
+        rq->bio = rq->biotail = bio;
+        blk_rq_bio_prep(q, rq, bio);
+
+        rq->rq_disk = bdev->bd_contains->bd_disk;
+
+        /* bio backed don't set data */
+        rq->buffer = rq->data = NULL;
+        /* rq data_len used for pc cmd's request_bufflen */
+        rq->data_len = bio->bi_size;
+
+        rq->sense = h->sense;
+        memset(rq->sense, 0, SCSI_SENSE_BUFFERSIZE);
+        rq->sense_len = 0;
+
+        memset(&rq->cmd, 0, BLK_MAX_CDB);
+
+        rq->timeout = EMC_FAILOVER_TIMEOUT;
+        rq->flags |= (REQ_BLOCK_PC | REQ_FAILFAST | REQ_NOMERGE);
+
+        return rq;
+}
+
+static struct request *emc_trespass_get(struct emc_handler *h,
+                                        struct path *path)
+{
+        struct bio *bio;
+        struct request *rq;
+        unsigned char *page22;
+        unsigned char long_trespass_pg[] = {
+                0, 0, 0, 0,
+                TRESPASS_PAGE,        /* Page code */
+                0x09,                 /* Page length - 2 */
+                h->hr ? 0x01 : 0x81,  /* Trespass code + Honor reservation bit */
+                0xff, 0xff,           /* Trespass target */
+                0, 0, 0, 0, 0, 0      /* Reserved bytes / unknown */
+                };
+        unsigned char short_trespass_pg[] = {
+                0, 0, 0, 0,
+                TRESPASS_PAGE,        /* Page code */
+                0x02,                 /* Page length - 2 */
+                h->hr ? 0x01 : 0x81,  /* Trespass code + Honor reservation bit */
+                0xff,                 /* Trespass target */
+                };
+        unsigned data_size = h->short_trespass ? sizeof(short_trespass_pg) :
+                                sizeof(long_trespass_pg);
+
+        /* get bio backing */
+        if (data_size > PAGE_SIZE)
+                /* this should never happen */
+                return NULL;
+
+        bio = get_failover_bio(path, data_size);
+        if (!bio) {
+                DMERR("dm-emc: emc_trespass_get: no bio");
+                return NULL;
+        }
+
+        page22 = (unsigned char *)bio_data(bio);
+        memset(page22, 0, data_size);
+
+        memcpy(page22, h->short_trespass ?
+                short_trespass_pg : long_trespass_pg, data_size);
+
+        /* get request for block layer packet command */
+        rq = get_failover_req(h, bio, path);
+        if (!rq) {
+                DMERR("dm-emc: emc_trespass_get: no rq");
+                free_bio(bio);
+                return NULL;
+        }
+
+        /* Prepare the command. */
+        rq->cmd[0] = MODE_SELECT;
+        rq->cmd[1] = 0x10;
+        rq->cmd[4] = data_size;
+        rq->cmd_len = COMMAND_SIZE(rq->cmd[0]);
+
+        return rq;
+}
+
+static void emc_pg_init(struct hw_handler *hwh, unsigned bypassed,
+                        struct path *path)
+{
+        struct request *rq;
+        struct request_queue *q = bdev_get_queue(path->dev->bdev);
+
+        /*
+         * We can either blindly init the pg (then look at the sense),
+         * or we can send some commands to get the state here (then
+         * possibly send the fo cmnd), or we can also have the
+         * initial state passed into us and then get an update here.
+         */
+        if (!q) {
+                DMINFO("dm-emc: emc_pg_init: no queue");
+                goto fail_path;
+        }
+
+        /* FIXME: The request should be pre-allocated. */
+        rq = emc_trespass_get(hwh->context, path);
+        if (!rq) {
+                DMERR("dm-emc: emc_pg_init: no rq");
+                goto fail_path;
+        }
+
+        DMINFO("dm-emc: emc_pg_init: sending switch-over command");
+        elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 1);
+        return;
+
+fail_path:
+        dm_pg_init_complete(path, MP_FAIL_PATH);
+}
+
+static struct emc_handler *alloc_emc_handler(void)
+{
+        struct emc_handler *h = kmalloc(sizeof(*h), GFP_KERNEL);
+
+        if (h)
+                spin_lock_init(&h->lock);
+
+        return h;
+}
+
+static int emc_create(struct hw_handler *hwh, unsigned argc, char **argv)
+{
+        struct emc_handler *h;
+        unsigned hr, short_trespass;
+
+        if (argc == 0) {
+                /* No arguments: use defaults */
+                hr = 0;
+                short_trespass = 0;
+        } else if (argc != 2) {
+                DMWARN("dm-emc hwhandler: incorrect number of arguments");
+                return -EINVAL;
+        } else {
+                if ((sscanf(argv[0], "%u", &short_trespass) != 1)
+                        || (short_trespass > 1)) {
+                        DMWARN("dm-emc: invalid trespass mode selected");
+                        return -EINVAL;
+                }
+
+                if ((sscanf(argv[1], "%u", &hr) != 1)
+                        || (hr > 1)) {
+                        DMWARN("dm-emc: invalid honor reservation flag selected");
+                        return -EINVAL;
+                }
+        }
+
+        h = alloc_emc_handler();
+        if (!h)
+                return -ENOMEM;
+
+        memset(h, 0, sizeof(*h));
+
+        hwh->context = h;
+
+        if ((h->short_trespass = short_trespass))
+                DMWARN("dm-emc: short trespass command will be send");
+        else
+                DMWARN("dm-emc: long trespass command will be send");
+
+        if ((h->hr = hr))
+                DMWARN("dm-emc: honor reservation bit will be set");
+        else
+                DMWARN("dm-emc: honor reservation bit will not be set (default)");
+
+        return 0;
+}
+
+static void emc_destroy(struct hw_handler *hwh)
+{
+        struct emc_handler *h = (struct emc_handler *) hwh->context;
+
+        kfree(h);
+        hwh->context = NULL;
+}
+
+static unsigned emc_error(struct hw_handler *hwh, struct bio *bio)
+{
+        /* FIXME: Patch from axboe still missing */
+#if 0
+        int sense;
+
+        if (bio->bi_error & BIO_SENSE) {
+                sense = bio->bi_error & 0xffffff; /* sense key / asc / ascq */
+
+                if (sense == 0x020403) {
+                        /* LUN Not Ready - Manual Intervention Required
+                         * indicates this is a passive path.
+                         *
+                         * FIXME: However, if this is seen and EVPD C0
+                         * indicates that this is due to a NDU in
+                         * progress, we should set FAIL_PATH too.
+                         * This indicates we might have to do a SCSI
+                         * inquiry in the end_io path. Ugh. */
+                        return MP_BYPASS_PG | MP_RETRY_IO;
+                } else if (sense == 0x052501) {
+                        /* An array based copy is in progress. Do not
+                         * fail the path, do not bypass to another PG,
+                         * do not retry. Fail the IO immediately.
+                         * (Actually this is the same conclusion as in
+                         * the default handler, but lets make sure.) */
+                        return 0;
+                } else if (sense == 0x062900) {
+                        /* Unit Attention Code. This is the first IO
+                         * to the new path, so just retry. */
+                        return MP_RETRY_IO;
+                }
+        }
+#endif
+
+        /* Try default handler */
+        return dm_scsi_err_handler(hwh, bio);
+}
+
+static struct hw_handler_type emc_hwh = {
+        .name = "emc",
+        .module = THIS_MODULE,
+        .create = emc_create,
+        .destroy = emc_destroy,
+        .pg_init = emc_pg_init,
+        .error = emc_error,
+};
+
+static int __init dm_emc_init(void)
+{
+        int r = dm_register_hw_handler(&emc_hwh);
+
+        if (r < 0)
+                DMERR("emc: register failed %d", r);
+
+        DMINFO("dm-emc version 0.0.3 loaded");
+
+        return r;
+}
+
+static void __exit dm_emc_exit(void)
+{
+        int r = dm_unregister_hw_handler(&emc_hwh);
+
+        if (r < 0)
+                DMERR("emc: unregister failed %d", r);
+}
+
+module_init(dm_emc_init);
+module_exit(dm_emc_exit);
+
+MODULE_DESCRIPTION(DM_NAME " EMC CX/AX/FC-family multipath");
+MODULE_AUTHOR("Lars Marowsky-Bree <lmb@suse.de>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-exception-store.c b/drivers/md/dm-exception-store.c
new file mode 100644
index 000000000000..17212b4201a1
--- /dev/null
+++ b/drivers/md/dm-exception-store.c
@@ -0,0 +1,648 @@
+/*
+ * dm-snapshot.c
+ *
+ * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+#include "dm-snap.h"
+#include "dm-io.h"
+#include "kcopyd.h"
+
+#include <linux/mm.h>
+#include <linux/pagemap.h>
+#include <linux/vmalloc.h>
+#include <linux/slab.h>
+
+/*-----------------------------------------------------------------
+ * Persistent snapshots, by persistent we mean that the snapshot
+ * will survive a reboot.
+ *---------------------------------------------------------------*/
+
+/*
+ * We need to store a record of which parts of the origin have
+ * been copied to the snapshot device.  The snapshot code
+ * requires that we copy exception chunks to chunk aligned areas
+ * of the COW store.  It makes sense therefore, to store the
+ * metadata in chunk size blocks.
+ *
+ * There is no backward or forward compatibility implemented,
+ * snapshots with different disk versions than the kernel will
+ * not be usable.  It is expected that "lvcreate" will blank out
+ * the start of a fresh COW device before calling the snapshot
+ * constructor.
+ *
+ * The first chunk of the COW device just contains the header.
+ * After this there is a chunk filled with exception metadata,
+ * followed by as many exception chunks as can fit in the
+ * metadata areas.
+ *
+ * All on disk structures are in little-endian format.  The end
+ * of the exceptions info is indicated by an exception with a
+ * new_chunk of 0, which is invalid since it would point to the
+ * header chunk.
+ */
+
+/*
+ * Magic for persistent snapshots: "SnAp" - Feeble isn't it.
+ */
+#define SNAP_MAGIC 0x70416e53
+
+/*
+ * The on-disk version of the metadata.
+ */
+#define SNAPSHOT_DISK_VERSION 1
+
+struct disk_header {
+        uint32_t magic;
+
+        /*
+         * Is this snapshot valid.  There is no way of recovering
+         * an invalid snapshot.
+         */
+        uint32_t valid;
+
+        /*
+         * Simple, incrementing version. no backward
+         * compatibility.
+         */
+        uint32_t version;
+
+        /* In sectors */
+        uint32_t chunk_size;
+};
+
+struct disk_exception {
+        uint64_t old_chunk;
+        uint64_t new_chunk;
+};
+
+struct commit_callback {
+        void (*callback)(void *, int success);
+        void *context;
+};
+
+/*
+ * The top level structure for a persistent exception store.
+ */
+struct pstore {
+        struct dm_snapshot *snap;       /* up pointer to my snapshot */
+        int version;
+        int valid;
+        uint32_t chunk_size;
+        uint32_t exceptions_per_area;
+
+        /*
+         * Now that we have an asynchronous kcopyd there is no
+         * need for large chunk sizes, so it wont hurt to have a
+         * whole chunks worth of metadata in memory at once.
+         */
+        void *area;
+
+        /*
+         * Used to keep track of which metadata area the data in
+         * 'chunk' refers to.
+         */
+        uint32_t current_area;
+
+        /*
+         * The next free chunk for an exception.
+         */
+        uint32_t next_free;
+
+        /*
+         * The index of next free exception in the current
+         * metadata area.
+         */
+        uint32_t current_committed;
+
+        atomic_t pending_count;
+        uint32_t callback_count;
+        struct commit_callback *callbacks;
+};
+
+static inline unsigned int sectors_to_pages(unsigned int sectors)
+{
+        return sectors / (PAGE_SIZE >> 9);
+}
+
+static int alloc_area(struct pstore *ps)
+{
+        int r = -ENOMEM;
+        size_t len;
+
+        len = ps->chunk_size << SECTOR_SHIFT;
+
+        /*
+         * Allocate the chunk_size block of memory that will hold
+         * a single metadata area.
+         */
+        ps->area = vmalloc(len);
+        if (!ps->area)
+                return r;
+
+        return 0;
+}
+
+static void free_area(struct pstore *ps)
+{
+        vfree(ps->area);
+}
+
+/*
+ * Read or write a chunk aligned and sized block of data from a device.
+ */
+static int chunk_io(struct pstore *ps, uint32_t chunk, int rw)
+{
+        struct io_region where;
+        unsigned long bits;
+
+        where.bdev = ps->snap->cow->bdev;
+        where.sector = ps->chunk_size * chunk;
+        where.count = ps->chunk_size;
+
+        return dm_io_sync_vm(1, &where, rw, ps->area, &bits);
+}
+
+/*
+ * Read or write a metadata area.  Remembering to skip the first
+ * chunk which holds the header.
+ */
+static int area_io(struct pstore *ps, uint32_t area, int rw)
+{
+        int r;
+        uint32_t chunk;
+
+        /* convert a metadata area index to a chunk index */
+        chunk = 1 + ((ps->exceptions_per_area + 1) * area);
+
+        r = chunk_io(ps, chunk, rw);
+        if (r)
+                return r;
+
+        ps->current_area = area;
+        return 0;
+}
+
+static int zero_area(struct pstore *ps, uint32_t area)
+{
+        memset(ps->area, 0, ps->chunk_size << SECTOR_SHIFT);
+        return area_io(ps, area, WRITE);
+}
+
+static int read_header(struct pstore *ps, int *new_snapshot)
+{
+        int r;
+        struct disk_header *dh;
+
+        r = chunk_io(ps, 0, READ);
+        if (r)
+                return r;
+
+        dh = (struct disk_header *) ps->area;
+
+        if (le32_to_cpu(dh->magic) == 0) {
+                *new_snapshot = 1;
+
+        } else if (le32_to_cpu(dh->magic) == SNAP_MAGIC) {
+                *new_snapshot = 0;
+                ps->valid = le32_to_cpu(dh->valid);
+                ps->version = le32_to_cpu(dh->version);
+                ps->chunk_size = le32_to_cpu(dh->chunk_size);
+
+        } else {
+                DMWARN("Invalid/corrupt snapshot");
+                r = -ENXIO;
+        }
+
+        return r;
+}
+
+static int write_header(struct pstore *ps)
+{
+        struct disk_header *dh;
+
+        memset(ps->area, 0, ps->chunk_size << SECTOR_SHIFT);
+
+        dh = (struct disk_header *) ps->area;
+        dh->magic = cpu_to_le32(SNAP_MAGIC);
+        dh->valid = cpu_to_le32(ps->valid);
+        dh->version = cpu_to_le32(ps->version);
+        dh->chunk_size = cpu_to_le32(ps->chunk_size);
+
+        return chunk_io(ps, 0, WRITE);
+}
+
+/*
+ * Access functions for the disk exceptions, these do the endian conversions.
+ */
+static struct disk_exception *get_exception(struct pstore *ps, uint32_t index)
+{
+        if (index >= ps->exceptions_per_area)
+                return NULL;
+
+        return ((struct disk_exception *) ps->area) + index;
+}
+
+static int read_exception(struct pstore *ps,
+                          uint32_t index, struct disk_exception *result)
+{
+        struct disk_exception *e;
+
+        e = get_exception(ps, index);
+        if (!e)
+                return -EINVAL;
+
+        /* copy it */
+        result->old_chunk = le64_to_cpu(e->old_chunk);
+        result->new_chunk = le64_to_cpu(e->new_chunk);
+
+        return 0;
+}
+
+static int write_exception(struct pstore *ps,
+                           uint32_t index, struct disk_exception *de)
+{
+        struct disk_exception *e;
+
+        e = get_exception(ps, index);
+        if (!e)
+                return -EINVAL;
+
+        /* copy it */
+        e->old_chunk = cpu_to_le64(de->old_chunk);
+        e->new_chunk = cpu_to_le64(de->new_chunk);
+
+        return 0;
+}
+
+/*
+ * Registers the exceptions that are present in the current area.
+ * 'full' is filled in to indicate if the area has been
+ * filled.
+ */
+static int insert_exceptions(struct pstore *ps, int *full)
+{
+        int r;
+        unsigned int i;
+        struct disk_exception de;
+
+        /* presume the area is full */
+        *full = 1;
+
+        for (i = 0; i < ps->exceptions_per_area; i++) {
+                r = read_exception(ps, i, &de);
+
+                if (r)
+                        return r;
+
+                /*
+                 * If the new_chunk is pointing at the start of
+                 * the COW device, where the first metadata area
+                 * is we know that we've hit the end of the
+                 * exceptions.  Therefore the area is not full.
+                 */
+                if (de.new_chunk == 0LL) {
+                        ps->current_committed = i;
+                        *full = 0;
+                        break;
+                }
+
+                /*
+                 * Keep track of the start of the free chunks.
+                 */
+                if (ps->next_free <= de.new_chunk)
+                        ps->next_free = de.new_chunk + 1;
+
+                /*
+                 * Otherwise we add the exception to the snapshot.
+                 */
+                r = dm_add_exception(ps->snap, de.old_chunk, de.new_chunk);
+                if (r)
+                        return r;
+        }
+
+        return 0;
+}
+
+static int read_exceptions(struct pstore *ps)
+{
+        uint32_t area;
+        int r, full = 1;
+
+        /*
+         * Keeping reading chunks and inserting exceptions until
+         * we find a partially full area.
+         */
+        for (area = 0; full; area++) {
+                r = area_io(ps, area, READ);
+                if (r)
+                        return r;
+
+                r = insert_exceptions(ps, &full);
+                if (r)
+                        return r;
+        }
+
+        return 0;
+}
+
+static inline struct pstore *get_info(struct exception_store *store)
+{
+        return (struct pstore *) store->context;
+}
+
+static void persistent_fraction_full(struct exception_store *store,
+                                     sector_t *numerator, sector_t *denominator)
+{
+        *numerator = get_info(store)->next_free * store->snap->chunk_size;
+        *denominator = get_dev_size(store->snap->cow->bdev);
+}
+
+static void persistent_destroy(struct exception_store *store)
+{
+        struct pstore *ps = get_info(store);
+
+        dm_io_put(sectors_to_pages(ps->chunk_size));
+        vfree(ps->callbacks);
+        free_area(ps);
+        kfree(ps);
+}
+
+static int persistent_read_metadata(struct exception_store *store)
+{
+        int r, new_snapshot;
+        struct pstore *ps = get_info(store);
+
+        /*
+         * Read the snapshot header.
+         */
+        r = read_header(ps, &new_snapshot);
+        if (r)
+                return r;
+
+        /*
+         * Do we need to setup a new snapshot ?
+         */
+        if (new_snapshot) {
+                r = write_header(ps);
+                if (r) {
+                        DMWARN("write_header failed");
+                        return r;
+                }
+
+                r = zero_area(ps, 0);
+                if (r) {
+                        DMWARN("zero_area(0) failed");
+                        return r;
+                }
+
+        } else {
+                /*
+                 * Sanity checks.
+                 */
+                if (!ps->valid) {
+                        DMWARN("snapshot is marked invalid");
+                        return -EINVAL;
+                }
+
+                if (ps->version != SNAPSHOT_DISK_VERSION) {
+                        DMWARN("unable to handle snapshot disk version %d",
+                               ps->version);
+                        return -EINVAL;
+                }
+
+                /*
+                 * Read the metadata.
+                 */
+                r = read_exceptions(ps);
+                if (r)
+                        return r;
+        }
+
+        return 0;
+}
+
+static int persistent_prepare(struct exception_store *store,
+                              struct exception *e)
+{
+        struct pstore *ps = get_info(store);
+        uint32_t stride;
+        sector_t size = get_dev_size(store->snap->cow->bdev);
+
+        /* Is there enough room ? */
+        if (size < ((ps->next_free + 1) * store->snap->chunk_size))
+                return -ENOSPC;
+
+        e->new_chunk = ps->next_free;
+
+        /*
+         * Move onto the next free pending, making sure to take
+         * into account the location of the metadata chunks.
+         */
+        stride = (ps->exceptions_per_area + 1);
+        if ((++ps->next_free % stride) == 1)
+                ps->next_free++;
+
+        atomic_inc(&ps->pending_count);
+        return 0;
+}
+
+static void persistent_commit(struct exception_store *store,
+                              struct exception *e,
+                              void (*callback) (void *, int success),
+                              void *callback_context)
+{
+        int r;
+        unsigned int i;
+        struct pstore *ps = get_info(store);
+        struct disk_exception de;
+        struct commit_callback *cb;
+
+        de.old_chunk = e->old_chunk;
+        de.new_chunk = e->new_chunk;
+        write_exception(ps, ps->current_committed++, &de);
+
+        /*
+         * Add the callback to the back of the array.  This code
+         * is the only place where the callback array is
+         * manipulated, and we know that it will never be called
+         * multiple times concurrently.
+         */
+        cb = ps->callbacks + ps->callback_count++;
+        cb->callback = callback;
+        cb->context = callback_context;
+
+        /*
+         * If there are no more exceptions in flight, or we have
+         * filled this metadata area we commit the exceptions to
+         * disk.
+         */
+        if (atomic_dec_and_test(&ps->pending_count) ||
+            (ps->current_committed == ps->exceptions_per_area)) {
+                r = area_io(ps, ps->current_area, WRITE);
+                if (r)
+                        ps->valid = 0;
+
+                for (i = 0; i < ps->callback_count; i++) {
+                        cb = ps->callbacks + i;
+                        cb->callback(cb->context, r == 0 ? 1 : 0);
+                }
+
+                ps->callback_count = 0;
+        }
+
+        /*
+         * Have we completely filled the current area ?
+         */
+        if (ps->current_committed == ps->exceptions_per_area) {
+                ps->current_committed = 0;
+                r = zero_area(ps, ps->current_area + 1);
+                if (r)
+                        ps->valid = 0;
+        }
+}
+
+static void persistent_drop(struct exception_store *store)
+{
+        struct pstore *ps = get_info(store);
+
+        ps->valid = 0;
+        if (write_header(ps))
+                DMWARN("write header failed");
+}
+
+int dm_create_persistent(struct exception_store *store, uint32_t chunk_size)
+{
+        int r;
+        struct pstore *ps;
+
+        r = dm_io_get(sectors_to_pages(chunk_size));
+        if (r)
+                return r;
+
+        /* allocate the pstore */
+        ps = kmalloc(sizeof(*ps), GFP_KERNEL);
+        if (!ps) {
+                r = -ENOMEM;
+                goto bad;
+        }
+
+        ps->snap = store->snap;
+        ps->valid = 1;
+        ps->version = SNAPSHOT_DISK_VERSION;
+        ps->chunk_size = chunk_size;
+        ps->exceptions_per_area = (chunk_size << SECTOR_SHIFT) /
+            sizeof(struct disk_exception);
+        ps->next_free = 2;      /* skipping the header and first area */
+        ps->current_committed = 0;
+
+        r = alloc_area(ps);
+        if (r)
+                goto bad;
+
+        /*
+         * Allocate space for all the callbacks.
+         */
+        ps->callback_count = 0;
+        atomic_set(&ps->pending_count, 0);
+        ps->callbacks = dm_vcalloc(ps->exceptions_per_area,
+                                   sizeof(*ps->callbacks));
+
+        if (!ps->callbacks) {
+                r = -ENOMEM;
+                goto bad;
+        }
+
+        store->destroy = persistent_destroy;
+        store->read_metadata = persistent_read_metadata;
+        store->prepare_exception = persistent_prepare;
+        store->commit_exception = persistent_commit;
+        store->drop_snapshot = persistent_drop;
+        store->fraction_full = persistent_fraction_full;
+        store->context = ps;
+
+        return 0;
+
+      bad:
+        dm_io_put(sectors_to_pages(chunk_size));
+        if (ps) {
+                if (ps->area)
+                        free_area(ps);
+
+                kfree(ps);
+        }
+        return r;
+}
+
+/*-----------------------------------------------------------------
+ * Implementation of the store for non-persistent snapshots.
+ *---------------------------------------------------------------*/
+struct transient_c {
+        sector_t next_free;
+};
+
+static void transient_destroy(struct exception_store *store)
+{
+        kfree(store->context);
+}
+
+static int transient_read_metadata(struct exception_store *store)
+{
+        return 0;
+}
+
+static int transient_prepare(struct exception_store *store, struct exception *e)
+{
+        struct transient_c *tc = (struct transient_c *) store->context;
+        sector_t size = get_dev_size(store->snap->cow->bdev);
+
+        if (size < (tc->next_free + store->snap->chunk_size))
+                return -1;
+
+        e->new_chunk = sector_to_chunk(store->snap, tc->next_free);
+        tc->next_free += store->snap->chunk_size;
+
+        return 0;
+}
+
+static void transient_commit(struct exception_store *store,
+                      struct exception *e,
+                      void (*callback) (void *, int success),
+                      void *callback_context)
+{
+        /* Just succeed */
+        callback(callback_context, 1);
+}
+
+static void transient_fraction_full(struct exception_store *store,
+                                    sector_t *numerator, sector_t *denominator)
+{
+        *numerator = ((struct transient_c *) store->context)->next_free;
+        *denominator = get_dev_size(store->snap->cow->bdev);
+}
+
+int dm_create_transient(struct exception_store *store,
+                        struct dm_snapshot *s, int blocksize)
+{
+        struct transient_c *tc;
+
+        memset(store, 0, sizeof(*store));
+        store->destroy = transient_destroy;
+        store->read_metadata = transient_read_metadata;
+        store->prepare_exception = transient_prepare;
+        store->commit_exception = transient_commit;
+        store->fraction_full = transient_fraction_full;
+        store->snap = s;
+
+        tc = kmalloc(sizeof(struct transient_c), GFP_KERNEL);
+        if (!tc)
+                return -ENOMEM;
+
+        tc->next_free = 0;
+        store->context = tc;
+
+        return 0;
+}
diff --git a/drivers/md/dm-hw-handler.c b/drivers/md/dm-hw-handler.c
new file mode 100644
index 000000000000..ae63772e44c9
--- /dev/null
+++ b/drivers/md/dm-hw-handler.c
@@ -0,0 +1,216 @@
+/*
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ *
+ * Multipath hardware handler registration.
+ */
+
+#include "dm.h"
+#include "dm-hw-handler.h"
+
+#include <linux/slab.h>
+
+struct hwh_internal {
+        struct hw_handler_type hwht;
+
+        struct list_head list;
+        long use;
+};
+
+#define hwht_to_hwhi(__hwht) container_of((__hwht), struct hwh_internal, hwht)
+
+static LIST_HEAD(_hw_handlers);
+static DECLARE_RWSEM(_hwh_lock);
+
+struct hwh_internal *__find_hw_handler_type(const char *name)
+{
+        struct hwh_internal *hwhi;
+
+        list_for_each_entry(hwhi, &_hw_handlers, list) {
+                if (!strcmp(name, hwhi->hwht.name))
+                        return hwhi;
+        }
+
+        return NULL;
+}
+
+static struct hwh_internal *get_hw_handler(const char *name)
+{
+        struct hwh_internal *hwhi;
+
+        down_read(&_hwh_lock);
+        hwhi = __find_hw_handler_type(name);
+        if (hwhi) {
+                if ((hwhi->use == 0) && !try_module_get(hwhi->hwht.module))
+                        hwhi = NULL;
+                else
+                        hwhi->use++;
+        }
+        up_read(&_hwh_lock);
+
+        return hwhi;
+}
+
+struct hw_handler_type *dm_get_hw_handler(const char *name)
+{
+        struct hwh_internal *hwhi;
+
+        if (!name)
+                return NULL;
+
+        hwhi = get_hw_handler(name);
+        if (!hwhi) {
+                request_module("dm-%s", name);
+                hwhi = get_hw_handler(name);
+        }
+
+        return hwhi ? &hwhi->hwht : NULL;
+}
+
+void dm_put_hw_handler(struct hw_handler_type *hwht)
+{
+        struct hwh_internal *hwhi;
+
+        if (!hwht)
+                return;
+
+        down_read(&_hwh_lock);
+        hwhi = __find_hw_handler_type(hwht->name);
+        if (!hwhi)
+                goto out;
+
+        if (--hwhi->use == 0)
+                module_put(hwhi->hwht.module);
+
+        if (hwhi->use < 0)
+                BUG();
+
+      out:
+        up_read(&_hwh_lock);
+}
+
+static struct hwh_internal *_alloc_hw_handler(struct hw_handler_type *hwht)
+{
+        struct hwh_internal *hwhi = kmalloc(sizeof(*hwhi), GFP_KERNEL);
+
+        if (hwhi) {
+                memset(hwhi, 0, sizeof(*hwhi));
+                hwhi->hwht = *hwht;
+        }
+
+        return hwhi;
+}
+
+int dm_register_hw_handler(struct hw_handler_type *hwht)
+{
+        int r = 0;
+        struct hwh_internal *hwhi = _alloc_hw_handler(hwht);
+
+        if (!hwhi)
+                return -ENOMEM;
+
+        down_write(&_hwh_lock);
+
+        if (__find_hw_handler_type(hwht->name)) {
+                kfree(hwhi);
+                r = -EEXIST;
+        } else
+                list_add(&hwhi->list, &_hw_handlers);
+
+        up_write(&_hwh_lock);
+
+        return r;
+}
+
+int dm_unregister_hw_handler(struct hw_handler_type *hwht)
+{
+        struct hwh_internal *hwhi;
+
+        down_write(&_hwh_lock);
+
+        hwhi = __find_hw_handler_type(hwht->name);
+        if (!hwhi) {
+                up_write(&_hwh_lock);
+                return -EINVAL;
+        }
+
+        if (hwhi->use) {
+                up_write(&_hwh_lock);
+                return -ETXTBSY;
+        }
+
+        list_del(&hwhi->list);
+
+        up_write(&_hwh_lock);
+
+        kfree(hwhi);
+
+        return 0;
+}
+
+unsigned dm_scsi_err_handler(struct hw_handler *hwh, struct bio *bio)
+{
+#if 0
+        int sense_key, asc, ascq;
+
+        if (bio->bi_error & BIO_SENSE) {
+                /* FIXME: This is just an initial guess. */
+                /* key / asc / ascq */
+                sense_key = (bio->bi_error >> 16) & 0xff;
+                asc = (bio->bi_error >> 8) & 0xff;
+                ascq = bio->bi_error & 0xff;
+
+                switch (sense_key) {
+                        /* This block as a whole comes from the device.
+                         * So no point retrying on another path. */
+                case 0x03:      /* Medium error */
+                case 0x05:      /* Illegal request */
+                case 0x07:      /* Data protect */
+                case 0x08:      /* Blank check */
+                case 0x0a:      /* copy aborted */
+                case 0x0c:      /* obsolete - no clue ;-) */
+                case 0x0d:      /* volume overflow */
+                case 0x0e:      /* data miscompare */
+                case 0x0f:      /* reserved - no idea either. */
+                        return MP_ERROR_IO;
+
+                        /* For these errors it's unclear whether they
+                         * come from the device or the controller.
+                         * So just lets try a different path, and if
+                         * it eventually succeeds, user-space will clear
+                         * the paths again... */
+                case 0x02:      /* Not ready */
+                case 0x04:      /* Hardware error */
+                case 0x09:      /* vendor specific */
+                case 0x0b:      /* Aborted command */
+                        return MP_FAIL_PATH;
+
+                case 0x06:      /* Unit attention - might want to decode */
+                        if (asc == 0x04 && ascq == 0x01)
+                                /* "Unit in the process of
+                                 * becoming ready" */
+                                return 0;
+                        return MP_FAIL_PATH;
+
+                        /* FIXME: For Unit Not Ready we may want
+                         * to have a generic pg activation
+                         * feature (START_UNIT). */
+
+                        /* Should these two ever end up in the
+                         * error path? I don't think so. */
+                case 0x00:      /* No sense */
+                case 0x01:      /* Recovered error */
+                        return 0;
+                }
+        }
+#endif
+
+        /* We got no idea how to decode the other kinds of errors ->
+         * assume generic error condition. */
+        return MP_FAIL_PATH;
+}
+
+EXPORT_SYMBOL_GPL(dm_register_hw_handler);
+EXPORT_SYMBOL_GPL(dm_unregister_hw_handler);
+EXPORT_SYMBOL_GPL(dm_scsi_err_handler);
diff --git a/drivers/md/dm-hw-handler.h b/drivers/md/dm-hw-handler.h
new file mode 100644
index 000000000000..15f5629e231a
--- /dev/null
+++ b/drivers/md/dm-hw-handler.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ *
+ * Multipath hardware handler registration.
+ */
+
+#ifndef DM_HW_HANDLER_H
+#define DM_HW_HANDLER_H
+
+#include <linux/device-mapper.h>
+
+#include "dm-mpath.h"
+
+struct hw_handler_type;
+struct hw_handler {
+        struct hw_handler_type *type;
+        void *context;
+};
+
+/*
+ * Constructs a hardware handler object, takes custom arguments
+ */
+/* Information about a hardware handler type */
+struct hw_handler_type {
+        char *name;
+        struct module *module;
+
+        int (*create) (struct hw_handler *handler, unsigned int argc,
+                       char **argv);
+        void (*destroy) (struct hw_handler *hwh);
+
+        void (*pg_init) (struct hw_handler *hwh, unsigned bypassed,
+                         struct path *path);
+        unsigned (*error) (struct hw_handler *hwh, struct bio *bio);
+        int (*status) (struct hw_handler *hwh, status_type_t type,
+                       char *result, unsigned int maxlen);
+};
+
+/* Register a hardware handler */
+int dm_register_hw_handler(struct hw_handler_type *type);
+
+/* Unregister a hardware handler */
+int dm_unregister_hw_handler(struct hw_handler_type *type);
+
+/* Returns a registered hardware handler type */
+struct hw_handler_type *dm_get_hw_handler(const char *name);
+
+/* Releases a hardware handler  */
+void dm_put_hw_handler(struct hw_handler_type *hwht);
+
+/* Default err function */
+unsigned dm_scsi_err_handler(struct hw_handler *hwh, struct bio *bio);
+
+/* Error flags for err and dm_pg_init_complete */
+#define MP_FAIL_PATH 1
+#define MP_BYPASS_PG 2
+#define MP_ERROR_IO  4  /* Don't retry this I/O */
+
+#endif
diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c
new file mode 100644
index 000000000000..45754bb6a799
--- /dev/null
+++ b/drivers/md/dm-io.c
@@ -0,0 +1,426 @@
+/*
+ * Copyright (C) 2003 Sistina Software
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm-io.h"
+
+#include <linux/bio.h>
+#include <linux/mempool.h>
+#include <linux/module.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+
+static struct bio_set *_bios;
+
+/* FIXME: can we shrink this ? */
+struct io {
+        unsigned long error;
+        atomic_t count;
+        struct task_struct *sleeper;
+        io_notify_fn callback;
+        void *context;
+};
+
+/*
+ * io contexts are only dynamically allocated for asynchronous
+ * io.  Since async io is likely to be the majority of io we'll
+ * have the same number of io contexts as buffer heads ! (FIXME:
+ * must reduce this).
+ */
+static unsigned _num_ios;
+static mempool_t *_io_pool;
+
+static void *alloc_io(unsigned int __nocast gfp_mask, void *pool_data)
+{
+        return kmalloc(sizeof(struct io), gfp_mask);
+}
+
+static void free_io(void *element, void *pool_data)
+{
+        kfree(element);
+}
+
+static unsigned int pages_to_ios(unsigned int pages)
+{
+        return 4 * pages;       /* too many ? */
+}
+
+static int resize_pool(unsigned int new_ios)
+{
+        int r = 0;
+
+        if (_io_pool) {
+                if (new_ios == 0) {
+                        /* free off the pool */
+                        mempool_destroy(_io_pool);
+                        _io_pool = NULL;
+                        bioset_free(_bios);
+
+                } else {
+                        /* resize the pool */
+                        r = mempool_resize(_io_pool, new_ios, GFP_KERNEL);
+                }
+
+        } else {
+                /* create new pool */
+                _io_pool = mempool_create(new_ios, alloc_io, free_io, NULL);
+                if (!_io_pool)
+                        return -ENOMEM;
+
+                _bios = bioset_create(16, 16, 4);
+                if (!_bios) {
+                        mempool_destroy(_io_pool);
+                        _io_pool = NULL;
+                        return -ENOMEM;
+                }
+        }
+
+        if (!r)
+                _num_ios = new_ios;
+
+        return r;
+}
+
+int dm_io_get(unsigned int num_pages)
+{
+        return resize_pool(_num_ios + pages_to_ios(num_pages));
+}
+
+void dm_io_put(unsigned int num_pages)
+{
+        resize_pool(_num_ios - pages_to_ios(num_pages));
+}
+
+/*-----------------------------------------------------------------
+ * We need to keep track of which region a bio is doing io for.
+ * In order to save a memory allocation we store this the last
+ * bvec which we know is unused (blech).
+ * XXX This is ugly and can OOPS with some configs... find another way.
+ *---------------------------------------------------------------*/
+static inline void bio_set_region(struct bio *bio, unsigned region)
+{
+        bio->bi_io_vec[bio->bi_max_vecs - 1].bv_len = region;
+}
+
+static inline unsigned bio_get_region(struct bio *bio)
+{
+        return bio->bi_io_vec[bio->bi_max_vecs - 1].bv_len;
+}
+
+/*-----------------------------------------------------------------
+ * We need an io object to keep track of the number of bios that
+ * have been dispatched for a particular io.
+ *---------------------------------------------------------------*/
+static void dec_count(struct io *io, unsigned int region, int error)
+{
+        if (error)
+                set_bit(region, &io->error);
+
+        if (atomic_dec_and_test(&io->count)) {
+                if (io->sleeper)
+                        wake_up_process(io->sleeper);
+
+                else {
+                        int r = io->error;
+                        io_notify_fn fn = io->callback;
+                        void *context = io->context;
+
+                        mempool_free(io, _io_pool);
+                        fn(r, context);
+                }
+        }
+}
+
+static int endio(struct bio *bio, unsigned int done, int error)
+{
+        struct io *io = (struct io *) bio->bi_private;
+
+        /* keep going until we've finished */
+        if (bio->bi_size)
+                return 1;
+
+        if (error && bio_data_dir(bio) == READ)
+                zero_fill_bio(bio);
+
+        dec_count(io, bio_get_region(bio), error);
+        bio_put(bio);
+
+        return 0;
+}
+
+/*-----------------------------------------------------------------
+ * These little objects provide an abstraction for getting a new
+ * destination page for io.
+ *---------------------------------------------------------------*/
+struct dpages {
+        void (*get_page)(struct dpages *dp,
+                         struct page **p, unsigned long *len, unsigned *offset);
+        void (*next_page)(struct dpages *dp);
+
+        unsigned context_u;
+        void *context_ptr;
+};
+
+/*
+ * Functions for getting the pages from a list.
+ */
+static void list_get_page(struct dpages *dp,
+                  struct page **p, unsigned long *len, unsigned *offset)
+{
+        unsigned o = dp->context_u;
+        struct page_list *pl = (struct page_list *) dp->context_ptr;
+
+        *p = pl->page;
+        *len = PAGE_SIZE - o;
+        *offset = o;
+}
+
+static void list_next_page(struct dpages *dp)
+{
+        struct page_list *pl = (struct page_list *) dp->context_ptr;
+        dp->context_ptr = pl->next;
+        dp->context_u = 0;
+}
+
+static void list_dp_init(struct dpages *dp, struct page_list *pl, unsigned offset)
+{
+        dp->get_page = list_get_page;
+        dp->next_page = list_next_page;
+        dp->context_u = offset;
+        dp->context_ptr = pl;
+}
+
+/*
+ * Functions for getting the pages from a bvec.
+ */
+static void bvec_get_page(struct dpages *dp,
+                  struct page **p, unsigned long *len, unsigned *offset)
+{
+        struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr;
+        *p = bvec->bv_page;
+        *len = bvec->bv_len;
+        *offset = bvec->bv_offset;
+}
+
+static void bvec_next_page(struct dpages *dp)
+{
+        struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr;
+        dp->context_ptr = bvec + 1;
+}
+
+static void bvec_dp_init(struct dpages *dp, struct bio_vec *bvec)
+{
+        dp->get_page = bvec_get_page;
+        dp->next_page = bvec_next_page;
+        dp->context_ptr = bvec;
+}
+
+static void vm_get_page(struct dpages *dp,
+                 struct page **p, unsigned long *len, unsigned *offset)
+{
+        *p = vmalloc_to_page(dp->context_ptr);
+        *offset = dp->context_u;
+        *len = PAGE_SIZE - dp->context_u;
+}
+
+static void vm_next_page(struct dpages *dp)
+{
+        dp->context_ptr += PAGE_SIZE - dp->context_u;
+        dp->context_u = 0;
+}
+
+static void vm_dp_init(struct dpages *dp, void *data)
+{
+        dp->get_page = vm_get_page;
+        dp->next_page = vm_next_page;
+        dp->context_u = ((unsigned long) data) & (PAGE_SIZE - 1);
+        dp->context_ptr = data;
+}
+
+/*-----------------------------------------------------------------
+ * IO routines that accept a list of pages.
+ *---------------------------------------------------------------*/
+static void do_region(int rw, unsigned int region, struct io_region *where,
+                      struct dpages *dp, struct io *io)
+{
+        struct bio *bio;
+        struct page *page;
+        unsigned long len;
+        unsigned offset;
+        unsigned num_bvecs;
+        sector_t remaining = where->count;
+
+        while (remaining) {
+                /*
+                 * Allocate a suitably sized bio, we add an extra
+                 * bvec for bio_get/set_region().
+                 */
+                num_bvecs = (remaining / (PAGE_SIZE >> 9)) + 2;
+                bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, _bios);
+                bio->bi_sector = where->sector + (where->count - remaining);
+                bio->bi_bdev = where->bdev;
+                bio->bi_end_io = endio;
+                bio->bi_private = io;
+                bio_set_region(bio, region);
+
+                /*
+                 * Try and add as many pages as possible.
+                 */
+                while (remaining) {
+                        dp->get_page(dp, &page, &len, &offset);
+                        len = min(len, to_bytes(remaining));
+                        if (!bio_add_page(bio, page, len, offset))
+                                break;
+
+                        offset = 0;
+                        remaining -= to_sector(len);
+                        dp->next_page(dp);
+                }
+
+                atomic_inc(&io->count);
+                submit_bio(rw, bio);
+        }
+}
+
+static void dispatch_io(int rw, unsigned int num_regions,
+                        struct io_region *where, struct dpages *dp,
+                        struct io *io, int sync)
+{
+        int i;
+        struct dpages old_pages = *dp;
+
+        if (sync)
+                rw |= (1 << BIO_RW_SYNC);
+
+        /*
+         * For multiple regions we need to be careful to rewind
+         * the dp object for each call to do_region.
+         */
+        for (i = 0; i < num_regions; i++) {
+                *dp = old_pages;
+                if (where[i].count)
+                        do_region(rw, i, where + i, dp, io);
+        }
+
+        /*
+         * Drop the extra refence that we were holding to avoid
+         * the io being completed too early.
+         */
+        dec_count(io, 0, 0);
+}
+
+static int sync_io(unsigned int num_regions, struct io_region *where,
+            int rw, struct dpages *dp, unsigned long *error_bits)
+{
+        struct io io;
+
+        if (num_regions > 1 && rw != WRITE) {
+                WARN_ON(1);
+                return -EIO;
+        }
+
+        io.error = 0;
+        atomic_set(&io.count, 1); /* see dispatch_io() */
+        io.sleeper = current;
+
+        dispatch_io(rw, num_regions, where, dp, &io, 1);
+
+        while (1) {
+                set_current_state(TASK_UNINTERRUPTIBLE);
+
+                if (!atomic_read(&io.count) || signal_pending(current))
+                        break;
+
+                io_schedule();
+        }
+        set_current_state(TASK_RUNNING);
+
+        if (atomic_read(&io.count))
+                return -EINTR;
+
+        *error_bits = io.error;
+        return io.error ? -EIO : 0;
+}
+
+static int async_io(unsigned int num_regions, struct io_region *where, int rw,
+             struct dpages *dp, io_notify_fn fn, void *context)
+{
+        struct io *io;
+
+        if (num_regions > 1 && rw != WRITE) {
+                WARN_ON(1);
+                fn(1, context);
+                return -EIO;
+        }
+
+        io = mempool_alloc(_io_pool, GFP_NOIO);
+        io->error = 0;
+        atomic_set(&io->count, 1); /* see dispatch_io() */
+        io->sleeper = NULL;
+        io->callback = fn;
+        io->context = context;
+
+        dispatch_io(rw, num_regions, where, dp, io, 0);
+        return 0;
+}
+
+int dm_io_sync(unsigned int num_regions, struct io_region *where, int rw,
+               struct page_list *pl, unsigned int offset,
+               unsigned long *error_bits)
+{
+        struct dpages dp;
+        list_dp_init(&dp, pl, offset);
+        return sync_io(num_regions, where, rw, &dp, error_bits);
+}
+
+int dm_io_sync_bvec(unsigned int num_regions, struct io_region *where, int rw,
+                    struct bio_vec *bvec, unsigned long *error_bits)
+{
+        struct dpages dp;
+        bvec_dp_init(&dp, bvec);
+        return sync_io(num_regions, where, rw, &dp, error_bits);
+}
+
+int dm_io_sync_vm(unsigned int num_regions, struct io_region *where, int rw,
+                  void *data, unsigned long *error_bits)
+{
+        struct dpages dp;
+        vm_dp_init(&dp, data);
+        return sync_io(num_regions, where, rw, &dp, error_bits);
+}
+
+int dm_io_async(unsigned int num_regions, struct io_region *where, int rw,
+                struct page_list *pl, unsigned int offset,
+                io_notify_fn fn, void *context)
+{
+        struct dpages dp;
+        list_dp_init(&dp, pl, offset);
+        return async_io(num_regions, where, rw, &dp, fn, context);
+}
+
+int dm_io_async_bvec(unsigned int num_regions, struct io_region *where, int rw,
+                     struct bio_vec *bvec, io_notify_fn fn, void *context)
+{
+        struct dpages dp;
+        bvec_dp_init(&dp, bvec);
+        return async_io(num_regions, where, rw, &dp, fn, context);
+}
+
+int dm_io_async_vm(unsigned int num_regions, struct io_region *where, int rw,
+                   void *data, io_notify_fn fn, void *context)
+{
+        struct dpages dp;
+        vm_dp_init(&dp, data);
+        return async_io(num_regions, where, rw, &dp, fn, context);
+}
+
+EXPORT_SYMBOL(dm_io_get);
+EXPORT_SYMBOL(dm_io_put);
+EXPORT_SYMBOL(dm_io_sync);
+EXPORT_SYMBOL(dm_io_async);
+EXPORT_SYMBOL(dm_io_sync_bvec);
+EXPORT_SYMBOL(dm_io_async_bvec);
+EXPORT_SYMBOL(dm_io_sync_vm);
+EXPORT_SYMBOL(dm_io_async_vm);
diff --git a/drivers/md/dm-io.h b/drivers/md/dm-io.h
new file mode 100644
index 000000000000..1a77f3265706
--- /dev/null
+++ b/drivers/md/dm-io.h
@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2003 Sistina Software
+ *
+ * This file is released under the GPL.
+ */
+
+#ifndef _DM_IO_H
+#define _DM_IO_H
+
+#include "dm.h"
+
+/* FIXME make this configurable */
+#define DM_MAX_IO_REGIONS 8
+
+struct io_region {
+        struct block_device *bdev;
+        sector_t sector;
+        sector_t count;
+};
+
+struct page_list {
+        struct page_list *next;
+        struct page *page;
+};
+
+
+/*
+ * 'error' is a bitset, with each bit indicating whether an error
+ * occurred doing io to the corresponding region.
+ */
+typedef void (*io_notify_fn)(unsigned long error, void *context);
+
+
+/*
+ * Before anyone uses the IO interface they should call
+ * dm_io_get(), specifying roughly how many pages they are
+ * expecting to perform io on concurrently.
+ *
+ * This function may block.
+ */
+int dm_io_get(unsigned int num_pages);
+void dm_io_put(unsigned int num_pages);
+
+/*
+ * Synchronous IO.
+ *
+ * Please ensure that the rw flag in the next two functions is
+ * either READ or WRITE, ie. we don't take READA.  Any
+ * regions with a zero count field will be ignored.
+ */
+int dm_io_sync(unsigned int num_regions, struct io_region *where, int rw,
+               struct page_list *pl, unsigned int offset,
+               unsigned long *error_bits);
+
+int dm_io_sync_bvec(unsigned int num_regions, struct io_region *where, int rw,
+                    struct bio_vec *bvec, unsigned long *error_bits);
+
+int dm_io_sync_vm(unsigned int num_regions, struct io_region *where, int rw,
+                  void *data, unsigned long *error_bits);
+
+/*
+ * Aynchronous IO.
+ *
+ * The 'where' array may be safely allocated on the stack since
+ * the function takes a copy.
+ */
+int dm_io_async(unsigned int num_regions, struct io_region *where, int rw,
+                struct page_list *pl, unsigned int offset,
+                io_notify_fn fn, void *context);
+
+int dm_io_async_bvec(unsigned int num_regions, struct io_region *where, int rw,
+                     struct bio_vec *bvec, io_notify_fn fn, void *context);
+
+int dm_io_async_vm(unsigned int num_regions, struct io_region *where, int rw,
+                   void *data, io_notify_fn fn, void *context);
+
+#endif
diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c
new file mode 100644
index 000000000000..ee3c869d9701
--- /dev/null
+++ b/drivers/md/dm-ioctl.c
@@ -0,0 +1,1416 @@
+/*
+ * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
+ * Copyright (C) 2004 - 2005 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+
+#include <linux/module.h>
+#include <linux/vmalloc.h>
+#include <linux/miscdevice.h>
+#include <linux/init.h>
+#include <linux/wait.h>
+#include <linux/slab.h>
+#include <linux/devfs_fs_kernel.h>
+#include <linux/dm-ioctl.h>
+
+#include <asm/uaccess.h>
+
+#define DM_DRIVER_EMAIL "dm-devel@redhat.com"
+
+/*-----------------------------------------------------------------
+ * The ioctl interface needs to be able to look up devices by
+ * name or uuid.
+ *---------------------------------------------------------------*/
+struct hash_cell {
+        struct list_head name_list;
+        struct list_head uuid_list;
+
+        char *name;
+        char *uuid;
+        struct mapped_device *md;
+        struct dm_table *new_map;
+};
+
+struct vers_iter {
+    size_t param_size;
+    struct dm_target_versions *vers, *old_vers;
+    char *end;
+    uint32_t flags;
+};
+
+
+#define NUM_BUCKETS 64
+#define MASK_BUCKETS (NUM_BUCKETS - 1)
+static struct list_head _name_buckets[NUM_BUCKETS];
+static struct list_head _uuid_buckets[NUM_BUCKETS];
+
+static void dm_hash_remove_all(void);
+
+/*
+ * Guards access to both hash tables.
+ */
+static DECLARE_RWSEM(_hash_lock);
+
+static void init_buckets(struct list_head *buckets)
+{
+        unsigned int i;
+
+        for (i = 0; i < NUM_BUCKETS; i++)
+                INIT_LIST_HEAD(buckets + i);
+}
+
+static int dm_hash_init(void)
+{
+        init_buckets(_name_buckets);
+        init_buckets(_uuid_buckets);
+        devfs_mk_dir(DM_DIR);
+        return 0;
+}
+
+static void dm_hash_exit(void)
+{
+        dm_hash_remove_all();
+        devfs_remove(DM_DIR);
+}
+
+/*-----------------------------------------------------------------
+ * Hash function:
+ * We're not really concerned with the str hash function being
+ * fast since it's only used by the ioctl interface.
+ *---------------------------------------------------------------*/
+static unsigned int hash_str(const char *str)
+{
+        const unsigned int hash_mult = 2654435387U;
+        unsigned int h = 0;
+
+        while (*str)
+                h = (h + (unsigned int) *str++) * hash_mult;
+
+        return h & MASK_BUCKETS;
+}
+
+/*-----------------------------------------------------------------
+ * Code for looking up a device by name
+ *---------------------------------------------------------------*/
+static struct hash_cell *__get_name_cell(const char *str)
+{
+        struct hash_cell *hc;
+        unsigned int h = hash_str(str);
+
+        list_for_each_entry (hc, _name_buckets + h, name_list)
+                if (!strcmp(hc->name, str))
+                        return hc;
+
+        return NULL;
+}
+
+static struct hash_cell *__get_uuid_cell(const char *str)
+{
+        struct hash_cell *hc;
+        unsigned int h = hash_str(str);
+
+        list_for_each_entry (hc, _uuid_buckets + h, uuid_list)
+                if (!strcmp(hc->uuid, str))
+                        return hc;
+
+        return NULL;
+}
+
+/*-----------------------------------------------------------------
+ * Inserting, removing and renaming a device.
+ *---------------------------------------------------------------*/
+static inline char *kstrdup(const char *str)
+{
+        char *r = kmalloc(strlen(str) + 1, GFP_KERNEL);
+        if (r)
+                strcpy(r, str);
+        return r;
+}
+
+static struct hash_cell *alloc_cell(const char *name, const char *uuid,
+                                    struct mapped_device *md)
+{
+        struct hash_cell *hc;
+
+        hc = kmalloc(sizeof(*hc), GFP_KERNEL);
+        if (!hc)
+                return NULL;
+
+        hc->name = kstrdup(name);
+        if (!hc->name) {
+                kfree(hc);
+                return NULL;
+        }
+
+        if (!uuid)
+                hc->uuid = NULL;
+
+        else {
+                hc->uuid = kstrdup(uuid);
+                if (!hc->uuid) {
+                        kfree(hc->name);
+                        kfree(hc);
+                        return NULL;
+                }
+        }
+
+        INIT_LIST_HEAD(&hc->name_list);
+        INIT_LIST_HEAD(&hc->uuid_list);
+        hc->md = md;
+        hc->new_map = NULL;
+        return hc;
+}
+
+static void free_cell(struct hash_cell *hc)
+{
+        if (hc) {
+                kfree(hc->name);
+                kfree(hc->uuid);
+                kfree(hc);
+        }
+}
+
+/*
+ * devfs stuff.
+ */
+static int register_with_devfs(struct hash_cell *hc)
+{
+        struct gendisk *disk = dm_disk(hc->md);
+
+        devfs_mk_bdev(MKDEV(disk->major, disk->first_minor),
+                      S_IFBLK | S_IRUSR | S_IWUSR | S_IRGRP,
+                      DM_DIR "/%s", hc->name);
+        return 0;
+}
+
+static int unregister_with_devfs(struct hash_cell *hc)
+{
+        devfs_remove(DM_DIR"/%s", hc->name);
+        return 0;
+}
+
+/*
+ * The kdev_t and uuid of a device can never change once it is
+ * initially inserted.
+ */
+static int dm_hash_insert(const char *name, const char *uuid, struct mapped_device *md)
+{
+        struct hash_cell *cell;
+
+        /*
+         * Allocate the new cells.
+         */
+        cell = alloc_cell(name, uuid, md);
+        if (!cell)
+                return -ENOMEM;
+
+        /*
+         * Insert the cell into both hash tables.
+         */
+        down_write(&_hash_lock);
+        if (__get_name_cell(name))
+                goto bad;
+
+        list_add(&cell->name_list, _name_buckets + hash_str(name));
+
+        if (uuid) {
+                if (__get_uuid_cell(uuid)) {
+                        list_del(&cell->name_list);
+                        goto bad;
+                }
+                list_add(&cell->uuid_list, _uuid_buckets + hash_str(uuid));
+        }
+        register_with_devfs(cell);
+        dm_get(md);
+        dm_set_mdptr(md, cell);
+        up_write(&_hash_lock);
+
+        return 0;
+
+ bad:
+        up_write(&_hash_lock);
+        free_cell(cell);
+        return -EBUSY;
+}
+
+static void __hash_remove(struct hash_cell *hc)
+{
+        /* remove from the dev hash */
+        list_del(&hc->uuid_list);
+        list_del(&hc->name_list);
+        unregister_with_devfs(hc);
+        dm_set_mdptr(hc->md, NULL);
+        dm_put(hc->md);
+        if (hc->new_map)
+                dm_table_put(hc->new_map);
+        free_cell(hc);
+}
+
+static void dm_hash_remove_all(void)
+{
+        int i;
+        struct hash_cell *hc;
+        struct list_head *tmp, *n;
+
+        down_write(&_hash_lock);
+        for (i = 0; i < NUM_BUCKETS; i++) {
+                list_for_each_safe (tmp, n, _name_buckets + i) {
+                        hc = list_entry(tmp, struct hash_cell, name_list);
+                        __hash_remove(hc);
+                }
+        }
+        up_write(&_hash_lock);
+}
+
+static int dm_hash_rename(const char *old, const char *new)
+{
+        char *new_name, *old_name;
+        struct hash_cell *hc;
+
+        /*
+         * duplicate new.
+         */
+        new_name = kstrdup(new);
+        if (!new_name)
+                return -ENOMEM;
+
+        down_write(&_hash_lock);
+
+        /*
+         * Is new free ?
+         */
+        hc = __get_name_cell(new);
+        if (hc) {
+                DMWARN("asked to rename to an already existing name %s -> %s",
+                       old, new);
+                up_write(&_hash_lock);
+                kfree(new_name);
+                return -EBUSY;
+        }
+
+        /*
+         * Is there such a device as 'old' ?
+         */
+        hc = __get_name_cell(old);
+        if (!hc) {
+                DMWARN("asked to rename a non existent device %s -> %s",
+                       old, new);
+                up_write(&_hash_lock);
+                kfree(new_name);
+                return -ENXIO;
+        }
+
+        /*
+         * rename and move the name cell.
+         */
+        unregister_with_devfs(hc);
+
+        list_del(&hc->name_list);
+        old_name = hc->name;
+        hc->name = new_name;
+        list_add(&hc->name_list, _name_buckets + hash_str(new_name));
+
+        /* rename the device node in devfs */
+        register_with_devfs(hc);
+
+        up_write(&_hash_lock);
+        kfree(old_name);
+        return 0;
+}
+
+/*-----------------------------------------------------------------
+ * Implementation of the ioctl commands
+ *---------------------------------------------------------------*/
+/*
+ * All the ioctl commands get dispatched to functions with this
+ * prototype.
+ */
+typedef int (*ioctl_fn)(struct dm_ioctl *param, size_t param_size);
+
+static int remove_all(struct dm_ioctl *param, size_t param_size)
+{
+        dm_hash_remove_all();
+        param->data_size = 0;
+        return 0;
+}
+
+/*
+ * Round up the ptr to an 8-byte boundary.
+ */
+#define ALIGN_MASK 7
+static inline void *align_ptr(void *ptr)
+{
+        return (void *) (((size_t) (ptr + ALIGN_MASK)) & ~ALIGN_MASK);
+}
+
+/*
+ * Retrieves the data payload buffer from an already allocated
+ * struct dm_ioctl.
+ */
+static void *get_result_buffer(struct dm_ioctl *param, size_t param_size,
+                               size_t *len)
+{
+        param->data_start = align_ptr(param + 1) - (void *) param;
+
+        if (param->data_start < param_size)
+                *len = param_size - param->data_start;
+        else
+                *len = 0;
+
+        return ((void *) param) + param->data_start;
+}
+
+static int list_devices(struct dm_ioctl *param, size_t param_size)
+{
+        unsigned int i;
+        struct hash_cell *hc;
+        size_t len, needed = 0;
+        struct gendisk *disk;
+        struct dm_name_list *nl, *old_nl = NULL;
+
+        down_write(&_hash_lock);
+
+        /*
+         * Loop through all the devices working out how much
+         * space we need.
+         */
+        for (i = 0; i < NUM_BUCKETS; i++) {
+                list_for_each_entry (hc, _name_buckets + i, name_list) {
+                        needed += sizeof(struct dm_name_list);
+                        needed += strlen(hc->name) + 1;
+                        needed += ALIGN_MASK;
+                }
+        }
+
+        /*
+         * Grab our output buffer.
+         */
+        nl = get_result_buffer(param, param_size, &len);
+        if (len < needed) {
+                param->flags |= DM_BUFFER_FULL_FLAG;
+                goto out;
+        }
+        param->data_size = param->data_start + needed;
+
+        nl->dev = 0;    /* Flags no data */
+
+        /*
+         * Now loop through filling out the names.
+         */
+        for (i = 0; i < NUM_BUCKETS; i++) {
+                list_for_each_entry (hc, _name_buckets + i, name_list) {
+                        if (old_nl)
+                                old_nl->next = (uint32_t) ((void *) nl -
+                                                           (void *) old_nl);
+                        disk = dm_disk(hc->md);
+                        nl->dev = huge_encode_dev(MKDEV(disk->major, disk->first_minor));
+                        nl->next = 0;
+                        strcpy(nl->name, hc->name);
+
+                        old_nl = nl;
+                        nl = align_ptr(((void *) ++nl) + strlen(hc->name) + 1);
+                }
+        }
+
+ out:
+        up_write(&_hash_lock);
+        return 0;
+}
+
+static void list_version_get_needed(struct target_type *tt, void *needed_param)
+{
+    size_t *needed = needed_param;
+
+    *needed += strlen(tt->name);
+    *needed += sizeof(tt->version);
+    *needed += ALIGN_MASK;
+}
+
+static void list_version_get_info(struct target_type *tt, void *param)
+{
+    struct vers_iter *info = param;
+
+    /* Check space - it might have changed since the first iteration */
+    if ((char *)info->vers + sizeof(tt->version) + strlen(tt->name) + 1 >
+        info->end) {
+
+        info->flags = DM_BUFFER_FULL_FLAG;
+        return;
+    }
+
+    if (info->old_vers)
+        info->old_vers->next = (uint32_t) ((void *)info->vers -
+                                           (void *)info->old_vers);
+    info->vers->version[0] = tt->version[0];
+    info->vers->version[1] = tt->version[1];
+    info->vers->version[2] = tt->version[2];
+    info->vers->next = 0;
+    strcpy(info->vers->name, tt->name);
+
+    info->old_vers = info->vers;
+    info->vers = align_ptr(((void *) ++info->vers) + strlen(tt->name) + 1);
+}
+
+static int list_versions(struct dm_ioctl *param, size_t param_size)
+{
+        size_t len, needed = 0;
+        struct dm_target_versions *vers;
+        struct vers_iter iter_info;
+
+        /*
+         * Loop through all the devices working out how much
+         * space we need.
+         */
+        dm_target_iterate(list_version_get_needed, &needed);
+
+        /*
+         * Grab our output buffer.
+         */
+        vers = get_result_buffer(param, param_size, &len);
+        if (len < needed) {
+                param->flags |= DM_BUFFER_FULL_FLAG;
+                goto out;
+        }
+        param->data_size = param->data_start + needed;
+
+        iter_info.param_size = param_size;
+        iter_info.old_vers = NULL;
+        iter_info.vers = vers;
+        iter_info.flags = 0;
+        iter_info.end = (char *)vers+len;
+
+        /*
+         * Now loop through filling out the names & versions.
+         */
+        dm_target_iterate(list_version_get_info, &iter_info);
+        param->flags |= iter_info.flags;
+
+ out:
+        return 0;
+}
+
+
+
+static int check_name(const char *name)
+{
+        if (strchr(name, '/')) {
+                DMWARN("invalid device name");
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+/*
+ * Fills in a dm_ioctl structure, ready for sending back to
+ * userland.
+ */
+static int __dev_status(struct mapped_device *md, struct dm_ioctl *param)
+{
+        struct gendisk *disk = dm_disk(md);
+        struct dm_table *table;
+        struct block_device *bdev;
+
+        param->flags &= ~(DM_SUSPEND_FLAG | DM_READONLY_FLAG |
+                          DM_ACTIVE_PRESENT_FLAG);
+
+        if (dm_suspended(md))
+                param->flags |= DM_SUSPEND_FLAG;
+
+        param->dev = huge_encode_dev(MKDEV(disk->major, disk->first_minor));
+
+        if (!(param->flags & DM_SKIP_BDGET_FLAG)) {
+                bdev = bdget_disk(disk, 0);
+                if (!bdev)
+                        return -ENXIO;
+
+                /*
+                 * Yes, this will be out of date by the time it gets back
+                 * to userland, but it is still very useful for
+                 * debugging.
+                 */
+                param->open_count = bdev->bd_openers;
+                bdput(bdev);
+        } else
+                param->open_count = -1;
+
+        if (disk->policy)
+                param->flags |= DM_READONLY_FLAG;
+
+        param->event_nr = dm_get_event_nr(md);
+
+        table = dm_get_table(md);
+        if (table) {
+                param->flags |= DM_ACTIVE_PRESENT_FLAG;
+                param->target_count = dm_table_get_num_targets(table);
+                dm_table_put(table);
+        } else
+                param->target_count = 0;
+
+        return 0;
+}
+
+static int dev_create(struct dm_ioctl *param, size_t param_size)
+{
+        int r;
+        struct mapped_device *md;
+
+        r = check_name(param->name);
+        if (r)
+                return r;
+
+        if (param->flags & DM_PERSISTENT_DEV_FLAG)
+                r = dm_create_with_minor(MINOR(huge_decode_dev(param->dev)), &md);
+        else
+                r = dm_create(&md);
+
+        if (r)
+                return r;
+
+        r = dm_hash_insert(param->name, *param->uuid ? param->uuid : NULL, md);
+        if (r) {
+                dm_put(md);
+                return r;
+        }
+
+        param->flags &= ~DM_INACTIVE_PRESENT_FLAG;
+
+        r = __dev_status(md, param);
+        dm_put(md);
+
+        return r;
+}
+
+/*
+ * Always use UUID for lookups if it's present, otherwise use name or dev.
+ */
+static inline struct hash_cell *__find_device_hash_cell(struct dm_ioctl *param)
+{
+        if (*param->uuid)
+                return __get_uuid_cell(param->uuid);
+        else if (*param->name)
+                return __get_name_cell(param->name);
+        else
+                return dm_get_mdptr(huge_decode_dev(param->dev));
+}
+
+static inline struct mapped_device *find_device(struct dm_ioctl *param)
+{
+        struct hash_cell *hc;
+        struct mapped_device *md = NULL;
+
+        down_read(&_hash_lock);
+        hc = __find_device_hash_cell(param);
+        if (hc) {
+                md = hc->md;
+                dm_get(md);
+
+                /*
+                 * Sneakily write in both the name and the uuid
+                 * while we have the cell.
+                 */
+                strncpy(param->name, hc->name, sizeof(param->name));
+                if (hc->uuid)
+                        strncpy(param->uuid, hc->uuid, sizeof(param->uuid)-1);
+                else
+                        param->uuid[0] = '\0';
+
+                if (hc->new_map)
+                        param->flags |= DM_INACTIVE_PRESENT_FLAG;
+                else
+                        param->flags &= ~DM_INACTIVE_PRESENT_FLAG;
+        }
+        up_read(&_hash_lock);
+
+        return md;
+}
+
+static int dev_remove(struct dm_ioctl *param, size_t param_size)
+{
+        struct hash_cell *hc;
+
+        down_write(&_hash_lock);
+        hc = __find_device_hash_cell(param);
+
+        if (!hc) {
+                DMWARN("device doesn't appear to be in the dev hash table.");
+                up_write(&_hash_lock);
+                return -ENXIO;
+        }
+
+        __hash_remove(hc);
+        up_write(&_hash_lock);
+        param->data_size = 0;
+        return 0;
+}
+
+/*
+ * Check a string doesn't overrun the chunk of
+ * memory we copied from userland.
+ */
+static int invalid_str(char *str, void *end)
+{
+        while ((void *) str < end)
+                if (!*str++)
+                        return 0;
+
+        return -EINVAL;
+}
+
+static int dev_rename(struct dm_ioctl *param, size_t param_size)
+{
+        int r;
+        char *new_name = (char *) param + param->data_start;
+
+        if (new_name < (char *) (param + 1) ||
+            invalid_str(new_name, (void *) param + param_size)) {
+                DMWARN("Invalid new logical volume name supplied.");
+                return -EINVAL;
+        }
+
+        r = check_name(new_name);
+        if (r)
+                return r;
+
+        param->data_size = 0;
+        return dm_hash_rename(param->name, new_name);
+}
+
+static int do_suspend(struct dm_ioctl *param)
+{
+        int r = 0;
+        struct mapped_device *md;
+
+        md = find_device(param);
+        if (!md)
+                return -ENXIO;
+
+        if (!dm_suspended(md))
+                r = dm_suspend(md);
+
+        if (!r)
+                r = __dev_status(md, param);
+
+        dm_put(md);
+        return r;
+}
+
+static int do_resume(struct dm_ioctl *param)
+{
+        int r = 0;
+        struct hash_cell *hc;
+        struct mapped_device *md;
+        struct dm_table *new_map;
+
+        down_write(&_hash_lock);
+
+        hc = __find_device_hash_cell(param);
+        if (!hc) {
+                DMWARN("device doesn't appear to be in the dev hash table.");
+                up_write(&_hash_lock);
+                return -ENXIO;
+        }
+
+        md = hc->md;
+        dm_get(md);
+
+        new_map = hc->new_map;
+        hc->new_map = NULL;
+        param->flags &= ~DM_INACTIVE_PRESENT_FLAG;
+
+        up_write(&_hash_lock);
+
+        /* Do we need to load a new map ? */
+        if (new_map) {
+                /* Suspend if it isn't already suspended */
+                if (!dm_suspended(md))
+                        dm_suspend(md);
+
+                r = dm_swap_table(md, new_map);
+                if (r) {
+                        dm_put(md);
+                        dm_table_put(new_map);
+                        return r;
+                }
+
+                if (dm_table_get_mode(new_map) & FMODE_WRITE)
+                        set_disk_ro(dm_disk(md), 0);
+                else
+                        set_disk_ro(dm_disk(md), 1);
+
+                dm_table_put(new_map);
+        }
+
+        if (dm_suspended(md))
+                r = dm_resume(md);
+
+        if (!r)
+                r = __dev_status(md, param);
+
+        dm_put(md);
+        return r;
+}
+
+/*
+ * Set or unset the suspension state of a device.
+ * If the device already is in the requested state we just return its status.
+ */
+static int dev_suspend(struct dm_ioctl *param, size_t param_size)
+{
+        if (param->flags & DM_SUSPEND_FLAG)
+                return do_suspend(param);
+
+        return do_resume(param);
+}
+
+/*
+ * Copies device info back to user space, used by
+ * the create and info ioctls.
+ */
+static int dev_status(struct dm_ioctl *param, size_t param_size)
+{
+        int r;
+        struct mapped_device *md;
+
+        md = find_device(param);
+        if (!md)
+                return -ENXIO;
+
+        r = __dev_status(md, param);
+        dm_put(md);
+        return r;
+}
+
+/*
+ * Build up the status struct for each target
+ */
+static void retrieve_status(struct dm_table *table,
+                            struct dm_ioctl *param, size_t param_size)
+{
+        unsigned int i, num_targets;
+        struct dm_target_spec *spec;
+        char *outbuf, *outptr;
+        status_type_t type;
+        size_t remaining, len, used = 0;
+
+        outptr = outbuf = get_result_buffer(param, param_size, &len);
+
+        if (param->flags & DM_STATUS_TABLE_FLAG)
+                type = STATUSTYPE_TABLE;
+        else
+                type = STATUSTYPE_INFO;
+
+        /* Get all the target info */
+        num_targets = dm_table_get_num_targets(table);
+        for (i = 0; i < num_targets; i++) {
+                struct dm_target *ti = dm_table_get_target(table, i);
+
+                remaining = len - (outptr - outbuf);
+                if (remaining <= sizeof(struct dm_target_spec)) {
+                        param->flags |= DM_BUFFER_FULL_FLAG;
+                        break;
+                }
+
+                spec = (struct dm_target_spec *) outptr;
+
+                spec->status = 0;
+                spec->sector_start = ti->begin;
+                spec->length = ti->len;
+                strncpy(spec->target_type, ti->type->name,
+                        sizeof(spec->target_type));
+
+                outptr += sizeof(struct dm_target_spec);
+                remaining = len - (outptr - outbuf);
+                if (remaining <= 0) {
+                        param->flags |= DM_BUFFER_FULL_FLAG;
+                        break;
+                }
+
+                /* Get the status/table string from the target driver */
+                if (ti->type->status) {
+                        if (ti->type->status(ti, type, outptr, remaining)) {
+                                param->flags |= DM_BUFFER_FULL_FLAG;
+                                break;
+                        }
+                } else
+                        outptr[0] = '\0';
+
+                outptr += strlen(outptr) + 1;
+                used = param->data_start + (outptr - outbuf);
+
+                outptr = align_ptr(outptr);
+                spec->next = outptr - outbuf;
+        }
+
+        if (used)
+                param->data_size = used;
+
+        param->target_count = num_targets;
+}
+
+/*
+ * Wait for a device to report an event
+ */
+static int dev_wait(struct dm_ioctl *param, size_t param_size)
+{
+        int r;
+        struct mapped_device *md;
+        struct dm_table *table;
+
+        md = find_device(param);
+        if (!md)
+                return -ENXIO;
+
+        /*
+         * Wait for a notification event
+         */
+        if (dm_wait_event(md, param->event_nr)) {
+                r = -ERESTARTSYS;
+                goto out;
+        }
+
+        /*
+         * The userland program is going to want to know what
+         * changed to trigger the event, so we may as well tell
+         * him and save an ioctl.
+         */
+        r = __dev_status(md, param);
+        if (r)
+                goto out;
+
+        table = dm_get_table(md);
+        if (table) {
+                retrieve_status(table, param, param_size);
+                dm_table_put(table);
+        }
+
+ out:
+        dm_put(md);
+        return r;
+}
+
+static inline int get_mode(struct dm_ioctl *param)
+{
+        int mode = FMODE_READ | FMODE_WRITE;
+
+        if (param->flags & DM_READONLY_FLAG)
+                mode = FMODE_READ;
+
+        return mode;
+}
+
+static int next_target(struct dm_target_spec *last, uint32_t next, void *end,
+                       struct dm_target_spec **spec, char **target_params)
+{
+        *spec = (struct dm_target_spec *) ((unsigned char *) last + next);
+        *target_params = (char *) (*spec + 1);
+
+        if (*spec < (last + 1))
+                return -EINVAL;
+
+        return invalid_str(*target_params, end);
+}
+
+static int populate_table(struct dm_table *table,
+                          struct dm_ioctl *param, size_t param_size)
+{
+        int r;
+        unsigned int i = 0;
+        struct dm_target_spec *spec = (struct dm_target_spec *) param;
+        uint32_t next = param->data_start;
+        void *end = (void *) param + param_size;
+        char *target_params;
+
+        if (!param->target_count) {
+                DMWARN("populate_table: no targets specified");
+                return -EINVAL;
+        }
+
+        for (i = 0; i < param->target_count; i++) {
+
+                r = next_target(spec, next, end, &spec, &target_params);
+                if (r) {
+                        DMWARN("unable to find target");
+                        return r;
+                }
+
+                r = dm_table_add_target(table, spec->target_type,
+                                        (sector_t) spec->sector_start,
+                                        (sector_t) spec->length,
+                                        target_params);
+                if (r) {
+                        DMWARN("error adding target to table");
+                        return r;
+                }
+
+                next = spec->next;
+        }
+
+        return dm_table_complete(table);
+}
+
+static int table_load(struct dm_ioctl *param, size_t param_size)
+{
+        int r;
+        struct hash_cell *hc;
+        struct dm_table *t;
+
+        r = dm_table_create(&t, get_mode(param), param->target_count);
+        if (r)
+                return r;
+
+        r = populate_table(t, param, param_size);
+        if (r) {
+                dm_table_put(t);
+                return r;
+        }
+
+        down_write(&_hash_lock);
+        hc = __find_device_hash_cell(param);
+        if (!hc) {
+                DMWARN("device doesn't appear to be in the dev hash table.");
+                up_write(&_hash_lock);
+                return -ENXIO;
+        }
+
+        if (hc->new_map)
+                dm_table_put(hc->new_map);
+        hc->new_map = t;
+        param->flags |= DM_INACTIVE_PRESENT_FLAG;
+
+        r = __dev_status(hc->md, param);
+        up_write(&_hash_lock);
+        return r;
+}
+
+static int table_clear(struct dm_ioctl *param, size_t param_size)
+{
+        int r;
+        struct hash_cell *hc;
+
+        down_write(&_hash_lock);
+
+        hc = __find_device_hash_cell(param);
+        if (!hc) {
+                DMWARN("device doesn't appear to be in the dev hash table.");
+                up_write(&_hash_lock);
+                return -ENXIO;
+        }
+
+        if (hc->new_map) {
+                dm_table_put(hc->new_map);
+                hc->new_map = NULL;
+        }
+
+        param->flags &= ~DM_INACTIVE_PRESENT_FLAG;
+
+        r = __dev_status(hc->md, param);
+        up_write(&_hash_lock);
+        return r;
+}
+
+/*
+ * Retrieves a list of devices used by a particular dm device.
+ */
+static void retrieve_deps(struct dm_table *table,
+                          struct dm_ioctl *param, size_t param_size)
+{
+        unsigned int count = 0;
+        struct list_head *tmp;
+        size_t len, needed;
+        struct dm_dev *dd;
+        struct dm_target_deps *deps;
+
+        deps = get_result_buffer(param, param_size, &len);
+
+        /*
+         * Count the devices.
+         */
+        list_for_each (tmp, dm_table_get_devices(table))
+                count++;
+
+        /*
+         * Check we have enough space.
+         */
+        needed = sizeof(*deps) + (sizeof(*deps->dev) * count);
+        if (len < needed) {
+                param->flags |= DM_BUFFER_FULL_FLAG;
+                return;
+        }
+
+        /*
+         * Fill in the devices.
+         */
+        deps->count = count;
+        count = 0;
+        list_for_each_entry (dd, dm_table_get_devices(table), list)
+                deps->dev[count++] = huge_encode_dev(dd->bdev->bd_dev);
+
+        param->data_size = param->data_start + needed;
+}
+
+static int table_deps(struct dm_ioctl *param, size_t param_size)
+{
+        int r = 0;
+        struct mapped_device *md;
+        struct dm_table *table;
+
+        md = find_device(param);
+        if (!md)
+                return -ENXIO;
+
+        r = __dev_status(md, param);
+        if (r)
+                goto out;
+
+        table = dm_get_table(md);
+        if (table) {
+                retrieve_deps(table, param, param_size);
+                dm_table_put(table);
+        }
+
+ out:
+        dm_put(md);
+        return r;
+}
+
+/*
+ * Return the status of a device as a text string for each
+ * target.
+ */
+static int table_status(struct dm_ioctl *param, size_t param_size)
+{
+        int r;
+        struct mapped_device *md;
+        struct dm_table *table;
+
+        md = find_device(param);
+        if (!md)
+                return -ENXIO;
+
+        r = __dev_status(md, param);
+        if (r)
+                goto out;
+
+        table = dm_get_table(md);
+        if (table) {
+                retrieve_status(table, param, param_size);
+                dm_table_put(table);
+        }
+
+ out:
+        dm_put(md);
+        return r;
+}
+
+/*
+ * Pass a message to the target that's at the supplied device offset.
+ */
+static int target_message(struct dm_ioctl *param, size_t param_size)
+{
+        int r, argc;
+        char **argv;
+        struct mapped_device *md;
+        struct dm_table *table;
+        struct dm_target *ti;
+        struct dm_target_msg *tmsg = (void *) param + param->data_start;
+
+        md = find_device(param);
+        if (!md)
+                return -ENXIO;
+
+        r = __dev_status(md, param);
+        if (r)
+                goto out;
+
+        if (tmsg < (struct dm_target_msg *) (param + 1) ||
+            invalid_str(tmsg->message, (void *) param + param_size)) {
+                DMWARN("Invalid target message parameters.");
+                r = -EINVAL;
+                goto out;
+        }
+
+        r = dm_split_args(&argc, &argv, tmsg->message);
+        if (r) {
+                DMWARN("Failed to split target message parameters");
+                goto out;
+        }
+
+        table = dm_get_table(md);
+        if (!table)
+                goto out_argv;
+
+        if (tmsg->sector >= dm_table_get_size(table)) {
+                DMWARN("Target message sector outside device.");
+                r = -EINVAL;
+                goto out_table;
+        }
+
+        ti = dm_table_find_target(table, tmsg->sector);
+        if (ti->type->message)
+                r = ti->type->message(ti, argc, argv);
+        else {
+                DMWARN("Target type does not support messages");
+                r = -EINVAL;
+        }
+
+ out_table:
+        dm_table_put(table);
+ out_argv:
+        kfree(argv);
+ out:
+        param->data_size = 0;
+        dm_put(md);
+        return r;
+}
+
+/*-----------------------------------------------------------------
+ * Implementation of open/close/ioctl on the special char
+ * device.
+ *---------------------------------------------------------------*/
+static ioctl_fn lookup_ioctl(unsigned int cmd)
+{
+        static struct {
+                int cmd;
+                ioctl_fn fn;
+        } _ioctls[] = {
+                {DM_VERSION_CMD, NULL}, /* version is dealt with elsewhere */
+                {DM_REMOVE_ALL_CMD, remove_all},
+                {DM_LIST_DEVICES_CMD, list_devices},
+
+                {DM_DEV_CREATE_CMD, dev_create},
+                {DM_DEV_REMOVE_CMD, dev_remove},
+                {DM_DEV_RENAME_CMD, dev_rename},
+                {DM_DEV_SUSPEND_CMD, dev_suspend},
+                {DM_DEV_STATUS_CMD, dev_status},
+                {DM_DEV_WAIT_CMD, dev_wait},
+
+                {DM_TABLE_LOAD_CMD, table_load},
+                {DM_TABLE_CLEAR_CMD, table_clear},
+                {DM_TABLE_DEPS_CMD, table_deps},
+                {DM_TABLE_STATUS_CMD, table_status},
+
+                {DM_LIST_VERSIONS_CMD, list_versions},
+
+                {DM_TARGET_MSG_CMD, target_message}
+        };
+
+        return (cmd >= ARRAY_SIZE(_ioctls)) ? NULL : _ioctls[cmd].fn;
+}
+
+/*
+ * As well as checking the version compatibility this always
+ * copies the kernel interface version out.
+ */
+static int check_version(unsigned int cmd, struct dm_ioctl __user *user)
+{
+        uint32_t version[3];
+        int r = 0;
+
+        if (copy_from_user(version, user->version, sizeof(version)))
+                return -EFAULT;
+
+        if ((DM_VERSION_MAJOR != version[0]) ||
+            (DM_VERSION_MINOR < version[1])) {
+                DMWARN("ioctl interface mismatch: "
+                       "kernel(%u.%u.%u), user(%u.%u.%u), cmd(%d)",
+                       DM_VERSION_MAJOR, DM_VERSION_MINOR,
+                       DM_VERSION_PATCHLEVEL,
+                       version[0], version[1], version[2], cmd);
+                r = -EINVAL;
+        }
+
+        /*
+         * Fill in the kernel version.
+         */
+        version[0] = DM_VERSION_MAJOR;
+        version[1] = DM_VERSION_MINOR;
+        version[2] = DM_VERSION_PATCHLEVEL;
+        if (copy_to_user(user->version, version, sizeof(version)))
+                return -EFAULT;
+
+        return r;
+}
+
+static void free_params(struct dm_ioctl *param)
+{
+        vfree(param);
+}
+
+static int copy_params(struct dm_ioctl __user *user, struct dm_ioctl **param)
+{
+        struct dm_ioctl tmp, *dmi;
+
+        if (copy_from_user(&tmp, user, sizeof(tmp)))
+                return -EFAULT;
+
+        if (tmp.data_size < sizeof(tmp))
+                return -EINVAL;
+
+        dmi = (struct dm_ioctl *) vmalloc(tmp.data_size);
+        if (!dmi)
+                return -ENOMEM;
+
+        if (copy_from_user(dmi, user, tmp.data_size)) {
+                vfree(dmi);
+                return -EFAULT;
+        }
+
+        *param = dmi;
+        return 0;
+}
+
+static int validate_params(uint cmd, struct dm_ioctl *param)
+{
+        /* Always clear this flag */
+        param->flags &= ~DM_BUFFER_FULL_FLAG;
+
+        /* Ignores parameters */
+        if (cmd == DM_REMOVE_ALL_CMD ||
+            cmd == DM_LIST_DEVICES_CMD ||
+            cmd == DM_LIST_VERSIONS_CMD)
+                return 0;
+
+        if ((cmd == DM_DEV_CREATE_CMD)) {
+                if (!*param->name) {
+                        DMWARN("name not supplied when creating device");
+                        return -EINVAL;
+                }
+        } else if ((*param->uuid && *param->name)) {
+                DMWARN("only supply one of name or uuid, cmd(%u)", cmd);
+                return -EINVAL;
+        }
+
+        /* Ensure strings are terminated */
+        param->name[DM_NAME_LEN - 1] = '\0';
+        param->uuid[DM_UUID_LEN - 1] = '\0';
+
+        return 0;
+}
+
+static int ctl_ioctl(struct inode *inode, struct file *file,
+                     uint command, ulong u)
+{
+        int r = 0;
+        unsigned int cmd;
+        struct dm_ioctl *param;
+        struct dm_ioctl __user *user = (struct dm_ioctl __user *) u;
+        ioctl_fn fn = NULL;
+        size_t param_size;
+
+        /* only root can play with this */
+        if (!capable(CAP_SYS_ADMIN))
+                return -EACCES;
+
+        if (_IOC_TYPE(command) != DM_IOCTL)
+                return -ENOTTY;
+
+        cmd = _IOC_NR(command);
+
+        /*
+         * Check the interface version passed in.  This also
+         * writes out the kernel's interface version.
+         */
+        r = check_version(cmd, user);
+        if (r)
+                return r;
+
+        /*
+         * Nothing more to do for the version command.
+         */
+        if (cmd == DM_VERSION_CMD)
+                return 0;
+
+        fn = lookup_ioctl(cmd);
+        if (!fn) {
+                DMWARN("dm_ctl_ioctl: unknown command 0x%x", command);
+                return -ENOTTY;
+        }
+
+        /*
+         * Trying to avoid low memory issues when a device is
+         * suspended.
+         */
+        current->flags |= PF_MEMALLOC;
+
+        /*
+         * Copy the parameters into kernel space.
+         */
+        r = copy_params(user, &param);
+        if (r) {
+                current->flags &= ~PF_MEMALLOC;
+                return r;
+        }
+
+        /*
+         * FIXME: eventually we will remove the PF_MEMALLOC flag
+         * here.  However the tools still do nasty things like
+         * 'load' while a device is suspended.
+         */
+
+        r = validate_params(cmd, param);
+        if (r)
+                goto out;
+
+        param_size = param->data_size;
+        param->data_size = sizeof(*param);
+        r = fn(param, param_size);
+
+        /*
+         * Copy the results back to userland.
+         */
+        if (!r && copy_to_user(user, param, param->data_size))
+                r = -EFAULT;
+
+ out:
+        free_params(param);
+        current->flags &= ~PF_MEMALLOC;
+        return r;
+}
+
+static struct file_operations _ctl_fops = {
+        .ioctl   = ctl_ioctl,
+        .owner   = THIS_MODULE,
+};
+
+static struct miscdevice _dm_misc = {
+        .minor          = MISC_DYNAMIC_MINOR,
+        .name           = DM_NAME,
+        .devfs_name     = "mapper/control",
+        .fops           = &_ctl_fops
+};
+
+/*
+ * Create misc character device and link to DM_DIR/control.
+ */
+int __init dm_interface_init(void)
+{
+        int r;
+
+        r = dm_hash_init();
+        if (r)
+                return r;
+
+        r = misc_register(&_dm_misc);
+        if (r) {
+                DMERR("misc_register failed for control device");
+                dm_hash_exit();
+                return r;
+        }
+
+        DMINFO("%d.%d.%d%s initialised: %s", DM_VERSION_MAJOR,
+               DM_VERSION_MINOR, DM_VERSION_PATCHLEVEL, DM_VERSION_EXTRA,
+               DM_DRIVER_EMAIL);
+        return 0;
+}
+
+void dm_interface_exit(void)
+{
+        if (misc_deregister(&_dm_misc) < 0)
+                DMERR("misc_deregister failed for control device");
+
+        dm_hash_exit();
+}
diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c
new file mode 100644
index 000000000000..6a2cd5dc8a63
--- /dev/null
+++ b/drivers/md/dm-linear.c
@@ -0,0 +1,123 @@
+/*
+ * Copyright (C) 2001-2003 Sistina Software (UK) Limited.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/blkdev.h>
+#include <linux/bio.h>
+#include <linux/slab.h>
+
+/*
+ * Linear: maps a linear range of a device.
+ */
+struct linear_c {
+        struct dm_dev *dev;
+        sector_t start;
+};
+
+/*
+ * Construct a linear mapping: <dev_path> <offset>
+ */
+static int linear_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+        struct linear_c *lc;
+
+        if (argc != 2) {
+                ti->error = "dm-linear: Invalid argument count";
+                return -EINVAL;
+        }
+
+        lc = kmalloc(sizeof(*lc), GFP_KERNEL);
+        if (lc == NULL) {
+                ti->error = "dm-linear: Cannot allocate linear context";
+                return -ENOMEM;
+        }
+
+        if (sscanf(argv[1], SECTOR_FORMAT, &lc->start) != 1) {
+                ti->error = "dm-linear: Invalid device sector";
+                goto bad;
+        }
+
+        if (dm_get_device(ti, argv[0], lc->start, ti->len,
+                          dm_table_get_mode(ti->table), &lc->dev)) {
+                ti->error = "dm-linear: Device lookup failed";
+                goto bad;
+        }
+
+        ti->private = lc;
+        return 0;
+
+      bad:
+        kfree(lc);
+        return -EINVAL;
+}
+
+static void linear_dtr(struct dm_target *ti)
+{
+        struct linear_c *lc = (struct linear_c *) ti->private;
+
+        dm_put_device(ti, lc->dev);
+        kfree(lc);
+}
+
+static int linear_map(struct dm_target *ti, struct bio *bio,
+                      union map_info *map_context)
+{
+        struct linear_c *lc = (struct linear_c *) ti->private;
+
+        bio->bi_bdev = lc->dev->bdev;
+        bio->bi_sector = lc->start + (bio->bi_sector - ti->begin);
+
+        return 1;
+}
+
+static int linear_status(struct dm_target *ti, status_type_t type,
+                         char *result, unsigned int maxlen)
+{
+        struct linear_c *lc = (struct linear_c *) ti->private;
+
+        switch (type) {
+        case STATUSTYPE_INFO:
+                result[0] = '\0';
+                break;
+
+        case STATUSTYPE_TABLE:
+                snprintf(result, maxlen, "%s " SECTOR_FORMAT, lc->dev->name,
+                         lc->start);
+                break;
+        }
+        return 0;
+}
+
+static struct target_type linear_target = {
+        .name   = "linear",
+        .version= {1, 0, 1},
+        .module = THIS_MODULE,
+        .ctr    = linear_ctr,
+        .dtr    = linear_dtr,
+        .map    = linear_map,
+        .status = linear_status,
+};
+
+int __init dm_linear_init(void)
+{
+        int r = dm_register_target(&linear_target);
+
+        if (r < 0)
+                DMERR("linear: register failed %d", r);
+
+        return r;
+}
+
+void dm_linear_exit(void)
+{
+        int r = dm_unregister_target(&linear_target);
+
+        if (r < 0)
+                DMERR("linear: unregister failed %d", r);
+}
diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c
new file mode 100644
index 000000000000..e110655eabdb
--- /dev/null
+++ b/drivers/md/dm-log.c
@@ -0,0 +1,711 @@
+/*
+ * Copyright (C) 2003 Sistina Software
+ *
+ * This file is released under the LGPL.
+ */
+
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/vmalloc.h>
+
+#include "dm-log.h"
+#include "dm-io.h"
+
+static LIST_HEAD(_log_types);
+static DEFINE_SPINLOCK(_lock);
+
+int dm_register_dirty_log_type(struct dirty_log_type *type)
+{
+        spin_lock(&_lock);
+        type->use_count = 0;
+        list_add(&type->list, &_log_types);
+        spin_unlock(&_lock);
+
+        return 0;
+}
+
+int dm_unregister_dirty_log_type(struct dirty_log_type *type)
+{
+        spin_lock(&_lock);
+
+        if (type->use_count)
+                DMWARN("Attempt to unregister a log type that is still in use");
+        else
+                list_del(&type->list);
+
+        spin_unlock(&_lock);
+
+        return 0;
+}
+
+static struct dirty_log_type *get_type(const char *type_name)
+{
+        struct dirty_log_type *type;
+
+        spin_lock(&_lock);
+        list_for_each_entry (type, &_log_types, list)
+                if (!strcmp(type_name, type->name)) {
+                        if (!type->use_count && !try_module_get(type->module)){
+                                spin_unlock(&_lock);
+                                return NULL;
+                        }
+                        type->use_count++;
+                        spin_unlock(&_lock);
+                        return type;
+                }
+
+        spin_unlock(&_lock);
+        return NULL;
+}
+
+static void put_type(struct dirty_log_type *type)
+{
+        spin_lock(&_lock);
+        if (!--type->use_count)
+                module_put(type->module);
+        spin_unlock(&_lock);
+}
+
+struct dirty_log *dm_create_dirty_log(const char *type_name, struct dm_target *ti,
+                                      unsigned int argc, char **argv)
+{
+        struct dirty_log_type *type;
+        struct dirty_log *log;
+
+        log = kmalloc(sizeof(*log), GFP_KERNEL);
+        if (!log)
+                return NULL;
+
+        type = get_type(type_name);
+        if (!type) {
+                kfree(log);
+                return NULL;
+        }
+
+        log->type = type;
+        if (type->ctr(log, ti, argc, argv)) {
+                kfree(log);
+                put_type(type);
+                return NULL;
+        }
+
+        return log;
+}
+
+void dm_destroy_dirty_log(struct dirty_log *log)
+{
+        log->type->dtr(log);
+        put_type(log->type);
+        kfree(log);
+}
+
+/*-----------------------------------------------------------------
+ * Persistent and core logs share a lot of their implementation.
+ * FIXME: need a reload method to be called from a resume
+ *---------------------------------------------------------------*/
+/*
+ * Magic for persistent mirrors: "MiRr"
+ */
+#define MIRROR_MAGIC 0x4D695272
+
+/*
+ * The on-disk version of the metadata.
+ */
+#define MIRROR_DISK_VERSION 1
+#define LOG_OFFSET 2
+
+struct log_header {
+        uint32_t magic;
+
+        /*
+         * Simple, incrementing version. no backward
+         * compatibility.
+         */
+        uint32_t version;
+        sector_t nr_regions;
+};
+
+struct log_c {
+        struct dm_target *ti;
+        int touched;
+        uint32_t region_size;
+        unsigned int region_count;
+        region_t sync_count;
+
+        unsigned bitset_uint32_count;
+        uint32_t *clean_bits;
+        uint32_t *sync_bits;
+        uint32_t *recovering_bits;      /* FIXME: this seems excessive */
+
+        int sync_search;
+
+        /* Resync flag */
+        enum sync {
+                DEFAULTSYNC,    /* Synchronize if necessary */
+                NOSYNC,         /* Devices known to be already in sync */
+                FORCESYNC,      /* Force a sync to happen */
+        } sync;
+
+        /*
+         * Disk log fields
+         */
+        struct dm_dev *log_dev;
+        struct log_header header;
+
+        struct io_region header_location;
+        struct log_header *disk_header;
+
+        struct io_region bits_location;
+        uint32_t *disk_bits;
+};
+
+/*
+ * The touched member needs to be updated every time we access
+ * one of the bitsets.
+ */
+static  inline int log_test_bit(uint32_t *bs, unsigned bit)
+{
+        return test_bit(bit, (unsigned long *) bs) ? 1 : 0;
+}
+
+static inline void log_set_bit(struct log_c *l,
+                               uint32_t *bs, unsigned bit)
+{
+        set_bit(bit, (unsigned long *) bs);
+        l->touched = 1;
+}
+
+static inline void log_clear_bit(struct log_c *l,
+                                 uint32_t *bs, unsigned bit)
+{
+        clear_bit(bit, (unsigned long *) bs);
+        l->touched = 1;
+}
+
+/*----------------------------------------------------------------
+ * Header IO
+ *--------------------------------------------------------------*/
+static void header_to_disk(struct log_header *core, struct log_header *disk)
+{
+        disk->magic = cpu_to_le32(core->magic);
+        disk->version = cpu_to_le32(core->version);
+        disk->nr_regions = cpu_to_le64(core->nr_regions);
+}
+
+static void header_from_disk(struct log_header *core, struct log_header *disk)
+{
+        core->magic = le32_to_cpu(disk->magic);
+        core->version = le32_to_cpu(disk->version);
+        core->nr_regions = le64_to_cpu(disk->nr_regions);
+}
+
+static int read_header(struct log_c *log)
+{
+        int r;
+        unsigned long ebits;
+
+        r = dm_io_sync_vm(1, &log->header_location, READ,
+                          log->disk_header, &ebits);
+        if (r)
+                return r;
+
+        header_from_disk(&log->header, log->disk_header);
+
+        /* New log required? */
+        if (log->sync != DEFAULTSYNC || log->header.magic != MIRROR_MAGIC) {
+                log->header.magic = MIRROR_MAGIC;
+                log->header.version = MIRROR_DISK_VERSION;
+                log->header.nr_regions = 0;
+        }
+
+        if (log->header.version != MIRROR_DISK_VERSION) {
+                DMWARN("incompatible disk log version");
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+static inline int write_header(struct log_c *log)
+{
+        unsigned long ebits;
+
+        header_to_disk(&log->header, log->disk_header);
+        return dm_io_sync_vm(1, &log->header_location, WRITE,
+                             log->disk_header, &ebits);
+}
+
+/*----------------------------------------------------------------
+ * Bits IO
+ *--------------------------------------------------------------*/
+static inline void bits_to_core(uint32_t *core, uint32_t *disk, unsigned count)
+{
+        unsigned i;
+
+        for (i = 0; i < count; i++)
+                core[i] = le32_to_cpu(disk[i]);
+}
+
+static inline void bits_to_disk(uint32_t *core, uint32_t *disk, unsigned count)
+{
+        unsigned i;
+
+        /* copy across the clean/dirty bitset */
+        for (i = 0; i < count; i++)
+                disk[i] = cpu_to_le32(core[i]);
+}
+
+static int read_bits(struct log_c *log)
+{
+        int r;
+        unsigned long ebits;
+
+        r = dm_io_sync_vm(1, &log->bits_location, READ,
+                          log->disk_bits, &ebits);
+        if (r)
+                return r;
+
+        bits_to_core(log->clean_bits, log->disk_bits,
+                     log->bitset_uint32_count);
+        return 0;
+}
+
+static int write_bits(struct log_c *log)
+{
+        unsigned long ebits;
+        bits_to_disk(log->clean_bits, log->disk_bits,
+                     log->bitset_uint32_count);
+        return dm_io_sync_vm(1, &log->bits_location, WRITE,
+                             log->disk_bits, &ebits);
+}
+
+/*----------------------------------------------------------------
+ * core log constructor/destructor
+ *
+ * argv contains region_size followed optionally by [no]sync
+ *--------------------------------------------------------------*/
+#define BYTE_SHIFT 3
+static int core_ctr(struct dirty_log *log, struct dm_target *ti,
+                    unsigned int argc, char **argv)
+{
+        enum sync sync = DEFAULTSYNC;
+
+        struct log_c *lc;
+        uint32_t region_size;
+        unsigned int region_count;
+        size_t bitset_size;
+
+        if (argc < 1 || argc > 2) {
+                DMWARN("wrong number of arguments to mirror log");
+                return -EINVAL;
+        }
+
+        if (argc > 1) {
+                if (!strcmp(argv[1], "sync"))
+                        sync = FORCESYNC;
+                else if (!strcmp(argv[1], "nosync"))
+                        sync = NOSYNC;
+                else {
+                        DMWARN("unrecognised sync argument to mirror log: %s",
+                               argv[1]);
+                        return -EINVAL;
+                }
+        }
+
+        if (sscanf(argv[0], "%u", &region_size) != 1) {
+                DMWARN("invalid region size string");
+                return -EINVAL;
+        }
+
+        region_count = dm_sector_div_up(ti->len, region_size);
+
+        lc = kmalloc(sizeof(*lc), GFP_KERNEL);
+        if (!lc) {
+                DMWARN("couldn't allocate core log");
+                return -ENOMEM;
+        }
+
+        lc->ti = ti;
+        lc->touched = 0;
+        lc->region_size = region_size;
+        lc->region_count = region_count;
+        lc->sync = sync;
+
+        /*
+         * Work out how many words we need to hold the bitset.
+         */
+        bitset_size = dm_round_up(region_count,
+                                  sizeof(*lc->clean_bits) << BYTE_SHIFT);
+        bitset_size >>= BYTE_SHIFT;
+
+        lc->bitset_uint32_count = bitset_size / 4;
+        lc->clean_bits = vmalloc(bitset_size);
+        if (!lc->clean_bits) {
+                DMWARN("couldn't allocate clean bitset");
+                kfree(lc);
+                return -ENOMEM;
+        }
+        memset(lc->clean_bits, -1, bitset_size);
+
+        lc->sync_bits = vmalloc(bitset_size);
+        if (!lc->sync_bits) {
+                DMWARN("couldn't allocate sync bitset");
+                vfree(lc->clean_bits);
+                kfree(lc);
+                return -ENOMEM;
+        }
+        memset(lc->sync_bits, (sync == NOSYNC) ? -1 : 0, bitset_size);
+        lc->sync_count = (sync == NOSYNC) ? region_count : 0;
+
+        lc->recovering_bits = vmalloc(bitset_size);
+        if (!lc->recovering_bits) {
+                DMWARN("couldn't allocate sync bitset");
+                vfree(lc->sync_bits);
+                vfree(lc->clean_bits);
+                kfree(lc);
+                return -ENOMEM;
+        }
+        memset(lc->recovering_bits, 0, bitset_size);
+        lc->sync_search = 0;
+        log->context = lc;
+        return 0;
+}
+
+static void core_dtr(struct dirty_log *log)
+{
+        struct log_c *lc = (struct log_c *) log->context;
+        vfree(lc->clean_bits);
+        vfree(lc->sync_bits);
+        vfree(lc->recovering_bits);
+        kfree(lc);
+}
+
+/*----------------------------------------------------------------
+ * disk log constructor/destructor
+ *
+ * argv contains log_device region_size followed optionally by [no]sync
+ *--------------------------------------------------------------*/
+static int disk_ctr(struct dirty_log *log, struct dm_target *ti,
+                    unsigned int argc, char **argv)
+{
+        int r;
+        size_t size;
+        struct log_c *lc;
+        struct dm_dev *dev;
+
+        if (argc < 2 || argc > 3) {
+                DMWARN("wrong number of arguments to disk mirror log");
+                return -EINVAL;
+        }
+
+        r = dm_get_device(ti, argv[0], 0, 0 /* FIXME */,
+                          FMODE_READ | FMODE_WRITE, &dev);
+        if (r)
+                return r;
+
+        r = core_ctr(log, ti, argc - 1, argv + 1);
+        if (r) {
+                dm_put_device(ti, dev);
+                return r;
+        }
+
+        lc = (struct log_c *) log->context;
+        lc->log_dev = dev;
+
+        /* setup the disk header fields */
+        lc->header_location.bdev = lc->log_dev->bdev;
+        lc->header_location.sector = 0;
+        lc->header_location.count = 1;
+
+        /*
+         * We can't read less than this amount, even though we'll
+         * not be using most of this space.
+         */
+        lc->disk_header = vmalloc(1 << SECTOR_SHIFT);
+        if (!lc->disk_header)
+                goto bad;
+
+        /* setup the disk bitset fields */
+        lc->bits_location.bdev = lc->log_dev->bdev;
+        lc->bits_location.sector = LOG_OFFSET;
+
+        size = dm_round_up(lc->bitset_uint32_count * sizeof(uint32_t),
+                           1 << SECTOR_SHIFT);
+        lc->bits_location.count = size >> SECTOR_SHIFT;
+        lc->disk_bits = vmalloc(size);
+        if (!lc->disk_bits) {
+                vfree(lc->disk_header);
+                goto bad;
+        }
+        return 0;
+
+ bad:
+        dm_put_device(ti, lc->log_dev);
+        core_dtr(log);
+        return -ENOMEM;
+}
+
+static void disk_dtr(struct dirty_log *log)
+{
+        struct log_c *lc = (struct log_c *) log->context;
+        dm_put_device(lc->ti, lc->log_dev);
+        vfree(lc->disk_header);
+        vfree(lc->disk_bits);
+        core_dtr(log);
+}
+
+static int count_bits32(uint32_t *addr, unsigned size)
+{
+        int count = 0, i;
+
+        for (i = 0; i < size; i++) {
+                count += hweight32(*(addr+i));
+        }
+        return count;
+}
+
+static int disk_resume(struct dirty_log *log)
+{
+        int r;
+        unsigned i;
+        struct log_c *lc = (struct log_c *) log->context;
+        size_t size = lc->bitset_uint32_count * sizeof(uint32_t);
+
+        /* read the disk header */
+        r = read_header(lc);
+        if (r)
+                return r;
+
+        /* read the bits */
+        r = read_bits(lc);
+        if (r)
+                return r;
+
+        /* set or clear any new bits */
+        if (lc->sync == NOSYNC)
+                for (i = lc->header.nr_regions; i < lc->region_count; i++)
+                        /* FIXME: amazingly inefficient */
+                        log_set_bit(lc, lc->clean_bits, i);
+        else
+                for (i = lc->header.nr_regions; i < lc->region_count; i++)
+                        /* FIXME: amazingly inefficient */
+                        log_clear_bit(lc, lc->clean_bits, i);
+
+        /* copy clean across to sync */
+        memcpy(lc->sync_bits, lc->clean_bits, size);
+        lc->sync_count = count_bits32(lc->clean_bits, lc->bitset_uint32_count);
+
+        /* write the bits */
+        r = write_bits(lc);
+        if (r)
+                return r;
+
+        /* set the correct number of regions in the header */
+        lc->header.nr_regions = lc->region_count;
+
+        /* write the new header */
+        return write_header(lc);
+}
+
+static uint32_t core_get_region_size(struct dirty_log *log)
+{
+        struct log_c *lc = (struct log_c *) log->context;
+        return lc->region_size;
+}
+
+static int core_is_clean(struct dirty_log *log, region_t region)
+{
+        struct log_c *lc = (struct log_c *) log->context;
+        return log_test_bit(lc->clean_bits, region);
+}
+
+static int core_in_sync(struct dirty_log *log, region_t region, int block)
+{
+        struct log_c *lc = (struct log_c *) log->context;
+        return log_test_bit(lc->sync_bits, region);
+}
+
+static int core_flush(struct dirty_log *log)
+{
+        /* no op */
+        return 0;
+}
+
+static int disk_flush(struct dirty_log *log)
+{
+        int r;
+        struct log_c *lc = (struct log_c *) log->context;
+
+        /* only write if the log has changed */
+        if (!lc->touched)
+                return 0;
+
+        r = write_bits(lc);
+        if (!r)
+                lc->touched = 0;
+
+        return r;
+}
+
+static void core_mark_region(struct dirty_log *log, region_t region)
+{
+        struct log_c *lc = (struct log_c *) log->context;
+        log_clear_bit(lc, lc->clean_bits, region);
+}
+
+static void core_clear_region(struct dirty_log *log, region_t region)
+{
+        struct log_c *lc = (struct log_c *) log->context;
+        log_set_bit(lc, lc->clean_bits, region);
+}
+
+static int core_get_resync_work(struct dirty_log *log, region_t *region)
+{
+        struct log_c *lc = (struct log_c *) log->context;
+
+        if (lc->sync_search >= lc->region_count)
+                return 0;
+
+        do {
+                *region = find_next_zero_bit((unsigned long *) lc->sync_bits,
+                                             lc->region_count,
+                                             lc->sync_search);
+                lc->sync_search = *region + 1;
+
+                if (*region == lc->region_count)
+                        return 0;
+
+        } while (log_test_bit(lc->recovering_bits, *region));
+
+        log_set_bit(lc, lc->recovering_bits, *region);
+        return 1;
+}
+
+static void core_complete_resync_work(struct dirty_log *log, region_t region,
+                                      int success)
+{
+        struct log_c *lc = (struct log_c *) log->context;
+
+        log_clear_bit(lc, lc->recovering_bits, region);
+        if (success) {
+                log_set_bit(lc, lc->sync_bits, region);
+                lc->sync_count++;
+        }
+}
+
+static region_t core_get_sync_count(struct dirty_log *log)
+{
+        struct log_c *lc = (struct log_c *) log->context;
+
+        return lc->sync_count;
+}
+
+#define DMEMIT_SYNC \
+        if (lc->sync != DEFAULTSYNC) \
+                DMEMIT("%ssync ", lc->sync == NOSYNC ? "no" : "")
+
+static int core_status(struct dirty_log *log, status_type_t status,
+                       char *result, unsigned int maxlen)
+{
+        int sz = 0;
+        struct log_c *lc = log->context;
+
+        switch(status) {
+        case STATUSTYPE_INFO:
+                break;
+
+        case STATUSTYPE_TABLE:
+                DMEMIT("%s %u %u ", log->type->name,
+                       lc->sync == DEFAULTSYNC ? 1 : 2, lc->region_size);
+                DMEMIT_SYNC;
+        }
+
+        return sz;
+}
+
+static int disk_status(struct dirty_log *log, status_type_t status,
+                       char *result, unsigned int maxlen)
+{
+        int sz = 0;
+        char buffer[16];
+        struct log_c *lc = log->context;
+
+        switch(status) {
+        case STATUSTYPE_INFO:
+                break;
+
+        case STATUSTYPE_TABLE:
+                format_dev_t(buffer, lc->log_dev->bdev->bd_dev);
+                DMEMIT("%s %u %s %u ", log->type->name,
+                       lc->sync == DEFAULTSYNC ? 2 : 3, buffer,
+                       lc->region_size);
+                DMEMIT_SYNC;
+        }
+
+        return sz;
+}
+
+static struct dirty_log_type _core_type = {
+        .name = "core",
+        .module = THIS_MODULE,
+        .ctr = core_ctr,
+        .dtr = core_dtr,
+        .get_region_size = core_get_region_size,
+        .is_clean = core_is_clean,
+        .in_sync = core_in_sync,
+        .flush = core_flush,
+        .mark_region = core_mark_region,
+        .clear_region = core_clear_region,
+        .get_resync_work = core_get_resync_work,
+        .complete_resync_work = core_complete_resync_work,
+        .get_sync_count = core_get_sync_count,
+        .status = core_status,
+};
+
+static struct dirty_log_type _disk_type = {
+        .name = "disk",
+        .module = THIS_MODULE,
+        .ctr = disk_ctr,
+        .dtr = disk_dtr,
+        .suspend = disk_flush,
+        .resume = disk_resume,
+        .get_region_size = core_get_region_size,
+        .is_clean = core_is_clean,
+        .in_sync = core_in_sync,
+        .flush = disk_flush,
+        .mark_region = core_mark_region,
+        .clear_region = core_clear_region,
+        .get_resync_work = core_get_resync_work,
+        .complete_resync_work = core_complete_resync_work,
+        .get_sync_count = core_get_sync_count,
+        .status = disk_status,
+};
+
+int __init dm_dirty_log_init(void)
+{
+        int r;
+
+        r = dm_register_dirty_log_type(&_core_type);
+        if (r)
+                DMWARN("couldn't register core log");
+
+        r = dm_register_dirty_log_type(&_disk_type);
+        if (r) {
+                DMWARN("couldn't register disk type");
+                dm_unregister_dirty_log_type(&_core_type);
+        }
+
+        return r;
+}
+
+void dm_dirty_log_exit(void)
+{
+        dm_unregister_dirty_log_type(&_disk_type);
+        dm_unregister_dirty_log_type(&_core_type);
+}
+
+EXPORT_SYMBOL(dm_register_dirty_log_type);
+EXPORT_SYMBOL(dm_unregister_dirty_log_type);
+EXPORT_SYMBOL(dm_create_dirty_log);
+EXPORT_SYMBOL(dm_destroy_dirty_log);
diff --git a/drivers/md/dm-log.h b/drivers/md/dm-log.h
new file mode 100644
index 000000000000..5ae5309ebf28
--- /dev/null
+++ b/drivers/md/dm-log.h
@@ -0,0 +1,130 @@
+/*
+ * Copyright (C) 2003 Sistina Software
+ *
+ * This file is released under the LGPL.
+ */
+
+#ifndef DM_DIRTY_LOG
+#define DM_DIRTY_LOG
+
+#include "dm.h"
+
+typedef sector_t region_t;
+
+struct dirty_log_type;
+
+struct dirty_log {
+        struct dirty_log_type *type;
+        void *context;
+};
+
+struct dirty_log_type {
+        struct list_head list;
+        const char *name;
+        struct module *module;
+        unsigned int use_count;
+
+        int (*ctr)(struct dirty_log *log, struct dm_target *ti,
+                   unsigned int argc, char **argv);
+        void (*dtr)(struct dirty_log *log);
+
+        /*
+         * There are times when we don't want the log to touch
+         * the disk.
+         */
+        int (*suspend)(struct dirty_log *log);
+        int (*resume)(struct dirty_log *log);
+
+        /*
+         * Retrieves the smallest size of region that the log can
+         * deal with.
+         */
+        uint32_t (*get_region_size)(struct dirty_log *log);
+
+        /*
+         * A predicate to say whether a region is clean or not.
+         * May block.
+         */
+        int (*is_clean)(struct dirty_log *log, region_t region);
+
+        /*
+         *  Returns: 0, 1, -EWOULDBLOCK, < 0
+         *
+         * A predicate function to check the area given by
+         * [sector, sector + len) is in sync.
+         *
+         * If -EWOULDBLOCK is returned the state of the region is
+         * unknown, typically this will result in a read being
+         * passed to a daemon to deal with, since a daemon is
+         * allowed to block.
+         */
+        int (*in_sync)(struct dirty_log *log, region_t region, int can_block);
+
+        /*
+         * Flush the current log state (eg, to disk).  This
+         * function may block.
+         */
+        int (*flush)(struct dirty_log *log);
+
+        /*
+         * Mark an area as clean or dirty.  These functions may
+         * block, though for performance reasons blocking should
+         * be extremely rare (eg, allocating another chunk of
+         * memory for some reason).
+         */
+        void (*mark_region)(struct dirty_log *log, region_t region);
+        void (*clear_region)(struct dirty_log *log, region_t region);
+
+        /*
+         * Returns: <0 (error), 0 (no region), 1 (region)
+         *
+         * The mirrord will need perform recovery on regions of
+         * the mirror that are in the NOSYNC state.  This
+         * function asks the log to tell the caller about the
+         * next region that this machine should recover.
+         *
+         * Do not confuse this function with 'in_sync()', one
+         * tells you if an area is synchronised, the other
+         * assigns recovery work.
+        */
+        int (*get_resync_work)(struct dirty_log *log, region_t *region);
+
+        /*
+         * This notifies the log that the resync of an area has
+         * been completed.  The log should then mark this region
+         * as CLEAN.
+         */
+        void (*complete_resync_work)(struct dirty_log *log,
+                                     region_t region, int success);
+
+        /*
+         * Returns the number of regions that are in sync.
+         */
+        region_t (*get_sync_count)(struct dirty_log *log);
+
+        /*
+         * Support function for mirror status requests.
+         */
+        int (*status)(struct dirty_log *log, status_type_t status_type,
+                      char *result, unsigned int maxlen);
+};
+
+int dm_register_dirty_log_type(struct dirty_log_type *type);
+int dm_unregister_dirty_log_type(struct dirty_log_type *type);
+
+
+/*
+ * Make sure you use these two functions, rather than calling
+ * type->constructor/destructor() directly.
+ */
+struct dirty_log *dm_create_dirty_log(const char *type_name, struct dm_target *ti,
+                                      unsigned int argc, char **argv);
+void dm_destroy_dirty_log(struct dirty_log *log);
+
+/*
+ * init/exit functions.
+ */
+int dm_dirty_log_init(void);
+void dm_dirty_log_exit(void);
+
+#endif
diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c
new file mode 100644
index 000000000000..43763a0bd096
--- /dev/null
+++ b/drivers/md/dm-mpath.c
@@ -0,0 +1,1302 @@
+/*
+ * Copyright (C) 2003 Sistina Software Limited.
+ * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+#include "dm-path-selector.h"
+#include "dm-hw-handler.h"
+#include "dm-bio-list.h"
+#include "dm-bio-record.h"
+
+#include <linux/ctype.h>
+#include <linux/init.h>
+#include <linux/mempool.h>
+#include <linux/module.h>
+#include <linux/pagemap.h>
+#include <linux/slab.h>
+#include <linux/time.h>
+#include <linux/workqueue.h>
+#include <asm/atomic.h>
+
+#define MESG_STR(x) x, sizeof(x)
+
+/* Path properties */
+struct pgpath {
+        struct list_head list;
+
+        struct priority_group *pg;      /* Owning PG */
+        unsigned fail_count;            /* Cumulative failure count */
+
+        struct path path;
+};
+
+#define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
+
+/*
+ * Paths are grouped into Priority Groups and numbered from 1 upwards.
+ * Each has a path selector which controls which path gets used.
+ */
+struct priority_group {
+        struct list_head list;
+
+        struct multipath *m;            /* Owning multipath instance */
+        struct path_selector ps;
+
+        unsigned pg_num;                /* Reference number */
+        unsigned bypassed;              /* Temporarily bypass this PG? */
+
+        unsigned nr_pgpaths;            /* Number of paths in PG */
+        struct list_head pgpaths;
+};
+
+/* Multipath context */
+struct multipath {
+        struct list_head list;
+        struct dm_target *ti;
+
+        spinlock_t lock;
+
+        struct hw_handler hw_handler;
+        unsigned nr_priority_groups;
+        struct list_head priority_groups;
+        unsigned pg_init_required;      /* pg_init needs calling? */
+
+        unsigned nr_valid_paths;        /* Total number of usable paths */
+        struct pgpath *current_pgpath;
+        struct priority_group *current_pg;
+        struct priority_group *next_pg; /* Switch to this PG if set */
+        unsigned repeat_count;          /* I/Os left before calling PS again */
+
+        unsigned queue_io;              /* Must we queue all I/O? */
+        unsigned queue_if_no_path;      /* Queue I/O if last path fails? */
+        unsigned suspended;             /* Has dm core suspended our I/O? */
+
+        struct work_struct process_queued_ios;
+        struct bio_list queued_ios;
+        unsigned queue_size;
+
+        struct work_struct trigger_event;
+
+        /*
+         * We must use a mempool of mpath_io structs so that we
+         * can resubmit bios on error.
+         */
+        mempool_t *mpio_pool;
+};
+
+/*
+ * Context information attached to each bio we process.
+ */
+struct mpath_io {
+        struct pgpath *pgpath;
+        struct dm_bio_details details;
+};
+
+typedef int (*action_fn) (struct pgpath *pgpath);
+
+#define MIN_IOS 256     /* Mempool size */
+
+static kmem_cache_t *_mpio_cache;
+
+static void process_queued_ios(void *data);
+static void trigger_event(void *data);
+
+
+/*-----------------------------------------------
+ * Allocation routines
+ *-----------------------------------------------*/
+
+static struct pgpath *alloc_pgpath(void)
+{
+        struct pgpath *pgpath = kmalloc(sizeof(*pgpath), GFP_KERNEL);
+
+        if (pgpath) {
+                memset(pgpath, 0, sizeof(*pgpath));
+                pgpath->path.is_active = 1;
+        }
+
+        return pgpath;
+}
+
+static inline void free_pgpath(struct pgpath *pgpath)
+{
+        kfree(pgpath);
+}
+
+static struct priority_group *alloc_priority_group(void)
+{
+        struct priority_group *pg;
+
+        pg = kmalloc(sizeof(*pg), GFP_KERNEL);
+        if (!pg)
+                return NULL;
+
+        memset(pg, 0, sizeof(*pg));
+        INIT_LIST_HEAD(&pg->pgpaths);
+
+        return pg;
+}
+
+static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
+{
+        struct pgpath *pgpath, *tmp;
+
+        list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
+                list_del(&pgpath->list);
+                dm_put_device(ti, pgpath->path.dev);
+                free_pgpath(pgpath);
+        }
+}
+
+static void free_priority_group(struct priority_group *pg,
+                                struct dm_target *ti)
+{
+        struct path_selector *ps = &pg->ps;
+
+        if (ps->type) {
+                ps->type->destroy(ps);
+                dm_put_path_selector(ps->type);
+        }
+
+        free_pgpaths(&pg->pgpaths, ti);
+        kfree(pg);
+}
+
+static struct multipath *alloc_multipath(void)
+{
+        struct multipath *m;
+
+        m = kmalloc(sizeof(*m), GFP_KERNEL);
+        if (m) {
+                memset(m, 0, sizeof(*m));
+                INIT_LIST_HEAD(&m->priority_groups);
+                spin_lock_init(&m->lock);
+                m->queue_io = 1;
+                INIT_WORK(&m->process_queued_ios, process_queued_ios, m);
+                INIT_WORK(&m->trigger_event, trigger_event, m);
+                m->mpio_pool = mempool_create(MIN_IOS, mempool_alloc_slab,
+                                              mempool_free_slab, _mpio_cache);
+                if (!m->mpio_pool) {
+                        kfree(m);
+                        return NULL;
+                }
+        }
+
+        return m;
+}
+
+static void free_multipath(struct multipath *m)
+{
+        struct priority_group *pg, *tmp;
+        struct hw_handler *hwh = &m->hw_handler;
+
+        list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
+                list_del(&pg->list);
+                free_priority_group(pg, m->ti);
+        }
+
+        if (hwh->type) {
+                hwh->type->destroy(hwh);
+                dm_put_hw_handler(hwh->type);
+        }
+
+        mempool_destroy(m->mpio_pool);
+        kfree(m);
+}
+
+
+/*-----------------------------------------------
+ * Path selection
+ *-----------------------------------------------*/
+
+static void __switch_pg(struct multipath *m, struct pgpath *pgpath)
+{
+        struct hw_handler *hwh = &m->hw_handler;
+
+        m->current_pg = pgpath->pg;
+
+        /* Must we initialise the PG first, and queue I/O till it's ready? */
+        if (hwh->type && hwh->type->pg_init) {
+                m->pg_init_required = 1;
+                m->queue_io = 1;
+        } else {
+                m->pg_init_required = 0;
+                m->queue_io = 0;
+        }
+}
+
+static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg)
+{
+        struct path *path;
+
+        path = pg->ps.type->select_path(&pg->ps, &m->repeat_count);
+        if (!path)
+                return -ENXIO;
+
+        m->current_pgpath = path_to_pgpath(path);
+
+        if (m->current_pg != pg)
+                __switch_pg(m, m->current_pgpath);
+
+        return 0;
+}
+
+static void __choose_pgpath(struct multipath *m)
+{
+        struct priority_group *pg;
+        unsigned bypassed = 1;
+
+        if (!m->nr_valid_paths)
+                goto failed;
+
+        /* Were we instructed to switch PG? */
+        if (m->next_pg) {
+                pg = m->next_pg;
+                m->next_pg = NULL;
+                if (!__choose_path_in_pg(m, pg))
+                        return;
+        }
+
+        /* Don't change PG until it has no remaining paths */
+        if (m->current_pg && !__choose_path_in_pg(m, m->current_pg))
+                return;
+
+        /*
+         * Loop through priority groups until we find a valid path.
+         * First time we skip PGs marked 'bypassed'.
+         * Second time we only try the ones we skipped.
+         */
+        do {
+                list_for_each_entry(pg, &m->priority_groups, list) {
+                        if (pg->bypassed == bypassed)
+                                continue;
+                        if (!__choose_path_in_pg(m, pg))
+                                return;
+                }
+        } while (bypassed--);
+
+failed:
+        m->current_pgpath = NULL;
+        m->current_pg = NULL;
+}
+
+static int map_io(struct multipath *m, struct bio *bio, struct mpath_io *mpio,
+                  unsigned was_queued)
+{
+        int r = 1;
+        unsigned long flags;
+        struct pgpath *pgpath;
+
+        spin_lock_irqsave(&m->lock, flags);
+
+        /* Do we need to select a new pgpath? */
+        if (!m->current_pgpath ||
+            (!m->queue_io && (m->repeat_count && --m->repeat_count == 0)))
+                __choose_pgpath(m);
+
+        pgpath = m->current_pgpath;
+
+        if (was_queued)
+                m->queue_size--;
+
+        if ((pgpath && m->queue_io) ||
+            (!pgpath && m->queue_if_no_path && !m->suspended)) {
+                /* Queue for the daemon to resubmit */
+                bio_list_add(&m->queued_ios, bio);
+                m->queue_size++;
+                if (m->pg_init_required || !m->queue_io)
+                        schedule_work(&m->process_queued_ios);
+                pgpath = NULL;
+                r = 0;
+        } else if (!pgpath)
+                r = -EIO;               /* Failed */
+        else
+                bio->bi_bdev = pgpath->path.dev->bdev;
+
+        mpio->pgpath = pgpath;
+
+        spin_unlock_irqrestore(&m->lock, flags);
+
+        return r;
+}
+
+/*
+ * If we run out of usable paths, should we queue I/O or error it?
+ */
+static int queue_if_no_path(struct multipath *m, unsigned queue_if_no_path)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&m->lock, flags);
+
+        m->queue_if_no_path = queue_if_no_path;
+        if (!m->queue_if_no_path)
+                schedule_work(&m->process_queued_ios);
+
+        spin_unlock_irqrestore(&m->lock, flags);
+
+        return 0;
+}
+
+/*-----------------------------------------------------------------
+ * The multipath daemon is responsible for resubmitting queued ios.
+ *---------------------------------------------------------------*/
+
+static void dispatch_queued_ios(struct multipath *m)
+{
+        int r;
+        unsigned long flags;
+        struct bio *bio = NULL, *next;
+        struct mpath_io *mpio;
+        union map_info *info;
+
+        spin_lock_irqsave(&m->lock, flags);
+        bio = bio_list_get(&m->queued_ios);
+        spin_unlock_irqrestore(&m->lock, flags);
+
+        while (bio) {
+                next = bio->bi_next;
+                bio->bi_next = NULL;
+
+                info = dm_get_mapinfo(bio);
+                mpio = info->ptr;
+
+                r = map_io(m, bio, mpio, 1);
+                if (r < 0)
+                        bio_endio(bio, bio->bi_size, r);
+                else if (r == 1)
+                        generic_make_request(bio);
+
+                bio = next;
+        }
+}
+
+static void process_queued_ios(void *data)
+{
+        struct multipath *m = (struct multipath *) data;
+        struct hw_handler *hwh = &m->hw_handler;
+        struct pgpath *pgpath;
+        unsigned init_required, must_queue = 0;
+        unsigned long flags;
+
+        spin_lock_irqsave(&m->lock, flags);
+
+        if (!m->current_pgpath)
+                __choose_pgpath(m);
+
+        pgpath = m->current_pgpath;
+
+        if ((pgpath && m->queue_io) ||
+            (!pgpath && m->queue_if_no_path && !m->suspended))
+                must_queue = 1;
+
+        init_required = m->pg_init_required;
+        if (init_required)
+                m->pg_init_required = 0;
+
+        spin_unlock_irqrestore(&m->lock, flags);
+
+        if (init_required)
+                hwh->type->pg_init(hwh, pgpath->pg->bypassed, &pgpath->path);
+
+        if (!must_queue)
+                dispatch_queued_ios(m);
+}
+
+/*
+ * An event is triggered whenever a path is taken out of use.
+ * Includes path failure and PG bypass.
+ */
+static void trigger_event(void *data)
+{
+        struct multipath *m = (struct multipath *) data;
+
+        dm_table_event(m->ti->table);
+}
+
+/*-----------------------------------------------------------------
+ * Constructor/argument parsing:
+ * <#multipath feature args> [<arg>]*
+ * <#hw_handler args> [hw_handler [<arg>]*]
+ * <#priority groups>
+ * <initial priority group>
+ *     [<selector> <#selector args> [<arg>]*
+ *      <#paths> <#per-path selector args>
+ *         [<path> [<arg>]* ]+ ]+
+ *---------------------------------------------------------------*/
+struct param {
+        unsigned min;
+        unsigned max;
+        char *error;
+};
+
+#define ESTR(s) ("dm-multipath: " s)
+
+static int read_param(struct param *param, char *str, unsigned *v, char **error)
+{
+        if (!str ||
+            (sscanf(str, "%u", v) != 1) ||
+            (*v < param->min) ||
+            (*v > param->max)) {
+                *error = param->error;
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+struct arg_set {
+        unsigned argc;
+        char **argv;
+};
+
+static char *shift(struct arg_set *as)
+{
+        char *r;
+
+        if (as->argc) {
+                as->argc--;
+                r = *as->argv;
+                as->argv++;
+                return r;
+        }
+
+        return NULL;
+}
+
+static void consume(struct arg_set *as, unsigned n)
+{
+        BUG_ON (as->argc < n);
+        as->argc -= n;
+        as->argv += n;
+}
+
+static int parse_path_selector(struct arg_set *as, struct priority_group *pg,
+                               struct dm_target *ti)
+{
+        int r;
+        struct path_selector_type *pst;
+        unsigned ps_argc;
+
+        static struct param _params[] = {
+                {0, 1024, ESTR("invalid number of path selector args")},
+        };
+
+        pst = dm_get_path_selector(shift(as));
+        if (!pst) {
+                ti->error = ESTR("unknown path selector type");
+                return -EINVAL;
+        }
+
+        r = read_param(_params, shift(as), &ps_argc, &ti->error);
+        if (r)
+                return -EINVAL;
+
+        r = pst->create(&pg->ps, ps_argc, as->argv);
+        if (r) {
+                dm_put_path_selector(pst);
+                ti->error = ESTR("path selector constructor failed");
+                return r;
+        }
+
+        pg->ps.type = pst;
+        consume(as, ps_argc);
+
+        return 0;
+}
+
+static struct pgpath *parse_path(struct arg_set *as, struct path_selector *ps,
+                               struct dm_target *ti)
+{
+        int r;
+        struct pgpath *p;
+
+        /* we need at least a path arg */
+        if (as->argc < 1) {
+                ti->error = ESTR("no device given");
+                return NULL;
+        }
+
+        p = alloc_pgpath();
+        if (!p)
+                return NULL;
+
+        r = dm_get_device(ti, shift(as), ti->begin, ti->len,
+                          dm_table_get_mode(ti->table), &p->path.dev);
+        if (r) {
+                ti->error = ESTR("error getting device");
+                goto bad;
+        }
+
+        r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
+        if (r) {
+                dm_put_device(ti, p->path.dev);
+                goto bad;
+        }
+
+        return p;
+
+ bad:
+        free_pgpath(p);
+        return NULL;
+}
+
+static struct priority_group *parse_priority_group(struct arg_set *as,
+                                                   struct multipath *m,
+                                                   struct dm_target *ti)
+{
+        static struct param _params[] = {
+                {1, 1024, ESTR("invalid number of paths")},
+                {0, 1024, ESTR("invalid number of selector args")}
+        };
+
+        int r;
+        unsigned i, nr_selector_args, nr_params;
+        struct priority_group *pg;
+
+        if (as->argc < 2) {
+                as->argc = 0;
+                ti->error = ESTR("not enough priority group aruments");
+                return NULL;
+        }
+
+        pg = alloc_priority_group();
+        if (!pg) {
+                ti->error = ESTR("couldn't allocate priority group");
+                return NULL;
+        }
+        pg->m = m;
+
+        r = parse_path_selector(as, pg, ti);
+        if (r)
+                goto bad;
+
+        /*
+         * read the paths
+         */
+        r = read_param(_params, shift(as), &pg->nr_pgpaths, &ti->error);
+        if (r)
+                goto bad;
+
+        r = read_param(_params + 1, shift(as), &nr_selector_args, &ti->error);
+        if (r)
+                goto bad;
+
+        nr_params = 1 + nr_selector_args;
+        for (i = 0; i < pg->nr_pgpaths; i++) {
+                struct pgpath *pgpath;
+                struct arg_set path_args;
+
+                if (as->argc < nr_params)
+                        goto bad;
+
+                path_args.argc = nr_params;
+                path_args.argv = as->argv;
+
+                pgpath = parse_path(&path_args, &pg->ps, ti);
+                if (!pgpath)
+                        goto bad;
+
+                pgpath->pg = pg;
+                list_add_tail(&pgpath->list, &pg->pgpaths);
+                consume(as, nr_params);
+        }
+
+        return pg;
+
+ bad:
+        free_priority_group(pg, ti);
+        return NULL;
+}
+
+static int parse_hw_handler(struct arg_set *as, struct multipath *m,
+                            struct dm_target *ti)
+{
+        int r;
+        struct hw_handler_type *hwht;
+        unsigned hw_argc;
+
+        static struct param _params[] = {
+                {0, 1024, ESTR("invalid number of hardware handler args")},
+        };
+
+        r = read_param(_params, shift(as), &hw_argc, &ti->error);
+        if (r)
+                return -EINVAL;
+
+        if (!hw_argc)
+                return 0;
+
+        hwht = dm_get_hw_handler(shift(as));
+        if (!hwht) {
+                ti->error = ESTR("unknown hardware handler type");
+                return -EINVAL;
+        }
+
+        r = hwht->create(&m->hw_handler, hw_argc - 1, as->argv);
+        if (r) {
+                dm_put_hw_handler(hwht);
+                ti->error = ESTR("hardware handler constructor failed");
+                return r;
+        }
+
+        m->hw_handler.type = hwht;
+        consume(as, hw_argc - 1);
+
+        return 0;
+}
+
+static int parse_features(struct arg_set *as, struct multipath *m,
+                          struct dm_target *ti)
+{
+        int r;
+        unsigned argc;
+
+        static struct param _params[] = {
+                {0, 1, ESTR("invalid number of feature args")},
+        };
+
+        r = read_param(_params, shift(as), &argc, &ti->error);
+        if (r)
+                return -EINVAL;
+
+        if (!argc)
+                return 0;
+
+        if (!strnicmp(shift(as), MESG_STR("queue_if_no_path")))
+                return queue_if_no_path(m, 1);
+        else {
+                ti->error = "Unrecognised multipath feature request";
+                return -EINVAL;
+        }
+}
+
+static int multipath_ctr(struct dm_target *ti, unsigned int argc,
+                         char **argv)
+{
+        /* target parameters */
+        static struct param _params[] = {
+                {1, 1024, ESTR("invalid number of priority groups")},
+                {1, 1024, ESTR("invalid initial priority group number")},
+        };
+
+        int r;
+        struct multipath *m;
+        struct arg_set as;
+        unsigned pg_count = 0;
+        unsigned next_pg_num;
+
+        as.argc = argc;
+        as.argv = argv;
+
+        m = alloc_multipath();
+        if (!m) {
+                ti->error = ESTR("can't allocate multipath");
+                return -EINVAL;
+        }
+
+        r = parse_features(&as, m, ti);
+        if (r)
+                goto bad;
+
+        r = parse_hw_handler(&as, m, ti);
+        if (r)
+                goto bad;
+
+        r = read_param(_params, shift(&as), &m->nr_priority_groups, &ti->error);
+        if (r)
+                goto bad;
+
+        r = read_param(_params + 1, shift(&as), &next_pg_num, &ti->error);
+        if (r)
+                goto bad;
+
+        /* parse the priority groups */
+        while (as.argc) {
+                struct priority_group *pg;
+
+                pg = parse_priority_group(&as, m, ti);
+                if (!pg) {
+                        r = -EINVAL;
+                        goto bad;
+                }
+
+                m->nr_valid_paths += pg->nr_pgpaths;
+                list_add_tail(&pg->list, &m->priority_groups);
+                pg_count++;
+                pg->pg_num = pg_count;
+                if (!--next_pg_num)
+                        m->next_pg = pg;
+        }
+
+        if (pg_count != m->nr_priority_groups) {
+                ti->error = ESTR("priority group count mismatch");
+                r = -EINVAL;
+                goto bad;
+        }
+
+        ti->private = m;
+        m->ti = ti;
+
+        return 0;
+
+ bad:
+        free_multipath(m);
+        return r;
+}
+
+static void multipath_dtr(struct dm_target *ti)
+{
+        struct multipath *m = (struct multipath *) ti->private;
+        free_multipath(m);
+}
+
+/*
+ * Map bios, recording original fields for later in case we have to resubmit
+ */
+static int multipath_map(struct dm_target *ti, struct bio *bio,
+                         union map_info *map_context)
+{
+        int r;
+        struct mpath_io *mpio;
+        struct multipath *m = (struct multipath *) ti->private;
+
+        mpio = mempool_alloc(m->mpio_pool, GFP_NOIO);
+        dm_bio_record(&mpio->details, bio);
+
+        map_context->ptr = mpio;
+        bio->bi_rw |= (1 << BIO_RW_FAILFAST);
+        r = map_io(m, bio, mpio, 0);
+        if (r < 0)
+                mempool_free(mpio, m->mpio_pool);
+
+        return r;
+}
+
+/*
+ * Take a path out of use.
+ */
+static int fail_path(struct pgpath *pgpath)
+{
+        unsigned long flags;
+        struct multipath *m = pgpath->pg->m;
+
+        spin_lock_irqsave(&m->lock, flags);
+
+        if (!pgpath->path.is_active)
+                goto out;
+
+        DMWARN("dm-multipath: Failing path %s.", pgpath->path.dev->name);
+
+        pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
+        pgpath->path.is_active = 0;
+        pgpath->fail_count++;
+
+        m->nr_valid_paths--;
+
+        if (pgpath == m->current_pgpath)
+                m->current_pgpath = NULL;
+
+        schedule_work(&m->trigger_event);
+
+out:
+        spin_unlock_irqrestore(&m->lock, flags);
+
+        return 0;
+}
+
+/*
+ * Reinstate a previously-failed path
+ */
+static int reinstate_path(struct pgpath *pgpath)
+{
+        int r = 0;
+        unsigned long flags;
+        struct multipath *m = pgpath->pg->m;
+
+        spin_lock_irqsave(&m->lock, flags);
+
+        if (pgpath->path.is_active)
+                goto out;
+
+        if (!pgpath->pg->ps.type) {
+                DMWARN("Reinstate path not supported by path selector %s",
+                       pgpath->pg->ps.type->name);
+                r = -EINVAL;
+                goto out;
+        }
+
+        r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
+        if (r)
+                goto out;
+
+        pgpath->path.is_active = 1;
+
+        m->current_pgpath = NULL;
+        if (!m->nr_valid_paths++)
+                schedule_work(&m->process_queued_ios);
+
+        schedule_work(&m->trigger_event);
+
+out:
+        spin_unlock_irqrestore(&m->lock, flags);
+
+        return r;
+}
+
+/*
+ * Fail or reinstate all paths that match the provided struct dm_dev.
+ */
+static int action_dev(struct multipath *m, struct dm_dev *dev,
+                      action_fn action)
+{
+        int r = 0;
+        struct pgpath *pgpath;
+        struct priority_group *pg;
+
+        list_for_each_entry(pg, &m->priority_groups, list) {
+                list_for_each_entry(pgpath, &pg->pgpaths, list) {
+                        if (pgpath->path.dev == dev)
+                                r = action(pgpath);
+                }
+        }
+
+        return r;
+}
+
+/*
+ * Temporarily try to avoid having to use the specified PG
+ */
+static void bypass_pg(struct multipath *m, struct priority_group *pg,
+                      int bypassed)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&m->lock, flags);
+
+        pg->bypassed = bypassed;
+        m->current_pgpath = NULL;
+        m->current_pg = NULL;
+
+        spin_unlock_irqrestore(&m->lock, flags);
+
+        schedule_work(&m->trigger_event);
+}
+
+/*
+ * Switch to using the specified PG from the next I/O that gets mapped
+ */
+static int switch_pg_num(struct multipath *m, const char *pgstr)
+{
+        struct priority_group *pg;
+        unsigned pgnum;
+        unsigned long flags;
+
+        if (!pgstr || (sscanf(pgstr, "%u", &pgnum) != 1) || !pgnum ||
+            (pgnum > m->nr_priority_groups)) {
+                DMWARN("invalid PG number supplied to switch_pg_num");
+                return -EINVAL;
+        }
+
+        spin_lock_irqsave(&m->lock, flags);
+        list_for_each_entry(pg, &m->priority_groups, list) {
+                pg->bypassed = 0;
+                if (--pgnum)
+                        continue;
+
+                m->current_pgpath = NULL;
+                m->current_pg = NULL;
+                m->next_pg = pg;
+        }
+        spin_unlock_irqrestore(&m->lock, flags);
+
+        schedule_work(&m->trigger_event);
+        return 0;
+}
+
+/*
+ * Set/clear bypassed status of a PG.
+ * PGs are numbered upwards from 1 in the order they were declared.
+ */
+static int bypass_pg_num(struct multipath *m, const char *pgstr, int bypassed)
+{
+        struct priority_group *pg;
+        unsigned pgnum;
+
+        if (!pgstr || (sscanf(pgstr, "%u", &pgnum) != 1) || !pgnum ||
+            (pgnum > m->nr_priority_groups)) {
+                DMWARN("invalid PG number supplied to bypass_pg");
+                return -EINVAL;
+        }
+
+        list_for_each_entry(pg, &m->priority_groups, list) {
+                if (!--pgnum)
+                        break;
+        }
+
+        bypass_pg(m, pg, bypassed);
+        return 0;
+}
+
+/*
+ * pg_init must call this when it has completed its initialisation
+ */
+void dm_pg_init_complete(struct path *path, unsigned err_flags)
+{
+        struct pgpath *pgpath = path_to_pgpath(path);
+        struct priority_group *pg = pgpath->pg;
+        struct multipath *m = pg->m;
+        unsigned long flags;
+
+        /* We insist on failing the path if the PG is already bypassed. */
+        if (err_flags && pg->bypassed)
+                err_flags |= MP_FAIL_PATH;
+
+        if (err_flags & MP_FAIL_PATH)
+                fail_path(pgpath);
+
+        if (err_flags & MP_BYPASS_PG)
+                bypass_pg(m, pg, 1);
+
+        spin_lock_irqsave(&m->lock, flags);
+        if (!err_flags)
+                m->queue_io = 0;
+        else {
+                m->current_pgpath = NULL;
+                m->current_pg = NULL;
+        }
+        schedule_work(&m->process_queued_ios);
+        spin_unlock_irqrestore(&m->lock, flags);
+}
+
+/*
+ * end_io handling
+ */
+static int do_end_io(struct multipath *m, struct bio *bio,
+                     int error, struct mpath_io *mpio)
+{
+        struct hw_handler *hwh = &m->hw_handler;
+        unsigned err_flags = MP_FAIL_PATH;      /* Default behavior */
+
+        if (!error)
+                return 0;       /* I/O complete */
+
+        spin_lock(&m->lock);
+        if (!m->nr_valid_paths) {
+                if (!m->queue_if_no_path || m->suspended) {
+                        spin_unlock(&m->lock);
+                        return -EIO;
+                } else {
+                        spin_unlock(&m->lock);
+                        goto requeue;
+                }
+        }
+        spin_unlock(&m->lock);
+
+        if (hwh->type && hwh->type->error)
+                err_flags = hwh->type->error(hwh, bio);
+
+        if (mpio->pgpath) {
+                if (err_flags & MP_FAIL_PATH)
+                        fail_path(mpio->pgpath);
+
+                if (err_flags & MP_BYPASS_PG)
+                        bypass_pg(m, mpio->pgpath->pg, 1);
+        }
+
+        if (err_flags & MP_ERROR_IO)
+                return -EIO;
+
+      requeue:
+        dm_bio_restore(&mpio->details, bio);
+
+        /* queue for the daemon to resubmit or fail */
+        spin_lock(&m->lock);
+        bio_list_add(&m->queued_ios, bio);
+        m->queue_size++;
+        if (!m->queue_io)
+                schedule_work(&m->process_queued_ios);
+        spin_unlock(&m->lock);
+
+        return 1;       /* io not complete */
+}
+
+static int multipath_end_io(struct dm_target *ti, struct bio *bio,
+                            int error, union map_info *map_context)
+{
+        struct multipath *m = (struct multipath *) ti->private;
+        struct mpath_io *mpio = (struct mpath_io *) map_context->ptr;
+        struct pgpath *pgpath = mpio->pgpath;
+        struct path_selector *ps;
+        int r;
+
+        r  = do_end_io(m, bio, error, mpio);
+        if (pgpath) {
+                ps = &pgpath->pg->ps;
+                if (ps->type->end_io)
+                        ps->type->end_io(ps, &pgpath->path);
+        }
+        if (r <= 0)
+                mempool_free(mpio, m->mpio_pool);
+
+        return r;
+}
+
+/*
+ * Suspend can't complete until all the I/O is processed so if
+ * the last path failed we will now error any queued I/O.
+ */
+static void multipath_presuspend(struct dm_target *ti)
+{
+        struct multipath *m = (struct multipath *) ti->private;
+        unsigned long flags;
+
+        spin_lock_irqsave(&m->lock, flags);
+        m->suspended = 1;
+        if (m->queue_if_no_path)
+                schedule_work(&m->process_queued_ios);
+        spin_unlock_irqrestore(&m->lock, flags);
+}
+
+static void multipath_resume(struct dm_target *ti)
+{
+        struct multipath *m = (struct multipath *) ti->private;
+        unsigned long flags;
+
+        spin_lock_irqsave(&m->lock, flags);
+        m->suspended = 0;
+        spin_unlock_irqrestore(&m->lock, flags);
+}
+
+/*
+ * Info output has the following format:
+ * num_multipath_feature_args [multipath_feature_args]*
+ * num_handler_status_args [handler_status_args]*
+ * num_groups init_group_number
+ *            [A|D|E num_ps_status_args [ps_status_args]*
+ *             num_paths num_selector_args
+ *             [path_dev A|F fail_count [selector_args]* ]+ ]+
+ *
+ * Table output has the following format (identical to the constructor string):
+ * num_feature_args [features_args]*
+ * num_handler_args hw_handler [hw_handler_args]*
+ * num_groups init_group_number
+ *     [priority selector-name num_ps_args [ps_args]*
+ *      num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
+ */
+static int multipath_status(struct dm_target *ti, status_type_t type,
+                            char *result, unsigned int maxlen)
+{
+        int sz = 0;
+        unsigned long flags;
+        struct multipath *m = (struct multipath *) ti->private;
+        struct hw_handler *hwh = &m->hw_handler;
+        struct priority_group *pg;
+        struct pgpath *p;
+        unsigned pg_num;
+        char state;
+
+        spin_lock_irqsave(&m->lock, flags);
+
+        /* Features */
+        if (type == STATUSTYPE_INFO)
+                DMEMIT("1 %u ", m->queue_size);
+        else if (m->queue_if_no_path)
+                DMEMIT("1 queue_if_no_path ");
+        else
+                DMEMIT("0 ");
+
+        if (hwh->type && hwh->type->status)
+                sz += hwh->type->status(hwh, type, result + sz, maxlen - sz);
+        else if (!hwh->type || type == STATUSTYPE_INFO)
+                DMEMIT("0 ");
+        else
+                DMEMIT("1 %s ", hwh->type->name);
+
+        DMEMIT("%u ", m->nr_priority_groups);
+
+        if (m->next_pg)
+                pg_num = m->next_pg->pg_num;
+        else if (m->current_pg)
+                pg_num = m->current_pg->pg_num;
+        else
+                        pg_num = 1;
+
+        DMEMIT("%u ", pg_num);
+
+        switch (type) {
+        case STATUSTYPE_INFO:
+                list_for_each_entry(pg, &m->priority_groups, list) {
+                        if (pg->bypassed)
+                                state = 'D';    /* Disabled */
+                        else if (pg == m->current_pg)
+                                state = 'A';    /* Currently Active */
+                        else
+                                state = 'E';    /* Enabled */
+
+                        DMEMIT("%c ", state);
+
+                        if (pg->ps.type->status)
+                                sz += pg->ps.type->status(&pg->ps, NULL, type,
+                                                          result + sz,
+                                                          maxlen - sz);
+                        else
+                                DMEMIT("0 ");
+
+                        DMEMIT("%u %u ", pg->nr_pgpaths,
+                               pg->ps.type->info_args);
+
+                        list_for_each_entry(p, &pg->pgpaths, list) {
+                                DMEMIT("%s %s %u ", p->path.dev->name,
+                                       p->path.is_active ? "A" : "F",
+                                       p->fail_count);
+                                if (pg->ps.type->status)
+                                        sz += pg->ps.type->status(&pg->ps,
+                                              &p->path, type, result + sz,
+                                              maxlen - sz);
+                        }
+                }
+                break;
+
+        case STATUSTYPE_TABLE:
+                list_for_each_entry(pg, &m->priority_groups, list) {
+                        DMEMIT("%s ", pg->ps.type->name);
+
+                        if (pg->ps.type->status)
+                                sz += pg->ps.type->status(&pg->ps, NULL, type,
+                                                          result + sz,
+                                                          maxlen - sz);
+                        else
+                                DMEMIT("0 ");
+
+                        DMEMIT("%u %u ", pg->nr_pgpaths,
+                               pg->ps.type->table_args);
+
+                        list_for_each_entry(p, &pg->pgpaths, list) {
+                                DMEMIT("%s ", p->path.dev->name);
+                                if (pg->ps.type->status)
+                                        sz += pg->ps.type->status(&pg->ps,
+                                              &p->path, type, result + sz,
+                                              maxlen - sz);
+                        }
+                }
+                break;
+        }
+
+        spin_unlock_irqrestore(&m->lock, flags);
+
+        return 0;
+}
+
+static int multipath_message(struct dm_target *ti, unsigned argc, char **argv)
+{
+        int r;
+        struct dm_dev *dev;
+        struct multipath *m = (struct multipath *) ti->private;
+        action_fn action;
+
+        if (argc == 1) {
+                if (!strnicmp(argv[0], MESG_STR("queue_if_no_path")))
+                        return queue_if_no_path(m, 1);
+                else if (!strnicmp(argv[0], MESG_STR("fail_if_no_path")))
+                        return queue_if_no_path(m, 0);
+        }
+
+        if (argc != 2)
+                goto error;
+
+        if (!strnicmp(argv[0], MESG_STR("disable_group")))
+                return bypass_pg_num(m, argv[1], 1);
+        else if (!strnicmp(argv[0], MESG_STR("enable_group")))
+                return bypass_pg_num(m, argv[1], 0);
+        else if (!strnicmp(argv[0], MESG_STR("switch_group")))
+                return switch_pg_num(m, argv[1]);
+        else if (!strnicmp(argv[0], MESG_STR("reinstate_path")))
+                action = reinstate_path;
+        else if (!strnicmp(argv[0], MESG_STR("fail_path")))
+                action = fail_path;
+        else
+                goto error;
+
+        r = dm_get_device(ti, argv[1], ti->begin, ti->len,
+                          dm_table_get_mode(ti->table), &dev);
+        if (r) {
+                DMWARN("dm-multipath message: error getting device %s",
+                       argv[1]);
+                return -EINVAL;
+        }
+
+        r = action_dev(m, dev, action);
+
+        dm_put_device(ti, dev);
+
+        return r;
+
+error:
+        DMWARN("Unrecognised multipath message received.");
+        return -EINVAL;
+}
+
+/*-----------------------------------------------------------------
+ * Module setup
+ *---------------------------------------------------------------*/
+static struct target_type multipath_target = {
+        .name = "multipath",
+        .version = {1, 0, 4},
+        .module = THIS_MODULE,
+        .ctr = multipath_ctr,
+        .dtr = multipath_dtr,
+        .map = multipath_map,
+        .end_io = multipath_end_io,
+        .presuspend = multipath_presuspend,
+        .resume = multipath_resume,
+        .status = multipath_status,
+        .message = multipath_message,
+};
+
+static int __init dm_multipath_init(void)
+{
+        int r;
+
+        /* allocate a slab for the dm_ios */
+        _mpio_cache = kmem_cache_create("dm_mpath", sizeof(struct mpath_io),
+                                        0, 0, NULL, NULL);
+        if (!_mpio_cache)
+                return -ENOMEM;
+
+        r = dm_register_target(&multipath_target);
+        if (r < 0) {
+                DMERR("%s: register failed %d", multipath_target.name, r);
+                kmem_cache_destroy(_mpio_cache);
+                return -EINVAL;
+        }
+
+        DMINFO("dm-multipath version %u.%u.%u loaded",
+               multipath_target.version[0], multipath_target.version[1],
+               multipath_target.version[2]);
+
+        return r;
+}
+
+static void __exit dm_multipath_exit(void)
+{
+        int r;
+
+        r = dm_unregister_target(&multipath_target);
+        if (r < 0)
+                DMERR("%s: target unregister failed %d",
+                      multipath_target.name, r);
+        kmem_cache_destroy(_mpio_cache);
+}
+
+EXPORT_SYMBOL_GPL(dm_pg_init_complete);
+
+module_init(dm_multipath_init);
+module_exit(dm_multipath_exit);
+
+MODULE_DESCRIPTION(DM_NAME " multipath target");
+MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-mpath.h b/drivers/md/dm-mpath.h
new file mode 100644
index 000000000000..8a4bf2b6d52e
--- /dev/null
+++ b/drivers/md/dm-mpath.h
@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ *
+ * Multipath.
+ */
+
+#ifndef DM_MPATH_H
+#define DM_MPATH_H
+
+struct dm_dev;
+
+struct path {
+        struct dm_dev *dev;     /* Read-only */
+        unsigned is_active;     /* Read-only */
+
+        void *pscontext;        /* For path-selector use */
+        void *hwhcontext;       /* For hw-handler use */
+};
+
+/* Callback for hwh_pg_init_fn to use when complete */
+void dm_pg_init_complete(struct path *path, unsigned err_flags);
+
+#endif
diff --git a/drivers/md/dm-path-selector.c b/drivers/md/dm-path-selector.c
new file mode 100644
index 000000000000..ac5c4bbec6c1
--- /dev/null
+++ b/drivers/md/dm-path-selector.c
@@ -0,0 +1,156 @@
+/*
+ * Copyright (C) 2003 Sistina Software.
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * Module Author: Heinz Mauelshagen
+ *
+ * This file is released under the GPL.
+ *
+ * Path selector registration.
+ */
+
+#include "dm.h"
+#include "dm-path-selector.h"
+
+#include <linux/slab.h>
+
+struct ps_internal {
+        struct path_selector_type pst;
+
+        struct list_head list;
+        long use;
+};
+
+#define pst_to_psi(__pst) container_of((__pst), struct ps_internal, pst)
+
+static LIST_HEAD(_path_selectors);
+static DECLARE_RWSEM(_ps_lock);
+
+struct ps_internal *__find_path_selector_type(const char *name)
+{
+        struct ps_internal *psi;
+
+        list_for_each_entry(psi, &_path_selectors, list) {
+                if (!strcmp(name, psi->pst.name))
+                        return psi;
+        }
+
+        return NULL;
+}
+
+static struct ps_internal *get_path_selector(const char *name)
+{
+        struct ps_internal *psi;
+
+        down_read(&_ps_lock);
+        psi = __find_path_selector_type(name);
+        if (psi) {
+                if ((psi->use == 0) && !try_module_get(psi->pst.module))
+                        psi = NULL;
+                else
+                        psi->use++;
+        }
+        up_read(&_ps_lock);
+
+        return psi;
+}
+
+struct path_selector_type *dm_get_path_selector(const char *name)
+{
+        struct ps_internal *psi;
+
+        if (!name)
+                return NULL;
+
+        psi = get_path_selector(name);
+        if (!psi) {
+                request_module("dm-%s", name);
+                psi = get_path_selector(name);
+        }
+
+        return psi ? &psi->pst : NULL;
+}
+
+void dm_put_path_selector(struct path_selector_type *pst)
+{
+        struct ps_internal *psi;
+
+        if (!pst)
+                return;
+
+        down_read(&_ps_lock);
+        psi = __find_path_selector_type(pst->name);
+        if (!psi)
+                goto out;
+
+        if (--psi->use == 0)
+                module_put(psi->pst.module);
+
+        if (psi->use < 0)
+                BUG();
+
+out:
+        up_read(&_ps_lock);
+}
+
+static struct ps_internal *_alloc_path_selector(struct path_selector_type *pst)
+{
+        struct ps_internal *psi = kmalloc(sizeof(*psi), GFP_KERNEL);
+
+        if (psi) {
+                memset(psi, 0, sizeof(*psi));
+                psi->pst = *pst;
+        }
+
+        return psi;
+}
+
+int dm_register_path_selector(struct path_selector_type *pst)
+{
+        int r = 0;
+        struct ps_internal *psi = _alloc_path_selector(pst);
+
+        if (!psi)
+                return -ENOMEM;
+
+        down_write(&_ps_lock);
+
+        if (__find_path_selector_type(pst->name)) {
+                kfree(psi);
+                r = -EEXIST;
+        } else
+                list_add(&psi->list, &_path_selectors);
+
+        up_write(&_ps_lock);
+
+        return r;
+}
+
+int dm_unregister_path_selector(struct path_selector_type *pst)
+{
+        struct ps_internal *psi;
+
+        down_write(&_ps_lock);
+
+        psi = __find_path_selector_type(pst->name);
+        if (!psi) {
+                up_write(&_ps_lock);
+                return -EINVAL;
+        }
+
+        if (psi->use) {
+                up_write(&_ps_lock);
+                return -ETXTBSY;
+        }
+
+        list_del(&psi->list);
+
+        up_write(&_ps_lock);
+
+        kfree(psi);
+
+        return 0;
+}
+
+EXPORT_SYMBOL_GPL(dm_register_path_selector);
+EXPORT_SYMBOL_GPL(dm_unregister_path_selector);
diff --git a/drivers/md/dm-path-selector.h b/drivers/md/dm-path-selector.h
new file mode 100644
index 000000000000..732d06a84f85
--- /dev/null
+++ b/drivers/md/dm-path-selector.h
@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) 2003 Sistina Software.
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * Module Author: Heinz Mauelshagen
+ *
+ * This file is released under the GPL.
+ *
+ * Path-Selector registration.
+ */
+
+#ifndef DM_PATH_SELECTOR_H
+#define DM_PATH_SELECTOR_H
+
+#include <linux/device-mapper.h>
+
+#include "dm-mpath.h"
+
+/*
+ * We provide an abstraction for the code that chooses which path
+ * to send some io down.
+ */
+struct path_selector_type;
+struct path_selector {
+        struct path_selector_type *type;
+        void *context;
+};
+
+/* Information about a path selector type */
+struct path_selector_type {
+        char *name;
+        struct module *module;
+
+        unsigned int table_args;
+        unsigned int info_args;
+
+        /*
+         * Constructs a path selector object, takes custom arguments
+         */
+        int (*create) (struct path_selector *ps, unsigned argc, char **argv);
+        void (*destroy) (struct path_selector *ps);
+
+        /*
+         * Add an opaque path object, along with some selector specific
+         * path args (eg, path priority).
+         */
+        int (*add_path) (struct path_selector *ps, struct path *path,
+                         int argc, char **argv, char **error);
+
+        /*
+         * Chooses a path for this io, if no paths are available then
+         * NULL will be returned.
+         *
+         * repeat_count is the number of times to use the path before
+         * calling the function again.  0 means don't call it again unless
+         * the path fails.
+         */
+        struct path *(*select_path) (struct path_selector *ps,
+                                     unsigned *repeat_count);
+
+        /*
+         * Notify the selector that a path has failed.
+         */
+        void (*fail_path) (struct path_selector *ps, struct path *p);
+
+        /*
+         * Ask selector to reinstate a path.
+         */
+        int (*reinstate_path) (struct path_selector *ps, struct path *p);
+
+        /*
+         * Table content based on parameters added in ps_add_path_fn
+         * or path selector status
+         */
+        int (*status) (struct path_selector *ps, struct path *path,
+                       status_type_t type, char *result, unsigned int maxlen);
+
+        int (*end_io) (struct path_selector *ps, struct path *path);
+};
+
+/* Register a path selector */
+int dm_register_path_selector(struct path_selector_type *type);
+
+/* Unregister a path selector */
+int dm_unregister_path_selector(struct path_selector_type *type);
+
+/* Returns a registered path selector type */
+struct path_selector_type *dm_get_path_selector(const char *name);
+
+/* Releases a path selector  */
+void dm_put_path_selector(struct path_selector_type *pst);
+
+#endif
diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c
new file mode 100644
index 000000000000..6e3cf7e13451
--- /dev/null
+++ b/drivers/md/dm-raid1.c
@@ -0,0 +1,1269 @@
+/*
+ * Copyright (C) 2003 Sistina Software Limited.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+#include "dm-bio-list.h"
+#include "dm-io.h"
+#include "dm-log.h"
+#include "kcopyd.h"
+
+#include <linux/ctype.h>
+#include <linux/init.h>
+#include <linux/mempool.h>
+#include <linux/module.h>
+#include <linux/pagemap.h>
+#include <linux/slab.h>
+#include <linux/time.h>
+#include <linux/vmalloc.h>
+#include <linux/workqueue.h>
+
+static struct workqueue_struct *_kmirrord_wq;
+static struct work_struct _kmirrord_work;
+
+static inline void wake(void)
+{
+        queue_work(_kmirrord_wq, &_kmirrord_work);
+}
+
+/*-----------------------------------------------------------------
+ * Region hash
+ *
+ * The mirror splits itself up into discrete regions.  Each
+ * region can be in one of three states: clean, dirty,
+ * nosync.  There is no need to put clean regions in the hash.
+ *
+ * In addition to being present in the hash table a region _may_
+ * be present on one of three lists.
+ *
+ *   clean_regions: Regions on this list have no io pending to
+ *   them, they are in sync, we are no longer interested in them,
+ *   they are dull.  rh_update_states() will remove them from the
+ *   hash table.
+ *
+ *   quiesced_regions: These regions have been spun down, ready
+ *   for recovery.  rh_recovery_start() will remove regions from
+ *   this list and hand them to kmirrord, which will schedule the
+ *   recovery io with kcopyd.
+ *
+ *   recovered_regions: Regions that kcopyd has successfully
+ *   recovered.  rh_update_states() will now schedule any delayed
+ *   io, up the recovery_count, and remove the region from the
+ *   hash.
+ *
+ * There are 2 locks:
+ *   A rw spin lock 'hash_lock' protects just the hash table,
+ *   this is never held in write mode from interrupt context,
+ *   which I believe means that we only have to disable irqs when
+ *   doing a write lock.
+ *
+ *   An ordinary spin lock 'region_lock' that protects the three
+ *   lists in the region_hash, with the 'state', 'list' and
+ *   'bhs_delayed' fields of the regions.  This is used from irq
+ *   context, so all other uses will have to suspend local irqs.
+ *---------------------------------------------------------------*/
+struct mirror_set;
+struct region_hash {
+        struct mirror_set *ms;
+        uint32_t region_size;
+        unsigned region_shift;
+
+        /* holds persistent region state */
+        struct dirty_log *log;
+
+        /* hash table */
+        rwlock_t hash_lock;
+        mempool_t *region_pool;
+        unsigned int mask;
+        unsigned int nr_buckets;
+        struct list_head *buckets;
+
+        spinlock_t region_lock;
+        struct semaphore recovery_count;
+        struct list_head clean_regions;
+        struct list_head quiesced_regions;
+        struct list_head recovered_regions;
+};
+
+enum {
+        RH_CLEAN,
+        RH_DIRTY,
+        RH_NOSYNC,
+        RH_RECOVERING
+};
+
+struct region {
+        struct region_hash *rh; /* FIXME: can we get rid of this ? */
+        region_t key;
+        int state;
+
+        struct list_head hash_list;
+        struct list_head list;
+
+        atomic_t pending;
+        struct bio_list delayed_bios;
+};
+
+/*
+ * Conversion fns
+ */
+static inline region_t bio_to_region(struct region_hash *rh, struct bio *bio)
+{
+        return bio->bi_sector >> rh->region_shift;
+}
+
+static inline sector_t region_to_sector(struct region_hash *rh, region_t region)
+{
+        return region << rh->region_shift;
+}
+
+/* FIXME move this */
+static void queue_bio(struct mirror_set *ms, struct bio *bio, int rw);
+
+static void *region_alloc(unsigned int __nocast gfp_mask, void *pool_data)
+{
+        return kmalloc(sizeof(struct region), gfp_mask);
+}
+
+static void region_free(void *element, void *pool_data)
+{
+        kfree(element);
+}
+
+#define MIN_REGIONS 64
+#define MAX_RECOVERY 1
+static int rh_init(struct region_hash *rh, struct mirror_set *ms,
+                   struct dirty_log *log, uint32_t region_size,
+                   region_t nr_regions)
+{
+        unsigned int nr_buckets, max_buckets;
+        size_t i;
+
+        /*
+         * Calculate a suitable number of buckets for our hash
+         * table.
+         */
+        max_buckets = nr_regions >> 6;
+        for (nr_buckets = 128u; nr_buckets < max_buckets; nr_buckets <<= 1)
+                ;
+        nr_buckets >>= 1;
+
+        rh->ms = ms;
+        rh->log = log;
+        rh->region_size = region_size;
+        rh->region_shift = ffs(region_size) - 1;
+        rwlock_init(&rh->hash_lock);
+        rh->mask = nr_buckets - 1;
+        rh->nr_buckets = nr_buckets;
+
+        rh->buckets = vmalloc(nr_buckets * sizeof(*rh->buckets));
+        if (!rh->buckets) {
+                DMERR("unable to allocate region hash memory");
+                return -ENOMEM;
+        }
+
+        for (i = 0; i < nr_buckets; i++)
+                INIT_LIST_HEAD(rh->buckets + i);
+
+        spin_lock_init(&rh->region_lock);
+        sema_init(&rh->recovery_count, 0);
+        INIT_LIST_HEAD(&rh->clean_regions);
+        INIT_LIST_HEAD(&rh->quiesced_regions);
+        INIT_LIST_HEAD(&rh->recovered_regions);
+
+        rh->region_pool = mempool_create(MIN_REGIONS, region_alloc,
+                                         region_free, NULL);
+        if (!rh->region_pool) {
+                vfree(rh->buckets);
+                rh->buckets = NULL;
+                return -ENOMEM;
+        }
+
+        return 0;
+}
+
+static void rh_exit(struct region_hash *rh)
+{
+        unsigned int h;
+        struct region *reg, *nreg;
+
+        BUG_ON(!list_empty(&rh->quiesced_regions));
+        for (h = 0; h < rh->nr_buckets; h++) {
+                list_for_each_entry_safe(reg, nreg, rh->buckets + h, hash_list) {
+                        BUG_ON(atomic_read(&reg->pending));
+                        mempool_free(reg, rh->region_pool);
+                }
+        }
+
+        if (rh->log)
+                dm_destroy_dirty_log(rh->log);
+        if (rh->region_pool)
+                mempool_destroy(rh->region_pool);
+        vfree(rh->buckets);
+}
+
+#define RH_HASH_MULT 2654435387U
+
+static inline unsigned int rh_hash(struct region_hash *rh, region_t region)
+{
+        return (unsigned int) ((region * RH_HASH_MULT) >> 12) & rh->mask;
+}
+
+static struct region *__rh_lookup(struct region_hash *rh, region_t region)
+{
+        struct region *reg;
+
+        list_for_each_entry (reg, rh->buckets + rh_hash(rh, region), hash_list)
+                if (reg->key == region)
+                        return reg;
+
+        return NULL;
+}
+
+static void __rh_insert(struct region_hash *rh, struct region *reg)
+{
+        unsigned int h = rh_hash(rh, reg->key);
+        list_add(&reg->hash_list, rh->buckets + h);
+}
+
+static struct region *__rh_alloc(struct region_hash *rh, region_t region)
+{
+        struct region *reg, *nreg;
+
+        read_unlock(&rh->hash_lock);
+        nreg = mempool_alloc(rh->region_pool, GFP_NOIO);
+        nreg->state = rh->log->type->in_sync(rh->log, region, 1) ?
+                RH_CLEAN : RH_NOSYNC;
+        nreg->rh = rh;
+        nreg->key = region;
+
+        INIT_LIST_HEAD(&nreg->list);
+
+        atomic_set(&nreg->pending, 0);
+        bio_list_init(&nreg->delayed_bios);
+        write_lock_irq(&rh->hash_lock);
+
+        reg = __rh_lookup(rh, region);
+        if (reg)
+                /* we lost the race */
+                mempool_free(nreg, rh->region_pool);
+
+        else {
+                __rh_insert(rh, nreg);
+                if (nreg->state == RH_CLEAN) {
+                        spin_lock(&rh->region_lock);
+                        list_add(&nreg->list, &rh->clean_regions);
+                        spin_unlock(&rh->region_lock);
+                }
+                reg = nreg;
+        }
+        write_unlock_irq(&rh->hash_lock);
+        read_lock(&rh->hash_lock);
+
+        return reg;
+}
+
+static inline struct region *__rh_find(struct region_hash *rh, region_t region)
+{
+        struct region *reg;
+
+        reg = __rh_lookup(rh, region);
+        if (!reg)
+                reg = __rh_alloc(rh, region);
+
+        return reg;
+}
+
+static int rh_state(struct region_hash *rh, region_t region, int may_block)
+{
+        int r;
+        struct region *reg;
+
+        read_lock(&rh->hash_lock);
+        reg = __rh_lookup(rh, region);
+        read_unlock(&rh->hash_lock);
+
+        if (reg)
+                return reg->state;
+
+        /*
+         * The region wasn't in the hash, so we fall back to the
+         * dirty log.
+         */
+        r = rh->log->type->in_sync(rh->log, region, may_block);
+
+        /*
+         * Any error from the dirty log (eg. -EWOULDBLOCK) gets
+         * taken as a RH_NOSYNC
+         */
+        return r == 1 ? RH_CLEAN : RH_NOSYNC;
+}
+
+static inline int rh_in_sync(struct region_hash *rh,
+                             region_t region, int may_block)
+{
+        int state = rh_state(rh, region, may_block);
+        return state == RH_CLEAN || state == RH_DIRTY;
+}
+
+static void dispatch_bios(struct mirror_set *ms, struct bio_list *bio_list)
+{
+        struct bio *bio;
+
+        while ((bio = bio_list_pop(bio_list))) {
+                queue_bio(ms, bio, WRITE);
+        }
+}
+
+static void rh_update_states(struct region_hash *rh)
+{
+        struct region *reg, *next;
+
+        LIST_HEAD(clean);
+        LIST_HEAD(recovered);
+
+        /*
+         * Quickly grab the lists.
+         */
+        write_lock_irq(&rh->hash_lock);
+        spin_lock(&rh->region_lock);
+        if (!list_empty(&rh->clean_regions)) {
+                list_splice(&rh->clean_regions, &clean);
+                INIT_LIST_HEAD(&rh->clean_regions);
+
+                list_for_each_entry (reg, &clean, list) {
+                        rh->log->type->clear_region(rh->log, reg->key);
+                        list_del(&reg->hash_list);
+                }
+        }
+
+        if (!list_empty(&rh->recovered_regions)) {
+                list_splice(&rh->recovered_regions, &recovered);
+                INIT_LIST_HEAD(&rh->recovered_regions);
+
+                list_for_each_entry (reg, &recovered, list)
+                        list_del(&reg->hash_list);
+        }
+        spin_unlock(&rh->region_lock);
+        write_unlock_irq(&rh->hash_lock);
+
+        /*
+         * All the regions on the recovered and clean lists have
+         * now been pulled out of the system, so no need to do
+         * any more locking.
+         */
+        list_for_each_entry_safe (reg, next, &recovered, list) {
+                rh->log->type->clear_region(rh->log, reg->key);
+                rh->log->type->complete_resync_work(rh->log, reg->key, 1);
+                dispatch_bios(rh->ms, &reg->delayed_bios);
+                up(&rh->recovery_count);
+                mempool_free(reg, rh->region_pool);
+        }
+
+        if (!list_empty(&recovered))
+                rh->log->type->flush(rh->log);
+
+        list_for_each_entry_safe (reg, next, &clean, list)
+                mempool_free(reg, rh->region_pool);
+}
+
+static void rh_inc(struct region_hash *rh, region_t region)
+{
+        struct region *reg;
+
+        read_lock(&rh->hash_lock);
+        reg = __rh_find(rh, region);
+        if (reg->state == RH_CLEAN) {
+                rh->log->type->mark_region(rh->log, reg->key);
+
+                spin_lock_irq(&rh->region_lock);
+                reg->state = RH_DIRTY;
+                list_del_init(&reg->list);      /* take off the clean list */
+                spin_unlock_irq(&rh->region_lock);
+        }
+
+        atomic_inc(&reg->pending);
+        read_unlock(&rh->hash_lock);
+}
+
+static void rh_inc_pending(struct region_hash *rh, struct bio_list *bios)
+{
+        struct bio *bio;
+
+        for (bio = bios->head; bio; bio = bio->bi_next)
+                rh_inc(rh, bio_to_region(rh, bio));
+}
+
+static void rh_dec(struct region_hash *rh, region_t region)
+{
+        unsigned long flags;
+        struct region *reg;
+        int should_wake = 0;
+
+        read_lock(&rh->hash_lock);
+        reg = __rh_lookup(rh, region);
+        read_unlock(&rh->hash_lock);
+
+        if (atomic_dec_and_test(&reg->pending)) {
+                spin_lock_irqsave(&rh->region_lock, flags);
+                if (reg->state == RH_RECOVERING) {
+                        list_add_tail(&reg->list, &rh->quiesced_regions);
+                } else {
+                        reg->state = RH_CLEAN;
+                        list_add(&reg->list, &rh->clean_regions);
+                }
+                spin_unlock_irqrestore(&rh->region_lock, flags);
+                should_wake = 1;
+        }
+
+        if (should_wake)
+                wake();
+}
+
+/*
+ * Starts quiescing a region in preparation for recovery.
+ */
+static int __rh_recovery_prepare(struct region_hash *rh)
+{
+        int r;
+        struct region *reg;
+        region_t region;
+
+        /*
+         * Ask the dirty log what's next.
+         */
+        r = rh->log->type->get_resync_work(rh->log, &region);
+        if (r <= 0)
+                return r;
+
+        /*
+         * Get this region, and start it quiescing by setting the
+         * recovering flag.
+         */
+        read_lock(&rh->hash_lock);
+        reg = __rh_find(rh, region);
+        read_unlock(&rh->hash_lock);
+
+        spin_lock_irq(&rh->region_lock);
+        reg->state = RH_RECOVERING;
+
+        /* Already quiesced ? */
+        if (atomic_read(&reg->pending))
+                list_del_init(&reg->list);
+
+        else {
+                list_del_init(&reg->list);
+                list_add(&reg->list, &rh->quiesced_regions);
+        }
+        spin_unlock_irq(&rh->region_lock);
+
+        return 1;
+}
+
+static void rh_recovery_prepare(struct region_hash *rh)
+{
+        while (!down_trylock(&rh->recovery_count))
+                if (__rh_recovery_prepare(rh) <= 0) {
+                        up(&rh->recovery_count);
+                        break;
+                }
+}
+
+/*
+ * Returns any quiesced regions.
+ */
+static struct region *rh_recovery_start(struct region_hash *rh)
+{
+        struct region *reg = NULL;
+
+        spin_lock_irq(&rh->region_lock);
+        if (!list_empty(&rh->quiesced_regions)) {
+                reg = list_entry(rh->quiesced_regions.next,
+                                 struct region, list);
+                list_del_init(&reg->list);      /* remove from the quiesced list */
+        }
+        spin_unlock_irq(&rh->region_lock);
+
+        return reg;
+}
+
+/* FIXME: success ignored for now */
+static void rh_recovery_end(struct region *reg, int success)
+{
+        struct region_hash *rh = reg->rh;
+
+        spin_lock_irq(&rh->region_lock);
+        list_add(&reg->list, &reg->rh->recovered_regions);
+        spin_unlock_irq(&rh->region_lock);
+
+        wake();
+}
+
+static void rh_flush(struct region_hash *rh)
+{
+        rh->log->type->flush(rh->log);
+}
+
+static void rh_delay(struct region_hash *rh, struct bio *bio)
+{
+        struct region *reg;
+
+        read_lock(&rh->hash_lock);
+        reg = __rh_find(rh, bio_to_region(rh, bio));
+        bio_list_add(&reg->delayed_bios, bio);
+        read_unlock(&rh->hash_lock);
+}
+
+static void rh_stop_recovery(struct region_hash *rh)
+{
+        int i;
+
+        /* wait for any recovering regions */
+        for (i = 0; i < MAX_RECOVERY; i++)
+                down(&rh->recovery_count);
+}
+
+static void rh_start_recovery(struct region_hash *rh)
+{
+        int i;
+
+        for (i = 0; i < MAX_RECOVERY; i++)
+                up(&rh->recovery_count);
+
+        wake();
+}
+
+/*-----------------------------------------------------------------
+ * Mirror set structures.
+ *---------------------------------------------------------------*/
+struct mirror {
+        atomic_t error_count;
+        struct dm_dev *dev;
+        sector_t offset;
+};
+
+struct mirror_set {
+        struct dm_target *ti;
+        struct list_head list;
+        struct region_hash rh;
+        struct kcopyd_client *kcopyd_client;
+
+        spinlock_t lock;        /* protects the next two lists */
+        struct bio_list reads;
+        struct bio_list writes;
+
+        /* recovery */
+        region_t nr_regions;
+        int in_sync;
+
+        unsigned int nr_mirrors;
+        struct mirror mirror[0];
+};
+
+/*
+ * Every mirror should look like this one.
+ */
+#define DEFAULT_MIRROR 0
+
+/*
+ * This is yucky.  We squirrel the mirror_set struct away inside
+ * bi_next for write buffers.  This is safe since the bh
+ * doesn't get submitted to the lower levels of block layer.
+ */
+static struct mirror_set *bio_get_ms(struct bio *bio)
+{
+        return (struct mirror_set *) bio->bi_next;
+}
+
+static void bio_set_ms(struct bio *bio, struct mirror_set *ms)
+{
+        bio->bi_next = (struct bio *) ms;
+}
+
+/*-----------------------------------------------------------------
+ * Recovery.
+ *
+ * When a mirror is first activated we may find that some regions
+ * are in the no-sync state.  We have to recover these by
+ * recopying from the default mirror to all the others.
+ *---------------------------------------------------------------*/
+static void recovery_complete(int read_err, unsigned int write_err,
+                              void *context)
+{
+        struct region *reg = (struct region *) context;
+
+        /* FIXME: better error handling */
+        rh_recovery_end(reg, read_err || write_err);
+}
+
+static int recover(struct mirror_set *ms, struct region *reg)
+{
+        int r;
+        unsigned int i;
+        struct io_region from, to[KCOPYD_MAX_REGIONS], *dest;
+        struct mirror *m;
+        unsigned long flags = 0;
+
+        /* fill in the source */
+        m = ms->mirror + DEFAULT_MIRROR;
+        from.bdev = m->dev->bdev;
+        from.sector = m->offset + region_to_sector(reg->rh, reg->key);
+        if (reg->key == (ms->nr_regions - 1)) {
+                /*
+                 * The final region may be smaller than
+                 * region_size.
+                 */
+                from.count = ms->ti->len & (reg->rh->region_size - 1);
+                if (!from.count)
+                        from.count = reg->rh->region_size;
+        } else
+                from.count = reg->rh->region_size;
+
+        /* fill in the destinations */
+        for (i = 0, dest = to; i < ms->nr_mirrors; i++) {
+                if (i == DEFAULT_MIRROR)
+                        continue;
+
+                m = ms->mirror + i;
+                dest->bdev = m->dev->bdev;
+                dest->sector = m->offset + region_to_sector(reg->rh, reg->key);
+                dest->count = from.count;
+                dest++;
+        }
+
+        /* hand to kcopyd */
+        set_bit(KCOPYD_IGNORE_ERROR, &flags);
+        r = kcopyd_copy(ms->kcopyd_client, &from, ms->nr_mirrors - 1, to, flags,
+                        recovery_complete, reg);
+
+        return r;
+}
+
+static void do_recovery(struct mirror_set *ms)
+{
+        int r;
+        struct region *reg;
+        struct dirty_log *log = ms->rh.log;
+
+        /*
+         * Start quiescing some regions.
+         */
+        rh_recovery_prepare(&ms->rh);
+
+        /*
+         * Copy any already quiesced regions.
+         */
+        while ((reg = rh_recovery_start(&ms->rh))) {
+                r = recover(ms, reg);
+                if (r)
+                        rh_recovery_end(reg, 0);
+        }
+
+        /*
+         * Update the in sync flag.
+         */
+        if (!ms->in_sync &&
+            (log->type->get_sync_count(log) == ms->nr_regions)) {
+                /* the sync is complete */
+                dm_table_event(ms->ti->table);
+                ms->in_sync = 1;
+        }
+}
+
+/*-----------------------------------------------------------------
+ * Reads
+ *---------------------------------------------------------------*/
+static struct mirror *choose_mirror(struct mirror_set *ms, sector_t sector)
+{
+        /* FIXME: add read balancing */
+        return ms->mirror + DEFAULT_MIRROR;
+}
+
+/*
+ * remap a buffer to a particular mirror.
+ */
+static void map_bio(struct mirror_set *ms, struct mirror *m, struct bio *bio)
+{
+        bio->bi_bdev = m->dev->bdev;
+        bio->bi_sector = m->offset + (bio->bi_sector - ms->ti->begin);
+}
+
+static void do_reads(struct mirror_set *ms, struct bio_list *reads)
+{
+        region_t region;
+        struct bio *bio;
+        struct mirror *m;
+
+        while ((bio = bio_list_pop(reads))) {
+                region = bio_to_region(&ms->rh, bio);
+
+                /*
+                 * We can only read balance if the region is in sync.
+                 */
+                if (rh_in_sync(&ms->rh, region, 0))
+                        m = choose_mirror(ms, bio->bi_sector);
+                else
+                        m = ms->mirror + DEFAULT_MIRROR;
+
+                map_bio(ms, m, bio);
+                generic_make_request(bio);
+        }
+}
+
+/*-----------------------------------------------------------------
+ * Writes.
+ *
+ * We do different things with the write io depending on the
+ * state of the region that it's in:
+ *
+ * SYNC:        increment pending, use kcopyd to write to *all* mirrors
+ * RECOVERING:  delay the io until recovery completes
+ * NOSYNC:      increment pending, just write to the default mirror
+ *---------------------------------------------------------------*/
+static void write_callback(unsigned long error, void *context)
+{
+        unsigned int i;
+        int uptodate = 1;
+        struct bio *bio = (struct bio *) context;
+        struct mirror_set *ms;
+
+        ms = bio_get_ms(bio);
+        bio_set_ms(bio, NULL);
+
+        /*
+         * NOTE: We don't decrement the pending count here,
+         * instead it is done by the targets endio function.
+         * This way we handle both writes to SYNC and NOSYNC
+         * regions with the same code.
+         */
+
+        if (error) {
+                /*
+                 * only error the io if all mirrors failed.
+                 * FIXME: bogus
+                 */
+                uptodate = 0;
+                for (i = 0; i < ms->nr_mirrors; i++)
+                        if (!test_bit(i, &error)) {
+                                uptodate = 1;
+                                break;
+                        }
+        }
+        bio_endio(bio, bio->bi_size, 0);
+}
+
+static void do_write(struct mirror_set *ms, struct bio *bio)
+{
+        unsigned int i;
+        struct io_region io[KCOPYD_MAX_REGIONS+1];
+        struct mirror *m;
+
+        for (i = 0; i < ms->nr_mirrors; i++) {
+                m = ms->mirror + i;
+
+                io[i].bdev = m->dev->bdev;
+                io[i].sector = m->offset + (bio->bi_sector - ms->ti->begin);
+                io[i].count = bio->bi_size >> 9;
+        }
+
+        bio_set_ms(bio, ms);
+        dm_io_async_bvec(ms->nr_mirrors, io, WRITE,
+                         bio->bi_io_vec + bio->bi_idx,
+                         write_callback, bio);
+}
+
+static void do_writes(struct mirror_set *ms, struct bio_list *writes)
+{
+        int state;
+        struct bio *bio;
+        struct bio_list sync, nosync, recover, *this_list = NULL;
+
+        if (!writes->head)
+                return;
+
+        /*
+         * Classify each write.
+         */
+        bio_list_init(&sync);
+        bio_list_init(&nosync);
+        bio_list_init(&recover);
+
+        while ((bio = bio_list_pop(writes))) {
+                state = rh_state(&ms->rh, bio_to_region(&ms->rh, bio), 1);
+                switch (state) {
+                case RH_CLEAN:
+                case RH_DIRTY:
+                        this_list = &sync;
+                        break;
+
+                case RH_NOSYNC:
+                        this_list = &nosync;
+                        break;
+
+                case RH_RECOVERING:
+                        this_list = &recover;
+                        break;
+                }
+
+                bio_list_add(this_list, bio);
+        }
+
+        /*
+         * Increment the pending counts for any regions that will
+         * be written to (writes to recover regions are going to
+         * be delayed).
+         */
+        rh_inc_pending(&ms->rh, &sync);
+        rh_inc_pending(&ms->rh, &nosync);
+        rh_flush(&ms->rh);
+
+        /*
+         * Dispatch io.
+         */
+        while ((bio = bio_list_pop(&sync)))
+                do_write(ms, bio);
+
+        while ((bio = bio_list_pop(&recover)))
+                rh_delay(&ms->rh, bio);
+
+        while ((bio = bio_list_pop(&nosync))) {
+                map_bio(ms, ms->mirror + DEFAULT_MIRROR, bio);
+                generic_make_request(bio);
+        }
+}
+
+/*-----------------------------------------------------------------
+ * kmirrord
+ *---------------------------------------------------------------*/
+static LIST_HEAD(_mirror_sets);
+static DECLARE_RWSEM(_mirror_sets_lock);
+
+static void do_mirror(struct mirror_set *ms)
+{
+        struct bio_list reads, writes;
+
+        spin_lock(&ms->lock);
+        reads = ms->reads;
+        writes = ms->writes;
+        bio_list_init(&ms->reads);
+        bio_list_init(&ms->writes);
+        spin_unlock(&ms->lock);
+
+        rh_update_states(&ms->rh);
+        do_recovery(ms);
+        do_reads(ms, &reads);
+        do_writes(ms, &writes);
+}
+
+static void do_work(void *ignored)
+{
+        struct mirror_set *ms;
+
+        down_read(&_mirror_sets_lock);
+        list_for_each_entry (ms, &_mirror_sets, list)
+                do_mirror(ms);
+        up_read(&_mirror_sets_lock);
+}
+
+/*-----------------------------------------------------------------
+ * Target functions
+ *---------------------------------------------------------------*/
+static struct mirror_set *alloc_context(unsigned int nr_mirrors,
+                                        uint32_t region_size,
+                                        struct dm_target *ti,
+                                        struct dirty_log *dl)
+{
+        size_t len;
+        struct mirror_set *ms = NULL;
+
+        if (array_too_big(sizeof(*ms), sizeof(ms->mirror[0]), nr_mirrors))
+                return NULL;
+
+        len = sizeof(*ms) + (sizeof(ms->mirror[0]) * nr_mirrors);
+
+        ms = kmalloc(len, GFP_KERNEL);
+        if (!ms) {
+                ti->error = "dm-mirror: Cannot allocate mirror context";
+                return NULL;
+        }
+
+        memset(ms, 0, len);
+        spin_lock_init(&ms->lock);
+
+        ms->ti = ti;
+        ms->nr_mirrors = nr_mirrors;
+        ms->nr_regions = dm_sector_div_up(ti->len, region_size);
+        ms->in_sync = 0;
+
+        if (rh_init(&ms->rh, ms, dl, region_size, ms->nr_regions)) {
+                ti->error = "dm-mirror: Error creating dirty region hash";
+                kfree(ms);
+                return NULL;
+        }
+
+        return ms;
+}
+
+static void free_context(struct mirror_set *ms, struct dm_target *ti,
+                         unsigned int m)
+{
+        while (m--)
+                dm_put_device(ti, ms->mirror[m].dev);
+
+        rh_exit(&ms->rh);
+        kfree(ms);
+}
+
+static inline int _check_region_size(struct dm_target *ti, uint32_t size)
+{
+        return !(size % (PAGE_SIZE >> 9) || (size & (size - 1)) ||
+                 size > ti->len);
+}
+
+static int get_mirror(struct mirror_set *ms, struct dm_target *ti,
+                      unsigned int mirror, char **argv)
+{
+        sector_t offset;
+
+        if (sscanf(argv[1], SECTOR_FORMAT, &offset) != 1) {
+                ti->error = "dm-mirror: Invalid offset";
+                return -EINVAL;
+        }
+
+        if (dm_get_device(ti, argv[0], offset, ti->len,
+                          dm_table_get_mode(ti->table),
+                          &ms->mirror[mirror].dev)) {
+                ti->error = "dm-mirror: Device lookup failure";
+                return -ENXIO;
+        }
+
+        ms->mirror[mirror].offset = offset;
+
+        return 0;
+}
+
+static int add_mirror_set(struct mirror_set *ms)
+{
+        down_write(&_mirror_sets_lock);
+        list_add_tail(&ms->list, &_mirror_sets);
+        up_write(&_mirror_sets_lock);
+        wake();
+
+        return 0;
+}
+
+static void del_mirror_set(struct mirror_set *ms)
+{
+        down_write(&_mirror_sets_lock);
+        list_del(&ms->list);
+        up_write(&_mirror_sets_lock);
+}
+
+/*
+ * Create dirty log: log_type #log_params <log_params>
+ */
+static struct dirty_log *create_dirty_log(struct dm_target *ti,
+                                          unsigned int argc, char **argv,
+                                          unsigned int *args_used)
+{
+        unsigned int param_count;
+        struct dirty_log *dl;
+
+        if (argc < 2) {
+                ti->error = "dm-mirror: Insufficient mirror log arguments";
+                return NULL;
+        }
+
+        if (sscanf(argv[1], "%u", &param_count) != 1) {
+                ti->error = "dm-mirror: Invalid mirror log argument count";
+                return NULL;
+        }
+
+        *args_used = 2 + param_count;
+
+        if (argc < *args_used) {
+                ti->error = "dm-mirror: Insufficient mirror log arguments";
+                return NULL;
+        }
+
+        dl = dm_create_dirty_log(argv[0], ti, param_count, argv + 2);
+        if (!dl) {
+                ti->error = "dm-mirror: Error creating mirror dirty log";
+                return NULL;
+        }
+
+        if (!_check_region_size(ti, dl->type->get_region_size(dl))) {
+                ti->error = "dm-mirror: Invalid region size";
+                dm_destroy_dirty_log(dl);
+                return NULL;
+        }
+
+        return dl;
+}
+
+/*
+ * Construct a mirror mapping:
+ *
+ * log_type #log_params <log_params>
+ * #mirrors [mirror_path offset]{2,}
+ *
+ * log_type is "core" or "disk"
+ * #log_params is between 1 and 3
+ */
+#define DM_IO_PAGES 64
+static int mirror_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+        int r;
+        unsigned int nr_mirrors, m, args_used;
+        struct mirror_set *ms;
+        struct dirty_log *dl;
+
+        dl = create_dirty_log(ti, argc, argv, &args_used);
+        if (!dl)
+                return -EINVAL;
+
+        argv += args_used;
+        argc -= args_used;
+
+        if (!argc || sscanf(argv[0], "%u", &nr_mirrors) != 1 ||
+            nr_mirrors < 2 || nr_mirrors > KCOPYD_MAX_REGIONS + 1) {
+                ti->error = "dm-mirror: Invalid number of mirrors";
+                dm_destroy_dirty_log(dl);
+                return -EINVAL;
+        }
+
+        argv++, argc--;
+
+        if (argc != nr_mirrors * 2) {
+                ti->error = "dm-mirror: Wrong number of mirror arguments";
+                dm_destroy_dirty_log(dl);
+                return -EINVAL;
+        }
+
+        ms = alloc_context(nr_mirrors, dl->type->get_region_size(dl), ti, dl);
+        if (!ms) {
+                dm_destroy_dirty_log(dl);
+                return -ENOMEM;
+        }
+
+        /* Get the mirror parameter sets */
+        for (m = 0; m < nr_mirrors; m++) {
+                r = get_mirror(ms, ti, m, argv);
+                if (r) {
+                        free_context(ms, ti, m);
+                        return r;
+                }
+                argv += 2;
+                argc -= 2;
+        }
+
+        ti->private = ms;
+
+        r = kcopyd_client_create(DM_IO_PAGES, &ms->kcopyd_client);
+        if (r) {
+                free_context(ms, ti, ms->nr_mirrors);
+                return r;
+        }
+
+        add_mirror_set(ms);
+        return 0;
+}
+
+static void mirror_dtr(struct dm_target *ti)
+{
+        struct mirror_set *ms = (struct mirror_set *) ti->private;
+
+        del_mirror_set(ms);
+        kcopyd_client_destroy(ms->kcopyd_client);
+        free_context(ms, ti, ms->nr_mirrors);
+}
+
+static void queue_bio(struct mirror_set *ms, struct bio *bio, int rw)
+{
+        int should_wake = 0;
+        struct bio_list *bl;
+
+        bl = (rw == WRITE) ? &ms->writes : &ms->reads;
+        spin_lock(&ms->lock);
+        should_wake = !(bl->head);
+        bio_list_add(bl, bio);
+        spin_unlock(&ms->lock);
+
+        if (should_wake)
+                wake();
+}
+
+/*
+ * Mirror mapping function
+ */
+static int mirror_map(struct dm_target *ti, struct bio *bio,
+                      union map_info *map_context)
+{
+        int r, rw = bio_rw(bio);
+        struct mirror *m;
+        struct mirror_set *ms = ti->private;
+
+        map_context->ll = bio->bi_sector >> ms->rh.region_shift;
+
+        if (rw == WRITE) {
+                queue_bio(ms, bio, rw);
+                return 0;
+        }
+
+        r = ms->rh.log->type->in_sync(ms->rh.log,
+                                      bio_to_region(&ms->rh, bio), 0);
+        if (r < 0 && r != -EWOULDBLOCK)
+                return r;
+
+        if (r == -EWOULDBLOCK)  /* FIXME: ugly */
+                r = 0;
+
+        /*
+         * We don't want to fast track a recovery just for a read
+         * ahead.  So we just let it silently fail.
+         * FIXME: get rid of this.
+         */
+        if (!r && rw == READA)
+                return -EIO;
+
+        if (!r) {
+                /* Pass this io over to the daemon */
+                queue_bio(ms, bio, rw);
+                return 0;
+        }
+
+        m = choose_mirror(ms, bio->bi_sector);
+        if (!m)
+                return -EIO;
+
+        map_bio(ms, m, bio);
+        return 1;
+}
+
+static int mirror_end_io(struct dm_target *ti, struct bio *bio,
+                         int error, union map_info *map_context)
+{
+        int rw = bio_rw(bio);
+        struct mirror_set *ms = (struct mirror_set *) ti->private;
+        region_t region = map_context->ll;
+
+        /*
+         * We need to dec pending if this was a write.
+         */
+        if (rw == WRITE)
+                rh_dec(&ms->rh, region);
+
+        return 0;
+}
+
+static void mirror_postsuspend(struct dm_target *ti)
+{
+        struct mirror_set *ms = (struct mirror_set *) ti->private;
+        struct dirty_log *log = ms->rh.log;
+
+        rh_stop_recovery(&ms->rh);
+        if (log->type->suspend && log->type->suspend(log))
+                /* FIXME: need better error handling */
+                DMWARN("log suspend failed");
+}
+
+static void mirror_resume(struct dm_target *ti)
+{
+        struct mirror_set *ms = (struct mirror_set *) ti->private;
+        struct dirty_log *log = ms->rh.log;
+        if (log->type->resume && log->type->resume(log))
+                /* FIXME: need better error handling */
+                DMWARN("log resume failed");
+        rh_start_recovery(&ms->rh);
+}
+
+static int mirror_status(struct dm_target *ti, status_type_t type,
+                         char *result, unsigned int maxlen)
+{
+        unsigned int m, sz;
+        struct mirror_set *ms = (struct mirror_set *) ti->private;
+
+        sz = ms->rh.log->type->status(ms->rh.log, type, result, maxlen);
+
+        switch (type) {
+        case STATUSTYPE_INFO:
+                DMEMIT("%d ", ms->nr_mirrors);
+                for (m = 0; m < ms->nr_mirrors; m++)
+                        DMEMIT("%s ", ms->mirror[m].dev->name);
+
+                DMEMIT(SECTOR_FORMAT "/" SECTOR_FORMAT,
+                       ms->rh.log->type->get_sync_count(ms->rh.log),
+                       ms->nr_regions);
+                break;
+
+        case STATUSTYPE_TABLE:
+                DMEMIT("%d ", ms->nr_mirrors);
+                for (m = 0; m < ms->nr_mirrors; m++)
+                        DMEMIT("%s " SECTOR_FORMAT " ",
+                               ms->mirror[m].dev->name, ms->mirror[m].offset);
+        }
+
+        return 0;
+}
+
+static struct target_type mirror_target = {
+        .name    = "mirror",
+        .version = {1, 0, 1},
+        .module  = THIS_MODULE,
+        .ctr     = mirror_ctr,
+        .dtr     = mirror_dtr,
+        .map     = mirror_map,
+        .end_io  = mirror_end_io,
+        .postsuspend = mirror_postsuspend,
+        .resume  = mirror_resume,
+        .status  = mirror_status,
+};
+
+static int __init dm_mirror_init(void)
+{
+        int r;
+
+        r = dm_dirty_log_init();
+        if (r)
+                return r;
+
+        _kmirrord_wq = create_workqueue("kmirrord");
+        if (!_kmirrord_wq) {
+                DMERR("couldn't start kmirrord");
+                dm_dirty_log_exit();
+                return r;
+        }
+        INIT_WORK(&_kmirrord_work, do_work, NULL);
+
+        r = dm_register_target(&mirror_target);
+        if (r < 0) {
+                DMERR("%s: Failed to register mirror target",
+                      mirror_target.name);
+                dm_dirty_log_exit();
+                destroy_workqueue(_kmirrord_wq);
+        }
+
+        return r;
+}
+
+static void __exit dm_mirror_exit(void)
+{
+        int r;
+
+        r = dm_unregister_target(&mirror_target);
+        if (r < 0)
+                DMERR("%s: unregister failed %d", mirror_target.name, r);
+
+        destroy_workqueue(_kmirrord_wq);
+        dm_dirty_log_exit();
+}
+
+/* Module hooks */
+module_init(dm_mirror_init);
+module_exit(dm_mirror_exit);
+
+MODULE_DESCRIPTION(DM_NAME " mirror target");
+MODULE_AUTHOR("Joe Thornber");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-round-robin.c b/drivers/md/dm-round-robin.c
new file mode 100644
index 000000000000..d0024865a789
--- /dev/null
+++ b/drivers/md/dm-round-robin.c
@@ -0,0 +1,214 @@
+/*
+ * Copyright (C) 2003 Sistina Software.
+ * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
+ *
+ * Module Author: Heinz Mauelshagen
+ *
+ * This file is released under the GPL.
+ *
+ * Round-robin path selector.
+ */
+
+#include "dm.h"
+#include "dm-path-selector.h"
+
+#include <linux/slab.h>
+
+/*-----------------------------------------------------------------
+ * Path-handling code, paths are held in lists
+ *---------------------------------------------------------------*/
+struct path_info {
+        struct list_head list;
+        struct path *path;
+        unsigned repeat_count;
+};
+
+static void free_paths(struct list_head *paths)
+{
+        struct path_info *pi, *next;
+
+        list_for_each_entry_safe(pi, next, paths, list) {
+                list_del(&pi->list);
+                kfree(pi);
+        }
+}
+
+/*-----------------------------------------------------------------
+ * Round-robin selector
+ *---------------------------------------------------------------*/
+
+#define RR_MIN_IO               1000
+
+struct selector {
+        struct list_head valid_paths;
+        struct list_head invalid_paths;
+};
+
+static struct selector *alloc_selector(void)
+{
+        struct selector *s = kmalloc(sizeof(*s), GFP_KERNEL);
+
+        if (s) {
+                INIT_LIST_HEAD(&s->valid_paths);
+                INIT_LIST_HEAD(&s->invalid_paths);
+        }
+
+        return s;
+}
+
+static int rr_create(struct path_selector *ps, unsigned argc, char **argv)
+{
+        struct selector *s;
+
+        s = alloc_selector();
+        if (!s)
+                return -ENOMEM;
+
+        ps->context = s;
+        return 0;
+}
+
+static void rr_destroy(struct path_selector *ps)
+{
+        struct selector *s = (struct selector *) ps->context;
+
+        free_paths(&s->valid_paths);
+        free_paths(&s->invalid_paths);
+        kfree(s);
+        ps->context = NULL;
+}
+
+static int rr_status(struct path_selector *ps, struct path *path,
+                     status_type_t type, char *result, unsigned int maxlen)
+{
+        struct path_info *pi;
+        int sz = 0;
+
+        if (!path)
+                DMEMIT("0 ");
+        else {
+                switch(type) {
+                case STATUSTYPE_INFO:
+                        break;
+                case STATUSTYPE_TABLE:
+                        pi = path->pscontext;
+                        DMEMIT("%u ", pi->repeat_count);
+                        break;
+                }
+        }
+
+        return sz;
+}
+
+/*
+ * Called during initialisation to register each path with an
+ * optional repeat_count.
+ */
+static int rr_add_path(struct path_selector *ps, struct path *path,
+                       int argc, char **argv, char **error)
+{
+        struct selector *s = (struct selector *) ps->context;
+        struct path_info *pi;
+        unsigned repeat_count = RR_MIN_IO;
+
+        if (argc > 1) {
+                *error = "round-robin ps: incorrect number of arguments";
+                return -EINVAL;
+        }
+
+        /* First path argument is number of I/Os before switching path */
+        if ((argc == 1) && (sscanf(argv[0], "%u", &repeat_count) != 1)) {
+                *error = "round-robin ps: invalid repeat count";
+                return -EINVAL;
+        }
+
+        /* allocate the path */
+        pi = kmalloc(sizeof(*pi), GFP_KERNEL);
+        if (!pi) {
+                *error = "round-robin ps: Error allocating path context";
+                return -ENOMEM;
+        }
+
+        pi->path = path;
+        pi->repeat_count = repeat_count;
+
+        path->pscontext = pi;
+
+        list_add(&pi->list, &s->valid_paths);
+
+        return 0;
+}
+
+static void rr_fail_path(struct path_selector *ps, struct path *p)
+{
+        struct selector *s = (struct selector *) ps->context;
+        struct path_info *pi = p->pscontext;
+
+        list_move(&pi->list, &s->invalid_paths);
+}
+
+static int rr_reinstate_path(struct path_selector *ps, struct path *p)
+{
+        struct selector *s = (struct selector *) ps->context;
+        struct path_info *pi = p->pscontext;
+
+        list_move(&pi->list, &s->valid_paths);
+
+        return 0;
+}
+
+static struct path *rr_select_path(struct path_selector *ps,
+                                   unsigned *repeat_count)
+{
+        struct selector *s = (struct selector *) ps->context;
+        struct path_info *pi = NULL;
+
+        if (!list_empty(&s->valid_paths)) {
+                pi = list_entry(s->valid_paths.next, struct path_info, list);
+                list_move_tail(&pi->list, &s->valid_paths);
+                *repeat_count = pi->repeat_count;
+        }
+
+        return pi ? pi->path : NULL;
+}
+
+static struct path_selector_type rr_ps = {
+        .name = "round-robin",
+        .module = THIS_MODULE,
+        .table_args = 1,
+        .info_args = 0,
+        .create = rr_create,
+        .destroy = rr_destroy,
+        .status = rr_status,
+        .add_path = rr_add_path,
+        .fail_path = rr_fail_path,
+        .reinstate_path = rr_reinstate_path,
+        .select_path = rr_select_path,
+};
+
+static int __init dm_rr_init(void)
+{
+        int r = dm_register_path_selector(&rr_ps);
+
+        if (r < 0)
+                DMERR("round-robin: register failed %d", r);
+
+        DMINFO("dm-round-robin version 1.0.0 loaded");
+
+        return r;
+}
+
+static void __exit dm_rr_exit(void)
+{
+        int r = dm_unregister_path_selector(&rr_ps);
+
+        if (r < 0)
+                DMERR("round-robin: unregister failed %d", r);
+}
+
+module_init(dm_rr_init);
+module_exit(dm_rr_exit);
+
+MODULE_DESCRIPTION(DM_NAME " round-robin multipath path selector");
+MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c
new file mode 100644
index 000000000000..7e691ab9a748
--- /dev/null
+++ b/drivers/md/dm-snap.c
@@ -0,0 +1,1208 @@
+/*
+ * dm-snapshot.c
+ *
+ * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
+ *
+ * This file is released under the GPL.
+ */
+
+#include <linux/blkdev.h>
+#include <linux/config.h>
+#include <linux/ctype.h>
+#include <linux/device-mapper.h>
+#include <linux/fs.h>
+#include <linux/init.h>
+#include <linux/kdev_t.h>
+#include <linux/list.h>
+#include <linux/mempool.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+
+#include "dm-snap.h"
+#include "dm-bio-list.h"
+#include "kcopyd.h"
+
+/*
+ * The percentage increment we will wake up users at
+ */
+#define WAKE_UP_PERCENT 5
+
+/*
+ * kcopyd priority of snapshot operations
+ */
+#define SNAPSHOT_COPY_PRIORITY 2
+
+/*
+ * Each snapshot reserves this many pages for io
+ */
+#define SNAPSHOT_PAGES 256
+
+struct pending_exception {
+        struct exception e;
+
+        /*
+         * Origin buffers waiting for this to complete are held
+         * in a bio list
+         */
+        struct bio_list origin_bios;
+        struct bio_list snapshot_bios;
+
+        /*
+         * Other pending_exceptions that are processing this
+         * chunk.  When this list is empty, we know we can
+         * complete the origins.
+         */
+        struct list_head siblings;
+
+        /* Pointer back to snapshot context */
+        struct dm_snapshot *snap;
+
+        /*
+         * 1 indicates the exception has already been sent to
+         * kcopyd.
+         */
+        int started;
+};
+
+/*
+ * Hash table mapping origin volumes to lists of snapshots and
+ * a lock to protect it
+ */
+static kmem_cache_t *exception_cache;
+static kmem_cache_t *pending_cache;
+static mempool_t *pending_pool;
+
+/*
+ * One of these per registered origin, held in the snapshot_origins hash
+ */
+struct origin {
+        /* The origin device */
+        struct block_device *bdev;
+
+        struct list_head hash_list;
+
+        /* List of snapshots for this origin */
+        struct list_head snapshots;
+};
+
+/*
+ * Size of the hash table for origin volumes. If we make this
+ * the size of the minors list then it should be nearly perfect
+ */
+#define ORIGIN_HASH_SIZE 256
+#define ORIGIN_MASK      0xFF
+static struct list_head *_origins;
+static struct rw_semaphore _origins_lock;
+
+static int init_origin_hash(void)
+{
+        int i;
+
+        _origins = kmalloc(ORIGIN_HASH_SIZE * sizeof(struct list_head),
+                           GFP_KERNEL);
+        if (!_origins) {
+                DMERR("Device mapper: Snapshot: unable to allocate memory");
+                return -ENOMEM;
+        }
+
+        for (i = 0; i < ORIGIN_HASH_SIZE; i++)
+                INIT_LIST_HEAD(_origins + i);
+        init_rwsem(&_origins_lock);
+
+        return 0;
+}
+
+static void exit_origin_hash(void)
+{
+        kfree(_origins);
+}
+
+static inline unsigned int origin_hash(struct block_device *bdev)
+{
+        return bdev->bd_dev & ORIGIN_MASK;
+}
+
+static struct origin *__lookup_origin(struct block_device *origin)
+{
+        struct list_head *ol;
+        struct origin *o;
+
+        ol = &_origins[origin_hash(origin)];
+        list_for_each_entry (o, ol, hash_list)
+                if (bdev_equal(o->bdev, origin))
+                        return o;
+
+        return NULL;
+}
+
+static void __insert_origin(struct origin *o)
+{
+        struct list_head *sl = &_origins[origin_hash(o->bdev)];
+        list_add_tail(&o->hash_list, sl);
+}
+
+/*
+ * Make a note of the snapshot and its origin so we can look it
+ * up when the origin has a write on it.
+ */
+static int register_snapshot(struct dm_snapshot *snap)
+{
+        struct origin *o;
+        struct block_device *bdev = snap->origin->bdev;
+
+        down_write(&_origins_lock);
+        o = __lookup_origin(bdev);
+
+        if (!o) {
+                /* New origin */
+                o = kmalloc(sizeof(*o), GFP_KERNEL);
+                if (!o) {
+                        up_write(&_origins_lock);
+                        return -ENOMEM;
+                }
+
+                /* Initialise the struct */
+                INIT_LIST_HEAD(&o->snapshots);
+                o->bdev = bdev;
+
+                __insert_origin(o);
+        }
+
+        list_add_tail(&snap->list, &o->snapshots);
+
+        up_write(&_origins_lock);
+        return 0;
+}
+
+static void unregister_snapshot(struct dm_snapshot *s)
+{
+        struct origin *o;
+
+        down_write(&_origins_lock);
+        o = __lookup_origin(s->origin->bdev);
+
+        list_del(&s->list);
+        if (list_empty(&o->snapshots)) {
+                list_del(&o->hash_list);
+                kfree(o);
+        }
+
+        up_write(&_origins_lock);
+}
+
+/*
+ * Implementation of the exception hash tables.
+ */
+static int init_exception_table(struct exception_table *et, uint32_t size)
+{
+        unsigned int i;
+
+        et->hash_mask = size - 1;
+        et->table = dm_vcalloc(size, sizeof(struct list_head));
+        if (!et->table)
+                return -ENOMEM;
+
+        for (i = 0; i < size; i++)
+                INIT_LIST_HEAD(et->table + i);
+
+        return 0;
+}
+
+static void exit_exception_table(struct exception_table *et, kmem_cache_t *mem)
+{
+        struct list_head *slot;
+        struct exception *ex, *next;
+        int i, size;
+
+        size = et->hash_mask + 1;
+        for (i = 0; i < size; i++) {
+                slot = et->table + i;
+
+                list_for_each_entry_safe (ex, next, slot, hash_list)
+                        kmem_cache_free(mem, ex);
+        }
+
+        vfree(et->table);
+}
+
+static inline uint32_t exception_hash(struct exception_table *et, chunk_t chunk)
+{
+        return chunk & et->hash_mask;
+}
+
+static void insert_exception(struct exception_table *eh, struct exception *e)
+{
+        struct list_head *l = &eh->table[exception_hash(eh, e->old_chunk)];
+        list_add(&e->hash_list, l);
+}
+
+static inline void remove_exception(struct exception *e)
+{
+        list_del(&e->hash_list);
+}
+
+/*
+ * Return the exception data for a sector, or NULL if not
+ * remapped.
+ */
+static struct exception *lookup_exception(struct exception_table *et,
+                                          chunk_t chunk)
+{
+        struct list_head *slot;
+        struct exception *e;
+
+        slot = &et->table[exception_hash(et, chunk)];
+        list_for_each_entry (e, slot, hash_list)
+                if (e->old_chunk == chunk)
+                        return e;
+
+        return NULL;
+}
+
+static inline struct exception *alloc_exception(void)
+{
+        struct exception *e;
+
+        e = kmem_cache_alloc(exception_cache, GFP_NOIO);
+        if (!e)
+                e = kmem_cache_alloc(exception_cache, GFP_ATOMIC);
+
+        return e;
+}
+
+static inline void free_exception(struct exception *e)
+{
+        kmem_cache_free(exception_cache, e);
+}
+
+static inline struct pending_exception *alloc_pending_exception(void)
+{
+        return mempool_alloc(pending_pool, GFP_NOIO);
+}
+
+static inline void free_pending_exception(struct pending_exception *pe)
+{
+        mempool_free(pe, pending_pool);
+}
+
+int dm_add_exception(struct dm_snapshot *s, chunk_t old, chunk_t new)
+{
+        struct exception *e;
+
+        e = alloc_exception();
+        if (!e)
+                return -ENOMEM;
+
+        e->old_chunk = old;
+        e->new_chunk = new;
+        insert_exception(&s->complete, e);
+        return 0;
+}
+
+/*
+ * Hard coded magic.
+ */
+static int calc_max_buckets(void)
+{
+        /* use a fixed size of 2MB */
+        unsigned long mem = 2 * 1024 * 1024;
+        mem /= sizeof(struct list_head);
+
+        return mem;
+}
+
+/*
+ * Rounds a number down to a power of 2.
+ */
+static inline uint32_t round_down(uint32_t n)
+{
+        while (n & (n - 1))
+                n &= (n - 1);
+        return n;
+}
+
+/*
+ * Allocate room for a suitable hash table.
+ */
+static int init_hash_tables(struct dm_snapshot *s)
+{
+        sector_t hash_size, cow_dev_size, origin_dev_size, max_buckets;
+
+        /*
+         * Calculate based on the size of the original volume or
+         * the COW volume...
+         */
+        cow_dev_size = get_dev_size(s->cow->bdev);
+        origin_dev_size = get_dev_size(s->origin->bdev);
+        max_buckets = calc_max_buckets();
+
+        hash_size = min(origin_dev_size, cow_dev_size) >> s->chunk_shift;
+        hash_size = min(hash_size, max_buckets);
+
+        /* Round it down to a power of 2 */
+        hash_size = round_down(hash_size);
+        if (init_exception_table(&s->complete, hash_size))
+                return -ENOMEM;
+
+        /*
+         * Allocate hash table for in-flight exceptions
+         * Make this smaller than the real hash table
+         */
+        hash_size >>= 3;
+        if (hash_size < 64)
+                hash_size = 64;
+
+        if (init_exception_table(&s->pending, hash_size)) {
+                exit_exception_table(&s->complete, exception_cache);
+                return -ENOMEM;
+        }
+
+        return 0;
+}
+
+/*
+ * Round a number up to the nearest 'size' boundary.  size must
+ * be a power of 2.
+ */
+static inline ulong round_up(ulong n, ulong size)
+{
+        size--;
+        return (n + size) & ~size;
+}
+
+/*
+ * Construct a snapshot mapping: <origin_dev> <COW-dev> <p/n> <chunk-size>
+ */
+static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+        struct dm_snapshot *s;
+        unsigned long chunk_size;
+        int r = -EINVAL;
+        char persistent;
+        char *origin_path;
+        char *cow_path;
+        char *value;
+        int blocksize;
+
+        if (argc < 4) {
+                ti->error = "dm-snapshot: requires exactly 4 arguments";
+                r = -EINVAL;
+                goto bad1;
+        }
+
+        origin_path = argv[0];
+        cow_path = argv[1];
+        persistent = toupper(*argv[2]);
+
+        if (persistent != 'P' && persistent != 'N') {
+                ti->error = "Persistent flag is not P or N";
+                r = -EINVAL;
+                goto bad1;
+        }
+
+        chunk_size = simple_strtoul(argv[3], &value, 10);
+        if (chunk_size == 0 || value == NULL) {
+                ti->error = "Invalid chunk size";
+                r = -EINVAL;
+                goto bad1;
+        }
+
+        s = kmalloc(sizeof(*s), GFP_KERNEL);
+        if (s == NULL) {
+                ti->error = "Cannot allocate snapshot context private "
+                    "structure";
+                r = -ENOMEM;
+                goto bad1;
+        }
+
+        r = dm_get_device(ti, origin_path, 0, ti->len, FMODE_READ, &s->origin);
+        if (r) {
+                ti->error = "Cannot get origin device";
+                goto bad2;
+        }
+
+        r = dm_get_device(ti, cow_path, 0, 0,
+                          FMODE_READ | FMODE_WRITE, &s->cow);
+        if (r) {
+                dm_put_device(ti, s->origin);
+                ti->error = "Cannot get COW device";
+                goto bad2;
+        }
+
+        /*
+         * Chunk size must be multiple of page size.  Silently
+         * round up if it's not.
+         */
+        chunk_size = round_up(chunk_size, PAGE_SIZE >> 9);
+
+        /* Validate the chunk size against the device block size */
+        blocksize = s->cow->bdev->bd_disk->queue->hardsect_size;
+        if (chunk_size % (blocksize >> 9)) {
+                ti->error = "Chunk size is not a multiple of device blocksize";
+                r = -EINVAL;
+                goto bad3;
+        }
+
+        /* Check chunk_size is a power of 2 */
+        if (chunk_size & (chunk_size - 1)) {
+                ti->error = "Chunk size is not a power of 2";
+                r = -EINVAL;
+                goto bad3;
+        }
+
+        s->chunk_size = chunk_size;
+        s->chunk_mask = chunk_size - 1;
+        s->type = persistent;
+        s->chunk_shift = ffs(chunk_size) - 1;
+
+        s->valid = 1;
+        s->have_metadata = 0;
+        s->last_percent = 0;
+        init_rwsem(&s->lock);
+        s->table = ti->table;
+
+        /* Allocate hash table for COW data */
+        if (init_hash_tables(s)) {
+                ti->error = "Unable to allocate hash table space";
+                r = -ENOMEM;
+                goto bad3;
+        }
+
+        /*
+         * Check the persistent flag - done here because we need the iobuf
+         * to check the LV header
+         */
+        s->store.snap = s;
+
+        if (persistent == 'P')
+                r = dm_create_persistent(&s->store, chunk_size);
+        else
+                r = dm_create_transient(&s->store, s, blocksize);
+
+        if (r) {
+                ti->error = "Couldn't create exception store";
+                r = -EINVAL;
+                goto bad4;
+        }
+
+        r = kcopyd_client_create(SNAPSHOT_PAGES, &s->kcopyd_client);
+        if (r) {
+                ti->error = "Could not create kcopyd client";
+                goto bad5;
+        }
+
+        /* Add snapshot to the list of snapshots for this origin */
+        if (register_snapshot(s)) {
+                r = -EINVAL;
+                ti->error = "Cannot register snapshot origin";
+                goto bad6;
+        }
+
+        ti->private = s;
+        ti->split_io = chunk_size;
+
+        return 0;
+
+ bad6:
+        kcopyd_client_destroy(s->kcopyd_client);
+
+ bad5:
+        s->store.destroy(&s->store);
+
+ bad4:
+        exit_exception_table(&s->pending, pending_cache);
+        exit_exception_table(&s->complete, exception_cache);
+
+ bad3:
+        dm_put_device(ti, s->cow);
+        dm_put_device(ti, s->origin);
+
+ bad2:
+        kfree(s);
+
+ bad1:
+        return r;
+}
+
+static void snapshot_dtr(struct dm_target *ti)
+{
+        struct dm_snapshot *s = (struct dm_snapshot *) ti->private;
+
+        unregister_snapshot(s);
+
+        exit_exception_table(&s->pending, pending_cache);
+        exit_exception_table(&s->complete, exception_cache);
+
+        /* Deallocate memory used */
+        s->store.destroy(&s->store);
+
+        dm_put_device(ti, s->origin);
+        dm_put_device(ti, s->cow);
+        kcopyd_client_destroy(s->kcopyd_client);
+        kfree(s);
+}
+
+/*
+ * Flush a list of buffers.
+ */
+static void flush_bios(struct bio *bio)
+{
+        struct bio *n;
+
+        while (bio) {
+                n = bio->bi_next;
+                bio->bi_next = NULL;
+                generic_make_request(bio);
+                bio = n;
+        }
+}
+
+/*
+ * Error a list of buffers.
+ */
+static void error_bios(struct bio *bio)
+{
+        struct bio *n;
+
+        while (bio) {
+                n = bio->bi_next;
+                bio->bi_next = NULL;
+                bio_io_error(bio, bio->bi_size);
+                bio = n;
+        }
+}
+
+static struct bio *__flush_bios(struct pending_exception *pe)
+{
+        struct pending_exception *sibling;
+
+        if (list_empty(&pe->siblings))
+                return bio_list_get(&pe->origin_bios);
+
+        sibling = list_entry(pe->siblings.next,
+                             struct pending_exception, siblings);
+
+        list_del(&pe->siblings);
+
+        /* This is fine as long as kcopyd is single-threaded. If kcopyd
+         * becomes multi-threaded, we'll need some locking here.
+         */
+        bio_list_merge(&sibling->origin_bios, &pe->origin_bios);
+
+        return NULL;
+}
+
+static void pending_complete(struct pending_exception *pe, int success)
+{
+        struct exception *e;
+        struct dm_snapshot *s = pe->snap;
+        struct bio *flush = NULL;
+
+        if (success) {
+                e = alloc_exception();
+                if (!e) {
+                        DMWARN("Unable to allocate exception.");
+                        down_write(&s->lock);
+                        s->store.drop_snapshot(&s->store);
+                        s->valid = 0;
+                        flush = __flush_bios(pe);
+                        up_write(&s->lock);
+
+                        error_bios(bio_list_get(&pe->snapshot_bios));
+                        goto out;
+                }
+                *e = pe->e;
+
+                /*
+                 * Add a proper exception, and remove the
+                 * in-flight exception from the list.
+                 */
+                down_write(&s->lock);
+                insert_exception(&s->complete, e);
+                remove_exception(&pe->e);
+                flush = __flush_bios(pe);
+
+                /* Submit any pending write bios */
+                up_write(&s->lock);
+
+                flush_bios(bio_list_get(&pe->snapshot_bios));
+        } else {
+                /* Read/write error - snapshot is unusable */
+                down_write(&s->lock);
+                if (s->valid)
+                        DMERR("Error reading/writing snapshot");
+                s->store.drop_snapshot(&s->store);
+                s->valid = 0;
+                remove_exception(&pe->e);
+                flush = __flush_bios(pe);
+                up_write(&s->lock);
+
+                error_bios(bio_list_get(&pe->snapshot_bios));
+
+                dm_table_event(s->table);
+        }
+
+ out:
+        free_pending_exception(pe);
+
+        if (flush)
+                flush_bios(flush);
+}
+
+static void commit_callback(void *context, int success)
+{
+        struct pending_exception *pe = (struct pending_exception *) context;
+        pending_complete(pe, success);
+}
+
+/*
+ * Called when the copy I/O has finished.  kcopyd actually runs
+ * this code so don't block.
+ */
+static void copy_callback(int read_err, unsigned int write_err, void *context)
+{
+        struct pending_exception *pe = (struct pending_exception *) context;
+        struct dm_snapshot *s = pe->snap;
+
+        if (read_err || write_err)
+                pending_complete(pe, 0);
+
+        else
+                /* Update the metadata if we are persistent */
+                s->store.commit_exception(&s->store, &pe->e, commit_callback,
+                                          pe);
+}
+
+/*
+ * Dispatches the copy operation to kcopyd.
+ */
+static inline void start_copy(struct pending_exception *pe)
+{
+        struct dm_snapshot *s = pe->snap;
+        struct io_region src, dest;
+        struct block_device *bdev = s->origin->bdev;
+        sector_t dev_size;
+
+        dev_size = get_dev_size(bdev);
+
+        src.bdev = bdev;
+        src.sector = chunk_to_sector(s, pe->e.old_chunk);
+        src.count = min(s->chunk_size, dev_size - src.sector);
+
+        dest.bdev = s->cow->bdev;
+        dest.sector = chunk_to_sector(s, pe->e.new_chunk);
+        dest.count = src.count;
+
+        /* Hand over to kcopyd */
+        kcopyd_copy(s->kcopyd_client,
+                    &src, 1, &dest, 0, copy_callback, pe);
+}
+
+/*
+ * Looks to see if this snapshot already has a pending exception
+ * for this chunk, otherwise it allocates a new one and inserts
+ * it into the pending table.
+ *
+ * NOTE: a write lock must be held on snap->lock before calling
+ * this.
+ */
+static struct pending_exception *
+__find_pending_exception(struct dm_snapshot *s, struct bio *bio)
+{
+        struct exception *e;
+        struct pending_exception *pe;
+        chunk_t chunk = sector_to_chunk(s, bio->bi_sector);
+
+        /*
+         * Is there a pending exception for this already ?
+         */
+        e = lookup_exception(&s->pending, chunk);
+        if (e) {
+                /* cast the exception to a pending exception */
+                pe = container_of(e, struct pending_exception, e);
+
+        } else {
+                /*
+                 * Create a new pending exception, we don't want
+                 * to hold the lock while we do this.
+                 */
+                up_write(&s->lock);
+                pe = alloc_pending_exception();
+                down_write(&s->lock);
+
+                e = lookup_exception(&s->pending, chunk);
+                if (e) {
+                        free_pending_exception(pe);
+                        pe = container_of(e, struct pending_exception, e);
+                } else {
+                        pe->e.old_chunk = chunk;
+                        bio_list_init(&pe->origin_bios);
+                        bio_list_init(&pe->snapshot_bios);
+                        INIT_LIST_HEAD(&pe->siblings);
+                        pe->snap = s;
+                        pe->started = 0;
+
+                        if (s->store.prepare_exception(&s->store, &pe->e)) {
+                                free_pending_exception(pe);
+                                s->valid = 0;
+                                return NULL;
+                        }
+
+                        insert_exception(&s->pending, &pe->e);
+                }
+        }
+
+        return pe;
+}
+
+static inline void remap_exception(struct dm_snapshot *s, struct exception *e,
+                                   struct bio *bio)
+{
+        bio->bi_bdev = s->cow->bdev;
+        bio->bi_sector = chunk_to_sector(s, e->new_chunk) +
+                (bio->bi_sector & s->chunk_mask);
+}
+
+static int snapshot_map(struct dm_target *ti, struct bio *bio,
+                        union map_info *map_context)
+{
+        struct exception *e;
+        struct dm_snapshot *s = (struct dm_snapshot *) ti->private;
+        int r = 1;
+        chunk_t chunk;
+        struct pending_exception *pe;
+
+        chunk = sector_to_chunk(s, bio->bi_sector);
+
+        /* Full snapshots are not usable */
+        if (!s->valid)
+                return -1;
+
+        /*
+         * Write to snapshot - higher level takes care of RW/RO
+         * flags so we should only get this if we are
+         * writeable.
+         */
+        if (bio_rw(bio) == WRITE) {
+
+                /* FIXME: should only take write lock if we need
+                 * to copy an exception */
+                down_write(&s->lock);
+
+                /* If the block is already remapped - use that, else remap it */
+                e = lookup_exception(&s->complete, chunk);
+                if (e) {
+                        remap_exception(s, e, bio);
+                        up_write(&s->lock);
+
+                } else {
+                        pe = __find_pending_exception(s, bio);
+
+                        if (!pe) {
+                                if (s->store.drop_snapshot)
+                                        s->store.drop_snapshot(&s->store);
+                                s->valid = 0;
+                                r = -EIO;
+                                up_write(&s->lock);
+                        } else {
+                                remap_exception(s, &pe->e, bio);
+                                bio_list_add(&pe->snapshot_bios, bio);
+
+                                if (!pe->started) {
+                                        /* this is protected by snap->lock */
+                                        pe->started = 1;
+                                        up_write(&s->lock);
+                                        start_copy(pe);
+                                } else
+                                        up_write(&s->lock);
+                                r = 0;
+                        }
+                }
+
+        } else {
+                /*
+                 * FIXME: this read path scares me because we
+                 * always use the origin when we have a pending
+                 * exception.  However I can't think of a
+                 * situation where this is wrong - ejt.
+                 */
+
+                /* Do reads */
+                down_read(&s->lock);
+
+                /* See if it it has been remapped */
+                e = lookup_exception(&s->complete, chunk);
+                if (e)
+                        remap_exception(s, e, bio);
+                else
+                        bio->bi_bdev = s->origin->bdev;
+
+                up_read(&s->lock);
+        }
+
+        return r;
+}
+
+static void snapshot_resume(struct dm_target *ti)
+{
+        struct dm_snapshot *s = (struct dm_snapshot *) ti->private;
+
+        if (s->have_metadata)
+                return;
+
+        if (s->store.read_metadata(&s->store)) {
+                down_write(&s->lock);
+                s->valid = 0;
+                up_write(&s->lock);
+        }
+
+        s->have_metadata = 1;
+}
+
+static int snapshot_status(struct dm_target *ti, status_type_t type,
+                           char *result, unsigned int maxlen)
+{
+        struct dm_snapshot *snap = (struct dm_snapshot *) ti->private;
+
+        switch (type) {
+        case STATUSTYPE_INFO:
+                if (!snap->valid)
+                        snprintf(result, maxlen, "Invalid");
+                else {
+                        if (snap->store.fraction_full) {
+                                sector_t numerator, denominator;
+                                snap->store.fraction_full(&snap->store,
+                                                          &numerator,
+                                                          &denominator);
+                                snprintf(result, maxlen,
+                                         SECTOR_FORMAT "/" SECTOR_FORMAT,
+                                         numerator, denominator);
+                        }
+                        else
+                                snprintf(result, maxlen, "Unknown");
+                }
+                break;
+
+        case STATUSTYPE_TABLE:
+                /*
+                 * kdevname returns a static pointer so we need
+                 * to make private copies if the output is to
+                 * make sense.
+                 */
+                snprintf(result, maxlen, "%s %s %c " SECTOR_FORMAT,
+                         snap->origin->name, snap->cow->name,
+                         snap->type, snap->chunk_size);
+                break;
+        }
+
+        return 0;
+}
+
+/*-----------------------------------------------------------------
+ * Origin methods
+ *---------------------------------------------------------------*/
+static void list_merge(struct list_head *l1, struct list_head *l2)
+{
+        struct list_head *l1_n, *l2_p;
+
+        l1_n = l1->next;
+        l2_p = l2->prev;
+
+        l1->next = l2;
+        l2->prev = l1;
+
+        l2_p->next = l1_n;
+        l1_n->prev = l2_p;
+}
+
+static int __origin_write(struct list_head *snapshots, struct bio *bio)
+{
+        int r = 1, first = 1;
+        struct dm_snapshot *snap;
+        struct exception *e;
+        struct pending_exception *pe, *last = NULL;
+        chunk_t chunk;
+
+        /* Do all the snapshots on this origin */
+        list_for_each_entry (snap, snapshots, list) {
+
+                /* Only deal with valid snapshots */
+                if (!snap->valid)
+                        continue;
+
+                down_write(&snap->lock);
+
+                /*
+                 * Remember, different snapshots can have
+                 * different chunk sizes.
+                 */
+                chunk = sector_to_chunk(snap, bio->bi_sector);
+
+                /*
+                 * Check exception table to see if block
+                 * is already remapped in this snapshot
+                 * and trigger an exception if not.
+                 */
+                e = lookup_exception(&snap->complete, chunk);
+                if (!e) {
+                        pe = __find_pending_exception(snap, bio);
+                        if (!pe) {
+                                snap->store.drop_snapshot(&snap->store);
+                                snap->valid = 0;
+
+                        } else {
+                                if (last)
+                                        list_merge(&pe->siblings,
+                                                   &last->siblings);
+
+                                last = pe;
+                                r = 0;
+                        }
+                }
+
+                up_write(&snap->lock);
+        }
+
+        /*
+         * Now that we have a complete pe list we can start the copying.
+         */
+        if (last) {
+                pe = last;
+                do {
+                        down_write(&pe->snap->lock);
+                        if (first)
+                                bio_list_add(&pe->origin_bios, bio);
+                        if (!pe->started) {
+                                pe->started = 1;
+                                up_write(&pe->snap->lock);
+                                start_copy(pe);
+                        } else
+                                up_write(&pe->snap->lock);
+                        first = 0;
+                        pe = list_entry(pe->siblings.next,
+                                        struct pending_exception, siblings);
+
+                } while (pe != last);
+        }
+
+        return r;
+}
+
+/*
+ * Called on a write from the origin driver.
+ */
+static int do_origin(struct dm_dev *origin, struct bio *bio)
+{
+        struct origin *o;
+        int r = 1;
+
+        down_read(&_origins_lock);
+        o = __lookup_origin(origin->bdev);
+        if (o)
+                r = __origin_write(&o->snapshots, bio);
+        up_read(&_origins_lock);
+
+        return r;
+}
+
+/*
+ * Origin: maps a linear range of a device, with hooks for snapshotting.
+ */
+
+/*
+ * Construct an origin mapping: <dev_path>
+ * The context for an origin is merely a 'struct dm_dev *'
+ * pointing to the real device.
+ */
+static int origin_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+        int r;
+        struct dm_dev *dev;
+
+        if (argc != 1) {
+                ti->error = "dm-origin: incorrect number of arguments";
+                return -EINVAL;
+        }
+
+        r = dm_get_device(ti, argv[0], 0, ti->len,
+                          dm_table_get_mode(ti->table), &dev);
+        if (r) {
+                ti->error = "Cannot get target device";
+                return r;
+        }
+
+        ti->private = dev;
+        return 0;
+}
+
+static void origin_dtr(struct dm_target *ti)
+{
+        struct dm_dev *dev = (struct dm_dev *) ti->private;
+        dm_put_device(ti, dev);
+}
+
+static int origin_map(struct dm_target *ti, struct bio *bio,
+                      union map_info *map_context)
+{
+        struct dm_dev *dev = (struct dm_dev *) ti->private;
+        bio->bi_bdev = dev->bdev;
+
+        /* Only tell snapshots if this is a write */
+        return (bio_rw(bio) == WRITE) ? do_origin(dev, bio) : 1;
+}
+
+#define min_not_zero(l, r) (l == 0) ? r : ((r == 0) ? l : min(l, r))
+
+/*
+ * Set the target "split_io" field to the minimum of all the snapshots'
+ * chunk sizes.
+ */
+static void origin_resume(struct dm_target *ti)
+{
+        struct dm_dev *dev = (struct dm_dev *) ti->private;
+        struct dm_snapshot *snap;
+        struct origin *o;
+        chunk_t chunk_size = 0;
+
+        down_read(&_origins_lock);
+        o = __lookup_origin(dev->bdev);
+        if (o)
+                list_for_each_entry (snap, &o->snapshots, list)
+                        chunk_size = min_not_zero(chunk_size, snap->chunk_size);
+        up_read(&_origins_lock);
+
+        ti->split_io = chunk_size;
+}
+
+static int origin_status(struct dm_target *ti, status_type_t type, char *result,
+                         unsigned int maxlen)
+{
+        struct dm_dev *dev = (struct dm_dev *) ti->private;
+
+        switch (type) {
+        case STATUSTYPE_INFO:
+                result[0] = '\0';
+                break;
+
+        case STATUSTYPE_TABLE:
+                snprintf(result, maxlen, "%s", dev->name);
+                break;
+        }
+
+        return 0;
+}
+
+static struct target_type origin_target = {
+        .name    = "snapshot-origin",
+        .version = {1, 0, 1},
+        .module  = THIS_MODULE,
+        .ctr     = origin_ctr,
+        .dtr     = origin_dtr,
+        .map     = origin_map,
+        .resume  = origin_resume,
+        .status  = origin_status,
+};
+
+static struct target_type snapshot_target = {
+        .name    = "snapshot",
+        .version = {1, 0, 1},
+        .module  = THIS_MODULE,
+        .ctr     = snapshot_ctr,
+        .dtr     = snapshot_dtr,
+        .map     = snapshot_map,
+        .resume  = snapshot_resume,
+        .status  = snapshot_status,
+};
+
+static int __init dm_snapshot_init(void)
+{
+        int r;
+
+        r = dm_register_target(&snapshot_target);
+        if (r) {
+                DMERR("snapshot target register failed %d", r);
+                return r;
+        }
+
+        r = dm_register_target(&origin_target);
+        if (r < 0) {
+                DMERR("Device mapper: Origin: register failed %d\n", r);
+                goto bad1;
+        }
+
+        r = init_origin_hash();
+        if (r) {
+                DMERR("init_origin_hash failed.");
+                goto bad2;
+        }
+
+        exception_cache = kmem_cache_create("dm-snapshot-ex",
+                                            sizeof(struct exception),
+                                            __alignof__(struct exception),
+                                            0, NULL, NULL);
+        if (!exception_cache) {
+                DMERR("Couldn't create exception cache.");
+                r = -ENOMEM;
+                goto bad3;
+        }
+
+        pending_cache =
+            kmem_cache_create("dm-snapshot-in",
+                              sizeof(struct pending_exception),
+                              __alignof__(struct pending_exception),
+                              0, NULL, NULL);
+        if (!pending_cache) {
+                DMERR("Couldn't create pending cache.");
+                r = -ENOMEM;
+                goto bad4;
+        }
+
+        pending_pool = mempool_create(128, mempool_alloc_slab,
+                                      mempool_free_slab, pending_cache);
+        if (!pending_pool) {
+                DMERR("Couldn't create pending pool.");
+                r = -ENOMEM;
+                goto bad5;
+        }
+
+        return 0;
+
+      bad5:
+        kmem_cache_destroy(pending_cache);
+      bad4:
+        kmem_cache_destroy(exception_cache);
+      bad3:
+        exit_origin_hash();
+      bad2:
+        dm_unregister_target(&origin_target);
+      bad1:
+        dm_unregister_target(&snapshot_target);
+        return r;
+}
+
+static void __exit dm_snapshot_exit(void)
+{
+        int r;
+
+        r = dm_unregister_target(&snapshot_target);
+        if (r)
+                DMERR("snapshot unregister failed %d", r);
+
+        r = dm_unregister_target(&origin_target);
+        if (r)
+                DMERR("origin unregister failed %d", r);
+
+        exit_origin_hash();
+        mempool_destroy(pending_pool);
+        kmem_cache_destroy(pending_cache);
+        kmem_cache_destroy(exception_cache);
+}
+
+/* Module hooks */
+module_init(dm_snapshot_init);
+module_exit(dm_snapshot_exit);
+
+MODULE_DESCRIPTION(DM_NAME " snapshot target");
+MODULE_AUTHOR("Joe Thornber");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-snap.h b/drivers/md/dm-snap.h
new file mode 100644
index 000000000000..375aa24d4d7d
--- /dev/null
+++ b/drivers/md/dm-snap.h
@@ -0,0 +1,161 @@
+/*
+ * dm-snapshot.c
+ *
+ * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
+ *
+ * This file is released under the GPL.
+ */
+
+#ifndef DM_SNAPSHOT_H
+#define DM_SNAPSHOT_H
+
+#include "dm.h"
+#include <linux/blkdev.h>
+
+struct exception_table {
+        uint32_t hash_mask;
+        struct list_head *table;
+};
+
+/*
+ * The snapshot code deals with largish chunks of the disk at a
+ * time. Typically 64k - 256k.
+ */
+/* FIXME: can we get away with limiting these to a uint32_t ? */
+typedef sector_t chunk_t;
+
+/*
+ * An exception is used where an old chunk of data has been
+ * replaced by a new one.
+ */
+struct exception {
+        struct list_head hash_list;
+
+        chunk_t old_chunk;
+        chunk_t new_chunk;
+};
+
+/*
+ * Abstraction to handle the meta/layout of exception stores (the
+ * COW device).
+ */
+struct exception_store {
+
+        /*
+         * Destroys this object when you've finished with it.
+         */
+        void (*destroy) (struct exception_store *store);
+
+        /*
+         * The target shouldn't read the COW device until this is
+         * called.
+         */
+        int (*read_metadata) (struct exception_store *store);
+
+        /*
+         * Find somewhere to store the next exception.
+         */
+        int (*prepare_exception) (struct exception_store *store,
+                                  struct exception *e);
+
+        /*
+         * Update the metadata with this exception.
+         */
+        void (*commit_exception) (struct exception_store *store,
+                                  struct exception *e,
+                                  void (*callback) (void *, int success),
+                                  void *callback_context);
+
+        /*
+         * The snapshot is invalid, note this in the metadata.
+         */
+        void (*drop_snapshot) (struct exception_store *store);
+
+        /*
+         * Return how full the snapshot is.
+         */
+        void (*fraction_full) (struct exception_store *store,
+                               sector_t *numerator,
+                               sector_t *denominator);
+
+        struct dm_snapshot *snap;
+        void *context;
+};
+
+struct dm_snapshot {
+        struct rw_semaphore lock;
+        struct dm_table *table;
+
+        struct dm_dev *origin;
+        struct dm_dev *cow;
+
+        /* List of snapshots per Origin */
+        struct list_head list;
+
+        /* Size of data blocks saved - must be a power of 2 */
+        chunk_t chunk_size;
+        chunk_t chunk_mask;
+        chunk_t chunk_shift;
+
+        /* You can't use a snapshot if this is 0 (e.g. if full) */
+        int valid;
+        int have_metadata;
+
+        /* Used for display of table */
+        char type;
+
+        /* The last percentage we notified */
+        int last_percent;
+
+        struct exception_table pending;
+        struct exception_table complete;
+
+        /* The on disk metadata handler */
+        struct exception_store store;
+
+        struct kcopyd_client *kcopyd_client;
+};
+
+/*
+ * Used by the exception stores to load exceptions hen
+ * initialising.
+ */
+int dm_add_exception(struct dm_snapshot *s, chunk_t old, chunk_t new);
+
+/*
+ * Constructor and destructor for the default persistent
+ * store.
+ */
+int dm_create_persistent(struct exception_store *store, uint32_t chunk_size);
+
+int dm_create_transient(struct exception_store *store,
+                        struct dm_snapshot *s, int blocksize);
+
+/*
+ * Return the number of sectors in the device.
+ */
+static inline sector_t get_dev_size(struct block_device *bdev)
+{
+        return bdev->bd_inode->i_size >> SECTOR_SHIFT;
+}
+
+static inline chunk_t sector_to_chunk(struct dm_snapshot *s, sector_t sector)
+{
+        return (sector & ~s->chunk_mask) >> s->chunk_shift;
+}
+
+static inline sector_t chunk_to_sector(struct dm_snapshot *s, chunk_t chunk)
+{
+        return chunk << s->chunk_shift;
+}
+
+static inline int bdev_equal(struct block_device *lhs, struct block_device *rhs)
+{
+        /*
+         * There is only ever one instance of a particular block
+         * device so we can compare pointers safely.
+         */
+        return lhs == rhs;
+}
+
+#endif
diff --git a/drivers/md/dm-stripe.c b/drivers/md/dm-stripe.c
new file mode 100644
index 000000000000..ab89278a56bf
--- /dev/null
+++ b/drivers/md/dm-stripe.c
@@ -0,0 +1,234 @@
+/*
+ * Copyright (C) 2001-2003 Sistina Software (UK) Limited.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/blkdev.h>
+#include <linux/bio.h>
+#include <linux/slab.h>
+
+struct stripe {
+        struct dm_dev *dev;
+        sector_t physical_start;
+};
+
+struct stripe_c {
+        uint32_t stripes;
+
+        /* The size of this target / num. stripes */
+        sector_t stripe_width;
+
+        /* stripe chunk size */
+        uint32_t chunk_shift;
+        sector_t chunk_mask;
+
+        struct stripe stripe[0];
+};
+
+static inline struct stripe_c *alloc_context(unsigned int stripes)
+{
+        size_t len;
+
+        if (array_too_big(sizeof(struct stripe_c), sizeof(struct stripe),
+                          stripes))
+                return NULL;
+
+        len = sizeof(struct stripe_c) + (sizeof(struct stripe) * stripes);
+
+        return kmalloc(len, GFP_KERNEL);
+}
+
+/*
+ * Parse a single <dev> <sector> pair
+ */
+static int get_stripe(struct dm_target *ti, struct stripe_c *sc,
+                      unsigned int stripe, char **argv)
+{
+        sector_t start;
+
+        if (sscanf(argv[1], SECTOR_FORMAT, &start) != 1)
+                return -EINVAL;
+
+        if (dm_get_device(ti, argv[0], start, sc->stripe_width,
+                          dm_table_get_mode(ti->table),
+                          &sc->stripe[stripe].dev))
+                return -ENXIO;
+
+        sc->stripe[stripe].physical_start = start;
+        return 0;
+}
+
+/*
+ * Construct a striped mapping.
+ * <number of stripes> <chunk size (2^^n)> [<dev_path> <offset>]+
+ */
+static int stripe_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+        struct stripe_c *sc;
+        sector_t width;
+        uint32_t stripes;
+        uint32_t chunk_size;
+        char *end;
+        int r;
+        unsigned int i;
+
+        if (argc < 2) {
+                ti->error = "dm-stripe: Not enough arguments";
+                return -EINVAL;
+        }
+
+        stripes = simple_strtoul(argv[0], &end, 10);
+        if (*end) {
+                ti->error = "dm-stripe: Invalid stripe count";
+                return -EINVAL;
+        }
+
+        chunk_size = simple_strtoul(argv[1], &end, 10);
+        if (*end) {
+                ti->error = "dm-stripe: Invalid chunk_size";
+                return -EINVAL;
+        }
+
+        /*
+         * chunk_size is a power of two
+         */
+        if (!chunk_size || (chunk_size & (chunk_size - 1)) ||
+            (chunk_size < (PAGE_SIZE >> SECTOR_SHIFT))) {
+                ti->error = "dm-stripe: Invalid chunk size";
+                return -EINVAL;
+        }
+
+        width = ti->len;
+        if (sector_div(width, stripes)) {
+                ti->error = "dm-stripe: Target length not divisable by "
+                    "number of stripes";
+                return -EINVAL;
+        }
+
+        /*
+         * Do we have enough arguments for that many stripes ?
+         */
+        if (argc != (2 + 2 * stripes)) {
+                ti->error = "dm-stripe: Not enough destinations "
+                        "specified";
+                return -EINVAL;
+        }
+
+        sc = alloc_context(stripes);
+        if (!sc) {
+                ti->error = "dm-stripe: Memory allocation for striped context "
+                    "failed";
+                return -ENOMEM;
+        }
+
+        sc->stripes = stripes;
+        sc->stripe_width = width;
+        ti->split_io = chunk_size;
+
+        sc->chunk_mask = ((sector_t) chunk_size) - 1;
+        for (sc->chunk_shift = 0; chunk_size; sc->chunk_shift++)
+                chunk_size >>= 1;
+        sc->chunk_shift--;
+
+        /*
+         * Get the stripe destinations.
+         */
+        for (i = 0; i < stripes; i++) {
+                argv += 2;
+
+                r = get_stripe(ti, sc, i, argv);
+                if (r < 0) {
+                        ti->error = "dm-stripe: Couldn't parse stripe "
+                                "destination";
+                        while (i--)
+                                dm_put_device(ti, sc->stripe[i].dev);
+                        kfree(sc);
+                        return r;
+                }
+        }
+
+        ti->private = sc;
+        return 0;
+}
+
+static void stripe_dtr(struct dm_target *ti)
+{
+        unsigned int i;
+        struct stripe_c *sc = (struct stripe_c *) ti->private;
+
+        for (i = 0; i < sc->stripes; i++)
+                dm_put_device(ti, sc->stripe[i].dev);
+
+        kfree(sc);
+}
+
+static int stripe_map(struct dm_target *ti, struct bio *bio,
+                      union map_info *map_context)
+{
+        struct stripe_c *sc = (struct stripe_c *) ti->private;
+
+        sector_t offset = bio->bi_sector - ti->begin;
+        sector_t chunk = offset >> sc->chunk_shift;
+        uint32_t stripe = sector_div(chunk, sc->stripes);
+
+        bio->bi_bdev = sc->stripe[stripe].dev->bdev;
+        bio->bi_sector = sc->stripe[stripe].physical_start +
+            (chunk << sc->chunk_shift) + (offset & sc->chunk_mask);
+        return 1;
+}
+
+static int stripe_status(struct dm_target *ti,
+                         status_type_t type, char *result, unsigned int maxlen)
+{
+        struct stripe_c *sc = (struct stripe_c *) ti->private;
+        unsigned int sz = 0;
+        unsigned int i;
+
+        switch (type) {
+        case STATUSTYPE_INFO:
+                result[0] = '\0';
+                break;
+
+        case STATUSTYPE_TABLE:
+                DMEMIT("%d " SECTOR_FORMAT, sc->stripes, sc->chunk_mask + 1);
+                for (i = 0; i < sc->stripes; i++)
+                        DMEMIT(" %s " SECTOR_FORMAT, sc->stripe[i].dev->name,
+                               sc->stripe[i].physical_start);
+                break;
+        }
+        return 0;
+}
+
+static struct target_type stripe_target = {
+        .name   = "striped",
+        .version= {1, 0, 2},
+        .module = THIS_MODULE,
+        .ctr    = stripe_ctr,
+        .dtr    = stripe_dtr,
+        .map    = stripe_map,
+        .status = stripe_status,
+};
+
+int __init dm_stripe_init(void)
+{
+        int r;
+
+        r = dm_register_target(&stripe_target);
+        if (r < 0)
+                DMWARN("striped target registration failed");
+
+        return r;
+}
+
+void dm_stripe_exit(void)
+{
+        if (dm_unregister_target(&stripe_target))
+                DMWARN("striped target unregistration failed");
+
+        return;
+}
diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c
new file mode 100644
index 000000000000..ee175d4906c4
--- /dev/null
+++ b/drivers/md/dm-table.c
@@ -0,0 +1,950 @@
+/*
+ * Copyright (C) 2001 Sistina Software (UK) Limited.
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+
+#include <linux/module.h>
+#include <linux/vmalloc.h>
+#include <linux/blkdev.h>
+#include <linux/namei.h>
+#include <linux/ctype.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <asm/atomic.h>
+
+#define MAX_DEPTH 16
+#define NODE_SIZE L1_CACHE_BYTES
+#define KEYS_PER_NODE (NODE_SIZE / sizeof(sector_t))
+#define CHILDREN_PER_NODE (KEYS_PER_NODE + 1)
+
+struct dm_table {
+        atomic_t holders;
+
+        /* btree table */
+        unsigned int depth;
+        unsigned int counts[MAX_DEPTH]; /* in nodes */
+        sector_t *index[MAX_DEPTH];
+
+        unsigned int num_targets;
+        unsigned int num_allocated;
+        sector_t *highs;
+        struct dm_target *targets;
+
+        /*
+         * Indicates the rw permissions for the new logical
+         * device.  This should be a combination of FMODE_READ
+         * and FMODE_WRITE.
+         */
+        int mode;
+
+        /* a list of devices used by this table */
+        struct list_head devices;
+
+        /*
+         * These are optimistic limits taken from all the
+         * targets, some targets will need smaller limits.
+         */
+        struct io_restrictions limits;
+
+        /* events get handed up using this callback */
+        void (*event_fn)(void *);
+        void *event_context;
+};
+
+/*
+ * Similar to ceiling(log_size(n))
+ */
+static unsigned int int_log(unsigned int n, unsigned int base)
+{
+        int result = 0;
+
+        while (n > 1) {
+                n = dm_div_up(n, base);
+                result++;
+        }
+
+        return result;
+}
+
+/*
+ * Returns the minimum that is _not_ zero, unless both are zero.
+ */
+#define min_not_zero(l, r) (l == 0) ? r : ((r == 0) ? l : min(l, r))
+
+/*
+ * Combine two io_restrictions, always taking the lower value.
+ */
+static void combine_restrictions_low(struct io_restrictions *lhs,
+                                     struct io_restrictions *rhs)
+{
+        lhs->max_sectors =
+                min_not_zero(lhs->max_sectors, rhs->max_sectors);
+
+        lhs->max_phys_segments =
+                min_not_zero(lhs->max_phys_segments, rhs->max_phys_segments);
+
+        lhs->max_hw_segments =
+                min_not_zero(lhs->max_hw_segments, rhs->max_hw_segments);
+
+        lhs->hardsect_size = max(lhs->hardsect_size, rhs->hardsect_size);
+
+        lhs->max_segment_size =
+                min_not_zero(lhs->max_segment_size, rhs->max_segment_size);
+
+        lhs->seg_boundary_mask =
+                min_not_zero(lhs->seg_boundary_mask, rhs->seg_boundary_mask);
+}
+
+/*
+ * Calculate the index of the child node of the n'th node k'th key.
+ */
+static inline unsigned int get_child(unsigned int n, unsigned int k)
+{
+        return (n * CHILDREN_PER_NODE) + k;
+}
+
+/*
+ * Return the n'th node of level l from table t.
+ */
+static inline sector_t *get_node(struct dm_table *t,
+                                 unsigned int l, unsigned int n)
+{
+        return t->index[l] + (n * KEYS_PER_NODE);
+}
+
+/*
+ * Return the highest key that you could lookup from the n'th
+ * node on level l of the btree.
+ */
+static sector_t high(struct dm_table *t, unsigned int l, unsigned int n)
+{
+        for (; l < t->depth - 1; l++)
+                n = get_child(n, CHILDREN_PER_NODE - 1);
+
+        if (n >= t->counts[l])
+                return (sector_t) - 1;
+
+        return get_node(t, l, n)[KEYS_PER_NODE - 1];
+}
+
+/*
+ * Fills in a level of the btree based on the highs of the level
+ * below it.
+ */
+static int setup_btree_index(unsigned int l, struct dm_table *t)
+{
+        unsigned int n, k;
+        sector_t *node;
+
+        for (n = 0U; n < t->counts[l]; n++) {
+                node = get_node(t, l, n);
+
+                for (k = 0U; k < KEYS_PER_NODE; k++)
+                        node[k] = high(t, l + 1, get_child(n, k));
+        }
+
+        return 0;
+}
+
+void *dm_vcalloc(unsigned long nmemb, unsigned long elem_size)
+{
+        unsigned long size;
+        void *addr;
+
+        /*
+         * Check that we're not going to overflow.
+         */
+        if (nmemb > (ULONG_MAX / elem_size))
+                return NULL;
+
+        size = nmemb * elem_size;
+        addr = vmalloc(size);
+        if (addr)
+                memset(addr, 0, size);
+
+        return addr;
+}
+
+/*
+ * highs, and targets are managed as dynamic arrays during a
+ * table load.
+ */
+static int alloc_targets(struct dm_table *t, unsigned int num)
+{
+        sector_t *n_highs;
+        struct dm_target *n_targets;
+        int n = t->num_targets;
+
+        /*
+         * Allocate both the target array and offset array at once.
+         */
+        n_highs = (sector_t *) dm_vcalloc(num, sizeof(struct dm_target) +
+                                          sizeof(sector_t));
+        if (!n_highs)
+                return -ENOMEM;
+
+        n_targets = (struct dm_target *) (n_highs + num);
+
+        if (n) {
+                memcpy(n_highs, t->highs, sizeof(*n_highs) * n);
+                memcpy(n_targets, t->targets, sizeof(*n_targets) * n);
+        }
+
+        memset(n_highs + n, -1, sizeof(*n_highs) * (num - n));
+        vfree(t->highs);
+
+        t->num_allocated = num;
+        t->highs = n_highs;
+        t->targets = n_targets;
+
+        return 0;
+}
+
+int dm_table_create(struct dm_table **result, int mode, unsigned num_targets)
+{
+        struct dm_table *t = kmalloc(sizeof(*t), GFP_KERNEL);
+
+        if (!t)
+                return -ENOMEM;
+
+        memset(t, 0, sizeof(*t));
+        INIT_LIST_HEAD(&t->devices);
+        atomic_set(&t->holders, 1);
+
+        if (!num_targets)
+                num_targets = KEYS_PER_NODE;
+
+        num_targets = dm_round_up(num_targets, KEYS_PER_NODE);
+
+        if (alloc_targets(t, num_targets)) {
+                kfree(t);
+                t = NULL;
+                return -ENOMEM;
+        }
+
+        t->mode = mode;
+        *result = t;
+        return 0;
+}
+
+static void free_devices(struct list_head *devices)
+{
+        struct list_head *tmp, *next;
+
+        for (tmp = devices->next; tmp != devices; tmp = next) {
+                struct dm_dev *dd = list_entry(tmp, struct dm_dev, list);
+                next = tmp->next;
+                kfree(dd);
+        }
+}
+
+void table_destroy(struct dm_table *t)
+{
+        unsigned int i;
+
+        /* free the indexes (see dm_table_complete) */
+        if (t->depth >= 2)
+                vfree(t->index[t->depth - 2]);
+
+        /* free the targets */
+        for (i = 0; i < t->num_targets; i++) {
+                struct dm_target *tgt = t->targets + i;
+
+                if (tgt->type->dtr)
+                        tgt->type->dtr(tgt);
+
+                dm_put_target_type(tgt->type);
+        }
+
+        vfree(t->highs);
+
+        /* free the device list */
+        if (t->devices.next != &t->devices) {
+                DMWARN("devices still present during destroy: "
+                       "dm_table_remove_device calls missing");
+
+                free_devices(&t->devices);
+        }
+
+        kfree(t);
+}
+
+void dm_table_get(struct dm_table *t)
+{
+        atomic_inc(&t->holders);
+}
+
+void dm_table_put(struct dm_table *t)
+{
+        if (!t)
+                return;
+
+        if (atomic_dec_and_test(&t->holders))
+                table_destroy(t);
+}
+
+/*
+ * Checks to see if we need to extend highs or targets.
+ */
+static inline int check_space(struct dm_table *t)
+{
+        if (t->num_targets >= t->num_allocated)
+                return alloc_targets(t, t->num_allocated * 2);
+
+        return 0;
+}
+
+/*
+ * Convert a device path to a dev_t.
+ */
+static int lookup_device(const char *path, dev_t *dev)
+{
+        int r;
+        struct nameidata nd;
+        struct inode *inode;
+
+        if ((r = path_lookup(path, LOOKUP_FOLLOW, &nd)))
+                return r;
+
+        inode = nd.dentry->d_inode;
+        if (!inode) {
+                r = -ENOENT;
+                goto out;
+        }
+
+        if (!S_ISBLK(inode->i_mode)) {
+                r = -ENOTBLK;
+                goto out;
+        }
+
+        *dev = inode->i_rdev;
+
+ out:
+        path_release(&nd);
+        return r;
+}
+
+/*
+ * See if we've already got a device in the list.
+ */
+static struct dm_dev *find_device(struct list_head *l, dev_t dev)
+{
+        struct dm_dev *dd;
+
+        list_for_each_entry (dd, l, list)
+                if (dd->bdev->bd_dev == dev)
+                        return dd;
+
+        return NULL;
+}
+
+/*
+ * Open a device so we can use it as a map destination.
+ */
+static int open_dev(struct dm_dev *d, dev_t dev)
+{
+        static char *_claim_ptr = "I belong to device-mapper";
+        struct block_device *bdev;
+
+        int r;
+
+        if (d->bdev)
+                BUG();
+
+        bdev = open_by_devnum(dev, d->mode);
+        if (IS_ERR(bdev))
+                return PTR_ERR(bdev);
+        r = bd_claim(bdev, _claim_ptr);
+        if (r)
+                blkdev_put(bdev);
+        else
+                d->bdev = bdev;
+        return r;
+}
+
+/*
+ * Close a device that we've been using.
+ */
+static void close_dev(struct dm_dev *d)
+{
+        if (!d->bdev)
+                return;
+
+        bd_release(d->bdev);
+        blkdev_put(d->bdev);
+        d->bdev = NULL;
+}
+
+/*
+ * If possible (ie. blk_size[major] is set), this checks an area
+ * of a destination device is valid.
+ */
+static int check_device_area(struct dm_dev *dd, sector_t start, sector_t len)
+{
+        sector_t dev_size;
+        dev_size = dd->bdev->bd_inode->i_size >> SECTOR_SHIFT;
+        return ((start < dev_size) && (len <= (dev_size - start)));
+}
+
+/*
+ * This upgrades the mode on an already open dm_dev.  Being
+ * careful to leave things as they were if we fail to reopen the
+ * device.
+ */
+static int upgrade_mode(struct dm_dev *dd, int new_mode)
+{
+        int r;
+        struct dm_dev dd_copy;
+        dev_t dev = dd->bdev->bd_dev;
+
+        dd_copy = *dd;
+
+        dd->mode |= new_mode;
+        dd->bdev = NULL;
+        r = open_dev(dd, dev);
+        if (!r)
+                close_dev(&dd_copy);
+        else
+                *dd = dd_copy;
+
+        return r;
+}
+
+/*
+ * Add a device to the list, or just increment the usage count if
+ * it's already present.
+ */
+static int __table_get_device(struct dm_table *t, struct dm_target *ti,
+                              const char *path, sector_t start, sector_t len,
+                              int mode, struct dm_dev **result)
+{
+        int r;
+        dev_t dev;
+        struct dm_dev *dd;
+        unsigned int major, minor;
+
+        if (!t)
+                BUG();
+
+        if (sscanf(path, "%u:%u", &major, &minor) == 2) {
+                /* Extract the major/minor numbers */
+                dev = MKDEV(major, minor);
+                if (MAJOR(dev) != major || MINOR(dev) != minor)
+                        return -EOVERFLOW;
+        } else {
+                /* convert the path to a device */
+                if ((r = lookup_device(path, &dev)))
+                        return r;
+        }
+
+        dd = find_device(&t->devices, dev);
+        if (!dd) {
+                dd = kmalloc(sizeof(*dd), GFP_KERNEL);
+                if (!dd)
+                        return -ENOMEM;
+
+                dd->mode = mode;
+                dd->bdev = NULL;
+
+                if ((r = open_dev(dd, dev))) {
+                        kfree(dd);
+                        return r;
+                }
+
+                format_dev_t(dd->name, dev);
+
+                atomic_set(&dd->count, 0);
+                list_add(&dd->list, &t->devices);
+
+        } else if (dd->mode != (mode | dd->mode)) {
+                r = upgrade_mode(dd, mode);
+                if (r)
+                        return r;
+        }
+        atomic_inc(&dd->count);
+
+        if (!check_device_area(dd, start, len)) {
+                DMWARN("device %s too small for target", path);
+                dm_put_device(ti, dd);
+                return -EINVAL;
+        }
+
+        *result = dd;
+
+        return 0;
+}
+
+
+int dm_get_device(struct dm_target *ti, const char *path, sector_t start,
+                  sector_t len, int mode, struct dm_dev **result)
+{
+        int r = __table_get_device(ti->table, ti, path,
+                                   start, len, mode, result);
+        if (!r) {
+                request_queue_t *q = bdev_get_queue((*result)->bdev);
+                struct io_restrictions *rs = &ti->limits;
+
+                /*
+                 * Combine the device limits low.
+                 *
+                 * FIXME: if we move an io_restriction struct
+                 *        into q this would just be a call to
+                 *        combine_restrictions_low()
+                 */
+                rs->max_sectors =
+                        min_not_zero(rs->max_sectors, q->max_sectors);
+
+                /* FIXME: Device-Mapper on top of RAID-0 breaks because DM
+                 *        currently doesn't honor MD's merge_bvec_fn routine.
+                 *        In this case, we'll force DM to use PAGE_SIZE or
+                 *        smaller I/O, just to be safe. A better fix is in the
+                 *        works, but add this for the time being so it will at
+                 *        least operate correctly.
+                 */
+                if (q->merge_bvec_fn)
+                        rs->max_sectors =
+                                min_not_zero(rs->max_sectors,
+                                             (unsigned short)(PAGE_SIZE >> 9));
+
+                rs->max_phys_segments =
+                        min_not_zero(rs->max_phys_segments,
+                                     q->max_phys_segments);
+
+                rs->max_hw_segments =
+                        min_not_zero(rs->max_hw_segments, q->max_hw_segments);
+
+                rs->hardsect_size = max(rs->hardsect_size, q->hardsect_size);
+
+                rs->max_segment_size =
+                        min_not_zero(rs->max_segment_size, q->max_segment_size);
+
+                rs->seg_boundary_mask =
+                        min_not_zero(rs->seg_boundary_mask,
+                                     q->seg_boundary_mask);
+        }
+
+        return r;
+}
+
+/*
+ * Decrement a devices use count and remove it if necessary.
+ */
+void dm_put_device(struct dm_target *ti, struct dm_dev *dd)
+{
+        if (atomic_dec_and_test(&dd->count)) {
+                close_dev(dd);
+                list_del(&dd->list);
+                kfree(dd);
+        }
+}
+
+/*
+ * Checks to see if the target joins onto the end of the table.
+ */
+static int adjoin(struct dm_table *table, struct dm_target *ti)
+{
+        struct dm_target *prev;
+
+        if (!table->num_targets)
+                return !ti->begin;
+
+        prev = &table->targets[table->num_targets - 1];
+        return (ti->begin == (prev->begin + prev->len));
+}
+
+/*
+ * Used to dynamically allocate the arg array.
+ */
+static char **realloc_argv(unsigned *array_size, char **old_argv)
+{
+        char **argv;
+        unsigned new_size;
+
+        new_size = *array_size ? *array_size * 2 : 64;
+        argv = kmalloc(new_size * sizeof(*argv), GFP_KERNEL);
+        if (argv) {
+                memcpy(argv, old_argv, *array_size * sizeof(*argv));
+                *array_size = new_size;
+        }
+
+        kfree(old_argv);
+        return argv;
+}
+
+/*
+ * Destructively splits up the argument list to pass to ctr.
+ */
+int dm_split_args(int *argc, char ***argvp, char *input)
+{
+        char *start, *end = input, *out, **argv = NULL;
+        unsigned array_size = 0;
+
+        *argc = 0;
+        argv = realloc_argv(&array_size, argv);
+        if (!argv)
+                return -ENOMEM;
+
+        while (1) {
+                start = end;
+
+                /* Skip whitespace */
+                while (*start && isspace(*start))
+                        start++;
+
+                if (!*start)
+                        break;  /* success, we hit the end */
+
+                /* 'out' is used to remove any back-quotes */
+                end = out = start;
+                while (*end) {
+                        /* Everything apart from '\0' can be quoted */
+                        if (*end == '\\' && *(end + 1)) {
+                                *out++ = *(end + 1);
+                                end += 2;
+                                continue;
+                        }
+
+                        if (isspace(*end))
+                                break;  /* end of token */
+
+                        *out++ = *end++;
+                }
+
+                /* have we already filled the array ? */
+                if ((*argc + 1) > array_size) {
+                        argv = realloc_argv(&array_size, argv);
+                        if (!argv)
+                                return -ENOMEM;
+                }
+
+                /* we know this is whitespace */
+                if (*end)
+                        end++;
+
+                /* terminate the string and put it in the array */
+                *out = '\0';
+                argv[*argc] = start;
+                (*argc)++;
+        }
+
+        *argvp = argv;
+        return 0;
+}
+
+static void check_for_valid_limits(struct io_restrictions *rs)
+{
+        if (!rs->max_sectors)
+                rs->max_sectors = MAX_SECTORS;
+        if (!rs->max_phys_segments)
+                rs->max_phys_segments = MAX_PHYS_SEGMENTS;
+        if (!rs->max_hw_segments)
+                rs->max_hw_segments = MAX_HW_SEGMENTS;
+        if (!rs->hardsect_size)
+                rs->hardsect_size = 1 << SECTOR_SHIFT;
+        if (!rs->max_segment_size)
+                rs->max_segment_size = MAX_SEGMENT_SIZE;
+        if (!rs->seg_boundary_mask)
+                rs->seg_boundary_mask = -1;
+}
+
+int dm_table_add_target(struct dm_table *t, const char *type,
+                        sector_t start, sector_t len, char *params)
+{
+        int r = -EINVAL, argc;
+        char **argv;
+        struct dm_target *tgt;
+
+        if ((r = check_space(t)))
+                return r;
+
+        tgt = t->targets + t->num_targets;
+        memset(tgt, 0, sizeof(*tgt));
+
+        if (!len) {
+                tgt->error = "zero-length target";
+                DMERR("%s", tgt->error);
+                return -EINVAL;
+        }
+
+        tgt->type = dm_get_target_type(type);
+        if (!tgt->type) {
+                tgt->error = "unknown target type";
+                DMERR("%s", tgt->error);
+                return -EINVAL;
+        }
+
+        tgt->table = t;
+        tgt->begin = start;
+        tgt->len = len;
+        tgt->error = "Unknown error";
+
+        /*
+         * Does this target adjoin the previous one ?
+         */
+        if (!adjoin(t, tgt)) {
+                tgt->error = "Gap in table";
+                r = -EINVAL;
+                goto bad;
+        }
+
+        r = dm_split_args(&argc, &argv, params);
+        if (r) {
+                tgt->error = "couldn't split parameters (insufficient memory)";
+                goto bad;
+        }
+
+        r = tgt->type->ctr(tgt, argc, argv);
+        kfree(argv);
+        if (r)
+                goto bad;
+
+        t->highs[t->num_targets++] = tgt->begin + tgt->len - 1;
+
+        /* FIXME: the plan is to combine high here and then have
+         * the merge fn apply the target level restrictions. */
+        combine_restrictions_low(&t->limits, &tgt->limits);
+        return 0;
+
+ bad:
+        DMERR("%s", tgt->error);
+        dm_put_target_type(tgt->type);
+        return r;
+}
+
+static int setup_indexes(struct dm_table *t)
+{
+        int i;
+        unsigned int total = 0;
+        sector_t *indexes;
+
+        /* allocate the space for *all* the indexes */
+        for (i = t->depth - 2; i >= 0; i--) {
+                t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE);
+                total += t->counts[i];
+        }
+
+        indexes = (sector_t *) dm_vcalloc(total, (unsigned long) NODE_SIZE);
+        if (!indexes)
+                return -ENOMEM;
+
+        /* set up internal nodes, bottom-up */
+        for (i = t->depth - 2, total = 0; i >= 0; i--) {
+                t->index[i] = indexes;
+                indexes += (KEYS_PER_NODE * t->counts[i]);
+                setup_btree_index(i, t);
+        }
+
+        return 0;
+}
+
+/*
+ * Builds the btree to index the map.
+ */
+int dm_table_complete(struct dm_table *t)
+{
+        int r = 0;
+        unsigned int leaf_nodes;
+
+        check_for_valid_limits(&t->limits);
+
+        /* how many indexes will the btree have ? */
+        leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE);
+        t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE);
+
+        /* leaf layer has already been set up */
+        t->counts[t->depth - 1] = leaf_nodes;
+        t->index[t->depth - 1] = t->highs;
+
+        if (t->depth >= 2)
+                r = setup_indexes(t);
+
+        return r;
+}
+
+static DECLARE_MUTEX(_event_lock);
+void dm_table_event_callback(struct dm_table *t,
+                             void (*fn)(void *), void *context)
+{
+        down(&_event_lock);
+        t->event_fn = fn;
+        t->event_context = context;
+        up(&_event_lock);
+}
+
+void dm_table_event(struct dm_table *t)
+{
+        /*
+         * You can no longer call dm_table_event() from interrupt
+         * context, use a bottom half instead.
+         */
+        BUG_ON(in_interrupt());
+
+        down(&_event_lock);
+        if (t->event_fn)
+                t->event_fn(t->event_context);
+        up(&_event_lock);
+}
+
+sector_t dm_table_get_size(struct dm_table *t)
+{
+        return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0;
+}
+
+struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index)
+{
+        if (index > t->num_targets)
+                return NULL;
+
+        return t->targets + index;
+}
+
+/*
+ * Search the btree for the correct target.
+ */
+struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector)
+{
+        unsigned int l, n = 0, k = 0;
+        sector_t *node;
+
+        for (l = 0; l < t->depth; l++) {
+                n = get_child(n, k);
+                node = get_node(t, l, n);
+
+                for (k = 0; k < KEYS_PER_NODE; k++)
+                        if (node[k] >= sector)
+                                break;
+        }
+
+        return &t->targets[(KEYS_PER_NODE * n) + k];
+}
+
+void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q)
+{
+        /*
+         * Make sure we obey the optimistic sub devices
+         * restrictions.
+         */
+        blk_queue_max_sectors(q, t->limits.max_sectors);
+        q->max_phys_segments = t->limits.max_phys_segments;
+        q->max_hw_segments = t->limits.max_hw_segments;
+        q->hardsect_size = t->limits.hardsect_size;
+        q->max_segment_size = t->limits.max_segment_size;
+        q->seg_boundary_mask = t->limits.seg_boundary_mask;
+}
+
+unsigned int dm_table_get_num_targets(struct dm_table *t)
+{
+        return t->num_targets;
+}
+
+struct list_head *dm_table_get_devices(struct dm_table *t)
+{
+        return &t->devices;
+}
+
+int dm_table_get_mode(struct dm_table *t)
+{
+        return t->mode;
+}
+
+static void suspend_targets(struct dm_table *t, unsigned postsuspend)
+{
+        int i = t->num_targets;
+        struct dm_target *ti = t->targets;
+
+        while (i--) {
+                if (postsuspend) {
+                        if (ti->type->postsuspend)
+                                ti->type->postsuspend(ti);
+                } else if (ti->type->presuspend)
+                        ti->type->presuspend(ti);
+
+                ti++;
+        }
+}
+
+void dm_table_presuspend_targets(struct dm_table *t)
+{
+        return suspend_targets(t, 0);
+}
+
+void dm_table_postsuspend_targets(struct dm_table *t)
+{
+        return suspend_targets(t, 1);
+}
+
+void dm_table_resume_targets(struct dm_table *t)
+{
+        int i;
+
+        for (i = 0; i < t->num_targets; i++) {
+                struct dm_target *ti = t->targets + i;
+
+                if (ti->type->resume)
+                        ti->type->resume(ti);
+        }
+}
+
+int dm_table_any_congested(struct dm_table *t, int bdi_bits)
+{
+        struct list_head *d, *devices;
+        int r = 0;
+
+        devices = dm_table_get_devices(t);
+        for (d = devices->next; d != devices; d = d->next) {
+                struct dm_dev *dd = list_entry(d, struct dm_dev, list);
+                request_queue_t *q = bdev_get_queue(dd->bdev);
+                r |= bdi_congested(&q->backing_dev_info, bdi_bits);
+        }
+
+        return r;
+}
+
+void dm_table_unplug_all(struct dm_table *t)
+{
+        struct list_head *d, *devices = dm_table_get_devices(t);
+
+        for (d = devices->next; d != devices; d = d->next) {
+                struct dm_dev *dd = list_entry(d, struct dm_dev, list);
+                request_queue_t *q = bdev_get_queue(dd->bdev);
+
+                if (q->unplug_fn)
+                        q->unplug_fn(q);
+        }
+}
+
+int dm_table_flush_all(struct dm_table *t)
+{
+        struct list_head *d, *devices = dm_table_get_devices(t);
+        int ret = 0;
+
+        for (d = devices->next; d != devices; d = d->next) {
+                struct dm_dev *dd = list_entry(d, struct dm_dev, list);
+                request_queue_t *q = bdev_get_queue(dd->bdev);
+                int err;
+
+                if (!q->issue_flush_fn)
+                        err = -EOPNOTSUPP;
+                else
+                        err = q->issue_flush_fn(q, dd->bdev->bd_disk, NULL);
+
+                if (!ret)
+                        ret = err;
+        }
+
+        return ret;
+}
+
+EXPORT_SYMBOL(dm_vcalloc);
+EXPORT_SYMBOL(dm_get_device);
+EXPORT_SYMBOL(dm_put_device);
+EXPORT_SYMBOL(dm_table_event);
+EXPORT_SYMBOL(dm_table_get_mode);
+EXPORT_SYMBOL(dm_table_put);
+EXPORT_SYMBOL(dm_table_get);
+EXPORT_SYMBOL(dm_table_unplug_all);
+EXPORT_SYMBOL(dm_table_flush_all);
diff --git a/drivers/md/dm-target.c b/drivers/md/dm-target.c
new file mode 100644
index 000000000000..aecd9e0c2616
--- /dev/null
+++ b/drivers/md/dm-target.c
@@ -0,0 +1,196 @@
+/*
+ * Copyright (C) 2001 Sistina Software (UK) Limited
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kmod.h>
+#include <linux/bio.h>
+#include <linux/slab.h>
+
+struct tt_internal {
+        struct target_type tt;
+
+        struct list_head list;
+        long use;
+};
+
+static LIST_HEAD(_targets);
+static DECLARE_RWSEM(_lock);
+
+#define DM_MOD_NAME_SIZE 32
+
+static inline struct tt_internal *__find_target_type(const char *name)
+{
+        struct tt_internal *ti;
+
+        list_for_each_entry (ti, &_targets, list)
+                if (!strcmp(name, ti->tt.name))
+                        return ti;
+
+        return NULL;
+}
+
+static struct tt_internal *get_target_type(const char *name)
+{
+        struct tt_internal *ti;
+
+        down_read(&_lock);
+
+        ti = __find_target_type(name);
+        if (ti) {
+                if ((ti->use == 0) && !try_module_get(ti->tt.module))
+                        ti = NULL;
+                else
+                        ti->use++;
+        }
+
+        up_read(&_lock);
+        return ti;
+}
+
+static void load_module(const char *name)
+{
+        request_module("dm-%s", name);
+}
+
+struct target_type *dm_get_target_type(const char *name)
+{
+        struct tt_internal *ti = get_target_type(name);
+
+        if (!ti) {
+                load_module(name);
+                ti = get_target_type(name);
+        }
+
+        return ti ? &ti->tt : NULL;
+}
+
+void dm_put_target_type(struct target_type *t)
+{
+        struct tt_internal *ti = (struct tt_internal *) t;
+
+        down_read(&_lock);
+        if (--ti->use == 0)
+                module_put(ti->tt.module);
+
+        if (ti->use < 0)
+                BUG();
+        up_read(&_lock);
+
+        return;
+}
+
+static struct tt_internal *alloc_target(struct target_type *t)
+{
+        struct tt_internal *ti = kmalloc(sizeof(*ti), GFP_KERNEL);
+
+        if (ti) {
+                memset(ti, 0, sizeof(*ti));
+                ti->tt = *t;
+        }
+
+        return ti;
+}
+
+
+int dm_target_iterate(void (*iter_func)(struct target_type *tt,
+                                        void *param), void *param)
+{
+        struct tt_internal *ti;
+
+        down_read(&_lock);
+        list_for_each_entry (ti, &_targets, list)
+                iter_func(&ti->tt, param);
+        up_read(&_lock);
+
+        return 0;
+}
+
+int dm_register_target(struct target_type *t)
+{
+        int rv = 0;
+        struct tt_internal *ti = alloc_target(t);
+
+        if (!ti)
+                return -ENOMEM;
+
+        down_write(&_lock);
+        if (__find_target_type(t->name))
+                rv = -EEXIST;
+        else
+                list_add(&ti->list, &_targets);
+
+        up_write(&_lock);
+        if (rv)
+                kfree(ti);
+        return rv;
+}
+
+int dm_unregister_target(struct target_type *t)
+{
+        struct tt_internal *ti;
+
+        down_write(&_lock);
+        if (!(ti = __find_target_type(t->name))) {
+                up_write(&_lock);
+                return -EINVAL;
+        }
+
+        if (ti->use) {
+                up_write(&_lock);
+                return -ETXTBSY;
+        }
+
+        list_del(&ti->list);
+        kfree(ti);
+
+        up_write(&_lock);
+        return 0;
+}
+
+/*
+ * io-err: always fails an io, useful for bringing
+ * up LVs that have holes in them.
+ */
+static int io_err_ctr(struct dm_target *ti, unsigned int argc, char **args)
+{
+        return 0;
+}
+
+static void io_err_dtr(struct dm_target *ti)
+{
+        /* empty */
+}
+
+static int io_err_map(struct dm_target *ti, struct bio *bio,
+                      union map_info *map_context)
+{
+        return -EIO;
+}
+
+static struct target_type error_target = {
+        .name = "error",
+        .version = {1, 0, 1},
+        .ctr  = io_err_ctr,
+        .dtr  = io_err_dtr,
+        .map  = io_err_map,
+};
+
+int __init dm_target_init(void)
+{
+        return dm_register_target(&error_target);
+}
+
+void dm_target_exit(void)
+{
+        if (dm_unregister_target(&error_target))
+                DMWARN("error target unregistration failed");
+}
+
+EXPORT_SYMBOL(dm_register_target);
+EXPORT_SYMBOL(dm_unregister_target);
diff --git a/drivers/md/dm-zero.c b/drivers/md/dm-zero.c
new file mode 100644
index 000000000000..7febc2cac73d
--- /dev/null
+++ b/drivers/md/dm-zero.c
@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2003 Christophe Saout <christophe@saout.de>
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/bio.h>
+
+/*
+ * Construct a dummy mapping that only returns zeros
+ */
+static int zero_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+        if (argc != 0) {
+                ti->error = "dm-zero: No arguments required";
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+/*
+ * Return zeros only on reads
+ */
+static int zero_map(struct dm_target *ti, struct bio *bio,
+                      union map_info *map_context)
+{
+        switch(bio_rw(bio)) {
+        case READ:
+                zero_fill_bio(bio);
+                break;
+        case READA:
+                /* readahead of null bytes only wastes buffer cache */
+                return -EIO;
+        case WRITE:
+                /* writes get silently dropped */
+                break;
+        }
+
+        bio_endio(bio, bio->bi_size, 0);
+
+        /* accepted bio, don't make new request */
+        return 0;
+}
+
+static struct target_type zero_target = {
+        .name   = "zero",
+        .version = {1, 0, 0},
+        .module = THIS_MODULE,
+        .ctr    = zero_ctr,
+        .map    = zero_map,
+};
+
+int __init dm_zero_init(void)
+{
+        int r = dm_register_target(&zero_target);
+
+        if (r < 0)
+                DMERR("zero: register failed %d", r);
+
+        return r;
+}
+
+void __exit dm_zero_exit(void)
+{
+        int r = dm_unregister_target(&zero_target);
+
+        if (r < 0)
+                DMERR("zero: unregister failed %d", r);
+}
+
+module_init(dm_zero_init)
+module_exit(dm_zero_exit)
+
+MODULE_AUTHOR("Christophe Saout <christophe@saout.de>");
+MODULE_DESCRIPTION(DM_NAME " dummy target returning zeros");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
new file mode 100644
index 000000000000..243ff6884e83
--- /dev/null
+++ b/drivers/md/dm.c
@@ -0,0 +1,1194 @@
+/*
+ * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+#include "dm-bio-list.h"
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/blkpg.h>
+#include <linux/bio.h>
+#include <linux/buffer_head.h>
+#include <linux/mempool.h>
+#include <linux/slab.h>
+#include <linux/idr.h>
+
+static const char *_name = DM_NAME;
+
+static unsigned int major = 0;
+static unsigned int _major = 0;
+
+/*
+ * One of these is allocated per bio.
+ */
+struct dm_io {
+        struct mapped_device *md;
+        int error;
+        struct bio *bio;
+        atomic_t io_count;
+};
+
+/*
+ * One of these is allocated per target within a bio.  Hopefully
+ * this will be simplified out one day.
+ */
+struct target_io {
+        struct dm_io *io;
+        struct dm_target *ti;
+        union map_info info;
+};
+
+union map_info *dm_get_mapinfo(struct bio *bio)
+{
+        if (bio && bio->bi_private)
+                return &((struct target_io *)bio->bi_private)->info;
+        return NULL;
+}
+
+/*
+ * Bits for the md->flags field.
+ */
+#define DMF_BLOCK_IO 0
+#define DMF_SUSPENDED 1
+#define DMF_FS_LOCKED 2
+
+struct mapped_device {
+        struct rw_semaphore lock;
+        rwlock_t map_lock;
+        atomic_t holders;
+
+        unsigned long flags;
+
+        request_queue_t *queue;
+        struct gendisk *disk;
+
+        void *interface_ptr;
+
+        /*
+         * A list of ios that arrived while we were suspended.
+         */
+        atomic_t pending;
+        wait_queue_head_t wait;
+        struct bio_list deferred;
+
+        /*
+         * The current mapping.
+         */
+        struct dm_table *map;
+
+        /*
+         * io objects are allocated from here.
+         */
+        mempool_t *io_pool;
+        mempool_t *tio_pool;
+
+        /*
+         * Event handling.
+         */
+        atomic_t event_nr;
+        wait_queue_head_t eventq;
+
+        /*
+         * freeze/thaw support require holding onto a super block
+         */
+        struct super_block *frozen_sb;
+};
+
+#define MIN_IOS 256
+static kmem_cache_t *_io_cache;
+static kmem_cache_t *_tio_cache;
+
+static struct bio_set *dm_set;
+
+static int __init local_init(void)
+{
+        int r;
+
+        dm_set = bioset_create(16, 16, 4);
+        if (!dm_set)
+                return -ENOMEM;
+
+        /* allocate a slab for the dm_ios */
+        _io_cache = kmem_cache_create("dm_io",
+                                      sizeof(struct dm_io), 0, 0, NULL, NULL);
+        if (!_io_cache)
+                return -ENOMEM;
+
+        /* allocate a slab for the target ios */
+        _tio_cache = kmem_cache_create("dm_tio", sizeof(struct target_io),
+                                       0, 0, NULL, NULL);
+        if (!_tio_cache) {
+                kmem_cache_destroy(_io_cache);
+                return -ENOMEM;
+        }
+
+        _major = major;
+        r = register_blkdev(_major, _name);
+        if (r < 0) {
+                kmem_cache_destroy(_tio_cache);
+                kmem_cache_destroy(_io_cache);
+                return r;
+        }
+
+        if (!_major)
+                _major = r;
+
+        return 0;
+}
+
+static void local_exit(void)
+{
+        kmem_cache_destroy(_tio_cache);
+        kmem_cache_destroy(_io_cache);
+
+        bioset_free(dm_set);
+
+        if (unregister_blkdev(_major, _name) < 0)
+                DMERR("devfs_unregister_blkdev failed");
+
+        _major = 0;
+
+        DMINFO("cleaned up");
+}
+
+int (*_inits[])(void) __initdata = {
+        local_init,
+        dm_target_init,
+        dm_linear_init,
+        dm_stripe_init,
+        dm_interface_init,
+};
+
+void (*_exits[])(void) = {
+        local_exit,
+        dm_target_exit,
+        dm_linear_exit,
+        dm_stripe_exit,
+        dm_interface_exit,
+};
+
+static int __init dm_init(void)
+{
+        const int count = ARRAY_SIZE(_inits);
+
+        int r, i;
+
+        for (i = 0; i < count; i++) {
+                r = _inits[i]();
+                if (r)
+                        goto bad;
+        }
+
+        return 0;
+
+      bad:
+        while (i--)
+                _exits[i]();
+
+        return r;
+}
+
+static void __exit dm_exit(void)
+{
+        int i = ARRAY_SIZE(_exits);
+
+        while (i--)
+                _exits[i]();
+}
+
+/*
+ * Block device functions
+ */
+static int dm_blk_open(struct inode *inode, struct file *file)
+{
+        struct mapped_device *md;
+
+        md = inode->i_bdev->bd_disk->private_data;
+        dm_get(md);
+        return 0;
+}
+
+static int dm_blk_close(struct inode *inode, struct file *file)
+{
+        struct mapped_device *md;
+
+        md = inode->i_bdev->bd_disk->private_data;
+        dm_put(md);
+        return 0;
+}
+
+static inline struct dm_io *alloc_io(struct mapped_device *md)
+{
+        return mempool_alloc(md->io_pool, GFP_NOIO);
+}
+
+static inline void free_io(struct mapped_device *md, struct dm_io *io)
+{
+        mempool_free(io, md->io_pool);
+}
+
+static inline struct target_io *alloc_tio(struct mapped_device *md)
+{
+        return mempool_alloc(md->tio_pool, GFP_NOIO);
+}
+
+static inline void free_tio(struct mapped_device *md, struct target_io *tio)
+{
+        mempool_free(tio, md->tio_pool);
+}
+
+/*
+ * Add the bio to the list of deferred io.
+ */
+static int queue_io(struct mapped_device *md, struct bio *bio)
+{
+        down_write(&md->lock);
+
+        if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
+                up_write(&md->lock);
+                return 1;
+        }
+
+        bio_list_add(&md->deferred, bio);
+
+        up_write(&md->lock);
+        return 0;               /* deferred successfully */
+}
+
+/*
+ * Everyone (including functions in this file), should use this
+ * function to access the md->map field, and make sure they call
+ * dm_table_put() when finished.
+ */
+struct dm_table *dm_get_table(struct mapped_device *md)
+{
+        struct dm_table *t;
+
+        read_lock(&md->map_lock);
+        t = md->map;
+        if (t)
+                dm_table_get(t);
+        read_unlock(&md->map_lock);
+
+        return t;
+}
+
+/*-----------------------------------------------------------------
+ * CRUD START:
+ *   A more elegant soln is in the works that uses the queue
+ *   merge fn, unfortunately there are a couple of changes to
+ *   the block layer that I want to make for this.  So in the
+ *   interests of getting something for people to use I give
+ *   you this clearly demarcated crap.
+ *---------------------------------------------------------------*/
+
+/*
+ * Decrements the number of outstanding ios that a bio has been
+ * cloned into, completing the original io if necc.
+ */
+static inline void dec_pending(struct dm_io *io, int error)
+{
+        if (error)
+                io->error = error;
+
+        if (atomic_dec_and_test(&io->io_count)) {
+                if (atomic_dec_and_test(&io->md->pending))
+                        /* nudge anyone waiting on suspend queue */
+                        wake_up(&io->md->wait);
+
+                bio_endio(io->bio, io->bio->bi_size, io->error);
+                free_io(io->md, io);
+        }
+}
+
+static int clone_endio(struct bio *bio, unsigned int done, int error)
+{
+        int r = 0;
+        struct target_io *tio = bio->bi_private;
+        struct dm_io *io = tio->io;
+        dm_endio_fn endio = tio->ti->type->end_io;
+
+        if (bio->bi_size)
+                return 1;
+
+        if (!bio_flagged(bio, BIO_UPTODATE) && !error)
+                error = -EIO;
+
+        if (endio) {
+                r = endio(tio->ti, bio, error, &tio->info);
+                if (r < 0)
+                        error = r;
+
+                else if (r > 0)
+                        /* the target wants another shot at the io */
+                        return 1;
+        }
+
+        free_tio(io->md, tio);
+        dec_pending(io, error);
+        bio_put(bio);
+        return r;
+}
+
+static sector_t max_io_len(struct mapped_device *md,
+                           sector_t sector, struct dm_target *ti)
+{
+        sector_t offset = sector - ti->begin;
+        sector_t len = ti->len - offset;
+
+        /*
+         * Does the target need to split even further ?
+         */
+        if (ti->split_io) {
+                sector_t boundary;
+                boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
+                           - offset;
+                if (len > boundary)
+                        len = boundary;
+        }
+
+        return len;
+}
+
+static void __map_bio(struct dm_target *ti, struct bio *clone,
+                      struct target_io *tio)
+{
+        int r;
+
+        /*
+         * Sanity checks.
+         */
+        BUG_ON(!clone->bi_size);
+
+        clone->bi_end_io = clone_endio;
+        clone->bi_private = tio;
+
+        /*
+         * Map the clone.  If r == 0 we don't need to do
+         * anything, the target has assumed ownership of
+         * this io.
+         */
+        atomic_inc(&tio->io->io_count);
+        r = ti->type->map(ti, clone, &tio->info);
+        if (r > 0)
+                /* the bio has been remapped so dispatch it */
+                generic_make_request(clone);
+
+        else if (r < 0) {
+                /* error the io and bail out */
+                struct dm_io *io = tio->io;
+                free_tio(tio->io->md, tio);
+                dec_pending(io, -EIO);
+                bio_put(clone);
+        }
+}
+
+struct clone_info {
+        struct mapped_device *md;
+        struct dm_table *map;
+        struct bio *bio;
+        struct dm_io *io;
+        sector_t sector;
+        sector_t sector_count;
+        unsigned short idx;
+};
+
+/*
+ * Creates a little bio that is just does part of a bvec.
+ */
+static struct bio *split_bvec(struct bio *bio, sector_t sector,
+                              unsigned short idx, unsigned int offset,
+                              unsigned int len)
+{
+        struct bio *clone;
+        struct bio_vec *bv = bio->bi_io_vec + idx;
+
+        clone = bio_alloc_bioset(GFP_NOIO, 1, dm_set);
+        *clone->bi_io_vec = *bv;
+
+        clone->bi_sector = sector;
+        clone->bi_bdev = bio->bi_bdev;
+        clone->bi_rw = bio->bi_rw;
+        clone->bi_vcnt = 1;
+        clone->bi_size = to_bytes(len);
+        clone->bi_io_vec->bv_offset = offset;
+        clone->bi_io_vec->bv_len = clone->bi_size;
+
+        return clone;
+}
+
+/*
+ * Creates a bio that consists of range of complete bvecs.
+ */
+static struct bio *clone_bio(struct bio *bio, sector_t sector,
+                             unsigned short idx, unsigned short bv_count,
+                             unsigned int len)
+{
+        struct bio *clone;
+
+        clone = bio_clone(bio, GFP_NOIO);
+        clone->bi_sector = sector;
+        clone->bi_idx = idx;
+        clone->bi_vcnt = idx + bv_count;
+        clone->bi_size = to_bytes(len);
+        clone->bi_flags &= ~(1 << BIO_SEG_VALID);
+
+        return clone;
+}
+
+static void __clone_and_map(struct clone_info *ci)
+{
+        struct bio *clone, *bio = ci->bio;
+        struct dm_target *ti = dm_table_find_target(ci->map, ci->sector);
+        sector_t len = 0, max = max_io_len(ci->md, ci->sector, ti);
+        struct target_io *tio;
+
+        /*
+         * Allocate a target io object.
+         */
+        tio = alloc_tio(ci->md);
+        tio->io = ci->io;
+        tio->ti = ti;
+        memset(&tio->info, 0, sizeof(tio->info));
+
+        if (ci->sector_count <= max) {
+                /*
+                 * Optimise for the simple case where we can do all of
+                 * the remaining io with a single clone.
+                 */
+                clone = clone_bio(bio, ci->sector, ci->idx,
+                                  bio->bi_vcnt - ci->idx, ci->sector_count);
+                __map_bio(ti, clone, tio);
+                ci->sector_count = 0;
+
+        } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
+                /*
+                 * There are some bvecs that don't span targets.
+                 * Do as many of these as possible.
+                 */
+                int i;
+                sector_t remaining = max;
+                sector_t bv_len;
+
+                for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
+                        bv_len = to_sector(bio->bi_io_vec[i].bv_len);
+
+                        if (bv_len > remaining)
+                                break;
+
+                        remaining -= bv_len;
+                        len += bv_len;
+                }
+
+                clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len);
+                __map_bio(ti, clone, tio);
+
+                ci->sector += len;
+                ci->sector_count -= len;
+                ci->idx = i;
+
+        } else {
+                /*
+                 * Create two copy bios to deal with io that has
+                 * been split across a target.
+                 */
+                struct bio_vec *bv = bio->bi_io_vec + ci->idx;
+
+                clone = split_bvec(bio, ci->sector, ci->idx,
+                                   bv->bv_offset, max);
+                __map_bio(ti, clone, tio);
+
+                ci->sector += max;
+                ci->sector_count -= max;
+                ti = dm_table_find_target(ci->map, ci->sector);
+
+                len = to_sector(bv->bv_len) - max;
+                clone = split_bvec(bio, ci->sector, ci->idx,
+                                   bv->bv_offset + to_bytes(max), len);
+                tio = alloc_tio(ci->md);
+                tio->io = ci->io;
+                tio->ti = ti;
+                memset(&tio->info, 0, sizeof(tio->info));
+                __map_bio(ti, clone, tio);
+
+                ci->sector += len;
+                ci->sector_count -= len;
+                ci->idx++;
+        }
+}
+
+/*
+ * Split the bio into several clones.
+ */
+static void __split_bio(struct mapped_device *md, struct bio *bio)
+{
+        struct clone_info ci;
+
+        ci.map = dm_get_table(md);
+        if (!ci.map) {
+                bio_io_error(bio, bio->bi_size);
+                return;
+        }
+
+        ci.md = md;
+        ci.bio = bio;
+        ci.io = alloc_io(md);
+        ci.io->error = 0;
+        atomic_set(&ci.io->io_count, 1);
+        ci.io->bio = bio;
+        ci.io->md = md;
+        ci.sector = bio->bi_sector;
+        ci.sector_count = bio_sectors(bio);
+        ci.idx = bio->bi_idx;
+
+        atomic_inc(&md->pending);
+        while (ci.sector_count)
+                __clone_and_map(&ci);
+
+        /* drop the extra reference count */
+        dec_pending(ci.io, 0);
+        dm_table_put(ci.map);
+}
+/*-----------------------------------------------------------------
+ * CRUD END
+ *---------------------------------------------------------------*/
+
+/*
+ * The request function that just remaps the bio built up by
+ * dm_merge_bvec.
+ */
+static int dm_request(request_queue_t *q, struct bio *bio)
+{
+        int r;
+        struct mapped_device *md = q->queuedata;
+
+        down_read(&md->lock);
+
+        /*
+         * If we're suspended we have to queue
+         * this io for later.
+         */
+        while (test_bit(DMF_BLOCK_IO, &md->flags)) {
+                up_read(&md->lock);
+
+                if (bio_rw(bio) == READA) {
+                        bio_io_error(bio, bio->bi_size);
+                        return 0;
+                }
+
+                r = queue_io(md, bio);
+                if (r < 0) {
+                        bio_io_error(bio, bio->bi_size);
+                        return 0;
+
+                } else if (r == 0)
+                        return 0;       /* deferred successfully */
+
+                /*
+                 * We're in a while loop, because someone could suspend
+                 * before we get to the following read lock.
+                 */
+                down_read(&md->lock);
+        }
+
+        __split_bio(md, bio);
+        up_read(&md->lock);
+        return 0;
+}
+
+static int dm_flush_all(request_queue_t *q, struct gendisk *disk,
+                        sector_t *error_sector)
+{
+        struct mapped_device *md = q->queuedata;
+        struct dm_table *map = dm_get_table(md);
+        int ret = -ENXIO;
+
+        if (map) {
+                ret = dm_table_flush_all(md->map);
+                dm_table_put(map);
+        }
+
+        return ret;
+}
+
+static void dm_unplug_all(request_queue_t *q)
+{
+        struct mapped_device *md = q->queuedata;
+        struct dm_table *map = dm_get_table(md);
+
+        if (map) {
+                dm_table_unplug_all(map);
+                dm_table_put(map);
+        }
+}
+
+static int dm_any_congested(void *congested_data, int bdi_bits)
+{
+        int r;
+        struct mapped_device *md = (struct mapped_device *) congested_data;
+        struct dm_table *map = dm_get_table(md);
+
+        if (!map || test_bit(DMF_BLOCK_IO, &md->flags))
+                r = bdi_bits;
+        else
+                r = dm_table_any_congested(map, bdi_bits);
+
+        dm_table_put(map);
+        return r;
+}
+
+/*-----------------------------------------------------------------
+ * An IDR is used to keep track of allocated minor numbers.
+ *---------------------------------------------------------------*/
+static DECLARE_MUTEX(_minor_lock);
+static DEFINE_IDR(_minor_idr);
+
+static void free_minor(unsigned int minor)
+{
+        down(&_minor_lock);
+        idr_remove(&_minor_idr, minor);
+        up(&_minor_lock);
+}
+
+/*
+ * See if the device with a specific minor # is free.
+ */
+static int specific_minor(struct mapped_device *md, unsigned int minor)
+{
+        int r, m;
+
+        if (minor >= (1 << MINORBITS))
+                return -EINVAL;
+
+        down(&_minor_lock);
+
+        if (idr_find(&_minor_idr, minor)) {
+                r = -EBUSY;
+                goto out;
+        }
+
+        r = idr_pre_get(&_minor_idr, GFP_KERNEL);
+        if (!r) {
+                r = -ENOMEM;
+                goto out;
+        }
+
+        r = idr_get_new_above(&_minor_idr, md, minor, &m);
+        if (r) {
+                goto out;
+        }
+
+        if (m != minor) {
+                idr_remove(&_minor_idr, m);
+                r = -EBUSY;
+                goto out;
+        }
+
+out:
+        up(&_minor_lock);
+        return r;
+}
+
+static int next_free_minor(struct mapped_device *md, unsigned int *minor)
+{
+        int r;
+        unsigned int m;
+
+        down(&_minor_lock);
+
+        r = idr_pre_get(&_minor_idr, GFP_KERNEL);
+        if (!r) {
+                r = -ENOMEM;
+                goto out;
+        }
+
+        r = idr_get_new(&_minor_idr, md, &m);
+        if (r) {
+                goto out;
+        }
+
+        if (m >= (1 << MINORBITS)) {
+                idr_remove(&_minor_idr, m);
+                r = -ENOSPC;
+                goto out;
+        }
+
+        *minor = m;
+
+out:
+        up(&_minor_lock);
+        return r;
+}
+
+static struct block_device_operations dm_blk_dops;
+
+/*
+ * Allocate and initialise a blank device with a given minor.
+ */
+static struct mapped_device *alloc_dev(unsigned int minor, int persistent)
+{
+        int r;
+        struct mapped_device *md = kmalloc(sizeof(*md), GFP_KERNEL);
+
+        if (!md) {
+                DMWARN("unable to allocate device, out of memory.");
+                return NULL;
+        }
+
+        /* get a minor number for the dev */
+        r = persistent ? specific_minor(md, minor) : next_free_minor(md, &minor);
+        if (r < 0)
+                goto bad1;
+
+        memset(md, 0, sizeof(*md));
+        init_rwsem(&md->lock);
+        rwlock_init(&md->map_lock);
+        atomic_set(&md->holders, 1);
+        atomic_set(&md->event_nr, 0);
+
+        md->queue = blk_alloc_queue(GFP_KERNEL);
+        if (!md->queue)
+                goto bad1;
+
+        md->queue->queuedata = md;
+        md->queue->backing_dev_info.congested_fn = dm_any_congested;
+        md->queue->backing_dev_info.congested_data = md;
+        blk_queue_make_request(md->queue, dm_request);
+        md->queue->unplug_fn = dm_unplug_all;
+        md->queue->issue_flush_fn = dm_flush_all;
+
+        md->io_pool = mempool_create(MIN_IOS, mempool_alloc_slab,
+                                     mempool_free_slab, _io_cache);
+        if (!md->io_pool)
+                goto bad2;
+
+        md->tio_pool = mempool_create(MIN_IOS, mempool_alloc_slab,
+                                      mempool_free_slab, _tio_cache);
+        if (!md->tio_pool)
+                goto bad3;
+
+        md->disk = alloc_disk(1);
+        if (!md->disk)
+                goto bad4;
+
+        md->disk->major = _major;
+        md->disk->first_minor = minor;
+        md->disk->fops = &dm_blk_dops;
+        md->disk->queue = md->queue;
+        md->disk->private_data = md;
+        sprintf(md->disk->disk_name, "dm-%d", minor);
+        add_disk(md->disk);
+
+        atomic_set(&md->pending, 0);
+        init_waitqueue_head(&md->wait);
+        init_waitqueue_head(&md->eventq);
+
+        return md;
+
+ bad4:
+        mempool_destroy(md->tio_pool);
+ bad3:
+        mempool_destroy(md->io_pool);
+ bad2:
+        blk_put_queue(md->queue);
+        free_minor(minor);
+ bad1:
+        kfree(md);
+        return NULL;
+}
+
+static void free_dev(struct mapped_device *md)
+{
+        free_minor(md->disk->first_minor);
+        mempool_destroy(md->tio_pool);
+        mempool_destroy(md->io_pool);
+        del_gendisk(md->disk);
+        put_disk(md->disk);
+        blk_put_queue(md->queue);
+        kfree(md);
+}
+
+/*
+ * Bind a table to the device.
+ */
+static void event_callback(void *context)
+{
+        struct mapped_device *md = (struct mapped_device *) context;
+
+        atomic_inc(&md->event_nr);
+        wake_up(&md->eventq);
+}
+
+static void __set_size(struct gendisk *disk, sector_t size)
+{
+        struct block_device *bdev;
+
+        set_capacity(disk, size);
+        bdev = bdget_disk(disk, 0);
+        if (bdev) {
+                down(&bdev->bd_inode->i_sem);
+                i_size_write(bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
+                up(&bdev->bd_inode->i_sem);
+                bdput(bdev);
+        }
+}
+
+static int __bind(struct mapped_device *md, struct dm_table *t)
+{
+        request_queue_t *q = md->queue;
+        sector_t size;
+
+        size = dm_table_get_size(t);
+        __set_size(md->disk, size);
+        if (size == 0)
+                return 0;
+
+        write_lock(&md->map_lock);
+        md->map = t;
+        write_unlock(&md->map_lock);
+
+        dm_table_get(t);
+        dm_table_event_callback(md->map, event_callback, md);
+        dm_table_set_restrictions(t, q);
+        return 0;
+}
+
+static void __unbind(struct mapped_device *md)
+{
+        struct dm_table *map = md->map;
+
+        if (!map)
+                return;
+
+        dm_table_event_callback(map, NULL, NULL);
+        write_lock(&md->map_lock);
+        md->map = NULL;
+        write_unlock(&md->map_lock);
+        dm_table_put(map);
+}
+
+/*
+ * Constructor for a new device.
+ */
+static int create_aux(unsigned int minor, int persistent,
+                      struct mapped_device **result)
+{
+        struct mapped_device *md;
+
+        md = alloc_dev(minor, persistent);
+        if (!md)
+                return -ENXIO;
+
+        *result = md;
+        return 0;
+}
+
+int dm_create(struct mapped_device **result)
+{
+        return create_aux(0, 0, result);
+}
+
+int dm_create_with_minor(unsigned int minor, struct mapped_device **result)
+{
+        return create_aux(minor, 1, result);
+}
+
+void *dm_get_mdptr(dev_t dev)
+{
+        struct mapped_device *md;
+        void *mdptr = NULL;
+        unsigned minor = MINOR(dev);
+
+        if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
+                return NULL;
+
+        down(&_minor_lock);
+
+        md = idr_find(&_minor_idr, minor);
+
+        if (md && (dm_disk(md)->first_minor == minor))
+                mdptr = md->interface_ptr;
+
+        up(&_minor_lock);
+
+        return mdptr;
+}
+
+void dm_set_mdptr(struct mapped_device *md, void *ptr)
+{
+        md->interface_ptr = ptr;
+}
+
+void dm_get(struct mapped_device *md)
+{
+        atomic_inc(&md->holders);
+}
+
+void dm_put(struct mapped_device *md)
+{
+        struct dm_table *map = dm_get_table(md);
+
+        if (atomic_dec_and_test(&md->holders)) {
+                if (!test_bit(DMF_SUSPENDED, &md->flags) && map) {
+                        dm_table_presuspend_targets(map);
+                        dm_table_postsuspend_targets(map);
+                }
+                __unbind(md);
+                free_dev(md);
+        }
+
+        dm_table_put(map);
+}
+
+/*
+ * Process the deferred bios
+ */
+static void __flush_deferred_io(struct mapped_device *md, struct bio *c)
+{
+        struct bio *n;
+
+        while (c) {
+                n = c->bi_next;
+                c->bi_next = NULL;
+                __split_bio(md, c);
+                c = n;
+        }
+}
+
+/*
+ * Swap in a new table (destroying old one).
+ */
+int dm_swap_table(struct mapped_device *md, struct dm_table *table)
+{
+        int r;
+
+        down_write(&md->lock);
+
+        /* device must be suspended */
+        if (!test_bit(DMF_SUSPENDED, &md->flags)) {
+                up_write(&md->lock);
+                return -EPERM;
+        }
+
+        __unbind(md);
+        r = __bind(md, table);
+        if (r)
+                return r;
+
+        up_write(&md->lock);
+        return 0;
+}
+
+/*
+ * Functions to lock and unlock any filesystem running on the
+ * device.
+ */
+static int __lock_fs(struct mapped_device *md)
+{
+        struct block_device *bdev;
+
+        if (test_and_set_bit(DMF_FS_LOCKED, &md->flags))
+                return 0;
+
+        bdev = bdget_disk(md->disk, 0);
+        if (!bdev) {
+                DMWARN("bdget failed in __lock_fs");
+                return -ENOMEM;
+        }
+
+        WARN_ON(md->frozen_sb);
+        md->frozen_sb = freeze_bdev(bdev);
+        /* don't bdput right now, we don't want the bdev
+         * to go away while it is locked.  We'll bdput
+         * in __unlock_fs
+         */
+        return 0;
+}
+
+static int __unlock_fs(struct mapped_device *md)
+{
+        struct block_device *bdev;
+
+        if (!test_and_clear_bit(DMF_FS_LOCKED, &md->flags))
+                return 0;
+
+        bdev = bdget_disk(md->disk, 0);
+        if (!bdev) {
+                DMWARN("bdget failed in __unlock_fs");
+                return -ENOMEM;
+        }
+
+        thaw_bdev(bdev, md->frozen_sb);
+        md->frozen_sb = NULL;
+        bdput(bdev);
+        bdput(bdev);
+        return 0;
+}
+
+/*
+ * We need to be able to change a mapping table under a mounted
+ * filesystem.  For example we might want to move some data in
+ * the background.  Before the table can be swapped with
+ * dm_bind_table, dm_suspend must be called to flush any in
+ * flight bios and ensure that any further io gets deferred.
+ */
+int dm_suspend(struct mapped_device *md)
+{
+        struct dm_table *map;
+        DECLARE_WAITQUEUE(wait, current);
+
+        /* Flush I/O to the device. */
+        down_read(&md->lock);
+        if (test_bit(DMF_BLOCK_IO, &md->flags)) {
+                up_read(&md->lock);
+                return -EINVAL;
+        }
+
+        map = dm_get_table(md);
+        if (map)
+                dm_table_presuspend_targets(map);
+        __lock_fs(md);
+
+        up_read(&md->lock);
+
+        /*
+         * First we set the BLOCK_IO flag so no more ios will be
+         * mapped.
+         */
+        down_write(&md->lock);
+        if (test_bit(DMF_BLOCK_IO, &md->flags)) {
+                /*
+                 * If we get here we know another thread is
+                 * trying to suspend as well, so we leave the fs
+                 * locked for this thread.
+                 */
+                up_write(&md->lock);
+                return -EINVAL;
+        }
+
+        set_bit(DMF_BLOCK_IO, &md->flags);
+        add_wait_queue(&md->wait, &wait);
+        up_write(&md->lock);
+
+        /* unplug */
+        if (map) {
+                dm_table_unplug_all(map);
+                dm_table_put(map);
+        }
+
+        /*
+         * Then we wait for the already mapped ios to
+         * complete.
+         */
+        while (1) {
+                set_current_state(TASK_INTERRUPTIBLE);
+
+                if (!atomic_read(&md->pending) || signal_pending(current))
+                        break;
+
+                io_schedule();
+        }
+        set_current_state(TASK_RUNNING);
+
+        down_write(&md->lock);
+        remove_wait_queue(&md->wait, &wait);
+
+        /* were we interrupted ? */
+        if (atomic_read(&md->pending)) {
+                __unlock_fs(md);
+                clear_bit(DMF_BLOCK_IO, &md->flags);
+                up_write(&md->lock);
+                return -EINTR;
+        }
+
+        set_bit(DMF_SUSPENDED, &md->flags);
+
+        map = dm_get_table(md);
+        if (map)
+                dm_table_postsuspend_targets(map);
+        dm_table_put(map);
+        up_write(&md->lock);
+
+        return 0;
+}
+
+int dm_resume(struct mapped_device *md)
+{
+        struct bio *def;
+        struct dm_table *map = dm_get_table(md);
+
+        down_write(&md->lock);
+        if (!map ||
+            !test_bit(DMF_SUSPENDED, &md->flags) ||
+            !dm_table_get_size(map)) {
+                up_write(&md->lock);
+                dm_table_put(map);
+                return -EINVAL;
+        }
+
+        dm_table_resume_targets(map);
+        clear_bit(DMF_SUSPENDED, &md->flags);
+        clear_bit(DMF_BLOCK_IO, &md->flags);
+
+        def = bio_list_get(&md->deferred);
+        __flush_deferred_io(md, def);
+        up_write(&md->lock);
+        __unlock_fs(md);
+        dm_table_unplug_all(map);
+        dm_table_put(map);
+
+        return 0;
+}
+
+/*-----------------------------------------------------------------
+ * Event notification.
+ *---------------------------------------------------------------*/
+uint32_t dm_get_event_nr(struct mapped_device *md)
+{
+        return atomic_read(&md->event_nr);
+}
+
+int dm_wait_event(struct mapped_device *md, int event_nr)
+{
+        return wait_event_interruptible(md->eventq,
+                        (event_nr != atomic_read(&md->event_nr)));
+}
+
+/*
+ * The gendisk is only valid as long as you have a reference
+ * count on 'md'.
+ */
+struct gendisk *dm_disk(struct mapped_device *md)
+{
+        return md->disk;
+}
+
+int dm_suspended(struct mapped_device *md)
+{
+        return test_bit(DMF_SUSPENDED, &md->flags);
+}
+
+static struct block_device_operations dm_blk_dops = {
+        .open = dm_blk_open,
+        .release = dm_blk_close,
+        .owner = THIS_MODULE
+};
+
+EXPORT_SYMBOL(dm_get_mapinfo);
+
+/*
+ * module hooks
+ */
+module_init(dm_init);
+module_exit(dm_exit);
+
+module_param(major, uint, 0);
+MODULE_PARM_DESC(major, "The major number of the device mapper");
+MODULE_DESCRIPTION(DM_NAME " driver");
+MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm.h b/drivers/md/dm.h
new file mode 100644
index 000000000000..e38c3fc1a1db
--- /dev/null
+++ b/drivers/md/dm.h
@@ -0,0 +1,195 @@
+/*
+ * Internal header file for device mapper
+ *
+ * Copyright (C) 2001, 2002 Sistina Software
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the LGPL.
+ */
+
+#ifndef DM_INTERNAL_H
+#define DM_INTERNAL_H
+
+#include <linux/fs.h>
+#include <linux/device-mapper.h>
+#include <linux/list.h>
+#include <linux/blkdev.h>
+
+#define DM_NAME "device-mapper"
+#define DMWARN(f, x...) printk(KERN_WARNING DM_NAME ": " f "\n" , ## x)
+#define DMERR(f, x...) printk(KERN_ERR DM_NAME ": " f "\n" , ## x)
+#define DMINFO(f, x...) printk(KERN_INFO DM_NAME ": " f "\n" , ## x)
+
+#define DMEMIT(x...) sz += ((sz >= maxlen) ? \
+                          0 : scnprintf(result + sz, maxlen - sz, x))
+
+/*
+ * FIXME: I think this should be with the definition of sector_t
+ * in types.h.
+ */
+#ifdef CONFIG_LBD
+#define SECTOR_FORMAT "%Lu"
+#else
+#define SECTOR_FORMAT "%lu"
+#endif
+
+#define SECTOR_SHIFT 9
+
+/*
+ * List of devices that a metadevice uses and should open/close.
+ */
+struct dm_dev {
+        struct list_head list;
+
+        atomic_t count;
+        int mode;
+        struct block_device *bdev;
+        char name[16];
+};
+
+struct dm_table;
+struct mapped_device;
+
+/*-----------------------------------------------------------------
+ * Functions for manipulating a struct mapped_device.
+ * Drop the reference with dm_put when you finish with the object.
+ *---------------------------------------------------------------*/
+int dm_create(struct mapped_device **md);
+int dm_create_with_minor(unsigned int minor, struct mapped_device **md);
+void dm_set_mdptr(struct mapped_device *md, void *ptr);
+void *dm_get_mdptr(dev_t dev);
+
+/*
+ * Reference counting for md.
+ */
+void dm_get(struct mapped_device *md);
+void dm_put(struct mapped_device *md);
+
+/*
+ * A device can still be used while suspended, but I/O is deferred.
+ */
+int dm_suspend(struct mapped_device *md);
+int dm_resume(struct mapped_device *md);
+
+/*
+ * The device must be suspended before calling this method.
+ */
+int dm_swap_table(struct mapped_device *md, struct dm_table *t);
+
+/*
+ * Drop a reference on the table when you've finished with the
+ * result.
+ */
+struct dm_table *dm_get_table(struct mapped_device *md);
+
+/*
+ * Event functions.
+ */
+uint32_t dm_get_event_nr(struct mapped_device *md);
+int dm_wait_event(struct mapped_device *md, int event_nr);
+
+/*
+ * Info functions.
+ */
+struct gendisk *dm_disk(struct mapped_device *md);
+int dm_suspended(struct mapped_device *md);
+
+/*-----------------------------------------------------------------
+ * Functions for manipulating a table.  Tables are also reference
+ * counted.
+ *---------------------------------------------------------------*/
+int dm_table_create(struct dm_table **result, int mode, unsigned num_targets);
+
+void dm_table_get(struct dm_table *t);
+void dm_table_put(struct dm_table *t);
+
+int dm_table_add_target(struct dm_table *t, const char *type,
+                        sector_t start, sector_t len, char *params);
+int dm_table_complete(struct dm_table *t);
+void dm_table_event_callback(struct dm_table *t,
+                             void (*fn)(void *), void *context);
+void dm_table_event(struct dm_table *t);
+sector_t dm_table_get_size(struct dm_table *t);
+struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index);
+struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector);
+void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q);
+unsigned int dm_table_get_num_targets(struct dm_table *t);
+struct list_head *dm_table_get_devices(struct dm_table *t);
+int dm_table_get_mode(struct dm_table *t);
+void dm_table_presuspend_targets(struct dm_table *t);
+void dm_table_postsuspend_targets(struct dm_table *t);
+void dm_table_resume_targets(struct dm_table *t);
+int dm_table_any_congested(struct dm_table *t, int bdi_bits);
+void dm_table_unplug_all(struct dm_table *t);
+int dm_table_flush_all(struct dm_table *t);
+
+/*-----------------------------------------------------------------
+ * A registry of target types.
+ *---------------------------------------------------------------*/
+int dm_target_init(void);
+void dm_target_exit(void);
+struct target_type *dm_get_target_type(const char *name);
+void dm_put_target_type(struct target_type *t);
+int dm_target_iterate(void (*iter_func)(struct target_type *tt,
+                                        void *param), void *param);
+
+
+/*-----------------------------------------------------------------
+ * Useful inlines.
+ *---------------------------------------------------------------*/
+static inline int array_too_big(unsigned long fixed, unsigned long obj,
+                                unsigned long num)
+{
+        return (num > (ULONG_MAX - fixed) / obj);
+}
+
+/*
+ * Ceiling(n / sz)
+ */
+#define dm_div_up(n, sz) (((n) + (sz) - 1) / (sz))
+
+#define dm_sector_div_up(n, sz) ( \
+{ \
+        sector_t _r = ((n) + (sz) - 1); \
+        sector_div(_r, (sz)); \
+        _r; \
+} \
+)
+
+/*
+ * ceiling(n / size) * size
+ */
+#define dm_round_up(n, sz) (dm_div_up((n), (sz)) * (sz))
+
+static inline sector_t to_sector(unsigned long n)
+{
+        return (n >> 9);
+}
+
+static inline unsigned long to_bytes(sector_t n)
+{
+        return (n << 9);
+}
+
+int dm_split_args(int *argc, char ***argvp, char *input);
+
+/*
+ * The device-mapper can be driven through one of two interfaces;
+ * ioctl or filesystem, depending which patch you have applied.
+ */
+int dm_interface_init(void);
+void dm_interface_exit(void);
+
+/*
+ * Targets for linear and striped mappings
+ */
+int dm_linear_init(void);
+void dm_linear_exit(void);
+
+int dm_stripe_init(void);
+void dm_stripe_exit(void);
+
+void *dm_vcalloc(unsigned long nmemb, unsigned long elem_size);
+union map_info *dm_get_mapinfo(struct bio *bio);
+
+#endif
diff --git a/drivers/md/faulty.c b/drivers/md/faulty.c
new file mode 100644
index 000000000000..0248f8e7eac0
--- /dev/null
+++ b/drivers/md/faulty.c
@@ -0,0 +1,343 @@
+/*
+ * faulty.c : Multiple Devices driver for Linux
+ *
+ * Copyright (C) 2004 Neil Brown
+ *
+ * fautly-device-simulator personality for md
+ *
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * (for example /usr/src/linux/COPYING); if not, write to the Free
+ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+
+/*
+ * The "faulty" personality causes some requests to fail.
+ *
+ * Possible failure modes are:
+ *   reads fail "randomly" but succeed on retry
+ *   writes fail "randomly" but succeed on retry
+ *   reads for some address fail and then persist until a write
+ *   reads for some address fail and then persist irrespective of write
+ *   writes for some address fail and persist
+ *   all writes fail
+ *
+ * Different modes can be active at a time, but only
+ * one can be set at array creation.  Others can be added later.
+ * A mode can be one-shot or recurrent with the recurrance being
+ * once in every N requests.
+ * The bottom 5 bits of the "layout" indicate the mode.  The
+ * remainder indicate a period, or 0 for one-shot.
+ *
+ * There is an implementation limit on the number of concurrently
+ * persisting-faulty blocks. When a new fault is requested that would
+ * exceed the limit, it is ignored.
+ * All current faults can be clear using a layout of "0".
+ *
+ * Requests are always sent to the device.  If they are to fail,
+ * we clone the bio and insert a new b_end_io into the chain.
+ */
+
+#define WriteTransient  0
+#define ReadTransient   1
+#define WritePersistent 2
+#define ReadPersistent  3
+#define WriteAll        4 /* doesn't go to device */
+#define ReadFixable     5
+#define Modes   6
+
+#define ClearErrors     31
+#define ClearFaults     30
+
+#define AllPersist      100 /* internal use only */
+#define NoPersist       101
+
+#define ModeMask        0x1f
+#define ModeShift       5
+
+#define MaxFault        50
+#include <linux/raid/md.h>
+
+
+static int faulty_fail(struct bio *bio, unsigned int bytes_done, int error)
+{
+        struct bio *b = bio->bi_private;
+
+        b->bi_size = bio->bi_size;
+        b->bi_sector = bio->bi_sector;
+
+        if (bio->bi_size == 0)
+                bio_put(bio);
+
+        clear_bit(BIO_UPTODATE, &b->bi_flags);
+        return (b->bi_end_io)(b, bytes_done, -EIO);
+}
+
+typedef struct faulty_conf {
+        int period[Modes];
+        atomic_t counters[Modes];
+        sector_t faults[MaxFault];
+        int     modes[MaxFault];
+        int nfaults;
+        mdk_rdev_t *rdev;
+} conf_t;
+
+static int check_mode(conf_t *conf, int mode)
+{
+        if (conf->period[mode] == 0 &&
+            atomic_read(&conf->counters[mode]) <= 0)
+                return 0; /* no failure, no decrement */
+
+
+        if (atomic_dec_and_test(&conf->counters[mode])) {
+                if (conf->period[mode])
+                        atomic_set(&conf->counters[mode], conf->period[mode]);
+                return 1;
+        }
+        return 0;
+}
+
+static int check_sector(conf_t *conf, sector_t start, sector_t end, int dir)
+{
+        /* If we find a ReadFixable sector, we fix it ... */
+        int i;
+        for (i=0; i<conf->nfaults; i++)
+                if (conf->faults[i] >= start &&
+                    conf->faults[i] < end) {
+                        /* found it ... */
+                        switch (conf->modes[i] * 2 + dir) {
+                        case WritePersistent*2+WRITE: return 1;
+                        case ReadPersistent*2+READ: return 1;
+                        case ReadFixable*2+READ: return 1;
+                        case ReadFixable*2+WRITE:
+                                conf->modes[i] = NoPersist;
+                                return 0;
+                        case AllPersist*2+READ:
+                        case AllPersist*2+WRITE: return 1;
+                        default:
+                                return 0;
+                        }
+                }
+        return 0;
+}
+
+static void add_sector(conf_t *conf, sector_t start, int mode)
+{
+        int i;
+        int n = conf->nfaults;
+        for (i=0; i<conf->nfaults; i++)
+                if (conf->faults[i] == start) {
+                        switch(mode) {
+                        case NoPersist: conf->modes[i] = mode; return;
+                        case WritePersistent:
+                                if (conf->modes[i] == ReadPersistent ||
+                                    conf->modes[i] == ReadFixable)
+                                        conf->modes[i] = AllPersist;
+                                else
+                                        conf->modes[i] = WritePersistent;
+                                return;
+                        case ReadPersistent:
+                                if (conf->modes[i] == WritePersistent)
+                                        conf->modes[i] = AllPersist;
+                                else
+                                        conf->modes[i] = ReadPersistent;
+                                return;
+                        case ReadFixable:
+                                if (conf->modes[i] == WritePersistent ||
+                                    conf->modes[i] == ReadPersistent)
+                                        conf->modes[i] = AllPersist;
+                                else
+                                        conf->modes[i] = ReadFixable;
+                                return;
+                        }
+                } else if (conf->modes[i] == NoPersist)
+                        n = i;
+
+        if (n >= MaxFault)
+                return;
+        conf->faults[n] = start;
+        conf->modes[n] = mode;
+        if (conf->nfaults == n)
+                conf->nfaults = n+1;
+}
+
+static int make_request(request_queue_t *q, struct bio *bio)
+{
+        mddev_t *mddev = q->queuedata;
+        conf_t *conf = (conf_t*)mddev->private;
+        int failit = 0;
+
+        if (bio->bi_rw & 1) {
+                /* write request */
+                if (atomic_read(&conf->counters[WriteAll])) {
+                        /* special case - don't decrement, don't generic_make_request,
+                         * just fail immediately
+                         */
+                        bio_endio(bio, bio->bi_size, -EIO);
+                        return 0;
+                }
+
+                if (check_sector(conf, bio->bi_sector, bio->bi_sector+(bio->bi_size>>9),
+                                 WRITE))
+                        failit = 1;
+                if (check_mode(conf, WritePersistent)) {
+                        add_sector(conf, bio->bi_sector, WritePersistent);
+                        failit = 1;
+                }
+                if (check_mode(conf, WriteTransient))
+                        failit = 1;
+        } else {
+                /* read request */
+                if (check_sector(conf, bio->bi_sector, bio->bi_sector + (bio->bi_size>>9),
+                                 READ))
+                        failit = 1;
+                if (check_mode(conf, ReadTransient))
+                        failit = 1;
+                if (check_mode(conf, ReadPersistent)) {
+                        add_sector(conf, bio->bi_sector, ReadPersistent);
+                        failit = 1;
+                }
+                if (check_mode(conf, ReadFixable)) {
+                        add_sector(conf, bio->bi_sector, ReadFixable);
+                        failit = 1;
+                }
+        }
+        if (failit) {
+                struct bio *b = bio_clone(bio, GFP_NOIO);
+                b->bi_bdev = conf->rdev->bdev;
+                b->bi_private = bio;
+                b->bi_end_io = faulty_fail;
+                generic_make_request(b);
+                return 0;
+        } else {
+                bio->bi_bdev = conf->rdev->bdev;
+                return 1;
+        }
+}
+
+static void status(struct seq_file *seq, mddev_t *mddev)
+{
+        conf_t *conf = (conf_t*)mddev->private;
+        int n;
+
+        if ((n=atomic_read(&conf->counters[WriteTransient])) != 0)
+                seq_printf(seq, " WriteTransient=%d(%d)",
+                           n, conf->period[WriteTransient]);
+
+        if ((n=atomic_read(&conf->counters[ReadTransient])) != 0)
+                seq_printf(seq, " ReadTransient=%d(%d)",
+                           n, conf->period[ReadTransient]);
+
+        if ((n=atomic_read(&conf->counters[WritePersistent])) != 0)
+                seq_printf(seq, " WritePersistent=%d(%d)",
+                           n, conf->period[WritePersistent]);
+
+        if ((n=atomic_read(&conf->counters[ReadPersistent])) != 0)
+                seq_printf(seq, " ReadPersistent=%d(%d)",
+                           n, conf->period[ReadPersistent]);
+
+
+        if ((n=atomic_read(&conf->counters[ReadFixable])) != 0)
+                seq_printf(seq, " ReadFixable=%d(%d)",
+                           n, conf->period[ReadFixable]);
+
+        if ((n=atomic_read(&conf->counters[WriteAll])) != 0)
+                seq_printf(seq, " WriteAll");
+
+        seq_printf(seq, " nfaults=%d", conf->nfaults);
+}
+
+
+static int reconfig(mddev_t *mddev, int layout, int chunk_size)
+{
+        int mode = layout & ModeMask;
+        int count = layout >> ModeShift;
+        conf_t *conf = mddev->private;
+
+        if (chunk_size != -1)
+                return -EINVAL;
+
+        /* new layout */
+        if (mode == ClearFaults)
+                conf->nfaults = 0;
+        else if (mode == ClearErrors) {
+                int i;
+                for (i=0 ; i < Modes ; i++) {
+                        conf->period[i] = 0;
+                        atomic_set(&conf->counters[i], 0);
+                }
+        } else if (mode < Modes) {
+                conf->period[mode] = count;
+                if (!count) count++;
+                atomic_set(&conf->counters[mode], count);
+        } else
+                return -EINVAL;
+        mddev->layout = -1; /* makes sure further changes come through */
+        return 0;
+}
+
+static int run(mddev_t *mddev)
+{
+        mdk_rdev_t *rdev;
+        struct list_head *tmp;
+        int i;
+
+        conf_t *conf = kmalloc(sizeof(*conf), GFP_KERNEL);
+
+        for (i=0; i<Modes; i++) {
+                atomic_set(&conf->counters[i], 0);
+                conf->period[i] = 0;
+        }
+        conf->nfaults = 0;
+
+        ITERATE_RDEV(mddev, rdev, tmp)
+                conf->rdev = rdev;
+
+        mddev->array_size = mddev->size;
+        mddev->private = conf;
+
+        reconfig(mddev, mddev->layout, -1);
+
+        return 0;
+}
+
+static int stop(mddev_t *mddev)
+{
+        conf_t *conf = (conf_t *)mddev->private;
+
+        kfree(conf);
+        mddev->private = NULL;
+        return 0;
+}
+
+static mdk_personality_t faulty_personality =
+{
+        .name           = "faulty",
+        .owner          = THIS_MODULE,
+        .make_request   = make_request,
+        .run            = run,
+        .stop           = stop,
+        .status         = status,
+        .reconfig       = reconfig,
+};
+
+static int __init raid_init(void)
+{
+        return register_md_personality(FAULTY, &faulty_personality);
+}
+
+static void raid_exit(void)
+{
+        unregister_md_personality(FAULTY);
+}
+
+module_init(raid_init);
+module_exit(raid_exit);
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("md-personality-10"); /* faulty */
diff --git a/drivers/md/kcopyd.c b/drivers/md/kcopyd.c
new file mode 100644
index 000000000000..eb7036485975
--- /dev/null
+++ b/drivers/md/kcopyd.c
@@ -0,0 +1,687 @@
+/*
+ * Copyright (C) 2002 Sistina Software (UK) Limited.
+ *
+ * This file is released under the GPL.
+ *
+ * Kcopyd provides a simple interface for copying an area of one
+ * block-device to one or more other block-devices, with an asynchronous
+ * completion notification.
+ */
+
+#include <asm/atomic.h>
+
+#include <linux/blkdev.h>
+#include <linux/config.h>
+#include <linux/fs.h>
+#include <linux/init.h>
+#include <linux/list.h>
+#include <linux/mempool.h>
+#include <linux/module.h>
+#include <linux/pagemap.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/workqueue.h>
+
+#include "kcopyd.h"
+
+static struct workqueue_struct *_kcopyd_wq;
+static struct work_struct _kcopyd_work;
+
+static inline void wake(void)
+{
+        queue_work(_kcopyd_wq, &_kcopyd_work);
+}
+
+/*-----------------------------------------------------------------
+ * Each kcopyd client has its own little pool of preallocated
+ * pages for kcopyd io.
+ *---------------------------------------------------------------*/
+struct kcopyd_client {
+        struct list_head list;
+
+        spinlock_t lock;
+        struct page_list *pages;
+        unsigned int nr_pages;
+        unsigned int nr_free_pages;
+};
+
+static struct page_list *alloc_pl(void)
+{
+        struct page_list *pl;
+
+        pl = kmalloc(sizeof(*pl), GFP_KERNEL);
+        if (!pl)
+                return NULL;
+
+        pl->page = alloc_page(GFP_KERNEL);
+        if (!pl->page) {
+                kfree(pl);
+                return NULL;
+        }
+
+        return pl;
+}
+
+static void free_pl(struct page_list *pl)
+{
+        __free_page(pl->page);
+        kfree(pl);
+}
+
+static int kcopyd_get_pages(struct kcopyd_client *kc,
+                            unsigned int nr, struct page_list **pages)
+{
+        struct page_list *pl;
+
+        spin_lock(&kc->lock);
+        if (kc->nr_free_pages < nr) {
+                spin_unlock(&kc->lock);
+                return -ENOMEM;
+        }
+
+        kc->nr_free_pages -= nr;
+        for (*pages = pl = kc->pages; --nr; pl = pl->next)
+                ;
+
+        kc->pages = pl->next;
+        pl->next = NULL;
+
+        spin_unlock(&kc->lock);
+
+        return 0;
+}
+
+static void kcopyd_put_pages(struct kcopyd_client *kc, struct page_list *pl)
+{
+        struct page_list *cursor;
+
+        spin_lock(&kc->lock);
+        for (cursor = pl; cursor->next; cursor = cursor->next)
+                kc->nr_free_pages++;
+
+        kc->nr_free_pages++;
+        cursor->next = kc->pages;
+        kc->pages = pl;
+        spin_unlock(&kc->lock);
+}
+
+/*
+ * These three functions resize the page pool.
+ */
+static void drop_pages(struct page_list *pl)
+{
+        struct page_list *next;
+
+        while (pl) {
+                next = pl->next;
+                free_pl(pl);
+                pl = next;
+        }
+}
+
+static int client_alloc_pages(struct kcopyd_client *kc, unsigned int nr)
+{
+        unsigned int i;
+        struct page_list *pl = NULL, *next;
+
+        for (i = 0; i < nr; i++) {
+                next = alloc_pl();
+                if (!next) {
+                        if (pl)
+                                drop_pages(pl);
+                        return -ENOMEM;
+                }
+                next->next = pl;
+                pl = next;
+        }
+
+        kcopyd_put_pages(kc, pl);
+        kc->nr_pages += nr;
+        return 0;
+}
+
+static void client_free_pages(struct kcopyd_client *kc)
+{
+        BUG_ON(kc->nr_free_pages != kc->nr_pages);
+        drop_pages(kc->pages);
+        kc->pages = NULL;
+        kc->nr_free_pages = kc->nr_pages = 0;
+}
+
+/*-----------------------------------------------------------------
+ * kcopyd_jobs need to be allocated by the *clients* of kcopyd,
+ * for this reason we use a mempool to prevent the client from
+ * ever having to do io (which could cause a deadlock).
+ *---------------------------------------------------------------*/
+struct kcopyd_job {
+        struct kcopyd_client *kc;
+        struct list_head list;
+        unsigned long flags;
+
+        /*
+         * Error state of the job.
+         */
+        int read_err;
+        unsigned int write_err;
+
+        /*
+         * Either READ or WRITE
+         */
+        int rw;
+        struct io_region source;
+
+        /*
+         * The destinations for the transfer.
+         */
+        unsigned int num_dests;
+        struct io_region dests[KCOPYD_MAX_REGIONS];
+
+        sector_t offset;
+        unsigned int nr_pages;
+        struct page_list *pages;
+
+        /*
+         * Set this to ensure you are notified when the job has
+         * completed.  'context' is for callback to use.
+         */
+        kcopyd_notify_fn fn;
+        void *context;
+
+        /*
+         * These fields are only used if the job has been split
+         * into more manageable parts.
+         */
+        struct semaphore lock;
+        atomic_t sub_jobs;
+        sector_t progress;
+};
+
+/* FIXME: this should scale with the number of pages */
+#define MIN_JOBS 512
+
+static kmem_cache_t *_job_cache;
+static mempool_t *_job_pool;
+
+/*
+ * We maintain three lists of jobs:
+ *
+ * i)   jobs waiting for pages
+ * ii)  jobs that have pages, and are waiting for the io to be issued.
+ * iii) jobs that have completed.
+ *
+ * All three of these are protected by job_lock.
+ */
+static DEFINE_SPINLOCK(_job_lock);
+
+static LIST_HEAD(_complete_jobs);
+static LIST_HEAD(_io_jobs);
+static LIST_HEAD(_pages_jobs);
+
+static int jobs_init(void)
+{
+        _job_cache = kmem_cache_create("kcopyd-jobs",
+                                       sizeof(struct kcopyd_job),
+                                       __alignof__(struct kcopyd_job),
+                                       0, NULL, NULL);
+        if (!_job_cache)
+                return -ENOMEM;
+
+        _job_pool = mempool_create(MIN_JOBS, mempool_alloc_slab,
+                                   mempool_free_slab, _job_cache);
+        if (!_job_pool) {
+                kmem_cache_destroy(_job_cache);
+                return -ENOMEM;
+        }
+
+        return 0;
+}
+
+static void jobs_exit(void)
+{
+        BUG_ON(!list_empty(&_complete_jobs));
+        BUG_ON(!list_empty(&_io_jobs));
+        BUG_ON(!list_empty(&_pages_jobs));
+
+        mempool_destroy(_job_pool);
+        kmem_cache_destroy(_job_cache);
+        _job_pool = NULL;
+        _job_cache = NULL;
+}
+
+/*
+ * Functions to push and pop a job onto the head of a given job
+ * list.
+ */
+static inline struct kcopyd_job *pop(struct list_head *jobs)
+{
+        struct kcopyd_job *job = NULL;
+        unsigned long flags;
+
+        spin_lock_irqsave(&_job_lock, flags);
+
+        if (!list_empty(jobs)) {
+                job = list_entry(jobs->next, struct kcopyd_job, list);
+                list_del(&job->list);
+        }
+        spin_unlock_irqrestore(&_job_lock, flags);
+
+        return job;
+}
+
+static inline void push(struct list_head *jobs, struct kcopyd_job *job)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&_job_lock, flags);
+        list_add_tail(&job->list, jobs);
+        spin_unlock_irqrestore(&_job_lock, flags);
+}
+
+/*
+ * These three functions process 1 item from the corresponding
+ * job list.
+ *
+ * They return:
+ * < 0: error
+ *   0: success
+ * > 0: can't process yet.
+ */
+static int run_complete_job(struct kcopyd_job *job)
+{
+        void *context = job->context;
+        int read_err = job->read_err;
+        unsigned int write_err = job->write_err;
+        kcopyd_notify_fn fn = job->fn;
+
+        kcopyd_put_pages(job->kc, job->pages);
+        mempool_free(job, _job_pool);
+        fn(read_err, write_err, context);
+        return 0;
+}
+
+static void complete_io(unsigned long error, void *context)
+{
+        struct kcopyd_job *job = (struct kcopyd_job *) context;
+
+        if (error) {
+                if (job->rw == WRITE)
+                        job->write_err &= error;
+                else
+                        job->read_err = 1;
+
+                if (!test_bit(KCOPYD_IGNORE_ERROR, &job->flags)) {
+                        push(&_complete_jobs, job);
+                        wake();
+                        return;
+                }
+        }
+
+        if (job->rw == WRITE)
+                push(&_complete_jobs, job);
+
+        else {
+                job->rw = WRITE;
+                push(&_io_jobs, job);
+        }
+
+        wake();
+}
+
+/*
+ * Request io on as many buffer heads as we can currently get for
+ * a particular job.
+ */
+static int run_io_job(struct kcopyd_job *job)
+{
+        int r;
+
+        if (job->rw == READ)
+                r = dm_io_async(1, &job->source, job->rw,
+                                job->pages,
+                                job->offset, complete_io, job);
+
+        else
+                r = dm_io_async(job->num_dests, job->dests, job->rw,
+                                job->pages,
+                                job->offset, complete_io, job);
+
+        return r;
+}
+
+static int run_pages_job(struct kcopyd_job *job)
+{
+        int r;
+
+        job->nr_pages = dm_div_up(job->dests[0].count + job->offset,
+                                  PAGE_SIZE >> 9);
+        r = kcopyd_get_pages(job->kc, job->nr_pages, &job->pages);
+        if (!r) {
+                /* this job is ready for io */
+                push(&_io_jobs, job);
+                return 0;
+        }
+
+        if (r == -ENOMEM)
+                /* can't complete now */
+                return 1;
+
+        return r;
+}
+
+/*
+ * Run through a list for as long as possible.  Returns the count
+ * of successful jobs.
+ */
+static int process_jobs(struct list_head *jobs, int (*fn) (struct kcopyd_job *))
+{
+        struct kcopyd_job *job;
+        int r, count = 0;
+
+        while ((job = pop(jobs))) {
+
+                r = fn(job);
+
+                if (r < 0) {
+                        /* error this rogue job */
+                        if (job->rw == WRITE)
+                                job->write_err = (unsigned int) -1;
+                        else
+                                job->read_err = 1;
+                        push(&_complete_jobs, job);
+                        break;
+                }
+
+                if (r > 0) {
+                        /*
+                         * We couldn't service this job ATM, so
+                         * push this job back onto the list.
+                         */
+                        push(jobs, job);
+                        break;
+                }
+
+                count++;
+        }
+
+        return count;
+}
+
+/*
+ * kcopyd does this every time it's woken up.
+ */
+static void do_work(void *ignored)
+{
+        /*
+         * The order that these are called is *very* important.
+         * complete jobs can free some pages for pages jobs.
+         * Pages jobs when successful will jump onto the io jobs
+         * list.  io jobs call wake when they complete and it all
+         * starts again.
+         */
+        process_jobs(&_complete_jobs, run_complete_job);
+        process_jobs(&_pages_jobs, run_pages_job);
+        process_jobs(&_io_jobs, run_io_job);
+}
+
+/*
+ * If we are copying a small region we just dispatch a single job
+ * to do the copy, otherwise the io has to be split up into many
+ * jobs.
+ */
+static void dispatch_job(struct kcopyd_job *job)
+{
+        push(&_pages_jobs, job);
+        wake();
+}
+
+#define SUB_JOB_SIZE 128
+static void segment_complete(int read_err,
+                             unsigned int write_err, void *context)
+{
+        /* FIXME: tidy this function */
+        sector_t progress = 0;
+        sector_t count = 0;
+        struct kcopyd_job *job = (struct kcopyd_job *) context;
+
+        down(&job->lock);
+
+        /* update the error */
+        if (read_err)
+                job->read_err = 1;
+
+        if (write_err)
+                job->write_err &= write_err;
+
+        /*
+         * Only dispatch more work if there hasn't been an error.
+         */
+        if ((!job->read_err && !job->write_err) ||
+            test_bit(KCOPYD_IGNORE_ERROR, &job->flags)) {
+                /* get the next chunk of work */
+                progress = job->progress;
+                count = job->source.count - progress;
+                if (count) {
+                        if (count > SUB_JOB_SIZE)
+                                count = SUB_JOB_SIZE;
+
+                        job->progress += count;
+                }
+        }
+        up(&job->lock);
+
+        if (count) {
+                int i;
+                struct kcopyd_job *sub_job = mempool_alloc(_job_pool, GFP_NOIO);
+
+                *sub_job = *job;
+                sub_job->source.sector += progress;
+                sub_job->source.count = count;
+
+                for (i = 0; i < job->num_dests; i++) {
+                        sub_job->dests[i].sector += progress;
+                        sub_job->dests[i].count = count;
+                }
+
+                sub_job->fn = segment_complete;
+                sub_job->context = job;
+                dispatch_job(sub_job);
+
+        } else if (atomic_dec_and_test(&job->sub_jobs)) {
+
+                /*
+                 * To avoid a race we must keep the job around
+                 * until after the notify function has completed.
+                 * Otherwise the client may try and stop the job
+                 * after we've completed.
+                 */
+                job->fn(read_err, write_err, job->context);
+                mempool_free(job, _job_pool);
+        }
+}
+
+/*
+ * Create some little jobs that will do the move between
+ * them.
+ */
+#define SPLIT_COUNT 8
+static void split_job(struct kcopyd_job *job)
+{
+        int i;
+
+        atomic_set(&job->sub_jobs, SPLIT_COUNT);
+        for (i = 0; i < SPLIT_COUNT; i++)
+                segment_complete(0, 0u, job);
+}
+
+int kcopyd_copy(struct kcopyd_client *kc, struct io_region *from,
+                unsigned int num_dests, struct io_region *dests,
+                unsigned int flags, kcopyd_notify_fn fn, void *context)
+{
+        struct kcopyd_job *job;
+
+        /*
+         * Allocate a new job.
+         */
+        job = mempool_alloc(_job_pool, GFP_NOIO);
+
+        /*
+         * set up for the read.
+         */
+        job->kc = kc;
+        job->flags = flags;
+        job->read_err = 0;
+        job->write_err = 0;
+        job->rw = READ;
+
+        job->source = *from;
+
+        job->num_dests = num_dests;
+        memcpy(&job->dests, dests, sizeof(*dests) * num_dests);
+
+        job->offset = 0;
+        job->nr_pages = 0;
+        job->pages = NULL;
+
+        job->fn = fn;
+        job->context = context;
+
+        if (job->source.count < SUB_JOB_SIZE)
+                dispatch_job(job);
+
+        else {
+                init_MUTEX(&job->lock);
+                job->progress = 0;
+                split_job(job);
+        }
+
+        return 0;
+}
+
+/*
+ * Cancels a kcopyd job, eg. someone might be deactivating a
+ * mirror.
+ */
+int kcopyd_cancel(struct kcopyd_job *job, int block)
+{
+        /* FIXME: finish */
+        return -1;
+}
+
+/*-----------------------------------------------------------------
+ * Unit setup
+ *---------------------------------------------------------------*/
+static DECLARE_MUTEX(_client_lock);
+static LIST_HEAD(_clients);
+
+static void client_add(struct kcopyd_client *kc)
+{
+        down(&_client_lock);
+        list_add(&kc->list, &_clients);
+        up(&_client_lock);
+}
+
+static void client_del(struct kcopyd_client *kc)
+{
+        down(&_client_lock);
+        list_del(&kc->list);
+        up(&_client_lock);
+}
+
+static DECLARE_MUTEX(kcopyd_init_lock);
+static int kcopyd_clients = 0;
+
+static int kcopyd_init(void)
+{
+        int r;
+
+        down(&kcopyd_init_lock);
+
+        if (kcopyd_clients) {
+                /* Already initialized. */
+                kcopyd_clients++;
+                up(&kcopyd_init_lock);
+                return 0;
+        }
+
+        r = jobs_init();
+        if (r) {
+                up(&kcopyd_init_lock);
+                return r;
+        }
+
+        _kcopyd_wq = create_singlethread_workqueue("kcopyd");
+        if (!_kcopyd_wq) {
+                jobs_exit();
+                up(&kcopyd_init_lock);
+                return -ENOMEM;
+        }
+
+        kcopyd_clients++;
+        INIT_WORK(&_kcopyd_work, do_work, NULL);
+        up(&kcopyd_init_lock);
+        return 0;
+}
+
+static void kcopyd_exit(void)
+{
+        down(&kcopyd_init_lock);
+        kcopyd_clients--;
+        if (!kcopyd_clients) {
+                jobs_exit();
+                destroy_workqueue(_kcopyd_wq);
+                _kcopyd_wq = NULL;
+        }
+        up(&kcopyd_init_lock);
+}
+
+int kcopyd_client_create(unsigned int nr_pages, struct kcopyd_client **result)
+{
+        int r = 0;
+        struct kcopyd_client *kc;
+
+        r = kcopyd_init();
+        if (r)
+                return r;
+
+        kc = kmalloc(sizeof(*kc), GFP_KERNEL);
+        if (!kc) {
+                kcopyd_exit();
+                return -ENOMEM;
+        }
+
+        spin_lock_init(&kc->lock);
+        kc->pages = NULL;
+        kc->nr_pages = kc->nr_free_pages = 0;
+        r = client_alloc_pages(kc, nr_pages);
+        if (r) {
+                kfree(kc);
+                kcopyd_exit();
+                return r;
+        }
+
+        r = dm_io_get(nr_pages);
+        if (r) {
+                client_free_pages(kc);
+                kfree(kc);
+                kcopyd_exit();
+                return r;
+        }
+
+        client_add(kc);
+        *result = kc;
+        return 0;
+}
+
+void kcopyd_client_destroy(struct kcopyd_client *kc)
+{
+        dm_io_put(kc->nr_pages);
+        client_free_pages(kc);
+        client_del(kc);
+        kfree(kc);
+        kcopyd_exit();
+}
+
+EXPORT_SYMBOL(kcopyd_client_create);
+EXPORT_SYMBOL(kcopyd_client_destroy);
+EXPORT_SYMBOL(kcopyd_copy);
+EXPORT_SYMBOL(kcopyd_cancel);
diff --git a/drivers/md/kcopyd.h b/drivers/md/kcopyd.h
new file mode 100644
index 000000000000..4621ea055c0e
--- /dev/null
+++ b/drivers/md/kcopyd.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2001 Sistina Software
+ *
+ * This file is released under the GPL.
+ *
+ * Kcopyd provides a simple interface for copying an area of one
+ * block-device to one or more other block-devices, with an asynchronous
+ * completion notification.
+ */
+
+#ifndef DM_KCOPYD_H
+#define DM_KCOPYD_H
+
+#include "dm-io.h"
+
+/* FIXME: make this configurable */
+#define KCOPYD_MAX_REGIONS 8
+
+#define KCOPYD_IGNORE_ERROR 1
+
+/*
+ * To use kcopyd you must first create a kcopyd client object.
+ */
+struct kcopyd_client;
+int kcopyd_client_create(unsigned int num_pages, struct kcopyd_client **result);
+void kcopyd_client_destroy(struct kcopyd_client *kc);
+
+/*
+ * Submit a copy job to kcopyd.  This is built on top of the
+ * previous three fns.
+ *
+ * read_err is a boolean,
+ * write_err is a bitset, with 1 bit for each destination region
+ */
+typedef void (*kcopyd_notify_fn)(int read_err,
+                                 unsigned int write_err, void *context);
+
+int kcopyd_copy(struct kcopyd_client *kc, struct io_region *from,
+                unsigned int num_dests, struct io_region *dests,
+                unsigned int flags, kcopyd_notify_fn fn, void *context);
+
+#endif
diff --git a/drivers/md/linear.c b/drivers/md/linear.c
new file mode 100644
index 000000000000..161e9aa87291
--- /dev/null
+++ b/drivers/md/linear.c
@@ -0,0 +1,343 @@
+/*
+   linear.c : Multiple Devices driver for Linux
+              Copyright (C) 1994-96 Marc ZYNGIER
+              <zyngier@ufr-info-p7.ibp.fr> or
+              <maz@gloups.fdn.fr>
+
+   Linear mode management functions.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+   
+   You should have received a copy of the GNU General Public License
+   (for example /usr/src/linux/COPYING); if not, write to the Free
+   Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  
+*/
+
+#include <linux/module.h>
+
+#include <linux/raid/md.h>
+#include <linux/slab.h>
+#include <linux/raid/linear.h>
+
+#define MAJOR_NR MD_MAJOR
+#define MD_DRIVER
+#define MD_PERSONALITY
+
+/*
+ * find which device holds a particular offset 
+ */
+static inline dev_info_t *which_dev(mddev_t *mddev, sector_t sector)
+{
+        dev_info_t *hash;
+        linear_conf_t *conf = mddev_to_conf(mddev);
+        sector_t block = sector >> 1;
+
+        /*
+         * sector_div(a,b) returns the remainer and sets a to a/b
+         */
+        (void)sector_div(block, conf->smallest->size);
+        hash = conf->hash_table[block];
+
+        while ((sector>>1) >= (hash->size + hash->offset))
+                hash++;
+        return hash;
+}
+
+/**
+ *      linear_mergeable_bvec -- tell bio layer if a two requests can be merged
+ *      @q: request queue
+ *      @bio: the buffer head that's been built up so far
+ *      @biovec: the request that could be merged to it.
+ *
+ *      Return amount of bytes we can take at this offset
+ */
+static int linear_mergeable_bvec(request_queue_t *q, struct bio *bio, struct bio_vec *biovec)
+{
+        mddev_t *mddev = q->queuedata;
+        dev_info_t *dev0;
+        unsigned long maxsectors, bio_sectors = bio->bi_size >> 9;
+        sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev);
+
+        dev0 = which_dev(mddev, sector);
+        maxsectors = (dev0->size << 1) - (sector - (dev0->offset<<1));
+
+        if (maxsectors < bio_sectors)
+                maxsectors = 0;
+        else
+                maxsectors -= bio_sectors;
+
+        if (maxsectors <= (PAGE_SIZE >> 9 ) && bio_sectors == 0)
+                return biovec->bv_len;
+        /* The bytes available at this offset could be really big,
+         * so we cap at 2^31 to avoid overflow */
+        if (maxsectors > (1 << (31-9)))
+                return 1<<31;
+        return maxsectors << 9;
+}
+
+static void linear_unplug(request_queue_t *q)
+{
+        mddev_t *mddev = q->queuedata;
+        linear_conf_t *conf = mddev_to_conf(mddev);
+        int i;
+
+        for (i=0; i < mddev->raid_disks; i++) {
+                request_queue_t *r_queue = bdev_get_queue(conf->disks[i].rdev->bdev);
+                if (r_queue->unplug_fn)
+                        r_queue->unplug_fn(r_queue);
+        }
+}
+
+static int linear_issue_flush(request_queue_t *q, struct gendisk *disk,
+                              sector_t *error_sector)
+{
+        mddev_t *mddev = q->queuedata;
+        linear_conf_t *conf = mddev_to_conf(mddev);
+        int i, ret = 0;
+
+        for (i=0; i < mddev->raid_disks && ret == 0; i++) {
+                struct block_device *bdev = conf->disks[i].rdev->bdev;
+                request_queue_t *r_queue = bdev_get_queue(bdev);
+
+                if (!r_queue->issue_flush_fn)
+                        ret = -EOPNOTSUPP;
+                else
+                        ret = r_queue->issue_flush_fn(r_queue, bdev->bd_disk, error_sector);
+        }
+        return ret;
+}
+
+static int linear_run (mddev_t *mddev)
+{
+        linear_conf_t *conf;
+        dev_info_t **table;
+        mdk_rdev_t *rdev;
+        int i, nb_zone, cnt;
+        sector_t start;
+        sector_t curr_offset;
+        struct list_head *tmp;
+
+        conf = kmalloc (sizeof (*conf) + mddev->raid_disks*sizeof(dev_info_t),
+                        GFP_KERNEL);
+        if (!conf)
+                goto out;
+        memset(conf, 0, sizeof(*conf) + mddev->raid_disks*sizeof(dev_info_t));
+        mddev->private = conf;
+
+        /*
+         * Find the smallest device.
+         */
+
+        conf->smallest = NULL;
+        cnt = 0;
+        mddev->array_size = 0;
+
+        ITERATE_RDEV(mddev,rdev,tmp) {
+                int j = rdev->raid_disk;
+                dev_info_t *disk = conf->disks + j;
+
+                if (j < 0 || j > mddev->raid_disks || disk->rdev) {
+                        printk("linear: disk numbering problem. Aborting!\n");
+                        goto out;
+                }
+
+                disk->rdev = rdev;
+
+                blk_queue_stack_limits(mddev->queue,
+                                       rdev->bdev->bd_disk->queue);
+                /* as we don't honour merge_bvec_fn, we must never risk
+                 * violating it, so limit ->max_sector to one PAGE, as
+                 * a one page request is never in violation.
+                 */
+                if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
+                    mddev->queue->max_sectors > (PAGE_SIZE>>9))
+                        blk_queue_max_sectors(mddev->queue, PAGE_SIZE>>9);
+
+                disk->size = rdev->size;
+                mddev->array_size += rdev->size;
+
+                if (!conf->smallest || (disk->size < conf->smallest->size))
+                        conf->smallest = disk;
+                cnt++;
+        }
+        if (cnt != mddev->raid_disks) {
+                printk("linear: not enough drives present. Aborting!\n");
+                goto out;
+        }
+
+        /*
+         * This code was restructured to work around a gcc-2.95.3 internal
+         * compiler error.  Alter it with care.
+         */
+        {
+                sector_t sz;
+                unsigned round;
+                unsigned long base;
+
+                sz = mddev->array_size;
+                base = conf->smallest->size;
+                round = sector_div(sz, base);
+                nb_zone = conf->nr_zones = sz + (round ? 1 : 0);
+        }
+                        
+        conf->hash_table = kmalloc (sizeof (dev_info_t*) * nb_zone,
+                                        GFP_KERNEL);
+        if (!conf->hash_table)
+                goto out;
+
+        /*
+         * Here we generate the linear hash table
+         */
+        table = conf->hash_table;
+        start = 0;
+        curr_offset = 0;
+        for (i = 0; i < cnt; i++) {
+                dev_info_t *disk = conf->disks + i;
+
+                disk->offset = curr_offset;
+                curr_offset += disk->size;
+
+                /* 'curr_offset' is the end of this disk
+                 * 'start' is the start of table
+                 */
+                while (start < curr_offset) {
+                        *table++ = disk;
+                        start += conf->smallest->size;
+                }
+        }
+        if (table-conf->hash_table != nb_zone)
+                BUG();
+
+        blk_queue_merge_bvec(mddev->queue, linear_mergeable_bvec);
+        mddev->queue->unplug_fn = linear_unplug;
+        mddev->queue->issue_flush_fn = linear_issue_flush;
+        return 0;
+
+out:
+        if (conf)
+                kfree(conf);
+        return 1;
+}
+
+static int linear_stop (mddev_t *mddev)
+{
+        linear_conf_t *conf = mddev_to_conf(mddev);
+  
+        blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
+        kfree(conf->hash_table);
+        kfree(conf);
+
+        return 0;
+}
+
+static int linear_make_request (request_queue_t *q, struct bio *bio)
+{
+        mddev_t *mddev = q->queuedata;
+        dev_info_t *tmp_dev;
+        sector_t block;
+
+        if (bio_data_dir(bio)==WRITE) {
+                disk_stat_inc(mddev->gendisk, writes);
+                disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bio));
+        } else {
+                disk_stat_inc(mddev->gendisk, reads);
+                disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bio));
+        }
+
+        tmp_dev = which_dev(mddev, bio->bi_sector);
+        block = bio->bi_sector >> 1;
+    
+        if (unlikely(block >= (tmp_dev->size + tmp_dev->offset)
+                     || block < tmp_dev->offset)) {
+                char b[BDEVNAME_SIZE];
+
+                printk("linear_make_request: Block %llu out of bounds on "
+                        "dev %s size %llu offset %llu\n",
+                        (unsigned long long)block,
+                        bdevname(tmp_dev->rdev->bdev, b),
+                        (unsigned long long)tmp_dev->size,
+                        (unsigned long long)tmp_dev->offset);
+                bio_io_error(bio, bio->bi_size);
+                return 0;
+        }
+        if (unlikely(bio->bi_sector + (bio->bi_size >> 9) >
+                     (tmp_dev->offset + tmp_dev->size)<<1)) {
+                /* This bio crosses a device boundary, so we have to
+                 * split it.
+                 */
+                struct bio_pair *bp;
+                bp = bio_split(bio, bio_split_pool, 
+                               (bio->bi_sector + (bio->bi_size >> 9) -
+                                (tmp_dev->offset + tmp_dev->size))<<1);
+                if (linear_make_request(q, &bp->bio1))
+                        generic_make_request(&bp->bio1);
+                if (linear_make_request(q, &bp->bio2))
+                        generic_make_request(&bp->bio2);
+                bio_pair_release(bp);
+                return 0;
+        }
+                    
+        bio->bi_bdev = tmp_dev->rdev->bdev;
+        bio->bi_sector = bio->bi_sector - (tmp_dev->offset << 1) + tmp_dev->rdev->data_offset;
+
+        return 1;
+}
+
+static void linear_status (struct seq_file *seq, mddev_t *mddev)
+{
+
+#undef MD_DEBUG
+#ifdef MD_DEBUG
+        int j;
+        linear_conf_t *conf = mddev_to_conf(mddev);
+        sector_t s = 0;
+  
+        seq_printf(seq, "      ");
+        for (j = 0; j < conf->nr_zones; j++)
+        {
+                char b[BDEVNAME_SIZE];
+                s += conf->smallest_size;
+                seq_printf(seq, "[%s",
+                           bdevname(conf->hash_table[j][0].rdev->bdev,b));
+
+                while (s > conf->hash_table[j][0].offset +
+                           conf->hash_table[j][0].size)
+                        seq_printf(seq, "/%s] ",
+                                   bdevname(conf->hash_table[j][1].rdev->bdev,b));
+                else
+                        seq_printf(seq, "] ");
+        }
+        seq_printf(seq, "\n");
+#endif
+        seq_printf(seq, " %dk rounding", mddev->chunk_size/1024);
+}
+
+
+static mdk_personality_t linear_personality=
+{
+        .name           = "linear",
+        .owner          = THIS_MODULE,
+        .make_request   = linear_make_request,
+        .run            = linear_run,
+        .stop           = linear_stop,
+        .status         = linear_status,
+};
+
+static int __init linear_init (void)
+{
+        return register_md_personality (LINEAR, &linear_personality);
+}
+
+static void linear_exit (void)
+{
+        unregister_md_personality (LINEAR);
+}
+
+
+module_init(linear_init);
+module_exit(linear_exit);
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("md-personality-1"); /* LINEAR */
diff --git a/drivers/md/md.c b/drivers/md/md.c
new file mode 100644
index 000000000000..04562add1920
--- /dev/null
+++ b/drivers/md/md.c
@@ -0,0 +1,3766 @@
+/*
+   md.c : Multiple Devices driver for Linux
+          Copyright (C) 1998, 1999, 2000 Ingo Molnar
+
+     completely rewritten, based on the MD driver code from Marc Zyngier
+
+   Changes:
+
+   - RAID-1/RAID-5 extensions by Miguel de Icaza, Gadi Oxman, Ingo Molnar
+   - RAID-6 extensions by H. Peter Anvin <hpa@zytor.com>
+   - boot support for linear and striped mode by Harald Hoyer <HarryH@Royal.Net>
+   - kerneld support by Boris Tobotras <boris@xtalk.msk.su>
+   - kmod support by: Cyrus Durgin
+   - RAID0 bugfixes: Mark Anthony Lisher <markal@iname.com>
+   - Devfs support by Richard Gooch <rgooch@atnf.csiro.au>
+
+   - lots of fixes and improvements to the RAID1/RAID5 and generic
+     RAID code (such as request based resynchronization):
+
+     Neil Brown <neilb@cse.unsw.edu.au>.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+
+   You should have received a copy of the GNU General Public License
+   (for example /usr/src/linux/COPYING); if not, write to the Free
+   Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+*/
+
+#include <linux/module.h>
+#include <linux/config.h>
+#include <linux/linkage.h>
+#include <linux/raid/md.h>
+#include <linux/sysctl.h>
+#include <linux/devfs_fs_kernel.h>
+#include <linux/buffer_head.h> /* for invalidate_bdev */
+#include <linux/suspend.h>
+
+#include <linux/init.h>
+
+#ifdef CONFIG_KMOD
+#include <linux/kmod.h>
+#endif
+
+#include <asm/unaligned.h>
+
+#define MAJOR_NR MD_MAJOR
+#define MD_DRIVER
+
+/* 63 partitions with the alternate major number (mdp) */
+#define MdpMinorShift 6
+
+#define DEBUG 0
+#define dprintk(x...) ((void)(DEBUG && printk(x)))
+
+
+#ifndef MODULE
+static void autostart_arrays (int part);
+#endif
+
+static mdk_personality_t *pers[MAX_PERSONALITY];
+static DEFINE_SPINLOCK(pers_lock);
+
+/*
+ * Current RAID-1,4,5 parallel reconstruction 'guaranteed speed limit'
+ * is 1000 KB/sec, so the extra system load does not show up that much.
+ * Increase it if you want to have more _guaranteed_ speed. Note that
+ * the RAID driver will use the maximum available bandwith if the IO
+ * subsystem is idle. There is also an 'absolute maximum' reconstruction
+ * speed limit - in case reconstruction slows down your system despite
+ * idle IO detection.
+ *
+ * you can change it via /proc/sys/dev/raid/speed_limit_min and _max.
+ */
+
+static int sysctl_speed_limit_min = 1000;
+static int sysctl_speed_limit_max = 200000;
+
+static struct ctl_table_header *raid_table_header;
+
+static ctl_table raid_table[] = {
+        {
+                .ctl_name       = DEV_RAID_SPEED_LIMIT_MIN,
+                .procname       = "speed_limit_min",
+                .data           = &sysctl_speed_limit_min,
+                .maxlen         = sizeof(int),
+                .mode           = 0644,
+                .proc_handler   = &proc_dointvec,
+        },
+        {
+                .ctl_name       = DEV_RAID_SPEED_LIMIT_MAX,
+                .procname       = "speed_limit_max",
+                .data           = &sysctl_speed_limit_max,
+                .maxlen         = sizeof(int),
+                .mode           = 0644,
+                .proc_handler   = &proc_dointvec,
+        },
+        { .ctl_name = 0 }
+};
+
+static ctl_table raid_dir_table[] = {
+        {
+                .ctl_name       = DEV_RAID,
+                .procname       = "raid",
+                .maxlen         = 0,
+                .mode           = 0555,
+                .child          = raid_table,
+        },
+        { .ctl_name = 0 }
+};
+
+static ctl_table raid_root_table[] = {
+        {
+                .ctl_name       = CTL_DEV,
+                .procname       = "dev",
+                .maxlen         = 0,
+                .mode           = 0555,
+                .child          = raid_dir_table,
+        },
+        { .ctl_name = 0 }
+};
+
+static struct block_device_operations md_fops;
+
+/*
+ * Enables to iterate over all existing md arrays
+ * all_mddevs_lock protects this list.
+ */
+static LIST_HEAD(all_mddevs);
+static DEFINE_SPINLOCK(all_mddevs_lock);
+
+
+/*
+ * iterates through all used mddevs in the system.
+ * We take care to grab the all_mddevs_lock whenever navigating
+ * the list, and to always hold a refcount when unlocked.
+ * Any code which breaks out of this loop while own
+ * a reference to the current mddev and must mddev_put it.
+ */
+#define ITERATE_MDDEV(mddev,tmp)                                        \
+                                                                        \
+        for (({ spin_lock(&all_mddevs_lock);                            \
+                tmp = all_mddevs.next;                                  \
+                mddev = NULL;});                                        \
+             ({ if (tmp != &all_mddevs)                                 \
+                        mddev_get(list_entry(tmp, mddev_t, all_mddevs));\
+                spin_unlock(&all_mddevs_lock);                          \
+                if (mddev) mddev_put(mddev);                            \
+                mddev = list_entry(tmp, mddev_t, all_mddevs);           \
+                tmp != &all_mddevs;});                                  \
+             ({ spin_lock(&all_mddevs_lock);                            \
+                tmp = tmp->next;})                                      \
+                )
+
+
+static int md_fail_request (request_queue_t *q, struct bio *bio)
+{
+        bio_io_error(bio, bio->bi_size);
+        return 0;
+}
+
+static inline mddev_t *mddev_get(mddev_t *mddev)
+{
+        atomic_inc(&mddev->active);
+        return mddev;
+}
+
+static void mddev_put(mddev_t *mddev)
+{
+        if (!atomic_dec_and_lock(&mddev->active, &all_mddevs_lock))
+                return;
+        if (!mddev->raid_disks && list_empty(&mddev->disks)) {
+                list_del(&mddev->all_mddevs);
+                blk_put_queue(mddev->queue);
+                kfree(mddev);
+        }
+        spin_unlock(&all_mddevs_lock);
+}
+
+static mddev_t * mddev_find(dev_t unit)
+{
+        mddev_t *mddev, *new = NULL;
+
+ retry:
+        spin_lock(&all_mddevs_lock);
+        list_for_each_entry(mddev, &all_mddevs, all_mddevs)
+                if (mddev->unit == unit) {
+                        mddev_get(mddev);
+                        spin_unlock(&all_mddevs_lock);
+                        if (new)
+                                kfree(new);
+                        return mddev;
+                }
+
+        if (new) {
+                list_add(&new->all_mddevs, &all_mddevs);
+                spin_unlock(&all_mddevs_lock);
+                return new;
+        }
+        spin_unlock(&all_mddevs_lock);
+
+        new = (mddev_t *) kmalloc(sizeof(*new), GFP_KERNEL);
+        if (!new)
+                return NULL;
+
+        memset(new, 0, sizeof(*new));
+
+        new->unit = unit;
+        if (MAJOR(unit) == MD_MAJOR)
+                new->md_minor = MINOR(unit);
+        else
+                new->md_minor = MINOR(unit) >> MdpMinorShift;
+
+        init_MUTEX(&new->reconfig_sem);
+        INIT_LIST_HEAD(&new->disks);
+        INIT_LIST_HEAD(&new->all_mddevs);
+        init_timer(&new->safemode_timer);
+        atomic_set(&new->active, 1);
+
+        new->queue = blk_alloc_queue(GFP_KERNEL);
+        if (!new->queue) {
+                kfree(new);
+                return NULL;
+        }
+
+        blk_queue_make_request(new->queue, md_fail_request);
+
+        goto retry;
+}
+
+static inline int mddev_lock(mddev_t * mddev)
+{
+        return down_interruptible(&mddev->reconfig_sem);
+}
+
+static inline void mddev_lock_uninterruptible(mddev_t * mddev)
+{
+        down(&mddev->reconfig_sem);
+}
+
+static inline int mddev_trylock(mddev_t * mddev)
+{
+        return down_trylock(&mddev->reconfig_sem);
+}
+
+static inline void mddev_unlock(mddev_t * mddev)
+{
+        up(&mddev->reconfig_sem);
+
+        if (mddev->thread)
+                md_wakeup_thread(mddev->thread);
+}
+
+mdk_rdev_t * find_rdev_nr(mddev_t *mddev, int nr)
+{
+        mdk_rdev_t * rdev;
+        struct list_head *tmp;
+
+        ITERATE_RDEV(mddev,rdev,tmp) {
+                if (rdev->desc_nr == nr)
+                        return rdev;
+        }
+        return NULL;
+}
+
+static mdk_rdev_t * find_rdev(mddev_t * mddev, dev_t dev)
+{
+        struct list_head *tmp;
+        mdk_rdev_t *rdev;
+
+        ITERATE_RDEV(mddev,rdev,tmp) {
+                if (rdev->bdev->bd_dev == dev)
+                        return rdev;
+        }
+        return NULL;
+}
+
+inline static sector_t calc_dev_sboffset(struct block_device *bdev)
+{
+        sector_t size = bdev->bd_inode->i_size >> BLOCK_SIZE_BITS;
+        return MD_NEW_SIZE_BLOCKS(size);
+}
+
+static sector_t calc_dev_size(mdk_rdev_t *rdev, unsigned chunk_size)
+{
+        sector_t size;
+
+        size = rdev->sb_offset;
+
+        if (chunk_size)
+                size &= ~((sector_t)chunk_size/1024 - 1);
+        return size;
+}
+
+static int alloc_disk_sb(mdk_rdev_t * rdev)
+{
+        if (rdev->sb_page)
+                MD_BUG();
+
+        rdev->sb_page = alloc_page(GFP_KERNEL);
+        if (!rdev->sb_page) {
+                printk(KERN_ALERT "md: out of memory.\n");
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+static void free_disk_sb(mdk_rdev_t * rdev)
+{
+        if (rdev->sb_page) {
+                page_cache_release(rdev->sb_page);
+                rdev->sb_loaded = 0;
+                rdev->sb_page = NULL;
+                rdev->sb_offset = 0;
+                rdev->size = 0;
+        }
+}
+
+
+static int bi_complete(struct bio *bio, unsigned int bytes_done, int error)
+{
+        if (bio->bi_size)
+                return 1;
+
+        complete((struct completion*)bio->bi_private);
+        return 0;
+}
+
+static int sync_page_io(struct block_device *bdev, sector_t sector, int size,
+                   struct page *page, int rw)
+{
+        struct bio *bio = bio_alloc(GFP_KERNEL, 1);
+        struct completion event;
+        int ret;
+
+        rw |= (1 << BIO_RW_SYNC);
+
+        bio->bi_bdev = bdev;
+        bio->bi_sector = sector;
+        bio_add_page(bio, page, size, 0);
+        init_completion(&event);
+        bio->bi_private = &event;
+        bio->bi_end_io = bi_complete;
+        submit_bio(rw, bio);
+        wait_for_completion(&event);
+
+        ret = test_bit(BIO_UPTODATE, &bio->bi_flags);
+        bio_put(bio);
+        return ret;
+}
+
+static int read_disk_sb(mdk_rdev_t * rdev)
+{
+        char b[BDEVNAME_SIZE];
+        if (!rdev->sb_page) {
+                MD_BUG();
+                return -EINVAL;
+        }
+        if (rdev->sb_loaded)
+                return 0;
+
+
+        if (!sync_page_io(rdev->bdev, rdev->sb_offset<<1, MD_SB_BYTES, rdev->sb_page, READ))
+                goto fail;
+        rdev->sb_loaded = 1;
+        return 0;
+
+fail:
+        printk(KERN_WARNING "md: disabled device %s, could not read superblock.\n",
+                bdevname(rdev->bdev,b));
+        return -EINVAL;
+}
+
+static int uuid_equal(mdp_super_t *sb1, mdp_super_t *sb2)
+{
+        if (    (sb1->set_uuid0 == sb2->set_uuid0) &&
+                (sb1->set_uuid1 == sb2->set_uuid1) &&
+                (sb1->set_uuid2 == sb2->set_uuid2) &&
+                (sb1->set_uuid3 == sb2->set_uuid3))
+
+                return 1;
+
+        return 0;
+}
+
+
+static int sb_equal(mdp_super_t *sb1, mdp_super_t *sb2)
+{
+        int ret;
+        mdp_super_t *tmp1, *tmp2;
+
+        tmp1 = kmalloc(sizeof(*tmp1),GFP_KERNEL);
+        tmp2 = kmalloc(sizeof(*tmp2),GFP_KERNEL);
+
+        if (!tmp1 || !tmp2) {
+                ret = 0;
+                printk(KERN_INFO "md.c: sb1 is not equal to sb2!\n");
+                goto abort;
+        }
+
+        *tmp1 = *sb1;
+        *tmp2 = *sb2;
+
+        /*
+         * nr_disks is not constant
+         */
+        tmp1->nr_disks = 0;
+        tmp2->nr_disks = 0;
+
+        if (memcmp(tmp1, tmp2, MD_SB_GENERIC_CONSTANT_WORDS * 4))
+                ret = 0;
+        else
+                ret = 1;
+
+abort:
+        if (tmp1)
+                kfree(tmp1);
+        if (tmp2)
+                kfree(tmp2);
+
+        return ret;
+}
+
+static unsigned int calc_sb_csum(mdp_super_t * sb)
+{
+        unsigned int disk_csum, csum;
+
+        disk_csum = sb->sb_csum;
+        sb->sb_csum = 0;
+        csum = csum_partial((void *)sb, MD_SB_BYTES, 0);
+        sb->sb_csum = disk_csum;
+        return csum;
+}
+
+
+/*
+ * Handle superblock details.
+ * We want to be able to handle multiple superblock formats
+ * so we have a common interface to them all, and an array of
+ * different handlers.
+ * We rely on user-space to write the initial superblock, and support
+ * reading and updating of superblocks.
+ * Interface methods are:
+ *   int load_super(mdk_rdev_t *dev, mdk_rdev_t *refdev, int minor_version)
+ *      loads and validates a superblock on dev.
+ *      if refdev != NULL, compare superblocks on both devices
+ *    Return:
+ *      0 - dev has a superblock that is compatible with refdev
+ *      1 - dev has a superblock that is compatible and newer than refdev
+ *          so dev should be used as the refdev in future
+ *     -EINVAL superblock incompatible or invalid
+ *     -othererror e.g. -EIO
+ *
+ *   int validate_super(mddev_t *mddev, mdk_rdev_t *dev)
+ *      Verify that dev is acceptable into mddev.
+ *       The first time, mddev->raid_disks will be 0, and data from
+ *       dev should be merged in.  Subsequent calls check that dev
+ *       is new enough.  Return 0 or -EINVAL
+ *
+ *   void sync_super(mddev_t *mddev, mdk_rdev_t *dev)
+ *     Update the superblock for rdev with data in mddev
+ *     This does not write to disc.
+ *
+ */
+
+struct super_type  {
+        char            *name;
+        struct module   *owner;
+        int             (*load_super)(mdk_rdev_t *rdev, mdk_rdev_t *refdev, int minor_version);
+        int             (*validate_super)(mddev_t *mddev, mdk_rdev_t *rdev);
+        void            (*sync_super)(mddev_t *mddev, mdk_rdev_t *rdev);
+};
+
+/*
+ * load_super for 0.90.0 
+ */
+static int super_90_load(mdk_rdev_t *rdev, mdk_rdev_t *refdev, int minor_version)
+{
+        char b[BDEVNAME_SIZE], b2[BDEVNAME_SIZE];
+        mdp_super_t *sb;
+        int ret;
+        sector_t sb_offset;
+
+        /*
+         * Calculate the position of the superblock,
+         * it's at the end of the disk.
+         *
+         * It also happens to be a multiple of 4Kb.
+         */
+        sb_offset = calc_dev_sboffset(rdev->bdev);
+        rdev->sb_offset = sb_offset;
+
+        ret = read_disk_sb(rdev);
+        if (ret) return ret;
+
+        ret = -EINVAL;
+
+        bdevname(rdev->bdev, b);
+        sb = (mdp_super_t*)page_address(rdev->sb_page);
+
+        if (sb->md_magic != MD_SB_MAGIC) {
+                printk(KERN_ERR "md: invalid raid superblock magic on %s\n",
+                       b);
+                goto abort;
+        }
+
+        if (sb->major_version != 0 ||
+            sb->minor_version != 90) {
+                printk(KERN_WARNING "Bad version number %d.%d on %s\n",
+                        sb->major_version, sb->minor_version,
+                        b);
+                goto abort;
+        }
+
+        if (sb->raid_disks <= 0)
+                goto abort;
+
+        if (csum_fold(calc_sb_csum(sb)) != csum_fold(sb->sb_csum)) {
+                printk(KERN_WARNING "md: invalid superblock checksum on %s\n",
+                        b);
+                goto abort;
+        }
+
+        rdev->preferred_minor = sb->md_minor;
+        rdev->data_offset = 0;
+
+        if (sb->level == LEVEL_MULTIPATH)
+                rdev->desc_nr = -1;
+        else
+                rdev->desc_nr = sb->this_disk.number;
+
+        if (refdev == 0)
+                ret = 1;
+        else {
+                __u64 ev1, ev2;
+                mdp_super_t *refsb = (mdp_super_t*)page_address(refdev->sb_page);
+                if (!uuid_equal(refsb, sb)) {
+                        printk(KERN_WARNING "md: %s has different UUID to %s\n",
+                                b, bdevname(refdev->bdev,b2));
+                        goto abort;
+                }
+                if (!sb_equal(refsb, sb)) {
+                        printk(KERN_WARNING "md: %s has same UUID"
+                               " but different superblock to %s\n",
+                               b, bdevname(refdev->bdev, b2));
+                        goto abort;
+                }
+                ev1 = md_event(sb);
+                ev2 = md_event(refsb);
+                if (ev1 > ev2)
+                        ret = 1;
+                else 
+                        ret = 0;
+        }
+        rdev->size = calc_dev_size(rdev, sb->chunk_size);
+
+ abort:
+        return ret;
+}
+
+/*
+ * validate_super for 0.90.0
+ */
+static int super_90_validate(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+        mdp_disk_t *desc;
+        mdp_super_t *sb = (mdp_super_t *)page_address(rdev->sb_page);
+
+        if (mddev->raid_disks == 0) {
+                mddev->major_version = 0;
+                mddev->minor_version = sb->minor_version;
+                mddev->patch_version = sb->patch_version;
+                mddev->persistent = ! sb->not_persistent;
+                mddev->chunk_size = sb->chunk_size;
+                mddev->ctime = sb->ctime;
+                mddev->utime = sb->utime;
+                mddev->level = sb->level;
+                mddev->layout = sb->layout;
+                mddev->raid_disks = sb->raid_disks;
+                mddev->size = sb->size;
+                mddev->events = md_event(sb);
+
+                if (sb->state & (1<<MD_SB_CLEAN))
+                        mddev->recovery_cp = MaxSector;
+                else {
+                        if (sb->events_hi == sb->cp_events_hi && 
+                                sb->events_lo == sb->cp_events_lo) {
+                                mddev->recovery_cp = sb->recovery_cp;
+                        } else
+                                mddev->recovery_cp = 0;
+                }
+
+                memcpy(mddev->uuid+0, &sb->set_uuid0, 4);
+                memcpy(mddev->uuid+4, &sb->set_uuid1, 4);
+                memcpy(mddev->uuid+8, &sb->set_uuid2, 4);
+                memcpy(mddev->uuid+12,&sb->set_uuid3, 4);
+
+                mddev->max_disks = MD_SB_DISKS;
+        } else {
+                __u64 ev1;
+                ev1 = md_event(sb);
+                ++ev1;
+                if (ev1 < mddev->events) 
+                        return -EINVAL;
+        }
+        if (mddev->level != LEVEL_MULTIPATH) {
+                rdev->raid_disk = -1;
+                rdev->in_sync = rdev->faulty = 0;
+                desc = sb->disks + rdev->desc_nr;
+
+                if (desc->state & (1<<MD_DISK_FAULTY))
+                        rdev->faulty = 1;
+                else if (desc->state & (1<<MD_DISK_SYNC) &&
+                         desc->raid_disk < mddev->raid_disks) {
+                        rdev->in_sync = 1;
+                        rdev->raid_disk = desc->raid_disk;
+                }
+        }
+        return 0;
+}
+
+/*
+ * sync_super for 0.90.0
+ */
+static void super_90_sync(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+        mdp_super_t *sb;
+        struct list_head *tmp;
+        mdk_rdev_t *rdev2;
+        int next_spare = mddev->raid_disks;
+
+        /* make rdev->sb match mddev data..
+         *
+         * 1/ zero out disks
+         * 2/ Add info for each disk, keeping track of highest desc_nr (next_spare);
+         * 3/ any empty disks < next_spare become removed
+         *
+         * disks[0] gets initialised to REMOVED because
+         * we cannot be sure from other fields if it has
+         * been initialised or not.
+         */
+        int i;
+        int active=0, working=0,failed=0,spare=0,nr_disks=0;
+
+        sb = (mdp_super_t*)page_address(rdev->sb_page);
+
+        memset(sb, 0, sizeof(*sb));
+
+        sb->md_magic = MD_SB_MAGIC;
+        sb->major_version = mddev->major_version;
+        sb->minor_version = mddev->minor_version;
+        sb->patch_version = mddev->patch_version;
+        sb->gvalid_words  = 0; /* ignored */
+        memcpy(&sb->set_uuid0, mddev->uuid+0, 4);
+        memcpy(&sb->set_uuid1, mddev->uuid+4, 4);
+        memcpy(&sb->set_uuid2, mddev->uuid+8, 4);
+        memcpy(&sb->set_uuid3, mddev->uuid+12,4);
+
+        sb->ctime = mddev->ctime;
+        sb->level = mddev->level;
+        sb->size  = mddev->size;
+        sb->raid_disks = mddev->raid_disks;
+        sb->md_minor = mddev->md_minor;
+        sb->not_persistent = !mddev->persistent;
+        sb->utime = mddev->utime;
+        sb->state = 0;
+        sb->events_hi = (mddev->events>>32);
+        sb->events_lo = (u32)mddev->events;
+
+        if (mddev->in_sync)
+        {
+                sb->recovery_cp = mddev->recovery_cp;
+                sb->cp_events_hi = (mddev->events>>32);
+                sb->cp_events_lo = (u32)mddev->events;
+                if (mddev->recovery_cp == MaxSector)
+                        sb->state = (1<< MD_SB_CLEAN);
+        } else
+                sb->recovery_cp = 0;
+
+        sb->layout = mddev->layout;
+        sb->chunk_size = mddev->chunk_size;
+
+        sb->disks[0].state = (1<<MD_DISK_REMOVED);
+        ITERATE_RDEV(mddev,rdev2,tmp) {
+                mdp_disk_t *d;
+                if (rdev2->raid_disk >= 0 && rdev2->in_sync && !rdev2->faulty)
+                        rdev2->desc_nr = rdev2->raid_disk;
+                else
+                        rdev2->desc_nr = next_spare++;
+                d = &sb->disks[rdev2->desc_nr];
+                nr_disks++;
+                d->number = rdev2->desc_nr;
+                d->major = MAJOR(rdev2->bdev->bd_dev);
+                d->minor = MINOR(rdev2->bdev->bd_dev);
+                if (rdev2->raid_disk >= 0 && rdev->in_sync && !rdev2->faulty)
+                        d->raid_disk = rdev2->raid_disk;
+                else
+                        d->raid_disk = rdev2->desc_nr; /* compatibility */
+                if (rdev2->faulty) {
+                        d->state = (1<<MD_DISK_FAULTY);
+                        failed++;
+                } else if (rdev2->in_sync) {
+                        d->state = (1<<MD_DISK_ACTIVE);
+                        d->state |= (1<<MD_DISK_SYNC);
+                        active++;
+                        working++;
+                } else {
+                        d->state = 0;
+                        spare++;
+                        working++;
+                }
+        }
+        
+        /* now set the "removed" and "faulty" bits on any missing devices */
+        for (i=0 ; i < mddev->raid_disks ; i++) {
+                mdp_disk_t *d = &sb->disks[i];
+                if (d->state == 0 && d->number == 0) {
+                        d->number = i;
+                        d->raid_disk = i;
+                        d->state = (1<<MD_DISK_REMOVED);
+                        d->state |= (1<<MD_DISK_FAULTY);
+                        failed++;
+                }
+        }
+        sb->nr_disks = nr_disks;
+        sb->active_disks = active;
+        sb->working_disks = working;
+        sb->failed_disks = failed;
+        sb->spare_disks = spare;
+
+        sb->this_disk = sb->disks[rdev->desc_nr];
+        sb->sb_csum = calc_sb_csum(sb);
+}
+
+/*
+ * version 1 superblock
+ */
+
+static unsigned int calc_sb_1_csum(struct mdp_superblock_1 * sb)
+{
+        unsigned int disk_csum, csum;
+        unsigned long long newcsum;
+        int size = 256 + le32_to_cpu(sb->max_dev)*2;
+        unsigned int *isuper = (unsigned int*)sb;
+        int i;
+
+        disk_csum = sb->sb_csum;
+        sb->sb_csum = 0;
+        newcsum = 0;
+        for (i=0; size>=4; size -= 4 )
+                newcsum += le32_to_cpu(*isuper++);
+
+        if (size == 2)
+                newcsum += le16_to_cpu(*(unsigned short*) isuper);
+
+        csum = (newcsum & 0xffffffff) + (newcsum >> 32);
+        sb->sb_csum = disk_csum;
+        return cpu_to_le32(csum);
+}
+
+static int super_1_load(mdk_rdev_t *rdev, mdk_rdev_t *refdev, int minor_version)
+{
+        struct mdp_superblock_1 *sb;
+        int ret;
+        sector_t sb_offset;
+        char b[BDEVNAME_SIZE], b2[BDEVNAME_SIZE];
+
+        /*
+         * Calculate the position of the superblock.
+         * It is always aligned to a 4K boundary and
+         * depeding on minor_version, it can be:
+         * 0: At least 8K, but less than 12K, from end of device
+         * 1: At start of device
+         * 2: 4K from start of device.
+         */
+        switch(minor_version) {
+        case 0:
+                sb_offset = rdev->bdev->bd_inode->i_size >> 9;
+                sb_offset -= 8*2;
+                sb_offset &= ~(4*2-1);
+                /* convert from sectors to K */
+                sb_offset /= 2;
+                break;
+        case 1:
+                sb_offset = 0;
+                break;
+        case 2:
+                sb_offset = 4;
+                break;
+        default:
+                return -EINVAL;
+        }
+        rdev->sb_offset = sb_offset;
+
+        ret = read_disk_sb(rdev);
+        if (ret) return ret;
+
+
+        sb = (struct mdp_superblock_1*)page_address(rdev->sb_page);
+
+        if (sb->magic != cpu_to_le32(MD_SB_MAGIC) ||
+            sb->major_version != cpu_to_le32(1) ||
+            le32_to_cpu(sb->max_dev) > (4096-256)/2 ||
+            le64_to_cpu(sb->super_offset) != (rdev->sb_offset<<1) ||
+            sb->feature_map != 0)
+                return -EINVAL;
+
+        if (calc_sb_1_csum(sb) != sb->sb_csum) {
+                printk("md: invalid superblock checksum on %s\n",
+                        bdevname(rdev->bdev,b));
+                return -EINVAL;
+        }
+        if (le64_to_cpu(sb->data_size) < 10) {
+                printk("md: data_size too small on %s\n",
+                       bdevname(rdev->bdev,b));
+                return -EINVAL;
+        }
+        rdev->preferred_minor = 0xffff;
+        rdev->data_offset = le64_to_cpu(sb->data_offset);
+
+        if (refdev == 0)
+                return 1;
+        else {
+                __u64 ev1, ev2;
+                struct mdp_superblock_1 *refsb = 
+                        (struct mdp_superblock_1*)page_address(refdev->sb_page);
+
+                if (memcmp(sb->set_uuid, refsb->set_uuid, 16) != 0 ||
+                    sb->level != refsb->level ||
+                    sb->layout != refsb->layout ||
+                    sb->chunksize != refsb->chunksize) {
+                        printk(KERN_WARNING "md: %s has strangely different"
+                                " superblock to %s\n",
+                                bdevname(rdev->bdev,b),
+                                bdevname(refdev->bdev,b2));
+                        return -EINVAL;
+                }
+                ev1 = le64_to_cpu(sb->events);
+                ev2 = le64_to_cpu(refsb->events);
+
+                if (ev1 > ev2)
+                        return 1;
+        }
+        if (minor_version) 
+                rdev->size = ((rdev->bdev->bd_inode->i_size>>9) - le64_to_cpu(sb->data_offset)) / 2;
+        else
+                rdev->size = rdev->sb_offset;
+        if (rdev->size < le64_to_cpu(sb->data_size)/2)
+                return -EINVAL;
+        rdev->size = le64_to_cpu(sb->data_size)/2;
+        if (le32_to_cpu(sb->chunksize))
+                rdev->size &= ~((sector_t)le32_to_cpu(sb->chunksize)/2 - 1);
+        return 0;
+}
+
+static int super_1_validate(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+        struct mdp_superblock_1 *sb = (struct mdp_superblock_1*)page_address(rdev->sb_page);
+
+        if (mddev->raid_disks == 0) {
+                mddev->major_version = 1;
+                mddev->patch_version = 0;
+                mddev->persistent = 1;
+                mddev->chunk_size = le32_to_cpu(sb->chunksize) << 9;
+                mddev->ctime = le64_to_cpu(sb->ctime) & ((1ULL << 32)-1);
+                mddev->utime = le64_to_cpu(sb->utime) & ((1ULL << 32)-1);
+                mddev->level = le32_to_cpu(sb->level);
+                mddev->layout = le32_to_cpu(sb->layout);
+                mddev->raid_disks = le32_to_cpu(sb->raid_disks);
+                mddev->size = le64_to_cpu(sb->size)/2;
+                mddev->events = le64_to_cpu(sb->events);
+                
+                mddev->recovery_cp = le64_to_cpu(sb->resync_offset);
+                memcpy(mddev->uuid, sb->set_uuid, 16);
+
+                mddev->max_disks =  (4096-256)/2;
+        } else {
+                __u64 ev1;
+                ev1 = le64_to_cpu(sb->events);
+                ++ev1;
+                if (ev1 < mddev->events)
+                        return -EINVAL;
+        }
+
+        if (mddev->level != LEVEL_MULTIPATH) {
+                int role;
+                rdev->desc_nr = le32_to_cpu(sb->dev_number);
+                role = le16_to_cpu(sb->dev_roles[rdev->desc_nr]);
+                switch(role) {
+                case 0xffff: /* spare */
+                        rdev->in_sync = 0;
+                        rdev->faulty = 0;
+                        rdev->raid_disk = -1;
+                        break;
+                case 0xfffe: /* faulty */
+                        rdev->in_sync = 0;
+                        rdev->faulty = 1;
+                        rdev->raid_disk = -1;
+                        break;
+                default:
+                        rdev->in_sync = 1;
+                        rdev->faulty = 0;
+                        rdev->raid_disk = role;
+                        break;
+                }
+        }
+        return 0;
+}
+
+static void super_1_sync(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+        struct mdp_superblock_1 *sb;
+        struct list_head *tmp;
+        mdk_rdev_t *rdev2;
+        int max_dev, i;
+        /* make rdev->sb match mddev and rdev data. */
+
+        sb = (struct mdp_superblock_1*)page_address(rdev->sb_page);
+
+        sb->feature_map = 0;
+        sb->pad0 = 0;
+        memset(sb->pad1, 0, sizeof(sb->pad1));
+        memset(sb->pad2, 0, sizeof(sb->pad2));
+        memset(sb->pad3, 0, sizeof(sb->pad3));
+
+        sb->utime = cpu_to_le64((__u64)mddev->utime);
+        sb->events = cpu_to_le64(mddev->events);
+        if (mddev->in_sync)
+                sb->resync_offset = cpu_to_le64(mddev->recovery_cp);
+        else
+                sb->resync_offset = cpu_to_le64(0);
+
+        max_dev = 0;
+        ITERATE_RDEV(mddev,rdev2,tmp)
+                if (rdev2->desc_nr+1 > max_dev)
+                        max_dev = rdev2->desc_nr+1;
+        
+        sb->max_dev = cpu_to_le32(max_dev);
+        for (i=0; i<max_dev;i++)
+                sb->dev_roles[i] = cpu_to_le16(0xfffe);
+        
+        ITERATE_RDEV(mddev,rdev2,tmp) {
+                i = rdev2->desc_nr;
+                if (rdev2->faulty)
+                        sb->dev_roles[i] = cpu_to_le16(0xfffe);
+                else if (rdev2->in_sync)
+                        sb->dev_roles[i] = cpu_to_le16(rdev2->raid_disk);
+                else
+                        sb->dev_roles[i] = cpu_to_le16(0xffff);
+        }
+
+        sb->recovery_offset = cpu_to_le64(0); /* not supported yet */
+        sb->sb_csum = calc_sb_1_csum(sb);
+}
+
+
+struct super_type super_types[] = {
+        [0] = {
+                .name   = "0.90.0",
+                .owner  = THIS_MODULE,
+                .load_super     = super_90_load,
+                .validate_super = super_90_validate,
+                .sync_super     = super_90_sync,
+        },
+        [1] = {
+                .name   = "md-1",
+                .owner  = THIS_MODULE,
+                .load_super     = super_1_load,
+                .validate_super = super_1_validate,
+                .sync_super     = super_1_sync,
+        },
+};
+        
+static mdk_rdev_t * match_dev_unit(mddev_t *mddev, mdk_rdev_t *dev)
+{
+        struct list_head *tmp;
+        mdk_rdev_t *rdev;
+
+        ITERATE_RDEV(mddev,rdev,tmp)
+                if (rdev->bdev->bd_contains == dev->bdev->bd_contains)
+                        return rdev;
+
+        return NULL;
+}
+
+static int match_mddev_units(mddev_t *mddev1, mddev_t *mddev2)
+{
+        struct list_head *tmp;
+        mdk_rdev_t *rdev;
+
+        ITERATE_RDEV(mddev1,rdev,tmp)
+                if (match_dev_unit(mddev2, rdev))
+                        return 1;
+
+        return 0;
+}
+
+static LIST_HEAD(pending_raid_disks);
+
+static int bind_rdev_to_array(mdk_rdev_t * rdev, mddev_t * mddev)
+{
+        mdk_rdev_t *same_pdev;
+        char b[BDEVNAME_SIZE], b2[BDEVNAME_SIZE];
+
+        if (rdev->mddev) {
+                MD_BUG();
+                return -EINVAL;
+        }
+        same_pdev = match_dev_unit(mddev, rdev);
+        if (same_pdev)
+                printk(KERN_WARNING
+                        "%s: WARNING: %s appears to be on the same physical"
+                        " disk as %s. True\n     protection against single-disk"
+                        " failure might be compromised.\n",
+                        mdname(mddev), bdevname(rdev->bdev,b),
+                        bdevname(same_pdev->bdev,b2));
+
+        /* Verify rdev->desc_nr is unique.
+         * If it is -1, assign a free number, else
+         * check number is not in use
+         */
+        if (rdev->desc_nr < 0) {
+                int choice = 0;
+                if (mddev->pers) choice = mddev->raid_disks;
+                while (find_rdev_nr(mddev, choice))
+                        choice++;
+                rdev->desc_nr = choice;
+        } else {
+                if (find_rdev_nr(mddev, rdev->desc_nr))
+                        return -EBUSY;
+        }
+                        
+        list_add(&rdev->same_set, &mddev->disks);
+        rdev->mddev = mddev;
+        printk(KERN_INFO "md: bind<%s>\n", bdevname(rdev->bdev,b));
+        return 0;
+}
+
+static void unbind_rdev_from_array(mdk_rdev_t * rdev)
+{
+        char b[BDEVNAME_SIZE];
+        if (!rdev->mddev) {
+                MD_BUG();
+                return;
+        }
+        list_del_init(&rdev->same_set);
+        printk(KERN_INFO "md: unbind<%s>\n", bdevname(rdev->bdev,b));
+        rdev->mddev = NULL;
+}
+
+/*
+ * prevent the device from being mounted, repartitioned or
+ * otherwise reused by a RAID array (or any other kernel
+ * subsystem), by bd_claiming the device.
+ */
+static int lock_rdev(mdk_rdev_t *rdev, dev_t dev)
+{
+        int err = 0;
+        struct block_device *bdev;
+        char b[BDEVNAME_SIZE];
+
+        bdev = open_by_devnum(dev, FMODE_READ|FMODE_WRITE);
+        if (IS_ERR(bdev)) {
+                printk(KERN_ERR "md: could not open %s.\n",
+                        __bdevname(dev, b));
+                return PTR_ERR(bdev);
+        }
+        err = bd_claim(bdev, rdev);
+        if (err) {
+                printk(KERN_ERR "md: could not bd_claim %s.\n",
+                        bdevname(bdev, b));
+                blkdev_put(bdev);
+                return err;
+        }
+        rdev->bdev = bdev;
+        return err;
+}
+
+static void unlock_rdev(mdk_rdev_t *rdev)
+{
+        struct block_device *bdev = rdev->bdev;
+        rdev->bdev = NULL;
+        if (!bdev)
+                MD_BUG();
+        bd_release(bdev);
+        blkdev_put(bdev);
+}
+
+void md_autodetect_dev(dev_t dev);
+
+static void export_rdev(mdk_rdev_t * rdev)
+{
+        char b[BDEVNAME_SIZE];
+        printk(KERN_INFO "md: export_rdev(%s)\n",
+                bdevname(rdev->bdev,b));
+        if (rdev->mddev)
+                MD_BUG();
+        free_disk_sb(rdev);
+        list_del_init(&rdev->same_set);
+#ifndef MODULE
+        md_autodetect_dev(rdev->bdev->bd_dev);
+#endif
+        unlock_rdev(rdev);
+        kfree(rdev);
+}
+
+static void kick_rdev_from_array(mdk_rdev_t * rdev)
+{
+        unbind_rdev_from_array(rdev);
+        export_rdev(rdev);
+}
+
+static void export_array(mddev_t *mddev)
+{
+        struct list_head *tmp;
+        mdk_rdev_t *rdev;
+
+        ITERATE_RDEV(mddev,rdev,tmp) {
+                if (!rdev->mddev) {
+                        MD_BUG();
+                        continue;
+                }
+                kick_rdev_from_array(rdev);
+        }
+        if (!list_empty(&mddev->disks))
+                MD_BUG();
+        mddev->raid_disks = 0;
+        mddev->major_version = 0;
+}
+
+static void print_desc(mdp_disk_t *desc)
+{
+        printk(" DISK<N:%d,(%d,%d),R:%d,S:%d>\n", desc->number,
+                desc->major,desc->minor,desc->raid_disk,desc->state);
+}
+
+static void print_sb(mdp_super_t *sb)
+{
+        int i;
+
+        printk(KERN_INFO 
+                "md:  SB: (V:%d.%d.%d) ID:<%08x.%08x.%08x.%08x> CT:%08x\n",
+                sb->major_version, sb->minor_version, sb->patch_version,
+                sb->set_uuid0, sb->set_uuid1, sb->set_uuid2, sb->set_uuid3,
+                sb->ctime);
+        printk(KERN_INFO "md:     L%d S%08d ND:%d RD:%d md%d LO:%d CS:%d\n",
+                sb->level, sb->size, sb->nr_disks, sb->raid_disks,
+                sb->md_minor, sb->layout, sb->chunk_size);
+        printk(KERN_INFO "md:     UT:%08x ST:%d AD:%d WD:%d"
+                " FD:%d SD:%d CSUM:%08x E:%08lx\n",
+                sb->utime, sb->state, sb->active_disks, sb->working_disks,
+                sb->failed_disks, sb->spare_disks,
+                sb->sb_csum, (unsigned long)sb->events_lo);
+
+        printk(KERN_INFO);
+        for (i = 0; i < MD_SB_DISKS; i++) {
+                mdp_disk_t *desc;
+
+                desc = sb->disks + i;
+                if (desc->number || desc->major || desc->minor ||
+                    desc->raid_disk || (desc->state && (desc->state != 4))) {
+                        printk("     D %2d: ", i);
+                        print_desc(desc);
+                }
+        }
+        printk(KERN_INFO "md:     THIS: ");
+        print_desc(&sb->this_disk);
+
+}
+
+static void print_rdev(mdk_rdev_t *rdev)
+{
+        char b[BDEVNAME_SIZE];
+        printk(KERN_INFO "md: rdev %s, SZ:%08llu F:%d S:%d DN:%u\n",
+                bdevname(rdev->bdev,b), (unsigned long long)rdev->size,
+                rdev->faulty, rdev->in_sync, rdev->desc_nr);
+        if (rdev->sb_loaded) {
+                printk(KERN_INFO "md: rdev superblock:\n");
+                print_sb((mdp_super_t*)page_address(rdev->sb_page));
+        } else
+                printk(KERN_INFO "md: no rdev superblock!\n");
+}
+
+void md_print_devices(void)
+{
+        struct list_head *tmp, *tmp2;
+        mdk_rdev_t *rdev;
+        mddev_t *mddev;
+        char b[BDEVNAME_SIZE];
+
+        printk("\n");
+        printk("md:     **********************************\n");
+        printk("md:     * <COMPLETE RAID STATE PRINTOUT> *\n");
+        printk("md:     **********************************\n");
+        ITERATE_MDDEV(mddev,tmp) {
+                printk("%s: ", mdname(mddev));
+
+                ITERATE_RDEV(mddev,rdev,tmp2)
+                        printk("<%s>", bdevname(rdev->bdev,b));
+                printk("\n");
+
+                ITERATE_RDEV(mddev,rdev,tmp2)
+                        print_rdev(rdev);
+        }
+        printk("md:     **********************************\n");
+        printk("\n");
+}
+
+
+static int write_disk_sb(mdk_rdev_t * rdev)
+{
+        char b[BDEVNAME_SIZE];
+        if (!rdev->sb_loaded) {
+                MD_BUG();
+                return 1;
+        }
+        if (rdev->faulty) {
+                MD_BUG();
+                return 1;
+        }
+
+        dprintk(KERN_INFO "(write) %s's sb offset: %llu\n",
+                bdevname(rdev->bdev,b),
+               (unsigned long long)rdev->sb_offset);
+  
+        if (sync_page_io(rdev->bdev, rdev->sb_offset<<1, MD_SB_BYTES, rdev->sb_page, WRITE))
+                return 0;
+
+        printk("md: write_disk_sb failed for device %s\n", 
+                bdevname(rdev->bdev,b));
+        return 1;
+}
+
+static void sync_sbs(mddev_t * mddev)
+{
+        mdk_rdev_t *rdev;
+        struct list_head *tmp;
+
+        ITERATE_RDEV(mddev,rdev,tmp) {
+                super_types[mddev->major_version].
+                        sync_super(mddev, rdev);
+                rdev->sb_loaded = 1;
+        }
+}
+
+static void md_update_sb(mddev_t * mddev)
+{
+        int err, count = 100;
+        struct list_head *tmp;
+        mdk_rdev_t *rdev;
+
+        mddev->sb_dirty = 0;
+repeat:
+        mddev->utime = get_seconds();
+        mddev->events ++;
+
+        if (!mddev->events) {
+                /*
+                 * oops, this 64-bit counter should never wrap.
+                 * Either we are in around ~1 trillion A.C., assuming
+                 * 1 reboot per second, or we have a bug:
+                 */
+                MD_BUG();
+                mddev->events --;
+        }
+        sync_sbs(mddev);
+
+        /*
+         * do not write anything to disk if using
+         * nonpersistent superblocks
+         */
+        if (!mddev->persistent)
+                return;
+
+        dprintk(KERN_INFO 
+                "md: updating %s RAID superblock on device (in sync %d)\n",
+                mdname(mddev),mddev->in_sync);
+
+        err = 0;
+        ITERATE_RDEV(mddev,rdev,tmp) {
+                char b[BDEVNAME_SIZE];
+                dprintk(KERN_INFO "md: ");
+                if (rdev->faulty)
+                        dprintk("(skipping faulty ");
+
+                dprintk("%s ", bdevname(rdev->bdev,b));
+                if (!rdev->faulty) {
+                        err += write_disk_sb(rdev);
+                } else
+                        dprintk(")\n");
+                if (!err && mddev->level == LEVEL_MULTIPATH)
+                        /* only need to write one superblock... */
+                        break;
+        }
+        if (err) {
+                if (--count) {
+                        printk(KERN_ERR "md: errors occurred during superblock"
+                                " update, repeating\n");
+                        goto repeat;
+                }
+                printk(KERN_ERR \
+                        "md: excessive errors occurred during superblock update, exiting\n");
+        }
+}
+
+/*
+ * Import a device. If 'super_format' >= 0, then sanity check the superblock
+ *
+ * mark the device faulty if:
+ *
+ *   - the device is nonexistent (zero size)
+ *   - the device has no valid superblock
+ *
+ * a faulty rdev _never_ has rdev->sb set.
+ */
+static mdk_rdev_t *md_import_device(dev_t newdev, int super_format, int super_minor)
+{
+        char b[BDEVNAME_SIZE];
+        int err;
+        mdk_rdev_t *rdev;
+        sector_t size;
+
+        rdev = (mdk_rdev_t *) kmalloc(sizeof(*rdev), GFP_KERNEL);
+        if (!rdev) {
+                printk(KERN_ERR "md: could not alloc mem for new device!\n");
+                return ERR_PTR(-ENOMEM);
+        }
+        memset(rdev, 0, sizeof(*rdev));
+
+        if ((err = alloc_disk_sb(rdev)))
+                goto abort_free;
+
+        err = lock_rdev(rdev, newdev);
+        if (err)
+                goto abort_free;
+
+        rdev->desc_nr = -1;
+        rdev->faulty = 0;
+        rdev->in_sync = 0;
+        rdev->data_offset = 0;
+        atomic_set(&rdev->nr_pending, 0);
+
+        size = rdev->bdev->bd_inode->i_size >> BLOCK_SIZE_BITS;
+        if (!size) {
+                printk(KERN_WARNING 
+                        "md: %s has zero or unknown size, marking faulty!\n",
+                        bdevname(rdev->bdev,b));
+                err = -EINVAL;
+                goto abort_free;
+        }
+
+        if (super_format >= 0) {
+                err = super_types[super_format].
+                        load_super(rdev, NULL, super_minor);
+                if (err == -EINVAL) {
+                        printk(KERN_WARNING 
+                                "md: %s has invalid sb, not importing!\n",
+                                bdevname(rdev->bdev,b));
+                        goto abort_free;
+                }
+                if (err < 0) {
+                        printk(KERN_WARNING 
+                                "md: could not read %s's sb, not importing!\n",
+                                bdevname(rdev->bdev,b));
+                        goto abort_free;
+                }
+        }
+        INIT_LIST_HEAD(&rdev->same_set);
+
+        return rdev;
+
+abort_free:
+        if (rdev->sb_page) {
+                if (rdev->bdev)
+                        unlock_rdev(rdev);
+                free_disk_sb(rdev);
+        }
+        kfree(rdev);
+        return ERR_PTR(err);
+}
+
+/*
+ * Check a full RAID array for plausibility
+ */
+
+
+static int analyze_sbs(mddev_t * mddev)
+{
+        int i;
+        struct list_head *tmp;
+        mdk_rdev_t *rdev, *freshest;
+        char b[BDEVNAME_SIZE];
+
+        freshest = NULL;
+        ITERATE_RDEV(mddev,rdev,tmp)
+                switch (super_types[mddev->major_version].
+                        load_super(rdev, freshest, mddev->minor_version)) {
+                case 1:
+                        freshest = rdev;
+                        break;
+                case 0:
+                        break;
+                default:
+                        printk( KERN_ERR \
+                                "md: fatal superblock inconsistency in %s"
+                                " -- removing from array\n", 
+                                bdevname(rdev->bdev,b));
+                        kick_rdev_from_array(rdev);
+                }
+
+
+        super_types[mddev->major_version].
+                validate_super(mddev, freshest);
+
+        i = 0;
+        ITERATE_RDEV(mddev,rdev,tmp) {
+                if (rdev != freshest)
+                        if (super_types[mddev->major_version].
+                            validate_super(mddev, rdev)) {
+                                printk(KERN_WARNING "md: kicking non-fresh %s"
+                                        " from array!\n",
+                                        bdevname(rdev->bdev,b));
+                                kick_rdev_from_array(rdev);
+                                continue;
+                        }
+                if (mddev->level == LEVEL_MULTIPATH) {
+                        rdev->desc_nr = i++;
+                        rdev->raid_disk = rdev->desc_nr;
+                        rdev->in_sync = 1;
+                }
+        }
+
+
+
+        if (mddev->recovery_cp != MaxSector &&
+            mddev->level >= 1)
+                printk(KERN_ERR "md: %s: raid array is not clean"
+                       " -- starting background reconstruction\n",
+                       mdname(mddev));
+
+        return 0;
+}
+
+int mdp_major = 0;
+
+static struct kobject *md_probe(dev_t dev, int *part, void *data)
+{
+        static DECLARE_MUTEX(disks_sem);
+        mddev_t *mddev = mddev_find(dev);
+        struct gendisk *disk;
+        int partitioned = (MAJOR(dev) != MD_MAJOR);
+        int shift = partitioned ? MdpMinorShift : 0;
+        int unit = MINOR(dev) >> shift;
+
+        if (!mddev)
+                return NULL;
+
+        down(&disks_sem);
+        if (mddev->gendisk) {
+                up(&disks_sem);
+                mddev_put(mddev);
+                return NULL;
+        }
+        disk = alloc_disk(1 << shift);
+        if (!disk) {
+                up(&disks_sem);
+                mddev_put(mddev);
+                return NULL;
+        }
+        disk->major = MAJOR(dev);
+        disk->first_minor = unit << shift;
+        if (partitioned) {
+                sprintf(disk->disk_name, "md_d%d", unit);
+                sprintf(disk->devfs_name, "md/d%d", unit);
+        } else {
+                sprintf(disk->disk_name, "md%d", unit);
+                sprintf(disk->devfs_name, "md/%d", unit);
+        }
+        disk->fops = &md_fops;
+        disk->private_data = mddev;
+        disk->queue = mddev->queue;
+        add_disk(disk);
+        mddev->gendisk = disk;
+        up(&disks_sem);
+        return NULL;
+}
+
+void md_wakeup_thread(mdk_thread_t *thread);
+
+static void md_safemode_timeout(unsigned long data)
+{
+        mddev_t *mddev = (mddev_t *) data;
+
+        mddev->safemode = 1;
+        md_wakeup_thread(mddev->thread);
+}
+
+
+static int do_md_run(mddev_t * mddev)
+{
+        int pnum, err;
+        int chunk_size;
+        struct list_head *tmp;
+        mdk_rdev_t *rdev;
+        struct gendisk *disk;
+        char b[BDEVNAME_SIZE];
+
+        if (list_empty(&mddev->disks)) {
+                MD_BUG();
+                return -EINVAL;
+        }
+
+        if (mddev->pers)
+                return -EBUSY;
+
+        /*
+         * Analyze all RAID superblock(s)
+         */
+        if (!mddev->raid_disks && analyze_sbs(mddev)) {
+                MD_BUG();
+                return -EINVAL;
+        }
+
+        chunk_size = mddev->chunk_size;
+        pnum = level_to_pers(mddev->level);
+
+        if ((pnum != MULTIPATH) && (pnum != RAID1)) {
+                if (!chunk_size) {
+                        /*
+                         * 'default chunksize' in the old md code used to
+                         * be PAGE_SIZE, baaad.
+                         * we abort here to be on the safe side. We don't
+                         * want to continue the bad practice.
+                         */
+                        printk(KERN_ERR 
+                                "no chunksize specified, see 'man raidtab'\n");
+                        return -EINVAL;
+                }
+                if (chunk_size > MAX_CHUNK_SIZE) {
+                        printk(KERN_ERR "too big chunk_size: %d > %d\n",
+                                chunk_size, MAX_CHUNK_SIZE);
+                        return -EINVAL;
+                }
+                /*
+                 * chunk-size has to be a power of 2 and multiples of PAGE_SIZE
+                 */
+                if ( (1 << ffz(~chunk_size)) != chunk_size) {
+                        MD_BUG();
+                        return -EINVAL;
+                }
+                if (chunk_size < PAGE_SIZE) {
+                        printk(KERN_ERR "too small chunk_size: %d < %ld\n",
+                                chunk_size, PAGE_SIZE);
+                        return -EINVAL;
+                }
+
+                /* devices must have minimum size of one chunk */
+                ITERATE_RDEV(mddev,rdev,tmp) {
+                        if (rdev->faulty)
+                                continue;
+                        if (rdev->size < chunk_size / 1024) {
+                                printk(KERN_WARNING
+                                        "md: Dev %s smaller than chunk_size:"
+                                        " %lluk < %dk\n",
+                                        bdevname(rdev->bdev,b),
+                                        (unsigned long long)rdev->size,
+                                        chunk_size / 1024);
+                                return -EINVAL;
+                        }
+                }
+        }
+
+        if (pnum >= MAX_PERSONALITY) {
+                MD_BUG();
+                return -EINVAL;
+        }
+
+#ifdef CONFIG_KMOD
+        if (!pers[pnum])
+        {
+                request_module("md-personality-%d", pnum);
+        }
+#endif
+
+        /*
+         * Drop all container device buffers, from now on
+         * the only valid external interface is through the md
+         * device.
+         * Also find largest hardsector size
+         */
+        ITERATE_RDEV(mddev,rdev,tmp) {
+                if (rdev->faulty)
+                        continue;
+                sync_blockdev(rdev->bdev);
+                invalidate_bdev(rdev->bdev, 0);
+        }
+
+        md_probe(mddev->unit, NULL, NULL);
+        disk = mddev->gendisk;
+        if (!disk)
+                return -ENOMEM;
+
+        spin_lock(&pers_lock);
+        if (!pers[pnum] || !try_module_get(pers[pnum]->owner)) {
+                spin_unlock(&pers_lock);
+                printk(KERN_WARNING "md: personality %d is not loaded!\n",
+                       pnum);
+                return -EINVAL;
+        }
+
+        mddev->pers = pers[pnum];
+        spin_unlock(&pers_lock);
+
+        mddev->resync_max_sectors = mddev->size << 1; /* may be over-ridden by personality */
+
+        err = mddev->pers->run(mddev);
+        if (err) {
+                printk(KERN_ERR "md: pers->run() failed ...\n");
+                module_put(mddev->pers->owner);
+                mddev->pers = NULL;
+                return -EINVAL;
+        }
+        atomic_set(&mddev->writes_pending,0);
+        mddev->safemode = 0;
+        mddev->safemode_timer.function = md_safemode_timeout;
+        mddev->safemode_timer.data = (unsigned long) mddev;
+        mddev->safemode_delay = (20 * HZ)/1000 +1; /* 20 msec delay */
+        mddev->in_sync = 1;
+        
+        set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+        
+        if (mddev->sb_dirty)
+                md_update_sb(mddev);
+
+        set_capacity(disk, mddev->array_size<<1);
+
+        /* If we call blk_queue_make_request here, it will
+         * re-initialise max_sectors etc which may have been
+         * refined inside -> run.  So just set the bits we need to set.
+         * Most initialisation happended when we called
+         * blk_queue_make_request(..., md_fail_request)
+         * earlier.
+         */
+        mddev->queue->queuedata = mddev;
+        mddev->queue->make_request_fn = mddev->pers->make_request;
+
+        mddev->changed = 1;
+        return 0;
+}
+
+static int restart_array(mddev_t *mddev)
+{
+        struct gendisk *disk = mddev->gendisk;
+        int err;
+
+        /*
+         * Complain if it has no devices
+         */
+        err = -ENXIO;
+        if (list_empty(&mddev->disks))
+                goto out;
+
+        if (mddev->pers) {
+                err = -EBUSY;
+                if (!mddev->ro)
+                        goto out;
+
+                mddev->safemode = 0;
+                mddev->ro = 0;
+                set_disk_ro(disk, 0);
+
+                printk(KERN_INFO "md: %s switched to read-write mode.\n",
+                        mdname(mddev));
+                /*
+                 * Kick recovery or resync if necessary
+                 */
+                set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+                md_wakeup_thread(mddev->thread);
+                err = 0;
+        } else {
+                printk(KERN_ERR "md: %s has no personality assigned.\n",
+                        mdname(mddev));
+                err = -EINVAL;
+        }
+
+out:
+        return err;
+}
+
+static int do_md_stop(mddev_t * mddev, int ro)
+{
+        int err = 0;
+        struct gendisk *disk = mddev->gendisk;
+
+        if (mddev->pers) {
+                if (atomic_read(&mddev->active)>2) {
+                        printk("md: %s still in use.\n",mdname(mddev));
+                        return -EBUSY;
+                }
+
+                if (mddev->sync_thread) {
+                        set_bit(MD_RECOVERY_INTR, &mddev->recovery);
+                        md_unregister_thread(mddev->sync_thread);
+                        mddev->sync_thread = NULL;
+                }
+
+                del_timer_sync(&mddev->safemode_timer);
+
+                invalidate_partition(disk, 0);
+
+                if (ro) {
+                        err  = -ENXIO;
+                        if (mddev->ro)
+                                goto out;
+                        mddev->ro = 1;
+                } else {
+                        if (mddev->ro)
+                                set_disk_ro(disk, 0);
+                        blk_queue_make_request(mddev->queue, md_fail_request);
+                        mddev->pers->stop(mddev);
+                        module_put(mddev->pers->owner);
+                        mddev->pers = NULL;
+                        if (mddev->ro)
+                                mddev->ro = 0;
+                }
+                if (!mddev->in_sync) {
+                        /* mark array as shutdown cleanly */
+                        mddev->in_sync = 1;
+                        md_update_sb(mddev);
+                }
+                if (ro)
+                        set_disk_ro(disk, 1);
+        }
+        /*
+         * Free resources if final stop
+         */
+        if (!ro) {
+                struct gendisk *disk;
+                printk(KERN_INFO "md: %s stopped.\n", mdname(mddev));
+
+                export_array(mddev);
+
+                mddev->array_size = 0;
+                disk = mddev->gendisk;
+                if (disk)
+                        set_capacity(disk, 0);
+                mddev->changed = 1;
+        } else
+                printk(KERN_INFO "md: %s switched to read-only mode.\n",
+                        mdname(mddev));
+        err = 0;
+out:
+        return err;
+}
+
+static void autorun_array(mddev_t *mddev)
+{
+        mdk_rdev_t *rdev;
+        struct list_head *tmp;
+        int err;
+
+        if (list_empty(&mddev->disks)) {
+                MD_BUG();
+                return;
+        }
+
+        printk(KERN_INFO "md: running: ");
+
+        ITERATE_RDEV(mddev,rdev,tmp) {
+                char b[BDEVNAME_SIZE];
+                printk("<%s>", bdevname(rdev->bdev,b));
+        }
+        printk("\n");
+
+        err = do_md_run (mddev);
+        if (err) {
+                printk(KERN_WARNING "md: do_md_run() returned %d\n", err);
+                do_md_stop (mddev, 0);
+        }
+}
+
+/*
+ * lets try to run arrays based on all disks that have arrived
+ * until now. (those are in pending_raid_disks)
+ *
+ * the method: pick the first pending disk, collect all disks with
+ * the same UUID, remove all from the pending list and put them into
+ * the 'same_array' list. Then order this list based on superblock
+ * update time (freshest comes first), kick out 'old' disks and
+ * compare superblocks. If everything's fine then run it.
+ *
+ * If "unit" is allocated, then bump its reference count
+ */
+static void autorun_devices(int part)
+{
+        struct list_head candidates;
+        struct list_head *tmp;
+        mdk_rdev_t *rdev0, *rdev;
+        mddev_t *mddev;
+        char b[BDEVNAME_SIZE];
+
+        printk(KERN_INFO "md: autorun ...\n");
+        while (!list_empty(&pending_raid_disks)) {
+                dev_t dev;
+                rdev0 = list_entry(pending_raid_disks.next,
+                                         mdk_rdev_t, same_set);
+
+                printk(KERN_INFO "md: considering %s ...\n",
+                        bdevname(rdev0->bdev,b));
+                INIT_LIST_HEAD(&candidates);
+                ITERATE_RDEV_PENDING(rdev,tmp)
+                        if (super_90_load(rdev, rdev0, 0) >= 0) {
+                                printk(KERN_INFO "md:  adding %s ...\n",
+                                        bdevname(rdev->bdev,b));
+                                list_move(&rdev->same_set, &candidates);
+                        }
+                /*
+                 * now we have a set of devices, with all of them having
+                 * mostly sane superblocks. It's time to allocate the
+                 * mddev.
+                 */
+                if (rdev0->preferred_minor < 0 || rdev0->preferred_minor >= MAX_MD_DEVS) {
+                        printk(KERN_INFO "md: unit number in %s is bad: %d\n",
+                               bdevname(rdev0->bdev, b), rdev0->preferred_minor);
+                        break;
+                }
+                if (part)
+                        dev = MKDEV(mdp_major,
+                                    rdev0->preferred_minor << MdpMinorShift);
+                else
+                        dev = MKDEV(MD_MAJOR, rdev0->preferred_minor);
+
+                md_probe(dev, NULL, NULL);
+                mddev = mddev_find(dev);
+                if (!mddev) {
+                        printk(KERN_ERR 
+                                "md: cannot allocate memory for md drive.\n");
+                        break;
+                }
+                if (mddev_lock(mddev)) 
+                        printk(KERN_WARNING "md: %s locked, cannot run\n",
+                               mdname(mddev));
+                else if (mddev->raid_disks || mddev->major_version
+                         || !list_empty(&mddev->disks)) {
+                        printk(KERN_WARNING 
+                                "md: %s already running, cannot run %s\n",
+                                mdname(mddev), bdevname(rdev0->bdev,b));
+                        mddev_unlock(mddev);
+                } else {
+                        printk(KERN_INFO "md: created %s\n", mdname(mddev));
+                        ITERATE_RDEV_GENERIC(candidates,rdev,tmp) {
+                                list_del_init(&rdev->same_set);
+                                if (bind_rdev_to_array(rdev, mddev))
+                                        export_rdev(rdev);
+                        }
+                        autorun_array(mddev);
+                        mddev_unlock(mddev);
+                }
+                /* on success, candidates will be empty, on error
+                 * it won't...
+                 */
+                ITERATE_RDEV_GENERIC(candidates,rdev,tmp)
+                        export_rdev(rdev);
+                mddev_put(mddev);
+        }
+        printk(KERN_INFO "md: ... autorun DONE.\n");
+}
+
+/*
+ * import RAID devices based on one partition
+ * if possible, the array gets run as well.
+ */
+
+static int autostart_array(dev_t startdev)
+{
+        char b[BDEVNAME_SIZE];
+        int err = -EINVAL, i;
+        mdp_super_t *sb = NULL;
+        mdk_rdev_t *start_rdev = NULL, *rdev;
+
+        start_rdev = md_import_device(startdev, 0, 0);
+        if (IS_ERR(start_rdev))
+                return err;
+
+
+        /* NOTE: this can only work for 0.90.0 superblocks */
+        sb = (mdp_super_t*)page_address(start_rdev->sb_page);
+        if (sb->major_version != 0 ||
+            sb->minor_version != 90 ) {
+                printk(KERN_WARNING "md: can only autostart 0.90.0 arrays\n");
+                export_rdev(start_rdev);
+                return err;
+        }
+
+        if (start_rdev->faulty) {
+                printk(KERN_WARNING 
+                        "md: can not autostart based on faulty %s!\n",
+                        bdevname(start_rdev->bdev,b));
+                export_rdev(start_rdev);
+                return err;
+        }
+        list_add(&start_rdev->same_set, &pending_raid_disks);
+
+        for (i = 0; i < MD_SB_DISKS; i++) {
+                mdp_disk_t *desc = sb->disks + i;
+                dev_t dev = MKDEV(desc->major, desc->minor);
+
+                if (!dev)
+                        continue;
+                if (dev == startdev)
+                        continue;
+                if (MAJOR(dev) != desc->major || MINOR(dev) != desc->minor)
+                        continue;
+                rdev = md_import_device(dev, 0, 0);
+                if (IS_ERR(rdev))
+                        continue;
+
+                list_add(&rdev->same_set, &pending_raid_disks);
+        }
+
+        /*
+         * possibly return codes
+         */
+        autorun_devices(0);
+        return 0;
+
+}
+
+
+static int get_version(void __user * arg)
+{
+        mdu_version_t ver;
+
+        ver.major = MD_MAJOR_VERSION;
+        ver.minor = MD_MINOR_VERSION;
+        ver.patchlevel = MD_PATCHLEVEL_VERSION;
+
+        if (copy_to_user(arg, &ver, sizeof(ver)))
+                return -EFAULT;
+
+        return 0;
+}
+
+static int get_array_info(mddev_t * mddev, void __user * arg)
+{
+        mdu_array_info_t info;
+        int nr,working,active,failed,spare;
+        mdk_rdev_t *rdev;
+        struct list_head *tmp;
+
+        nr=working=active=failed=spare=0;
+        ITERATE_RDEV(mddev,rdev,tmp) {
+                nr++;
+                if (rdev->faulty)
+                        failed++;
+                else {
+                        working++;
+                        if (rdev->in_sync)
+                                active++;       
+                        else
+                                spare++;
+                }
+        }
+
+        info.major_version = mddev->major_version;
+        info.minor_version = mddev->minor_version;
+        info.patch_version = MD_PATCHLEVEL_VERSION;
+        info.ctime         = mddev->ctime;
+        info.level         = mddev->level;
+        info.size          = mddev->size;
+        info.nr_disks      = nr;
+        info.raid_disks    = mddev->raid_disks;
+        info.md_minor      = mddev->md_minor;
+        info.not_persistent= !mddev->persistent;
+
+        info.utime         = mddev->utime;
+        info.state         = 0;
+        if (mddev->in_sync)
+                info.state = (1<<MD_SB_CLEAN);
+        info.active_disks  = active;
+        info.working_disks = working;
+        info.failed_disks  = failed;
+        info.spare_disks   = spare;
+
+        info.layout        = mddev->layout;
+        info.chunk_size    = mddev->chunk_size;
+
+        if (copy_to_user(arg, &info, sizeof(info)))
+                return -EFAULT;
+
+        return 0;
+}
+
+static int get_disk_info(mddev_t * mddev, void __user * arg)
+{
+        mdu_disk_info_t info;
+        unsigned int nr;
+        mdk_rdev_t *rdev;
+
+        if (copy_from_user(&info, arg, sizeof(info)))
+                return -EFAULT;
+
+        nr = info.number;
+
+        rdev = find_rdev_nr(mddev, nr);
+        if (rdev) {
+                info.major = MAJOR(rdev->bdev->bd_dev);
+                info.minor = MINOR(rdev->bdev->bd_dev);
+                info.raid_disk = rdev->raid_disk;
+                info.state = 0;
+                if (rdev->faulty)
+                        info.state |= (1<<MD_DISK_FAULTY);
+                else if (rdev->in_sync) {
+                        info.state |= (1<<MD_DISK_ACTIVE);
+                        info.state |= (1<<MD_DISK_SYNC);
+                }
+        } else {
+                info.major = info.minor = 0;
+                info.raid_disk = -1;
+                info.state = (1<<MD_DISK_REMOVED);
+        }
+
+        if (copy_to_user(arg, &info, sizeof(info)))
+                return -EFAULT;
+
+        return 0;
+}
+
+static int add_new_disk(mddev_t * mddev, mdu_disk_info_t *info)
+{
+        char b[BDEVNAME_SIZE], b2[BDEVNAME_SIZE];
+        mdk_rdev_t *rdev;
+        dev_t dev = MKDEV(info->major,info->minor);
+
+        if (info->major != MAJOR(dev) || info->minor != MINOR(dev))
+                return -EOVERFLOW;
+
+        if (!mddev->raid_disks) {
+                int err;
+                /* expecting a device which has a superblock */
+                rdev = md_import_device(dev, mddev->major_version, mddev->minor_version);
+                if (IS_ERR(rdev)) {
+                        printk(KERN_WARNING 
+                                "md: md_import_device returned %ld\n",
+                                PTR_ERR(rdev));
+                        return PTR_ERR(rdev);
+                }
+                if (!list_empty(&mddev->disks)) {
+                        mdk_rdev_t *rdev0 = list_entry(mddev->disks.next,
+                                                        mdk_rdev_t, same_set);
+                        int err = super_types[mddev->major_version]
+                                .load_super(rdev, rdev0, mddev->minor_version);
+                        if (err < 0) {
+                                printk(KERN_WARNING 
+                                        "md: %s has different UUID to %s\n",
+                                        bdevname(rdev->bdev,b), 
+                                        bdevname(rdev0->bdev,b2));
+                                export_rdev(rdev);
+                                return -EINVAL;
+                        }
+                }
+                err = bind_rdev_to_array(rdev, mddev);
+                if (err)
+                        export_rdev(rdev);
+                return err;
+        }
+
+        /*
+         * add_new_disk can be used once the array is assembled
+         * to add "hot spares".  They must already have a superblock
+         * written
+         */
+        if (mddev->pers) {
+                int err;
+                if (!mddev->pers->hot_add_disk) {
+                        printk(KERN_WARNING 
+                                "%s: personality does not support diskops!\n",
+                               mdname(mddev));
+                        return -EINVAL;
+                }
+                rdev = md_import_device(dev, mddev->major_version,
+                                        mddev->minor_version);
+                if (IS_ERR(rdev)) {
+                        printk(KERN_WARNING 
+                                "md: md_import_device returned %ld\n",
+                                PTR_ERR(rdev));
+                        return PTR_ERR(rdev);
+                }
+                rdev->in_sync = 0; /* just to be sure */
+                rdev->raid_disk = -1;
+                err = bind_rdev_to_array(rdev, mddev);
+                if (err)
+                        export_rdev(rdev);
+                if (mddev->thread)
+                        md_wakeup_thread(mddev->thread);
+                return err;
+        }
+
+        /* otherwise, add_new_disk is only allowed
+         * for major_version==0 superblocks
+         */
+        if (mddev->major_version != 0) {
+                printk(KERN_WARNING "%s: ADD_NEW_DISK not supported\n",
+                       mdname(mddev));
+                return -EINVAL;
+        }
+
+        if (!(info->state & (1<<MD_DISK_FAULTY))) {
+                int err;
+                rdev = md_import_device (dev, -1, 0);
+                if (IS_ERR(rdev)) {
+                        printk(KERN_WARNING 
+                                "md: error, md_import_device() returned %ld\n",
+                                PTR_ERR(rdev));
+                        return PTR_ERR(rdev);
+                }
+                rdev->desc_nr = info->number;
+                if (info->raid_disk < mddev->raid_disks)
+                        rdev->raid_disk = info->raid_disk;
+                else
+                        rdev->raid_disk = -1;
+
+                rdev->faulty = 0;
+                if (rdev->raid_disk < mddev->raid_disks)
+                        rdev->in_sync = (info->state & (1<<MD_DISK_SYNC));
+                else
+                        rdev->in_sync = 0;
+
+                err = bind_rdev_to_array(rdev, mddev);
+                if (err) {
+                        export_rdev(rdev);
+                        return err;
+                }
+
+                if (!mddev->persistent) {
+                        printk(KERN_INFO "md: nonpersistent superblock ...\n");
+                        rdev->sb_offset = rdev->bdev->bd_inode->i_size >> BLOCK_SIZE_BITS;
+                } else 
+                        rdev->sb_offset = calc_dev_sboffset(rdev->bdev);
+                rdev->size = calc_dev_size(rdev, mddev->chunk_size);
+
+                if (!mddev->size || (mddev->size > rdev->size))
+                        mddev->size = rdev->size;
+        }
+
+        return 0;
+}
+
+static int hot_remove_disk(mddev_t * mddev, dev_t dev)
+{
+        char b[BDEVNAME_SIZE];
+        mdk_rdev_t *rdev;
+
+        if (!mddev->pers)
+                return -ENODEV;
+
+        rdev = find_rdev(mddev, dev);
+        if (!rdev)
+                return -ENXIO;
+
+        if (rdev->raid_disk >= 0)
+                goto busy;
+
+        kick_rdev_from_array(rdev);
+        md_update_sb(mddev);
+
+        return 0;
+busy:
+        printk(KERN_WARNING "md: cannot remove active disk %s from %s ... \n",
+                bdevname(rdev->bdev,b), mdname(mddev));
+        return -EBUSY;
+}
+
+static int hot_add_disk(mddev_t * mddev, dev_t dev)
+{
+        char b[BDEVNAME_SIZE];
+        int err;
+        unsigned int size;
+        mdk_rdev_t *rdev;
+
+        if (!mddev->pers)
+                return -ENODEV;
+
+        if (mddev->major_version != 0) {
+                printk(KERN_WARNING "%s: HOT_ADD may only be used with"
+                        " version-0 superblocks.\n",
+                        mdname(mddev));
+                return -EINVAL;
+        }
+        if (!mddev->pers->hot_add_disk) {
+                printk(KERN_WARNING 
+                        "%s: personality does not support diskops!\n",
+                        mdname(mddev));
+                return -EINVAL;
+        }
+
+        rdev = md_import_device (dev, -1, 0);
+        if (IS_ERR(rdev)) {
+                printk(KERN_WARNING 
+                        "md: error, md_import_device() returned %ld\n",
+                        PTR_ERR(rdev));
+                return -EINVAL;
+        }
+
+        if (mddev->persistent)
+                rdev->sb_offset = calc_dev_sboffset(rdev->bdev);
+        else
+                rdev->sb_offset =
+                        rdev->bdev->bd_inode->i_size >> BLOCK_SIZE_BITS;
+
+        size = calc_dev_size(rdev, mddev->chunk_size);
+        rdev->size = size;
+
+        if (size < mddev->size) {
+                printk(KERN_WARNING 
+                        "%s: disk size %llu blocks < array size %llu\n",
+                        mdname(mddev), (unsigned long long)size,
+                        (unsigned long long)mddev->size);
+                err = -ENOSPC;
+                goto abort_export;
+        }
+
+        if (rdev->faulty) {
+                printk(KERN_WARNING 
+                        "md: can not hot-add faulty %s disk to %s!\n",
+                        bdevname(rdev->bdev,b), mdname(mddev));
+                err = -EINVAL;
+                goto abort_export;
+        }
+        rdev->in_sync = 0;
+        rdev->desc_nr = -1;
+        bind_rdev_to_array(rdev, mddev);
+
+        /*
+         * The rest should better be atomic, we can have disk failures
+         * noticed in interrupt contexts ...
+         */
+
+        if (rdev->desc_nr == mddev->max_disks) {
+                printk(KERN_WARNING "%s: can not hot-add to full array!\n",
+                        mdname(mddev));
+                err = -EBUSY;
+                goto abort_unbind_export;
+        }
+
+        rdev->raid_disk = -1;
+
+        md_update_sb(mddev);
+
+        /*
+         * Kick recovery, maybe this spare has to be added to the
+         * array immediately.
+         */
+        set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+        md_wakeup_thread(mddev->thread);
+
+        return 0;
+
+abort_unbind_export:
+        unbind_rdev_from_array(rdev);
+
+abort_export:
+        export_rdev(rdev);
+        return err;
+}
+
+/*
+ * set_array_info is used two different ways
+ * The original usage is when creating a new array.
+ * In this usage, raid_disks is > 0 and it together with
+ *  level, size, not_persistent,layout,chunksize determine the
+ *  shape of the array.
+ *  This will always create an array with a type-0.90.0 superblock.
+ * The newer usage is when assembling an array.
+ *  In this case raid_disks will be 0, and the major_version field is
+ *  use to determine which style super-blocks are to be found on the devices.
+ *  The minor and patch _version numbers are also kept incase the
+ *  super_block handler wishes to interpret them.
+ */
+static int set_array_info(mddev_t * mddev, mdu_array_info_t *info)
+{
+
+        if (info->raid_disks == 0) {
+                /* just setting version number for superblock loading */
+                if (info->major_version < 0 ||
+                    info->major_version >= sizeof(super_types)/sizeof(super_types[0]) ||
+                    super_types[info->major_version].name == NULL) {
+                        /* maybe try to auto-load a module? */
+                        printk(KERN_INFO 
+                                "md: superblock version %d not known\n",
+                                info->major_version);
+                        return -EINVAL;
+                }
+                mddev->major_version = info->major_version;
+                mddev->minor_version = info->minor_version;
+                mddev->patch_version = info->patch_version;
+                return 0;
+        }
+        mddev->major_version = MD_MAJOR_VERSION;
+        mddev->minor_version = MD_MINOR_VERSION;
+        mddev->patch_version = MD_PATCHLEVEL_VERSION;
+        mddev->ctime         = get_seconds();
+
+        mddev->level         = info->level;
+        mddev->size          = info->size;
+        mddev->raid_disks    = info->raid_disks;
+        /* don't set md_minor, it is determined by which /dev/md* was
+         * openned
+         */
+        if (info->state & (1<<MD_SB_CLEAN))
+                mddev->recovery_cp = MaxSector;
+        else
+                mddev->recovery_cp = 0;
+        mddev->persistent    = ! info->not_persistent;
+
+        mddev->layout        = info->layout;
+        mddev->chunk_size    = info->chunk_size;
+
+        mddev->max_disks     = MD_SB_DISKS;
+
+        mddev->sb_dirty      = 1;
+
+        /*
+         * Generate a 128 bit UUID
+         */
+        get_random_bytes(mddev->uuid, 16);
+
+        return 0;
+}
+
+/*
+ * update_array_info is used to change the configuration of an
+ * on-line array.
+ * The version, ctime,level,size,raid_disks,not_persistent, layout,chunk_size
+ * fields in the info are checked against the array.
+ * Any differences that cannot be handled will cause an error.
+ * Normally, only one change can be managed at a time.
+ */
+static int update_array_info(mddev_t *mddev, mdu_array_info_t *info)
+{
+        int rv = 0;
+        int cnt = 0;
+
+        if (mddev->major_version != info->major_version ||
+            mddev->minor_version != info->minor_version ||
+/*          mddev->patch_version != info->patch_version || */
+            mddev->ctime         != info->ctime         ||
+            mddev->level         != info->level         ||
+/*          mddev->layout        != info->layout        || */
+            !mddev->persistent   != info->not_persistent||
+            mddev->chunk_size    != info->chunk_size    )
+                return -EINVAL;
+        /* Check there is only one change */
+        if (mddev->size != info->size) cnt++;
+        if (mddev->raid_disks != info->raid_disks) cnt++;
+        if (mddev->layout != info->layout) cnt++;
+        if (cnt == 0) return 0;
+        if (cnt > 1) return -EINVAL;
+
+        if (mddev->layout != info->layout) {
+                /* Change layout
+                 * we don't need to do anything at the md level, the
+                 * personality will take care of it all.
+                 */
+                if (mddev->pers->reconfig == NULL)
+                        return -EINVAL;
+                else
+                        return mddev->pers->reconfig(mddev, info->layout, -1);
+        }
+        if (mddev->size != info->size) {
+                mdk_rdev_t * rdev;
+                struct list_head *tmp;
+                if (mddev->pers->resize == NULL)
+                        return -EINVAL;
+                /* The "size" is the amount of each device that is used.
+                 * This can only make sense for arrays with redundancy.
+                 * linear and raid0 always use whatever space is available
+                 * We can only consider changing the size if no resync
+                 * or reconstruction is happening, and if the new size
+                 * is acceptable. It must fit before the sb_offset or,
+                 * if that is <data_offset, it must fit before the
+                 * size of each device.
+                 * If size is zero, we find the largest size that fits.
+                 */
+                if (mddev->sync_thread)
+                        return -EBUSY;
+                ITERATE_RDEV(mddev,rdev,tmp) {
+                        sector_t avail;
+                        int fit = (info->size == 0);
+                        if (rdev->sb_offset > rdev->data_offset)
+                                avail = (rdev->sb_offset*2) - rdev->data_offset;
+                        else
+                                avail = get_capacity(rdev->bdev->bd_disk)
+                                        - rdev->data_offset;
+                        if (fit && (info->size == 0 || info->size > avail/2))
+                                info->size = avail/2;
+                        if (avail < ((sector_t)info->size << 1))
+                                return -ENOSPC;
+                }
+                rv = mddev->pers->resize(mddev, (sector_t)info->size *2);
+                if (!rv) {
+                        struct block_device *bdev;
+
+                        bdev = bdget_disk(mddev->gendisk, 0);
+                        if (bdev) {
+                                down(&bdev->bd_inode->i_sem);
+                                i_size_write(bdev->bd_inode, mddev->array_size << 10);
+                                up(&bdev->bd_inode->i_sem);
+                                bdput(bdev);
+                        }
+                }
+        }
+        if (mddev->raid_disks    != info->raid_disks) {
+                /* change the number of raid disks */
+                if (mddev->pers->reshape == NULL)
+                        return -EINVAL;
+                if (info->raid_disks <= 0 ||
+                    info->raid_disks >= mddev->max_disks)
+                        return -EINVAL;
+                if (mddev->sync_thread)
+                        return -EBUSY;
+                rv = mddev->pers->reshape(mddev, info->raid_disks);
+                if (!rv) {
+                        struct block_device *bdev;
+
+                        bdev = bdget_disk(mddev->gendisk, 0);
+                        if (bdev) {
+                                down(&bdev->bd_inode->i_sem);
+                                i_size_write(bdev->bd_inode, mddev->array_size << 10);
+                                up(&bdev->bd_inode->i_sem);
+                                bdput(bdev);
+                        }
+                }
+        }
+        md_update_sb(mddev);
+        return rv;
+}
+
+static int set_disk_faulty(mddev_t *mddev, dev_t dev)
+{
+        mdk_rdev_t *rdev;
+
+        if (mddev->pers == NULL)
+                return -ENODEV;
+
+        rdev = find_rdev(mddev, dev);
+        if (!rdev)
+                return -ENODEV;
+
+        md_error(mddev, rdev);
+        return 0;
+}
+
+static int md_ioctl(struct inode *inode, struct file *file,
+                        unsigned int cmd, unsigned long arg)
+{
+        int err = 0;
+        void __user *argp = (void __user *)arg;
+        struct hd_geometry __user *loc = argp;
+        mddev_t *mddev = NULL;
+
+        if (!capable(CAP_SYS_ADMIN))
+                return -EACCES;
+
+        /*
+         * Commands dealing with the RAID driver but not any
+         * particular array:
+         */
+        switch (cmd)
+        {
+                case RAID_VERSION:
+                        err = get_version(argp);
+                        goto done;
+
+                case PRINT_RAID_DEBUG:
+                        err = 0;
+                        md_print_devices();
+                        goto done;
+
+#ifndef MODULE
+                case RAID_AUTORUN:
+                        err = 0;
+                        autostart_arrays(arg);
+                        goto done;
+#endif
+                default:;
+        }
+
+        /*
+         * Commands creating/starting a new array:
+         */
+
+        mddev = inode->i_bdev->bd_disk->private_data;
+
+        if (!mddev) {
+                BUG();
+                goto abort;
+        }
+
+
+        if (cmd == START_ARRAY) {
+                /* START_ARRAY doesn't need to lock the array as autostart_array
+                 * does the locking, and it could even be a different array
+                 */
+                static int cnt = 3;
+                if (cnt > 0 ) {
+                        printk(KERN_WARNING
+                               "md: %s(pid %d) used deprecated START_ARRAY ioctl. "
+                               "This will not be supported beyond 2.6\n",
+                               current->comm, current->pid);
+                        cnt--;
+                }
+                err = autostart_array(new_decode_dev(arg));
+                if (err) {
+                        printk(KERN_WARNING "md: autostart failed!\n");
+                        goto abort;
+                }
+                goto done;
+        }
+
+        err = mddev_lock(mddev);
+        if (err) {
+                printk(KERN_INFO 
+                        "md: ioctl lock interrupted, reason %d, cmd %d\n",
+                        err, cmd);
+                goto abort;
+        }
+
+        switch (cmd)
+        {
+                case SET_ARRAY_INFO:
+                        {
+                                mdu_array_info_t info;
+                                if (!arg)
+                                        memset(&info, 0, sizeof(info));
+                                else if (copy_from_user(&info, argp, sizeof(info))) {
+                                        err = -EFAULT;
+                                        goto abort_unlock;
+                                }
+                                if (mddev->pers) {
+                                        err = update_array_info(mddev, &info);
+                                        if (err) {
+                                                printk(KERN_WARNING "md: couldn't update"
+                                                       " array info. %d\n", err);
+                                                goto abort_unlock;
+                                        }
+                                        goto done_unlock;
+                                }
+                                if (!list_empty(&mddev->disks)) {
+                                        printk(KERN_WARNING
+                                               "md: array %s already has disks!\n",
+                                               mdname(mddev));
+                                        err = -EBUSY;
+                                        goto abort_unlock;
+                                }
+                                if (mddev->raid_disks) {
+                                        printk(KERN_WARNING
+                                               "md: array %s already initialised!\n",
+                                               mdname(mddev));
+                                        err = -EBUSY;
+                                        goto abort_unlock;
+                                }
+                                err = set_array_info(mddev, &info);
+                                if (err) {
+                                        printk(KERN_WARNING "md: couldn't set"
+                                               " array info. %d\n", err);
+                                        goto abort_unlock;
+                                }
+                        }
+                        goto done_unlock;
+
+                default:;
+        }
+
+        /*
+         * Commands querying/configuring an existing array:
+         */
+        /* if we are initialised yet, only ADD_NEW_DISK or STOP_ARRAY is allowed */
+        if (!mddev->raid_disks && cmd != ADD_NEW_DISK && cmd != STOP_ARRAY && cmd != RUN_ARRAY) {
+                err = -ENODEV;
+                goto abort_unlock;
+        }
+
+        /*
+         * Commands even a read-only array can execute:
+         */
+        switch (cmd)
+        {
+                case GET_ARRAY_INFO:
+                        err = get_array_info(mddev, argp);
+                        goto done_unlock;
+
+                case GET_DISK_INFO:
+                        err = get_disk_info(mddev, argp);
+                        goto done_unlock;
+
+                case RESTART_ARRAY_RW:
+                        err = restart_array(mddev);
+                        goto done_unlock;
+
+                case STOP_ARRAY:
+                        err = do_md_stop (mddev, 0);
+                        goto done_unlock;
+
+                case STOP_ARRAY_RO:
+                        err = do_md_stop (mddev, 1);
+                        goto done_unlock;
+
+        /*
+         * We have a problem here : there is no easy way to give a CHS
+         * virtual geometry. We currently pretend that we have a 2 heads
+         * 4 sectors (with a BIG number of cylinders...). This drives
+         * dosfs just mad... ;-)
+         */
+                case HDIO_GETGEO:
+                        if (!loc) {
+                                err = -EINVAL;
+                                goto abort_unlock;
+                        }
+                        err = put_user (2, (char __user *) &loc->heads);
+                        if (err)
+                                goto abort_unlock;
+                        err = put_user (4, (char __user *) &loc->sectors);
+                        if (err)
+                                goto abort_unlock;
+                        err = put_user(get_capacity(mddev->gendisk)/8,
+                                        (short __user *) &loc->cylinders);
+                        if (err)
+                                goto abort_unlock;
+                        err = put_user (get_start_sect(inode->i_bdev),
+                                                (long __user *) &loc->start);
+                        goto done_unlock;
+        }
+
+        /*
+         * The remaining ioctls are changing the state of the
+         * superblock, so we do not allow read-only arrays
+         * here:
+         */
+        if (mddev->ro) {
+                err = -EROFS;
+                goto abort_unlock;
+        }
+
+        switch (cmd)
+        {
+                case ADD_NEW_DISK:
+                {
+                        mdu_disk_info_t info;
+                        if (copy_from_user(&info, argp, sizeof(info)))
+                                err = -EFAULT;
+                        else
+                                err = add_new_disk(mddev, &info);
+                        goto done_unlock;
+                }
+
+                case HOT_REMOVE_DISK:
+                        err = hot_remove_disk(mddev, new_decode_dev(arg));
+                        goto done_unlock;
+
+                case HOT_ADD_DISK:
+                        err = hot_add_disk(mddev, new_decode_dev(arg));
+                        goto done_unlock;
+
+                case SET_DISK_FAULTY:
+                        err = set_disk_faulty(mddev, new_decode_dev(arg));
+                        goto done_unlock;
+
+                case RUN_ARRAY:
+                        err = do_md_run (mddev);
+                        goto done_unlock;
+
+                default:
+                        if (_IOC_TYPE(cmd) == MD_MAJOR)
+                                printk(KERN_WARNING "md: %s(pid %d) used"
+                                        " obsolete MD ioctl, upgrade your"
+                                        " software to use new ictls.\n",
+                                        current->comm, current->pid);
+                        err = -EINVAL;
+                        goto abort_unlock;
+        }
+
+done_unlock:
+abort_unlock:
+        mddev_unlock(mddev);
+
+        return err;
+done:
+        if (err)
+                MD_BUG();
+abort:
+        return err;
+}
+
+static int md_open(struct inode *inode, struct file *file)
+{
+        /*
+         * Succeed if we can lock the mddev, which confirms that
+         * it isn't being stopped right now.
+         */
+        mddev_t *mddev = inode->i_bdev->bd_disk->private_data;
+        int err;
+
+        if ((err = mddev_lock(mddev)))
+                goto out;
+
+        err = 0;
+        mddev_get(mddev);
+        mddev_unlock(mddev);
+
+        check_disk_change(inode->i_bdev);
+ out:
+        return err;
+}
+
+static int md_release(struct inode *inode, struct file * file)
+{
+        mddev_t *mddev = inode->i_bdev->bd_disk->private_data;
+
+        if (!mddev)
+                BUG();
+        mddev_put(mddev);
+
+        return 0;
+}
+
+static int md_media_changed(struct gendisk *disk)
+{
+        mddev_t *mddev = disk->private_data;
+
+        return mddev->changed;
+}
+
+static int md_revalidate(struct gendisk *disk)
+{
+        mddev_t *mddev = disk->private_data;
+
+        mddev->changed = 0;
+        return 0;
+}
+static struct block_device_operations md_fops =
+{
+        .owner          = THIS_MODULE,
+        .open           = md_open,
+        .release        = md_release,
+        .ioctl          = md_ioctl,
+        .media_changed  = md_media_changed,
+        .revalidate_disk= md_revalidate,
+};
+
+int md_thread(void * arg)
+{
+        mdk_thread_t *thread = arg;
+
+        lock_kernel();
+
+        /*
+         * Detach thread
+         */
+
+        daemonize(thread->name, mdname(thread->mddev));
+
+        current->exit_signal = SIGCHLD;
+        allow_signal(SIGKILL);
+        thread->tsk = current;
+
+        /*
+         * md_thread is a 'system-thread', it's priority should be very
+         * high. We avoid resource deadlocks individually in each
+         * raid personality. (RAID5 does preallocation) We also use RR and
+         * the very same RT priority as kswapd, thus we will never get
+         * into a priority inversion deadlock.
+         *
+         * we definitely have to have equal or higher priority than
+         * bdflush, otherwise bdflush will deadlock if there are too
+         * many dirty RAID5 blocks.
+         */
+        unlock_kernel();
+
+        complete(thread->event);
+        while (thread->run) {
+                void (*run)(mddev_t *);
+
+                wait_event_interruptible(thread->wqueue,
+                                         test_bit(THREAD_WAKEUP, &thread->flags));
+                if (current->flags & PF_FREEZE)
+                        refrigerator(PF_FREEZE);
+
+                clear_bit(THREAD_WAKEUP, &thread->flags);
+
+                run = thread->run;
+                if (run)
+                        run(thread->mddev);
+
+                if (signal_pending(current))
+                        flush_signals(current);
+        }
+        complete(thread->event);
+        return 0;
+}
+
+void md_wakeup_thread(mdk_thread_t *thread)
+{
+        if (thread) {
+                dprintk("md: waking up MD thread %s.\n", thread->tsk->comm);
+                set_bit(THREAD_WAKEUP, &thread->flags);
+                wake_up(&thread->wqueue);
+        }
+}
+
+mdk_thread_t *md_register_thread(void (*run) (mddev_t *), mddev_t *mddev,
+                                 const char *name)
+{
+        mdk_thread_t *thread;
+        int ret;
+        struct completion event;
+
+        thread = (mdk_thread_t *) kmalloc
+                                (sizeof(mdk_thread_t), GFP_KERNEL);
+        if (!thread)
+                return NULL;
+
+        memset(thread, 0, sizeof(mdk_thread_t));
+        init_waitqueue_head(&thread->wqueue);
+
+        init_completion(&event);
+        thread->event = &event;
+        thread->run = run;
+        thread->mddev = mddev;
+        thread->name = name;
+        ret = kernel_thread(md_thread, thread, 0);
+        if (ret < 0) {
+                kfree(thread);
+                return NULL;
+        }
+        wait_for_completion(&event);
+        return thread;
+}
+
+static void md_interrupt_thread(mdk_thread_t *thread)
+{
+        if (!thread->tsk) {
+                MD_BUG();
+                return;
+        }
+        dprintk("interrupting MD-thread pid %d\n", thread->tsk->pid);
+        send_sig(SIGKILL, thread->tsk, 1);
+}
+
+void md_unregister_thread(mdk_thread_t *thread)
+{
+        struct completion event;
+
+        init_completion(&event);
+
+        thread->event = &event;
+        thread->run = NULL;
+        thread->name = NULL;
+        md_interrupt_thread(thread);
+        wait_for_completion(&event);
+        kfree(thread);
+}
+
+void md_error(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+        if (!mddev) {
+                MD_BUG();
+                return;
+        }
+
+        if (!rdev || rdev->faulty)
+                return;
+
+        dprintk("md_error dev:%s, rdev:(%d:%d), (caller: %p,%p,%p,%p).\n",
+                mdname(mddev),
+                MAJOR(rdev->bdev->bd_dev), MINOR(rdev->bdev->bd_dev),
+                __builtin_return_address(0),__builtin_return_address(1),
+                __builtin_return_address(2),__builtin_return_address(3));
+
+        if (!mddev->pers->error_handler)
+                return;
+        mddev->pers->error_handler(mddev,rdev);
+        set_bit(MD_RECOVERY_INTR, &mddev->recovery);
+        set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+        md_wakeup_thread(mddev->thread);
+}
+
+/* seq_file implementation /proc/mdstat */
+
+static void status_unused(struct seq_file *seq)
+{
+        int i = 0;
+        mdk_rdev_t *rdev;
+        struct list_head *tmp;
+
+        seq_printf(seq, "unused devices: ");
+
+        ITERATE_RDEV_PENDING(rdev,tmp) {
+                char b[BDEVNAME_SIZE];
+                i++;
+                seq_printf(seq, "%s ",
+                              bdevname(rdev->bdev,b));
+        }
+        if (!i)
+                seq_printf(seq, "<none>");
+
+        seq_printf(seq, "\n");
+}
+
+
+static void status_resync(struct seq_file *seq, mddev_t * mddev)
+{
+        unsigned long max_blocks, resync, res, dt, db, rt;
+
+        resync = (mddev->curr_resync - atomic_read(&mddev->recovery_active))/2;
+
+        if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
+                max_blocks = mddev->resync_max_sectors >> 1;
+        else
+                max_blocks = mddev->size;
+
+        /*
+         * Should not happen.
+         */
+        if (!max_blocks) {
+                MD_BUG();
+                return;
+        }
+        res = (resync/1024)*1000/(max_blocks/1024 + 1);
+        {
+                int i, x = res/50, y = 20-x;
+                seq_printf(seq, "[");
+                for (i = 0; i < x; i++)
+                        seq_printf(seq, "=");
+                seq_printf(seq, ">");
+                for (i = 0; i < y; i++)
+                        seq_printf(seq, ".");
+                seq_printf(seq, "] ");
+        }
+        seq_printf(seq, " %s =%3lu.%lu%% (%lu/%lu)",
+                      (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ?
+                       "resync" : "recovery"),
+                      res/10, res % 10, resync, max_blocks);
+
+        /*
+         * We do not want to overflow, so the order of operands and
+         * the * 100 / 100 trick are important. We do a +1 to be
+         * safe against division by zero. We only estimate anyway.
+         *
+         * dt: time from mark until now
+         * db: blocks written from mark until now
+         * rt: remaining time
+         */
+        dt = ((jiffies - mddev->resync_mark) / HZ);
+        if (!dt) dt++;
+        db = resync - (mddev->resync_mark_cnt/2);
+        rt = (dt * ((max_blocks-resync) / (db/100+1)))/100;
+
+        seq_printf(seq, " finish=%lu.%lumin", rt / 60, (rt % 60)/6);
+
+        seq_printf(seq, " speed=%ldK/sec", db/dt);
+}
+
+static void *md_seq_start(struct seq_file *seq, loff_t *pos)
+{
+        struct list_head *tmp;
+        loff_t l = *pos;
+        mddev_t *mddev;
+
+        if (l >= 0x10000)
+                return NULL;
+        if (!l--)
+                /* header */
+                return (void*)1;
+
+        spin_lock(&all_mddevs_lock);
+        list_for_each(tmp,&all_mddevs)
+                if (!l--) {
+                        mddev = list_entry(tmp, mddev_t, all_mddevs);
+                        mddev_get(mddev);
+                        spin_unlock(&all_mddevs_lock);
+                        return mddev;
+                }
+        spin_unlock(&all_mddevs_lock);
+        if (!l--)
+                return (void*)2;/* tail */
+        return NULL;
+}
+
+static void *md_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+        struct list_head *tmp;
+        mddev_t *next_mddev, *mddev = v;
+        
+        ++*pos;
+        if (v == (void*)2)
+                return NULL;
+
+        spin_lock(&all_mddevs_lock);
+        if (v == (void*)1)
+                tmp = all_mddevs.next;
+        else
+                tmp = mddev->all_mddevs.next;
+        if (tmp != &all_mddevs)
+                next_mddev = mddev_get(list_entry(tmp,mddev_t,all_mddevs));
+        else {
+                next_mddev = (void*)2;
+                *pos = 0x10000;
+        }               
+        spin_unlock(&all_mddevs_lock);
+
+        if (v != (void*)1)
+                mddev_put(mddev);
+        return next_mddev;
+
+}
+
+static void md_seq_stop(struct seq_file *seq, void *v)
+{
+        mddev_t *mddev = v;
+
+        if (mddev && v != (void*)1 && v != (void*)2)
+                mddev_put(mddev);
+}
+
+static int md_seq_show(struct seq_file *seq, void *v)
+{
+        mddev_t *mddev = v;
+        sector_t size;
+        struct list_head *tmp2;
+        mdk_rdev_t *rdev;
+        int i;
+
+        if (v == (void*)1) {
+                seq_printf(seq, "Personalities : ");
+                spin_lock(&pers_lock);
+                for (i = 0; i < MAX_PERSONALITY; i++)
+                        if (pers[i])
+                                seq_printf(seq, "[%s] ", pers[i]->name);
+
+                spin_unlock(&pers_lock);
+                seq_printf(seq, "\n");
+                return 0;
+        }
+        if (v == (void*)2) {
+                status_unused(seq);
+                return 0;
+        }
+
+        if (mddev_lock(mddev)!=0) 
+                return -EINTR;
+        if (mddev->pers || mddev->raid_disks || !list_empty(&mddev->disks)) {
+                seq_printf(seq, "%s : %sactive", mdname(mddev),
+                                                mddev->pers ? "" : "in");
+                if (mddev->pers) {
+                        if (mddev->ro)
+                                seq_printf(seq, " (read-only)");
+                        seq_printf(seq, " %s", mddev->pers->name);
+                }
+
+                size = 0;
+                ITERATE_RDEV(mddev,rdev,tmp2) {
+                        char b[BDEVNAME_SIZE];
+                        seq_printf(seq, " %s[%d]",
+                                bdevname(rdev->bdev,b), rdev->desc_nr);
+                        if (rdev->faulty) {
+                                seq_printf(seq, "(F)");
+                                continue;
+                        }
+                        size += rdev->size;
+                }
+
+                if (!list_empty(&mddev->disks)) {
+                        if (mddev->pers)
+                                seq_printf(seq, "\n      %llu blocks",
+                                        (unsigned long long)mddev->array_size);
+                        else
+                                seq_printf(seq, "\n      %llu blocks",
+                                        (unsigned long long)size);
+                }
+
+                if (mddev->pers) {
+                        mddev->pers->status (seq, mddev);
+                        seq_printf(seq, "\n      ");
+                        if (mddev->curr_resync > 2)
+                                status_resync (seq, mddev);
+                        else if (mddev->curr_resync == 1 || mddev->curr_resync == 2)
+                                seq_printf(seq, "       resync=DELAYED");
+                }
+
+                seq_printf(seq, "\n");
+        }
+        mddev_unlock(mddev);
+        
+        return 0;
+}
+
+static struct seq_operations md_seq_ops = {
+        .start  = md_seq_start,
+        .next   = md_seq_next,
+        .stop   = md_seq_stop,
+        .show   = md_seq_show,
+};
+
+static int md_seq_open(struct inode *inode, struct file *file)
+{
+        int error;
+
+        error = seq_open(file, &md_seq_ops);
+        return error;
+}
+
+static struct file_operations md_seq_fops = {
+        .open           = md_seq_open,
+        .read           = seq_read,
+        .llseek         = seq_lseek,
+        .release        = seq_release,
+};
+
+int register_md_personality(int pnum, mdk_personality_t *p)
+{
+        if (pnum >= MAX_PERSONALITY) {
+                printk(KERN_ERR
+                       "md: tried to install personality %s as nr %d, but max is %lu\n",
+                       p->name, pnum, MAX_PERSONALITY-1);
+                return -EINVAL;
+        }
+
+        spin_lock(&pers_lock);
+        if (pers[pnum]) {
+                spin_unlock(&pers_lock);
+                MD_BUG();
+                return -EBUSY;
+        }
+
+        pers[pnum] = p;
+        printk(KERN_INFO "md: %s personality registered as nr %d\n", p->name, pnum);
+        spin_unlock(&pers_lock);
+        return 0;
+}
+
+int unregister_md_personality(int pnum)
+{
+        if (pnum >= MAX_PERSONALITY) {
+                MD_BUG();
+                return -EINVAL;
+        }
+
+        printk(KERN_INFO "md: %s personality unregistered\n", pers[pnum]->name);
+        spin_lock(&pers_lock);
+        pers[pnum] = NULL;
+        spin_unlock(&pers_lock);
+        return 0;
+}
+
+static int is_mddev_idle(mddev_t *mddev)
+{
+        mdk_rdev_t * rdev;
+        struct list_head *tmp;
+        int idle;
+        unsigned long curr_events;
+
+        idle = 1;
+        ITERATE_RDEV(mddev,rdev,tmp) {
+                struct gendisk *disk = rdev->bdev->bd_contains->bd_disk;
+                curr_events = disk_stat_read(disk, read_sectors) + 
+                                disk_stat_read(disk, write_sectors) - 
+                                atomic_read(&disk->sync_io);
+                /* Allow some slack between valud of curr_events and last_events,
+                 * as there are some uninteresting races.
+                 * Note: the following is an unsigned comparison.
+                 */
+                if ((curr_events - rdev->last_events + 32) > 64) {
+                        rdev->last_events = curr_events;
+                        idle = 0;
+                }
+        }
+        return idle;
+}
+
+void md_done_sync(mddev_t *mddev, int blocks, int ok)
+{
+        /* another "blocks" (512byte) blocks have been synced */
+        atomic_sub(blocks, &mddev->recovery_active);
+        wake_up(&mddev->recovery_wait);
+        if (!ok) {
+                set_bit(MD_RECOVERY_ERR, &mddev->recovery);
+                md_wakeup_thread(mddev->thread);
+                // stop recovery, signal do_sync ....
+        }
+}
+
+
+void md_write_start(mddev_t *mddev)
+{
+        if (!atomic_read(&mddev->writes_pending)) {
+                mddev_lock_uninterruptible(mddev);
+                if (mddev->in_sync) {
+                        mddev->in_sync = 0;
+                        del_timer(&mddev->safemode_timer);
+                        md_update_sb(mddev);
+                }
+                atomic_inc(&mddev->writes_pending);
+                mddev_unlock(mddev);
+        } else
+                atomic_inc(&mddev->writes_pending);
+}
+
+void md_write_end(mddev_t *mddev)
+{
+        if (atomic_dec_and_test(&mddev->writes_pending)) {
+                if (mddev->safemode == 2)
+                        md_wakeup_thread(mddev->thread);
+                else
+                        mod_timer(&mddev->safemode_timer, jiffies + mddev->safemode_delay);
+        }
+}
+
+static inline void md_enter_safemode(mddev_t *mddev)
+{
+        if (!mddev->safemode) return;
+        if (mddev->safemode == 2 &&
+            (atomic_read(&mddev->writes_pending) || mddev->in_sync ||
+                    mddev->recovery_cp != MaxSector))
+                return; /* avoid the lock */
+        mddev_lock_uninterruptible(mddev);
+        if (mddev->safemode && !atomic_read(&mddev->writes_pending) &&
+            !mddev->in_sync && mddev->recovery_cp == MaxSector) {
+                mddev->in_sync = 1;
+                md_update_sb(mddev);
+        }
+        mddev_unlock(mddev);
+
+        if (mddev->safemode == 1)
+                mddev->safemode = 0;
+}
+
+void md_handle_safemode(mddev_t *mddev)
+{
+        if (signal_pending(current)) {
+                printk(KERN_INFO "md: %s in immediate safe mode\n",
+                        mdname(mddev));
+                mddev->safemode = 2;
+                flush_signals(current);
+        }
+        md_enter_safemode(mddev);
+}
+
+
+DECLARE_WAIT_QUEUE_HEAD(resync_wait);
+
+#define SYNC_MARKS      10
+#define SYNC_MARK_STEP  (3*HZ)
+static void md_do_sync(mddev_t *mddev)
+{
+        mddev_t *mddev2;
+        unsigned int currspeed = 0,
+                 window;
+        sector_t max_sectors,j;
+        unsigned long mark[SYNC_MARKS];
+        sector_t mark_cnt[SYNC_MARKS];
+        int last_mark,m;
+        struct list_head *tmp;
+        sector_t last_check;
+
+        /* just incase thread restarts... */
+        if (test_bit(MD_RECOVERY_DONE, &mddev->recovery))
+                return;
+
+        /* we overload curr_resync somewhat here.
+         * 0 == not engaged in resync at all
+         * 2 == checking that there is no conflict with another sync
+         * 1 == like 2, but have yielded to allow conflicting resync to
+         *              commense
+         * other == active in resync - this many blocks
+         *
+         * Before starting a resync we must have set curr_resync to
+         * 2, and then checked that every "conflicting" array has curr_resync
+         * less than ours.  When we find one that is the same or higher
+         * we wait on resync_wait.  To avoid deadlock, we reduce curr_resync
+         * to 1 if we choose to yield (based arbitrarily on address of mddev structure).
+         * This will mean we have to start checking from the beginning again.
+         *
+         */
+
+        do {
+                mddev->curr_resync = 2;
+
+        try_again:
+                if (signal_pending(current)) {
+                        flush_signals(current);
+                        goto skip;
+                }
+                ITERATE_MDDEV(mddev2,tmp) {
+                        printk(".");
+                        if (mddev2 == mddev)
+                                continue;
+                        if (mddev2->curr_resync && 
+                            match_mddev_units(mddev,mddev2)) {
+                                DEFINE_WAIT(wq);
+                                if (mddev < mddev2 && mddev->curr_resync == 2) {
+                                        /* arbitrarily yield */
+                                        mddev->curr_resync = 1;
+                                        wake_up(&resync_wait);
+                                }
+                                if (mddev > mddev2 && mddev->curr_resync == 1)
+                                        /* no need to wait here, we can wait the next
+                                         * time 'round when curr_resync == 2
+                                         */
+                                        continue;
+                                prepare_to_wait(&resync_wait, &wq, TASK_INTERRUPTIBLE);
+                                if (!signal_pending(current)
+                                    && mddev2->curr_resync >= mddev->curr_resync) {
+                                        printk(KERN_INFO "md: delaying resync of %s"
+                                               " until %s has finished resync (they"
+                                               " share one or more physical units)\n",
+                                               mdname(mddev), mdname(mddev2));
+                                        mddev_put(mddev2);
+                                        schedule();
+                                        finish_wait(&resync_wait, &wq);
+                                        goto try_again;
+                                }
+                                finish_wait(&resync_wait, &wq);
+                        }
+                }
+        } while (mddev->curr_resync < 2);
+
+        if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
+                /* resync follows the size requested by the personality,
+                 * which default to physical size, but can be virtual size
+                 */
+                max_sectors = mddev->resync_max_sectors;
+        else
+                /* recovery follows the physical size of devices */
+                max_sectors = mddev->size << 1;
+
+        printk(KERN_INFO "md: syncing RAID array %s\n", mdname(mddev));
+        printk(KERN_INFO "md: minimum _guaranteed_ reconstruction speed:"
+                " %d KB/sec/disc.\n", sysctl_speed_limit_min);
+        printk(KERN_INFO "md: using maximum available idle IO bandwith "
+               "(but not more than %d KB/sec) for reconstruction.\n",
+               sysctl_speed_limit_max);
+
+        is_mddev_idle(mddev); /* this also initializes IO event counters */
+        if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
+                j = mddev->recovery_cp;
+        else
+                j = 0;
+        for (m = 0; m < SYNC_MARKS; m++) {
+                mark[m] = jiffies;
+                mark_cnt[m] = j;
+        }
+        last_mark = 0;
+        mddev->resync_mark = mark[last_mark];
+        mddev->resync_mark_cnt = mark_cnt[last_mark];
+
+        /*
+         * Tune reconstruction:
+         */
+        window = 32*(PAGE_SIZE/512);
+        printk(KERN_INFO "md: using %dk window, over a total of %llu blocks.\n",
+                window/2,(unsigned long long) max_sectors/2);
+
+        atomic_set(&mddev->recovery_active, 0);
+        init_waitqueue_head(&mddev->recovery_wait);
+        last_check = 0;
+
+        if (j>2) {
+                printk(KERN_INFO 
+                        "md: resuming recovery of %s from checkpoint.\n",
+                        mdname(mddev));
+                mddev->curr_resync = j;
+        }
+
+        while (j < max_sectors) {
+                int sectors;
+
+                sectors = mddev->pers->sync_request(mddev, j, currspeed < sysctl_speed_limit_min);
+                if (sectors < 0) {
+                        set_bit(MD_RECOVERY_ERR, &mddev->recovery);
+                        goto out;
+                }
+                atomic_add(sectors, &mddev->recovery_active);
+                j += sectors;
+                if (j>1) mddev->curr_resync = j;
+
+                if (last_check + window > j || j == max_sectors)
+                        continue;
+
+                last_check = j;
+
+                if (test_bit(MD_RECOVERY_INTR, &mddev->recovery) ||
+                    test_bit(MD_RECOVERY_ERR, &mddev->recovery))
+                        break;
+
+        repeat:
+                if (time_after_eq(jiffies, mark[last_mark] + SYNC_MARK_STEP )) {
+                        /* step marks */
+                        int next = (last_mark+1) % SYNC_MARKS;
+
+                        mddev->resync_mark = mark[next];
+                        mddev->resync_mark_cnt = mark_cnt[next];
+                        mark[next] = jiffies;
+                        mark_cnt[next] = j - atomic_read(&mddev->recovery_active);
+                        last_mark = next;
+                }
+
+
+                if (signal_pending(current)) {
+                        /*
+                         * got a signal, exit.
+                         */
+                        printk(KERN_INFO 
+                                "md: md_do_sync() got signal ... exiting\n");
+                        flush_signals(current);
+                        set_bit(MD_RECOVERY_INTR, &mddev->recovery);
+                        goto out;
+                }
+
+                /*
+                 * this loop exits only if either when we are slower than
+                 * the 'hard' speed limit, or the system was IO-idle for
+                 * a jiffy.
+                 * the system might be non-idle CPU-wise, but we only care
+                 * about not overloading the IO subsystem. (things like an
+                 * e2fsck being done on the RAID array should execute fast)
+                 */
+                mddev->queue->unplug_fn(mddev->queue);
+                cond_resched();
+
+                currspeed = ((unsigned long)(j-mddev->resync_mark_cnt))/2/((jiffies-mddev->resync_mark)/HZ +1) +1;
+
+                if (currspeed > sysctl_speed_limit_min) {
+                        if ((currspeed > sysctl_speed_limit_max) ||
+                                        !is_mddev_idle(mddev)) {
+                                msleep_interruptible(250);
+                                goto repeat;
+                        }
+                }
+        }
+        printk(KERN_INFO "md: %s: sync done.\n",mdname(mddev));
+        /*
+         * this also signals 'finished resyncing' to md_stop
+         */
+ out:
+        mddev->queue->unplug_fn(mddev->queue);
+
+        wait_event(mddev->recovery_wait, !atomic_read(&mddev->recovery_active));
+
+        /* tell personality that we are finished */
+        mddev->pers->sync_request(mddev, max_sectors, 1);
+
+        if (!test_bit(MD_RECOVERY_ERR, &mddev->recovery) &&
+            mddev->curr_resync > 2 &&
+            mddev->curr_resync >= mddev->recovery_cp) {
+                if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
+                        printk(KERN_INFO 
+                                "md: checkpointing recovery of %s.\n",
+                                mdname(mddev));
+                        mddev->recovery_cp = mddev->curr_resync;
+                } else
+                        mddev->recovery_cp = MaxSector;
+        }
+
+        md_enter_safemode(mddev);
+ skip:
+        mddev->curr_resync = 0;
+        wake_up(&resync_wait);
+        set_bit(MD_RECOVERY_DONE, &mddev->recovery);
+        md_wakeup_thread(mddev->thread);
+}
+
+
+/*
+ * This routine is regularly called by all per-raid-array threads to
+ * deal with generic issues like resync and super-block update.
+ * Raid personalities that don't have a thread (linear/raid0) do not
+ * need this as they never do any recovery or update the superblock.
+ *
+ * It does not do any resync itself, but rather "forks" off other threads
+ * to do that as needed.
+ * When it is determined that resync is needed, we set MD_RECOVERY_RUNNING in
+ * "->recovery" and create a thread at ->sync_thread.
+ * When the thread finishes it sets MD_RECOVERY_DONE (and might set MD_RECOVERY_ERR)
+ * and wakeups up this thread which will reap the thread and finish up.
+ * This thread also removes any faulty devices (with nr_pending == 0).
+ *
+ * The overall approach is:
+ *  1/ if the superblock needs updating, update it.
+ *  2/ If a recovery thread is running, don't do anything else.
+ *  3/ If recovery has finished, clean up, possibly marking spares active.
+ *  4/ If there are any faulty devices, remove them.
+ *  5/ If array is degraded, try to add spares devices
+ *  6/ If array has spares or is not in-sync, start a resync thread.
+ */
+void md_check_recovery(mddev_t *mddev)
+{
+        mdk_rdev_t *rdev;
+        struct list_head *rtmp;
+
+
+        dprintk(KERN_INFO "md: recovery thread got woken up ...\n");
+
+        if (mddev->ro)
+                return;
+        if ( ! (
+                mddev->sb_dirty ||
+                test_bit(MD_RECOVERY_NEEDED, &mddev->recovery) ||
+                test_bit(MD_RECOVERY_DONE, &mddev->recovery)
+                ))
+                return;
+        if (mddev_trylock(mddev)==0) {
+                int spares =0;
+                if (mddev->sb_dirty)
+                        md_update_sb(mddev);
+                if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) &&
+                    !test_bit(MD_RECOVERY_DONE, &mddev->recovery)) {
+                        /* resync/recovery still happening */
+                        clear_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+                        goto unlock;
+                }
+                if (mddev->sync_thread) {
+                        /* resync has finished, collect result */
+                        md_unregister_thread(mddev->sync_thread);
+                        mddev->sync_thread = NULL;
+                        if (!test_bit(MD_RECOVERY_ERR, &mddev->recovery) &&
+                            !test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
+                                /* success...*/
+                                /* activate any spares */
+                                mddev->pers->spare_active(mddev);
+                        }
+                        md_update_sb(mddev);
+                        mddev->recovery = 0;
+                        /* flag recovery needed just to double check */
+                        set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+                        goto unlock;
+                }
+                if (mddev->recovery)
+                        /* probably just the RECOVERY_NEEDED flag */
+                        mddev->recovery = 0;
+
+                /* no recovery is running.
+                 * remove any failed drives, then
+                 * add spares if possible.
+                 * Spare are also removed and re-added, to allow
+                 * the personality to fail the re-add.
+                 */
+                ITERATE_RDEV(mddev,rdev,rtmp)
+                        if (rdev->raid_disk >= 0 &&
+                            (rdev->faulty || ! rdev->in_sync) &&
+                            atomic_read(&rdev->nr_pending)==0) {
+                                if (mddev->pers->hot_remove_disk(mddev, rdev->raid_disk)==0)
+                                        rdev->raid_disk = -1;
+                        }
+
+                if (mddev->degraded) {
+                        ITERATE_RDEV(mddev,rdev,rtmp)
+                                if (rdev->raid_disk < 0
+                                    && !rdev->faulty) {
+                                        if (mddev->pers->hot_add_disk(mddev,rdev))
+                                                spares++;
+                                        else
+                                                break;
+                                }
+                }
+
+                if (!spares && (mddev->recovery_cp == MaxSector )) {
+                        /* nothing we can do ... */
+                        goto unlock;
+                }
+                if (mddev->pers->sync_request) {
+                        set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
+                        if (!spares)
+                                set_bit(MD_RECOVERY_SYNC, &mddev->recovery);
+                        mddev->sync_thread = md_register_thread(md_do_sync,
+                                                                mddev,
+                                                                "%s_resync");
+                        if (!mddev->sync_thread) {
+                                printk(KERN_ERR "%s: could not start resync"
+                                        " thread...\n", 
+                                        mdname(mddev));
+                                /* leave the spares where they are, it shouldn't hurt */
+                                mddev->recovery = 0;
+                        } else {
+                                md_wakeup_thread(mddev->sync_thread);
+                        }
+                }
+        unlock:
+                mddev_unlock(mddev);
+        }
+}
+
+int md_notify_reboot(struct notifier_block *this,
+                                        unsigned long code, void *x)
+{
+        struct list_head *tmp;
+        mddev_t *mddev;
+
+        if ((code == SYS_DOWN) || (code == SYS_HALT) || (code == SYS_POWER_OFF)) {
+
+                printk(KERN_INFO "md: stopping all md devices.\n");
+
+                ITERATE_MDDEV(mddev,tmp)
+                        if (mddev_trylock(mddev)==0)
+                                do_md_stop (mddev, 1);
+                /*
+                 * certain more exotic SCSI devices are known to be
+                 * volatile wrt too early system reboots. While the
+                 * right place to handle this issue is the given
+                 * driver, we do want to have a safe RAID driver ...
+                 */
+                mdelay(1000*1);
+        }
+        return NOTIFY_DONE;
+}
+
+struct notifier_block md_notifier = {
+        .notifier_call  = md_notify_reboot,
+        .next           = NULL,
+        .priority       = INT_MAX, /* before any real devices */
+};
+
+static void md_geninit(void)
+{
+        struct proc_dir_entry *p;
+
+        dprintk("md: sizeof(mdp_super_t) = %d\n", (int)sizeof(mdp_super_t));
+
+        p = create_proc_entry("mdstat", S_IRUGO, NULL);
+        if (p)
+                p->proc_fops = &md_seq_fops;
+}
+
+int __init md_init(void)
+{
+        int minor;
+
+        printk(KERN_INFO "md: md driver %d.%d.%d MAX_MD_DEVS=%d,"
+                        " MD_SB_DISKS=%d\n",
+                        MD_MAJOR_VERSION, MD_MINOR_VERSION,
+                        MD_PATCHLEVEL_VERSION, MAX_MD_DEVS, MD_SB_DISKS);
+
+        if (register_blkdev(MAJOR_NR, "md"))
+                return -1;
+        if ((mdp_major=register_blkdev(0, "mdp"))<=0) {
+                unregister_blkdev(MAJOR_NR, "md");
+                return -1;
+        }
+        devfs_mk_dir("md");
+        blk_register_region(MKDEV(MAJOR_NR, 0), MAX_MD_DEVS, THIS_MODULE,
+                                md_probe, NULL, NULL);
+        blk_register_region(MKDEV(mdp_major, 0), MAX_MD_DEVS<<MdpMinorShift, THIS_MODULE,
+                            md_probe, NULL, NULL);
+
+        for (minor=0; minor < MAX_MD_DEVS; ++minor)
+                devfs_mk_bdev(MKDEV(MAJOR_NR, minor),
+                                S_IFBLK|S_IRUSR|S_IWUSR,
+                                "md/%d", minor);
+
+        for (minor=0; minor < MAX_MD_DEVS; ++minor)
+                devfs_mk_bdev(MKDEV(mdp_major, minor<<MdpMinorShift),
+                              S_IFBLK|S_IRUSR|S_IWUSR,
+                              "md/mdp%d", minor);
+
+
+        register_reboot_notifier(&md_notifier);
+        raid_table_header = register_sysctl_table(raid_root_table, 1);
+
+        md_geninit();
+        return (0);
+}
+
+
+#ifndef MODULE
+
+/*
+ * Searches all registered partitions for autorun RAID arrays
+ * at boot time.
+ */
+static dev_t detected_devices[128];
+static int dev_cnt;
+
+void md_autodetect_dev(dev_t dev)
+{
+        if (dev_cnt >= 0 && dev_cnt < 127)
+                detected_devices[dev_cnt++] = dev;
+}
+
+
+static void autostart_arrays(int part)
+{
+        mdk_rdev_t *rdev;
+        int i;
+
+        printk(KERN_INFO "md: Autodetecting RAID arrays.\n");
+
+        for (i = 0; i < dev_cnt; i++) {
+                dev_t dev = detected_devices[i];
+
+                rdev = md_import_device(dev,0, 0);
+                if (IS_ERR(rdev))
+                        continue;
+
+                if (rdev->faulty) {
+                        MD_BUG();
+                        continue;
+                }
+                list_add(&rdev->same_set, &pending_raid_disks);
+        }
+        dev_cnt = 0;
+
+        autorun_devices(part);
+}
+
+#endif
+
+static __exit void md_exit(void)
+{
+        mddev_t *mddev;
+        struct list_head *tmp;
+        int i;
+        blk_unregister_region(MKDEV(MAJOR_NR,0), MAX_MD_DEVS);
+        blk_unregister_region(MKDEV(mdp_major,0), MAX_MD_DEVS << MdpMinorShift);
+        for (i=0; i < MAX_MD_DEVS; i++)
+                devfs_remove("md/%d", i);
+        for (i=0; i < MAX_MD_DEVS; i++)
+                devfs_remove("md/d%d", i);
+
+        devfs_remove("md");
+
+        unregister_blkdev(MAJOR_NR,"md");
+        unregister_blkdev(mdp_major, "mdp");
+        unregister_reboot_notifier(&md_notifier);
+        unregister_sysctl_table(raid_table_header);
+        remove_proc_entry("mdstat", NULL);
+        ITERATE_MDDEV(mddev,tmp) {
+                struct gendisk *disk = mddev->gendisk;
+                if (!disk)
+                        continue;
+                export_array(mddev);
+                del_gendisk(disk);
+                put_disk(disk);
+                mddev->gendisk = NULL;
+                mddev_put(mddev);
+        }
+}
+
+module_init(md_init)
+module_exit(md_exit)
+
+EXPORT_SYMBOL(register_md_personality);
+EXPORT_SYMBOL(unregister_md_personality);
+EXPORT_SYMBOL(md_error);
+EXPORT_SYMBOL(md_done_sync);
+EXPORT_SYMBOL(md_write_start);
+EXPORT_SYMBOL(md_write_end);
+EXPORT_SYMBOL(md_handle_safemode);
+EXPORT_SYMBOL(md_register_thread);
+EXPORT_SYMBOL(md_unregister_thread);
+EXPORT_SYMBOL(md_wakeup_thread);
+EXPORT_SYMBOL(md_print_devices);
+EXPORT_SYMBOL(md_check_recovery);
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/mktables.c b/drivers/md/mktables.c
new file mode 100644
index 000000000000..adef299908cf
--- /dev/null
+++ b/drivers/md/mktables.c
@@ -0,0 +1,125 @@
+#ident "$Id: mktables.c,v 1.2 2002/12/12 22:41:27 hpa Exp $"
+/* ----------------------------------------------------------------------- *
+ *
+ *   Copyright 2002 H. Peter Anvin - All Rights Reserved
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+ *   Bostom MA 02111-1307, USA; either version 2 of the License, or
+ *   (at your option) any later version; incorporated herein by reference.
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * mktables.c
+ *
+ * Make RAID-6 tables.  This is a host user space program to be run at
+ * compile time.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <inttypes.h>
+#include <stdlib.h>
+#include <time.h>
+
+static uint8_t gfmul(uint8_t a, uint8_t b)
+{
+  uint8_t v = 0;
+
+  while ( b ) {
+    if ( b & 1 ) v ^= a;
+    a = (a << 1) ^ (a & 0x80 ? 0x1d : 0);
+    b >>= 1;
+  }
+  return v;
+}
+
+static uint8_t gfpow(uint8_t a, int b)
+{
+  uint8_t v = 1;
+
+  b %= 255;
+  if ( b < 0 )
+    b += 255;
+
+  while ( b ) {
+    if ( b & 1 ) v = gfmul(v,a);
+    a = gfmul(a,a);
+    b >>= 1;
+  }
+  return v;
+}
+
+int main(int argc, char *argv[])
+{
+  int i, j, k;
+  uint8_t v;
+  uint8_t exptbl[256], invtbl[256];
+
+  printf("#include \"raid6.h\"\n");
+
+  /* Compute multiplication table */
+  printf("\nconst u8  __attribute__((aligned(256)))\n"
+         "raid6_gfmul[256][256] =\n"
+         "{\n");
+  for ( i = 0 ; i < 256 ; i++ ) {
+    printf("\t{\n");
+    for ( j = 0 ; j < 256 ; j += 8 ) {
+      printf("\t\t");
+      for ( k = 0 ; k < 8 ; k++ ) {
+        printf("0x%02x, ", gfmul(i,j+k));
+      }
+      printf("\n");
+    }
+    printf("\t},\n");
+  }
+  printf("};\n");
+
+  /* Compute power-of-2 table (exponent) */
+  v = 1;
+  printf("\nconst u8 __attribute__((aligned(256)))\n"
+         "raid6_gfexp[256] =\n"
+         "{\n");
+  for ( i = 0 ; i < 256 ; i += 8 ) {
+    printf("\t");
+    for ( j = 0 ; j < 8 ; j++ ) {
+      exptbl[i+j] = v;
+      printf("0x%02x, ", v);
+      v = gfmul(v,2);
+      if ( v == 1 ) v = 0;      /* For entry 255, not a real entry */
+    }
+    printf("\n");
+  }
+  printf("};\n");
+
+  /* Compute inverse table x^-1 == x^254 */
+  printf("\nconst u8 __attribute__((aligned(256)))\n"
+         "raid6_gfinv[256] =\n"
+         "{\n");
+  for ( i = 0 ; i < 256 ; i += 8 ) {
+    printf("\t");
+    for ( j = 0 ; j < 8 ; j++ ) {
+      invtbl[i+j] = v = gfpow(i+j,254);
+      printf("0x%02x, ", v);
+    }
+    printf("\n");
+  }
+  printf("};\n");
+
+  /* Compute inv(2^x + 1) (exponent-xor-inverse) table */
+  printf("\nconst u8 __attribute__((aligned(256)))\n"
+         "raid6_gfexi[256] =\n"
+         "{\n");
+  for ( i = 0 ; i < 256 ; i += 8 ) {
+    printf("\t");
+    for ( j = 0 ; j < 8 ; j++ ) {
+      printf("0x%02x, ", invtbl[exptbl[i+j]^1]);
+    }
+    printf("\n");
+  }
+  printf("};\n\n");
+
+  return 0;
+}
diff --git a/drivers/md/multipath.c b/drivers/md/multipath.c
new file mode 100644
index 000000000000..c9b134cd1532
--- /dev/null
+++ b/drivers/md/multipath.c
@@ -0,0 +1,584 @@
+/*
+ * multipath.c : Multiple Devices driver for Linux
+ *
+ * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat
+ *
+ * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman
+ *
+ * MULTIPATH management functions.
+ *
+ * derived from raid1.c.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * (for example /usr/src/linux/COPYING); if not, write to the Free
+ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/raid/multipath.h>
+#include <linux/buffer_head.h>
+#include <asm/atomic.h>
+
+#define MAJOR_NR MD_MAJOR
+#define MD_DRIVER
+#define MD_PERSONALITY
+
+#define MAX_WORK_PER_DISK 128
+
+#define NR_RESERVED_BUFS        32
+
+
+static mdk_personality_t multipath_personality;
+
+
+static void *mp_pool_alloc(unsigned int __nocast gfp_flags, void *data)
+{
+        struct multipath_bh *mpb;
+        mpb = kmalloc(sizeof(*mpb), gfp_flags);
+        if (mpb) 
+                memset(mpb, 0, sizeof(*mpb));
+        return mpb;
+}
+
+static void mp_pool_free(void *mpb, void *data)
+{
+        kfree(mpb);
+}
+
+static int multipath_map (multipath_conf_t *conf)
+{
+        int i, disks = conf->raid_disks;
+
+        /*
+         * Later we do read balancing on the read side 
+         * now we use the first available disk.
+         */
+
+        rcu_read_lock();
+        for (i = 0; i < disks; i++) {
+                mdk_rdev_t *rdev = conf->multipaths[i].rdev;
+                if (rdev && rdev->in_sync) {
+                        atomic_inc(&rdev->nr_pending);
+                        rcu_read_unlock();
+                        return i;
+                }
+        }
+        rcu_read_unlock();
+
+        printk(KERN_ERR "multipath_map(): no more operational IO paths?\n");
+        return (-1);
+}
+
+static void multipath_reschedule_retry (struct multipath_bh *mp_bh)
+{
+        unsigned long flags;
+        mddev_t *mddev = mp_bh->mddev;
+        multipath_conf_t *conf = mddev_to_conf(mddev);
+
+        spin_lock_irqsave(&conf->device_lock, flags);
+        list_add(&mp_bh->retry_list, &conf->retry_list);
+        spin_unlock_irqrestore(&conf->device_lock, flags);
+        md_wakeup_thread(mddev->thread);
+}
+
+
+/*
+ * multipath_end_bh_io() is called when we have finished servicing a multipathed
+ * operation and are ready to return a success/failure code to the buffer
+ * cache layer.
+ */
+static void multipath_end_bh_io (struct multipath_bh *mp_bh, int err)
+{
+        struct bio *bio = mp_bh->master_bio;
+        multipath_conf_t *conf = mddev_to_conf(mp_bh->mddev);
+
+        bio_endio(bio, bio->bi_size, err);
+        mempool_free(mp_bh, conf->pool);
+}
+
+int multipath_end_request(struct bio *bio, unsigned int bytes_done, int error)
+{
+        int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
+        struct multipath_bh * mp_bh = (struct multipath_bh *)(bio->bi_private);
+        multipath_conf_t *conf = mddev_to_conf(mp_bh->mddev);
+        mdk_rdev_t *rdev = conf->multipaths[mp_bh->path].rdev;
+
+        if (bio->bi_size)
+                return 1;
+
+        if (uptodate)
+                multipath_end_bh_io(mp_bh, 0);
+        else if (!bio_rw_ahead(bio)) {
+                /*
+                 * oops, IO error:
+                 */
+                char b[BDEVNAME_SIZE];
+                md_error (mp_bh->mddev, rdev);
+                printk(KERN_ERR "multipath: %s: rescheduling sector %llu\n", 
+                       bdevname(rdev->bdev,b), 
+                       (unsigned long long)bio->bi_sector);
+                multipath_reschedule_retry(mp_bh);
+        } else
+                multipath_end_bh_io(mp_bh, error);
+        rdev_dec_pending(rdev, conf->mddev);
+        return 0;
+}
+
+static void unplug_slaves(mddev_t *mddev)
+{
+        multipath_conf_t *conf = mddev_to_conf(mddev);
+        int i;
+
+        rcu_read_lock();
+        for (i=0; i<mddev->raid_disks; i++) {
+                mdk_rdev_t *rdev = conf->multipaths[i].rdev;
+                if (rdev && !rdev->faulty && atomic_read(&rdev->nr_pending)) {
+                        request_queue_t *r_queue = bdev_get_queue(rdev->bdev);
+
+                        atomic_inc(&rdev->nr_pending);
+                        rcu_read_unlock();
+
+                        if (r_queue->unplug_fn)
+                                r_queue->unplug_fn(r_queue);
+
+                        rdev_dec_pending(rdev, mddev);
+                        rcu_read_lock();
+                }
+        }
+        rcu_read_unlock();
+}
+
+static void multipath_unplug(request_queue_t *q)
+{
+        unplug_slaves(q->queuedata);
+}
+
+
+static int multipath_make_request (request_queue_t *q, struct bio * bio)
+{
+        mddev_t *mddev = q->queuedata;
+        multipath_conf_t *conf = mddev_to_conf(mddev);
+        struct multipath_bh * mp_bh;
+        struct multipath_info *multipath;
+
+        mp_bh = mempool_alloc(conf->pool, GFP_NOIO);
+
+        mp_bh->master_bio = bio;
+        mp_bh->mddev = mddev;
+
+        if (bio_data_dir(bio)==WRITE) {
+                disk_stat_inc(mddev->gendisk, writes);
+                disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bio));
+        } else {
+                disk_stat_inc(mddev->gendisk, reads);
+                disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bio));
+        }
+
+        mp_bh->path = multipath_map(conf);
+        if (mp_bh->path < 0) {
+                bio_endio(bio, bio->bi_size, -EIO);
+                mempool_free(mp_bh, conf->pool);
+                return 0;
+        }
+        multipath = conf->multipaths + mp_bh->path;
+
+        mp_bh->bio = *bio;
+        mp_bh->bio.bi_sector += multipath->rdev->data_offset;
+        mp_bh->bio.bi_bdev = multipath->rdev->bdev;
+        mp_bh->bio.bi_rw |= (1 << BIO_RW_FAILFAST);
+        mp_bh->bio.bi_end_io = multipath_end_request;
+        mp_bh->bio.bi_private = mp_bh;
+        generic_make_request(&mp_bh->bio);
+        return 0;
+}
+
+static void multipath_status (struct seq_file *seq, mddev_t *mddev)
+{
+        multipath_conf_t *conf = mddev_to_conf(mddev);
+        int i;
+        
+        seq_printf (seq, " [%d/%d] [", conf->raid_disks,
+                                                 conf->working_disks);
+        for (i = 0; i < conf->raid_disks; i++)
+                seq_printf (seq, "%s",
+                               conf->multipaths[i].rdev && 
+                               conf->multipaths[i].rdev->in_sync ? "U" : "_");
+        seq_printf (seq, "]");
+}
+
+static int multipath_issue_flush(request_queue_t *q, struct gendisk *disk,
+                                 sector_t *error_sector)
+{
+        mddev_t *mddev = q->queuedata;
+        multipath_conf_t *conf = mddev_to_conf(mddev);
+        int i, ret = 0;
+
+        rcu_read_lock();
+        for (i=0; i<mddev->raid_disks && ret == 0; i++) {
+                mdk_rdev_t *rdev = conf->multipaths[i].rdev;
+                if (rdev && !rdev->faulty) {
+                        struct block_device *bdev = rdev->bdev;
+                        request_queue_t *r_queue = bdev_get_queue(bdev);
+
+                        if (!r_queue->issue_flush_fn)
+                                ret = -EOPNOTSUPP;
+                        else {
+                                atomic_inc(&rdev->nr_pending);
+                                rcu_read_unlock();
+                                ret = r_queue->issue_flush_fn(r_queue, bdev->bd_disk,
+                                                              error_sector);
+                                rdev_dec_pending(rdev, mddev);
+                                rcu_read_lock();
+                        }
+                }
+        }
+        rcu_read_unlock();
+        return ret;
+}
+
+/*
+ * Careful, this can execute in IRQ contexts as well!
+ */
+static void multipath_error (mddev_t *mddev, mdk_rdev_t *rdev)
+{
+        multipath_conf_t *conf = mddev_to_conf(mddev);
+
+        if (conf->working_disks <= 1) {
+                /*
+                 * Uh oh, we can do nothing if this is our last path, but
+                 * first check if this is a queued request for a device
+                 * which has just failed.
+                 */
+                printk(KERN_ALERT 
+                        "multipath: only one IO path left and IO error.\n");
+                /* leave it active... it's all we have */
+        } else {
+                /*
+                 * Mark disk as unusable
+                 */
+                if (!rdev->faulty) {
+                        char b[BDEVNAME_SIZE];
+                        rdev->in_sync = 0;
+                        rdev->faulty = 1;
+                        mddev->sb_dirty = 1;
+                        conf->working_disks--;
+                        printk(KERN_ALERT "multipath: IO failure on %s,"
+                                " disabling IO path. \n Operation continuing"
+                                " on %d IO paths.\n",
+                                bdevname (rdev->bdev,b),
+                                conf->working_disks);
+                }
+        }
+}
+
+static void print_multipath_conf (multipath_conf_t *conf)
+{
+        int i;
+        struct multipath_info *tmp;
+
+        printk("MULTIPATH conf printout:\n");
+        if (!conf) {
+                printk("(conf==NULL)\n");
+                return;
+        }
+        printk(" --- wd:%d rd:%d\n", conf->working_disks,
+                         conf->raid_disks);
+
+        for (i = 0; i < conf->raid_disks; i++) {
+                char b[BDEVNAME_SIZE];
+                tmp = conf->multipaths + i;
+                if (tmp->rdev)
+                        printk(" disk%d, o:%d, dev:%s\n",
+                                i,!tmp->rdev->faulty,
+                               bdevname(tmp->rdev->bdev,b));
+        }
+}
+
+
+static int multipath_add_disk(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+        multipath_conf_t *conf = mddev->private;
+        int found = 0;
+        int path;
+        struct multipath_info *p;
+
+        print_multipath_conf(conf);
+
+        for (path=0; path<mddev->raid_disks; path++) 
+                if ((p=conf->multipaths+path)->rdev == NULL) {
+                        blk_queue_stack_limits(mddev->queue,
+                                               rdev->bdev->bd_disk->queue);
+
+                /* as we don't honour merge_bvec_fn, we must never risk
+                 * violating it, so limit ->max_sector to one PAGE, as
+                 * a one page request is never in violation.
+                 * (Note: it is very unlikely that a device with
+                 * merge_bvec_fn will be involved in multipath.)
+                 */
+                        if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
+                            mddev->queue->max_sectors > (PAGE_SIZE>>9))
+                                blk_queue_max_sectors(mddev->queue, PAGE_SIZE>>9);
+
+                        conf->working_disks++;
+                        rdev->raid_disk = path;
+                        rdev->in_sync = 1;
+                        p->rdev = rdev;
+                        found = 1;
+                }
+
+        print_multipath_conf(conf);
+        return found;
+}
+
+static int multipath_remove_disk(mddev_t *mddev, int number)
+{
+        multipath_conf_t *conf = mddev->private;
+        int err = 0;
+        mdk_rdev_t *rdev;
+        struct multipath_info *p = conf->multipaths + number;
+
+        print_multipath_conf(conf);
+
+        rdev = p->rdev;
+        if (rdev) {
+                if (rdev->in_sync ||
+                    atomic_read(&rdev->nr_pending)) {
+                        printk(KERN_ERR "hot-remove-disk, slot %d is identified"                                " but is still operational!\n", number);
+                        err = -EBUSY;
+                        goto abort;
+                }
+                p->rdev = NULL;
+                synchronize_kernel();
+                if (atomic_read(&rdev->nr_pending)) {
+                        /* lost the race, try later */
+                        err = -EBUSY;
+                        p->rdev = rdev;
+                }
+        }
+abort:
+
+        print_multipath_conf(conf);
+        return err;
+}
+
+
+
+/*
+ * This is a kernel thread which:
+ *
+ *      1.      Retries failed read operations on working multipaths.
+ *      2.      Updates the raid superblock when problems encounter.
+ *      3.      Performs writes following reads for array syncronising.
+ */
+
+static void multipathd (mddev_t *mddev)
+{
+        struct multipath_bh *mp_bh;
+        struct bio *bio;
+        unsigned long flags;
+        multipath_conf_t *conf = mddev_to_conf(mddev);
+        struct list_head *head = &conf->retry_list;
+
+        md_check_recovery(mddev);
+        for (;;) {
+                char b[BDEVNAME_SIZE];
+                spin_lock_irqsave(&conf->device_lock, flags);
+                if (list_empty(head))
+                        break;
+                mp_bh = list_entry(head->prev, struct multipath_bh, retry_list);
+                list_del(head->prev);
+                spin_unlock_irqrestore(&conf->device_lock, flags);
+
+                bio = &mp_bh->bio;
+                bio->bi_sector = mp_bh->master_bio->bi_sector;
+                
+                if ((mp_bh->path = multipath_map (conf))<0) {
+                        printk(KERN_ALERT "multipath: %s: unrecoverable IO read"
+                                " error for block %llu\n",
+                                bdevname(bio->bi_bdev,b),
+                                (unsigned long long)bio->bi_sector);
+                        multipath_end_bh_io(mp_bh, -EIO);
+                } else {
+                        printk(KERN_ERR "multipath: %s: redirecting sector %llu"
+                                " to another IO path\n",
+                                bdevname(bio->bi_bdev,b),
+                                (unsigned long long)bio->bi_sector);
+                        *bio = *(mp_bh->master_bio);
+                        bio->bi_sector += conf->multipaths[mp_bh->path].rdev->data_offset;
+                        bio->bi_bdev = conf->multipaths[mp_bh->path].rdev->bdev;
+                        bio->bi_rw |= (1 << BIO_RW_FAILFAST);
+                        bio->bi_end_io = multipath_end_request;
+                        bio->bi_private = mp_bh;
+                        generic_make_request(bio);
+                }
+        }
+        spin_unlock_irqrestore(&conf->device_lock, flags);
+}
+
+static int multipath_run (mddev_t *mddev)
+{
+        multipath_conf_t *conf;
+        int disk_idx;
+        struct multipath_info *disk;
+        mdk_rdev_t *rdev;
+        struct list_head *tmp;
+
+        if (mddev->level != LEVEL_MULTIPATH) {
+                printk("multipath: %s: raid level not set to multipath IO (%d)\n",
+                       mdname(mddev), mddev->level);
+                goto out;
+        }
+        /*
+         * copy the already verified devices into our private MULTIPATH
+         * bookkeeping area. [whatever we allocate in multipath_run(),
+         * should be freed in multipath_stop()]
+         */
+
+        conf = kmalloc(sizeof(multipath_conf_t), GFP_KERNEL);
+        mddev->private = conf;
+        if (!conf) {
+                printk(KERN_ERR 
+                        "multipath: couldn't allocate memory for %s\n",
+                        mdname(mddev));
+                goto out;
+        }
+        memset(conf, 0, sizeof(*conf));
+
+        conf->multipaths = kmalloc(sizeof(struct multipath_info)*mddev->raid_disks,
+                                   GFP_KERNEL);
+        if (!conf->multipaths) {
+                printk(KERN_ERR 
+                        "multipath: couldn't allocate memory for %s\n",
+                        mdname(mddev));
+                goto out_free_conf;
+        }
+        memset(conf->multipaths, 0, sizeof(struct multipath_info)*mddev->raid_disks);
+
+        mddev->queue->unplug_fn = multipath_unplug;
+
+        mddev->queue->issue_flush_fn = multipath_issue_flush;
+
+        conf->working_disks = 0;
+        ITERATE_RDEV(mddev,rdev,tmp) {
+                disk_idx = rdev->raid_disk;
+                if (disk_idx < 0 ||
+                    disk_idx >= mddev->raid_disks)
+                        continue;
+
+                disk = conf->multipaths + disk_idx;
+                disk->rdev = rdev;
+
+                blk_queue_stack_limits(mddev->queue,
+                                       rdev->bdev->bd_disk->queue);
+                /* as we don't honour merge_bvec_fn, we must never risk
+                 * violating it, not that we ever expect a device with
+                 * a merge_bvec_fn to be involved in multipath */
+                if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
+                    mddev->queue->max_sectors > (PAGE_SIZE>>9))
+                        blk_queue_max_sectors(mddev->queue, PAGE_SIZE>>9);
+
+                if (!rdev->faulty) 
+                        conf->working_disks++;
+        }
+
+        conf->raid_disks = mddev->raid_disks;
+        mddev->sb_dirty = 1;
+        conf->mddev = mddev;
+        spin_lock_init(&conf->device_lock);
+        INIT_LIST_HEAD(&conf->retry_list);
+
+        if (!conf->working_disks) {
+                printk(KERN_ERR "multipath: no operational IO paths for %s\n",
+                        mdname(mddev));
+                goto out_free_conf;
+        }
+        mddev->degraded = conf->raid_disks = conf->working_disks;
+
+        conf->pool = mempool_create(NR_RESERVED_BUFS,
+                                    mp_pool_alloc, mp_pool_free,
+                                    NULL);
+        if (conf->pool == NULL) {
+                printk(KERN_ERR 
+                        "multipath: couldn't allocate memory for %s\n",
+                        mdname(mddev));
+                goto out_free_conf;
+        }
+
+        {
+                mddev->thread = md_register_thread(multipathd, mddev, "%s_multipath");
+                if (!mddev->thread) {
+                        printk(KERN_ERR "multipath: couldn't allocate thread"
+                                " for %s\n", mdname(mddev));
+                        goto out_free_conf;
+                }
+        }
+
+        printk(KERN_INFO 
+                "multipath: array %s active with %d out of %d IO paths\n",
+                mdname(mddev), conf->working_disks, mddev->raid_disks);
+        /*
+         * Ok, everything is just fine now
+         */
+        mddev->array_size = mddev->size;
+        return 0;
+
+out_free_conf:
+        if (conf->pool)
+                mempool_destroy(conf->pool);
+        if (conf->multipaths)
+                kfree(conf->multipaths);
+        kfree(conf);
+        mddev->private = NULL;
+out:
+        return -EIO;
+}
+
+
+static int multipath_stop (mddev_t *mddev)
+{
+        multipath_conf_t *conf = mddev_to_conf(mddev);
+
+        md_unregister_thread(mddev->thread);
+        mddev->thread = NULL;
+        blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
+        mempool_destroy(conf->pool);
+        kfree(conf->multipaths);
+        kfree(conf);
+        mddev->private = NULL;
+        return 0;
+}
+
+static mdk_personality_t multipath_personality=
+{
+        .name           = "multipath",
+        .owner          = THIS_MODULE,
+        .make_request   = multipath_make_request,
+        .run            = multipath_run,
+        .stop           = multipath_stop,
+        .status         = multipath_status,
+        .error_handler  = multipath_error,
+        .hot_add_disk   = multipath_add_disk,
+        .hot_remove_disk= multipath_remove_disk,
+};
+
+static int __init multipath_init (void)
+{
+        return register_md_personality (MULTIPATH, &multipath_personality);
+}
+
+static void __exit multipath_exit (void)
+{
+        unregister_md_personality (MULTIPATH);
+}
+
+module_init(multipath_init);
+module_exit(multipath_exit);
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("md-personality-7"); /* MULTIPATH */
diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c
new file mode 100644
index 000000000000..e7d934eca06f
--- /dev/null
+++ b/drivers/md/raid0.c
@@ -0,0 +1,539 @@
+/*
+   raid0.c : Multiple Devices driver for Linux
+             Copyright (C) 1994-96 Marc ZYNGIER
+             <zyngier@ufr-info-p7.ibp.fr> or
+             <maz@gloups.fdn.fr>
+             Copyright (C) 1999, 2000 Ingo Molnar, Red Hat
+
+
+   RAID-0 management functions.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2, or (at your option)
+   any later version.
+   
+   You should have received a copy of the GNU General Public License
+   (for example /usr/src/linux/COPYING); if not, write to the Free
+   Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.  
+*/
+
+#include <linux/module.h>
+#include <linux/raid/raid0.h>
+
+#define MAJOR_NR MD_MAJOR
+#define MD_DRIVER
+#define MD_PERSONALITY
+
+static void raid0_unplug(request_queue_t *q)
+{
+        mddev_t *mddev = q->queuedata;
+        raid0_conf_t *conf = mddev_to_conf(mddev);
+        mdk_rdev_t **devlist = conf->strip_zone[0].dev;
+        int i;
+
+        for (i=0; i<mddev->raid_disks; i++) {
+                request_queue_t *r_queue = bdev_get_queue(devlist[i]->bdev);
+
+                if (r_queue->unplug_fn)
+                        r_queue->unplug_fn(r_queue);
+        }
+}
+
+static int raid0_issue_flush(request_queue_t *q, struct gendisk *disk,
+                             sector_t *error_sector)
+{
+        mddev_t *mddev = q->queuedata;
+        raid0_conf_t *conf = mddev_to_conf(mddev);
+        mdk_rdev_t **devlist = conf->strip_zone[0].dev;
+        int i, ret = 0;
+
+        for (i=0; i<mddev->raid_disks && ret == 0; i++) {
+                struct block_device *bdev = devlist[i]->bdev;
+                request_queue_t *r_queue = bdev_get_queue(bdev);
+
+                if (!r_queue->issue_flush_fn)
+                        ret = -EOPNOTSUPP;
+                else
+                        ret = r_queue->issue_flush_fn(r_queue, bdev->bd_disk, error_sector);
+        }
+        return ret;
+}
+
+
+static int create_strip_zones (mddev_t *mddev)
+{
+        int i, c, j;
+        sector_t current_offset, curr_zone_offset;
+        sector_t min_spacing;
+        raid0_conf_t *conf = mddev_to_conf(mddev);
+        mdk_rdev_t *smallest, *rdev1, *rdev2, *rdev;
+        struct list_head *tmp1, *tmp2;
+        struct strip_zone *zone;
+        int cnt;
+        char b[BDEVNAME_SIZE];
+ 
+        /*
+         * The number of 'same size groups'
+         */
+        conf->nr_strip_zones = 0;
+ 
+        ITERATE_RDEV(mddev,rdev1,tmp1) {
+                printk("raid0: looking at %s\n",
+                        bdevname(rdev1->bdev,b));
+                c = 0;
+                ITERATE_RDEV(mddev,rdev2,tmp2) {
+                        printk("raid0:   comparing %s(%llu)",
+                               bdevname(rdev1->bdev,b),
+                               (unsigned long long)rdev1->size);
+                        printk(" with %s(%llu)\n",
+                               bdevname(rdev2->bdev,b),
+                               (unsigned long long)rdev2->size);
+                        if (rdev2 == rdev1) {
+                                printk("raid0:   END\n");
+                                break;
+                        }
+                        if (rdev2->size == rdev1->size)
+                        {
+                                /*
+                                 * Not unique, don't count it as a new
+                                 * group
+                                 */
+                                printk("raid0:   EQUAL\n");
+                                c = 1;
+                                break;
+                        }
+                        printk("raid0:   NOT EQUAL\n");
+                }
+                if (!c) {
+                        printk("raid0:   ==> UNIQUE\n");
+                        conf->nr_strip_zones++;
+                        printk("raid0: %d zones\n", conf->nr_strip_zones);
+                }
+        }
+        printk("raid0: FINAL %d zones\n", conf->nr_strip_zones);
+
+        conf->strip_zone = kmalloc(sizeof(struct strip_zone)*
+                                conf->nr_strip_zones, GFP_KERNEL);
+        if (!conf->strip_zone)
+                return 1;
+        conf->devlist = kmalloc(sizeof(mdk_rdev_t*)*
+                                conf->nr_strip_zones*mddev->raid_disks,
+                                GFP_KERNEL);
+        if (!conf->devlist)
+                return 1;
+
+        memset(conf->strip_zone, 0,sizeof(struct strip_zone)*
+                                   conf->nr_strip_zones);
+        memset(conf->devlist, 0,
+               sizeof(mdk_rdev_t*) * conf->nr_strip_zones * mddev->raid_disks);
+
+        /* The first zone must contain all devices, so here we check that
+         * there is a proper alignment of slots to devices and find them all
+         */
+        zone = &conf->strip_zone[0];
+        cnt = 0;
+        smallest = NULL;
+        zone->dev = conf->devlist;
+        ITERATE_RDEV(mddev, rdev1, tmp1) {
+                int j = rdev1->raid_disk;
+
+                if (j < 0 || j >= mddev->raid_disks) {
+                        printk("raid0: bad disk number %d - aborting!\n", j);
+                        goto abort;
+                }
+                if (zone->dev[j]) {
+                        printk("raid0: multiple devices for %d - aborting!\n",
+                                j);
+                        goto abort;
+                }
+                zone->dev[j] = rdev1;
+
+                blk_queue_stack_limits(mddev->queue,
+                                       rdev1->bdev->bd_disk->queue);
+                /* as we don't honour merge_bvec_fn, we must never risk
+                 * violating it, so limit ->max_sector to one PAGE, as
+                 * a one page request is never in violation.
+                 */
+
+                if (rdev1->bdev->bd_disk->queue->merge_bvec_fn &&
+                    mddev->queue->max_sectors > (PAGE_SIZE>>9))
+                        blk_queue_max_sectors(mddev->queue, PAGE_SIZE>>9);
+
+                if (!smallest || (rdev1->size <smallest->size))
+                        smallest = rdev1;
+                cnt++;
+        }
+        if (cnt != mddev->raid_disks) {
+                printk("raid0: too few disks (%d of %d) - aborting!\n",
+                        cnt, mddev->raid_disks);
+                goto abort;
+        }
+        zone->nb_dev = cnt;
+        zone->size = smallest->size * cnt;
+        zone->zone_offset = 0;
+
+        current_offset = smallest->size;
+        curr_zone_offset = zone->size;
+
+        /* now do the other zones */
+        for (i = 1; i < conf->nr_strip_zones; i++)
+        {
+                zone = conf->strip_zone + i;
+                zone->dev = conf->strip_zone[i-1].dev + mddev->raid_disks;
+
+                printk("raid0: zone %d\n", i);
+                zone->dev_offset = current_offset;
+                smallest = NULL;
+                c = 0;
+
+                for (j=0; j<cnt; j++) {
+                        char b[BDEVNAME_SIZE];
+                        rdev = conf->strip_zone[0].dev[j];
+                        printk("raid0: checking %s ...", bdevname(rdev->bdev,b));
+                        if (rdev->size > current_offset)
+                        {
+                                printk(" contained as device %d\n", c);
+                                zone->dev[c] = rdev;
+                                c++;
+                                if (!smallest || (rdev->size <smallest->size)) {
+                                        smallest = rdev;
+                                        printk("  (%llu) is smallest!.\n", 
+                                                (unsigned long long)rdev->size);
+                                }
+                        } else
+                                printk(" nope.\n");
+                }
+
+                zone->nb_dev = c;
+                zone->size = (smallest->size - current_offset) * c;
+                printk("raid0: zone->nb_dev: %d, size: %llu\n",
+                        zone->nb_dev, (unsigned long long)zone->size);
+
+                zone->zone_offset = curr_zone_offset;
+                curr_zone_offset += zone->size;
+
+                current_offset = smallest->size;
+                printk("raid0: current zone offset: %llu\n",
+                        (unsigned long long)current_offset);
+        }
+
+        /* Now find appropriate hash spacing.
+         * We want a number which causes most hash entries to cover
+         * at most two strips, but the hash table must be at most
+         * 1 PAGE.  We choose the smallest strip, or contiguous collection
+         * of strips, that has big enough size.  We never consider the last
+         * strip though as it's size has no bearing on the efficacy of the hash
+         * table.
+         */
+        conf->hash_spacing = curr_zone_offset;
+        min_spacing = curr_zone_offset;
+        sector_div(min_spacing, PAGE_SIZE/sizeof(struct strip_zone*));
+        for (i=0; i < conf->nr_strip_zones-1; i++) {
+                sector_t sz = 0;
+                for (j=i; j<conf->nr_strip_zones-1 &&
+                             sz < min_spacing ; j++)
+                        sz += conf->strip_zone[j].size;
+                if (sz >= min_spacing && sz < conf->hash_spacing)
+                        conf->hash_spacing = sz;
+        }
+
+        mddev->queue->unplug_fn = raid0_unplug;
+
+        mddev->queue->issue_flush_fn = raid0_issue_flush;
+
+        printk("raid0: done.\n");
+        return 0;
+ abort:
+        return 1;
+}
+
+/**
+ *      raid0_mergeable_bvec -- tell bio layer if a two requests can be merged
+ *      @q: request queue
+ *      @bio: the buffer head that's been built up so far
+ *      @biovec: the request that could be merged to it.
+ *
+ *      Return amount of bytes we can accept at this offset
+ */
+static int raid0_mergeable_bvec(request_queue_t *q, struct bio *bio, struct bio_vec *biovec)
+{
+        mddev_t *mddev = q->queuedata;
+        sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev);
+        int max;
+        unsigned int chunk_sectors = mddev->chunk_size >> 9;
+        unsigned int bio_sectors = bio->bi_size >> 9;
+
+        max =  (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
+        if (max < 0) max = 0; /* bio_add cannot handle a negative return */
+        if (max <= biovec->bv_len && bio_sectors == 0)
+                return biovec->bv_len;
+        else 
+                return max;
+}
+
+static int raid0_run (mddev_t *mddev)
+{
+        unsigned  cur=0, i=0, nb_zone;
+        s64 size;
+        raid0_conf_t *conf;
+        mdk_rdev_t *rdev;
+        struct list_head *tmp;
+
+        printk("%s: setting max_sectors to %d, segment boundary to %d\n",
+               mdname(mddev),
+               mddev->chunk_size >> 9,
+               (mddev->chunk_size>>1)-1);
+        blk_queue_max_sectors(mddev->queue, mddev->chunk_size >> 9);
+        blk_queue_segment_boundary(mddev->queue, (mddev->chunk_size>>1) - 1);
+
+        conf = kmalloc(sizeof (raid0_conf_t), GFP_KERNEL);
+        if (!conf)
+                goto out;
+        mddev->private = (void *)conf;
+ 
+        conf->strip_zone = NULL;
+        conf->devlist = NULL;
+        if (create_strip_zones (mddev)) 
+                goto out_free_conf;
+
+        /* calculate array device size */
+        mddev->array_size = 0;
+        ITERATE_RDEV(mddev,rdev,tmp)
+                mddev->array_size += rdev->size;
+
+        printk("raid0 : md_size is %llu blocks.\n", 
+                (unsigned long long)mddev->array_size);
+        printk("raid0 : conf->hash_spacing is %llu blocks.\n",
+                (unsigned long long)conf->hash_spacing);
+        {
+#if __GNUC__ < 3
+                volatile
+#endif
+                sector_t s = mddev->array_size;
+                sector_t space = conf->hash_spacing;
+                int round;
+                conf->preshift = 0;
+                if (sizeof(sector_t) > sizeof(unsigned long)) {
+                        /*shift down space and s so that sector_div will work */
+                        while (space > (sector_t) (~(unsigned long)0)) {
+                                s >>= 1;
+                                space >>= 1;
+                                s += 1; /* force round-up */
+                                conf->preshift++;
+                        }
+                }
+                round = sector_div(s, (unsigned long)space) ? 1 : 0;
+                nb_zone = s + round;
+        }
+        printk("raid0 : nb_zone is %d.\n", nb_zone);
+
+        printk("raid0 : Allocating %Zd bytes for hash.\n",
+                                nb_zone*sizeof(struct strip_zone*));
+        conf->hash_table = kmalloc (sizeof (struct strip_zone *)*nb_zone, GFP_KERNEL);
+        if (!conf->hash_table)
+                goto out_free_conf;
+        size = conf->strip_zone[cur].size;
+
+        for (i=0; i< nb_zone; i++) {
+                conf->hash_table[i] = conf->strip_zone + cur;
+                while (size <= conf->hash_spacing) {
+                        cur++;
+                        size += conf->strip_zone[cur].size;
+                }
+                size -= conf->hash_spacing;
+        }
+        if (conf->preshift) {
+                conf->hash_spacing >>= conf->preshift;
+                /* round hash_spacing up so when we divide by it, we
+                 * err on the side of too-low, which is safest
+                 */
+                conf->hash_spacing++;
+        }
+
+        /* calculate the max read-ahead size.
+         * For read-ahead of large files to be effective, we need to
+         * readahead at least twice a whole stripe. i.e. number of devices
+         * multiplied by chunk size times 2.
+         * If an individual device has an ra_pages greater than the
+         * chunk size, then we will not drive that device as hard as it
+         * wants.  We consider this a configuration error: a larger
+         * chunksize should be used in that case.
+         */
+        {
+                int stripe = mddev->raid_disks * mddev->chunk_size / PAGE_CACHE_SIZE;
+                if (mddev->queue->backing_dev_info.ra_pages < 2* stripe)
+                        mddev->queue->backing_dev_info.ra_pages = 2* stripe;
+        }
+
+
+        blk_queue_merge_bvec(mddev->queue, raid0_mergeable_bvec);
+        return 0;
+
+out_free_conf:
+        if (conf->strip_zone)
+                kfree(conf->strip_zone);
+        if (conf->devlist)
+                kfree (conf->devlist);
+        kfree(conf);
+        mddev->private = NULL;
+out:
+        return 1;
+}
+
+static int raid0_stop (mddev_t *mddev)
+{
+        raid0_conf_t *conf = mddev_to_conf(mddev);
+
+        blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
+        kfree (conf->hash_table);
+        conf->hash_table = NULL;
+        kfree (conf->strip_zone);
+        conf->strip_zone = NULL;
+        kfree (conf);
+        mddev->private = NULL;
+
+        return 0;
+}
+
+static int raid0_make_request (request_queue_t *q, struct bio *bio)
+{
+        mddev_t *mddev = q->queuedata;
+        unsigned int sect_in_chunk, chunksize_bits,  chunk_size, chunk_sects;
+        raid0_conf_t *conf = mddev_to_conf(mddev);
+        struct strip_zone *zone;
+        mdk_rdev_t *tmp_dev;
+        unsigned long chunk;
+        sector_t block, rsect;
+
+        if (bio_data_dir(bio)==WRITE) {
+                disk_stat_inc(mddev->gendisk, writes);
+                disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bio));
+        } else {
+                disk_stat_inc(mddev->gendisk, reads);
+                disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bio));
+        }
+
+        chunk_size = mddev->chunk_size >> 10;
+        chunk_sects = mddev->chunk_size >> 9;
+        chunksize_bits = ffz(~chunk_size);
+        block = bio->bi_sector >> 1;
+        
+
+        if (unlikely(chunk_sects < (bio->bi_sector & (chunk_sects - 1)) + (bio->bi_size >> 9))) {
+                struct bio_pair *bp;
+                /* Sanity check -- queue functions should prevent this happening */
+                if (bio->bi_vcnt != 1 ||
+                    bio->bi_idx != 0)
+                        goto bad_map;
+                /* This is a one page bio that upper layers
+                 * refuse to split for us, so we need to split it.
+                 */
+                bp = bio_split(bio, bio_split_pool, chunk_sects - (bio->bi_sector & (chunk_sects - 1)) );
+                if (raid0_make_request(q, &bp->bio1))
+                        generic_make_request(&bp->bio1);
+                if (raid0_make_request(q, &bp->bio2))
+                        generic_make_request(&bp->bio2);
+
+                bio_pair_release(bp);
+                return 0;
+        }
+ 
+
+        {
+#if __GNUC__ < 3
+                volatile
+#endif
+                sector_t x = block >> conf->preshift;
+                sector_div(x, (unsigned long)conf->hash_spacing);
+                zone = conf->hash_table[x];
+        }
+ 
+        while (block >= (zone->zone_offset + zone->size)) 
+                zone++;
+    
+        sect_in_chunk = bio->bi_sector & ((chunk_size<<1) -1);
+
+
+        {
+                sector_t x =  (block - zone->zone_offset) >> chunksize_bits;
+
+                sector_div(x, zone->nb_dev);
+                chunk = x;
+                BUG_ON(x != (sector_t)chunk);
+
+                x = block >> chunksize_bits;
+                tmp_dev = zone->dev[sector_div(x, zone->nb_dev)];
+        }
+        rsect = (((chunk << chunksize_bits) + zone->dev_offset)<<1)
+                + sect_in_chunk;
+ 
+        bio->bi_bdev = tmp_dev->bdev;
+        bio->bi_sector = rsect + tmp_dev->data_offset;
+
+        /*
+         * Let the main block layer submit the IO and resolve recursion:
+         */
+        return 1;
+
+bad_map:
+        printk("raid0_make_request bug: can't convert block across chunks"
+                " or bigger than %dk %llu %d\n", chunk_size, 
+                (unsigned long long)bio->bi_sector, bio->bi_size >> 10);
+
+        bio_io_error(bio, bio->bi_size);
+        return 0;
+}
+                           
+static void raid0_status (struct seq_file *seq, mddev_t *mddev)
+{
+#undef MD_DEBUG
+#ifdef MD_DEBUG
+        int j, k, h;
+        char b[BDEVNAME_SIZE];
+        raid0_conf_t *conf = mddev_to_conf(mddev);
+  
+        h = 0;
+        for (j = 0; j < conf->nr_strip_zones; j++) {
+                seq_printf(seq, "      z%d", j);
+                if (conf->hash_table[h] == conf->strip_zone+j)
+                        seq_printf("(h%d)", h++);
+                seq_printf(seq, "=[");
+                for (k = 0; k < conf->strip_zone[j].nb_dev; k++)
+                        seq_printf (seq, "%s/", bdevname(
+                                conf->strip_zone[j].dev[k]->bdev,b));
+
+                seq_printf (seq, "] zo=%d do=%d s=%d\n",
+                                conf->strip_zone[j].zone_offset,
+                                conf->strip_zone[j].dev_offset,
+                                conf->strip_zone[j].size);
+        }
+#endif
+        seq_printf(seq, " %dk chunks", mddev->chunk_size/1024);
+        return;
+}
+
+static mdk_personality_t raid0_personality=
+{
+        .name           = "raid0",
+        .owner          = THIS_MODULE,
+        .make_request   = raid0_make_request,
+        .run            = raid0_run,
+        .stop           = raid0_stop,
+        .status         = raid0_status,
+};
+
+static int __init raid0_init (void)
+{
+        return register_md_personality (RAID0, &raid0_personality);
+}
+
+static void raid0_exit (void)
+{
+        unregister_md_personality (RAID0);
+}
+
+module_init(raid0_init);
+module_exit(raid0_exit);
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("md-personality-2"); /* RAID0 */
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
new file mode 100644
index 000000000000..a389394b52f6
--- /dev/null
+++ b/drivers/md/raid1.c
@@ -0,0 +1,1449 @@
+/*
+ * raid1.c : Multiple Devices driver for Linux
+ *
+ * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat
+ *
+ * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman
+ *
+ * RAID-1 management functions.
+ *
+ * Better read-balancing code written by Mika Kuoppala <miku@iki.fi>, 2000
+ *
+ * Fixes to reconstruction by Jakob Østergaard" <jakob@ostenfeld.dk>
+ * Various fixes by Neil Brown <neilb@cse.unsw.edu.au>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * (for example /usr/src/linux/COPYING); if not, write to the Free
+ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/raid/raid1.h>
+
+/*
+ * Number of guaranteed r1bios in case of extreme VM load:
+ */
+#define NR_RAID1_BIOS 256
+
+static mdk_personality_t raid1_personality;
+
+static void unplug_slaves(mddev_t *mddev);
+
+
+static void * r1bio_pool_alloc(unsigned int __nocast gfp_flags, void *data)
+{
+        struct pool_info *pi = data;
+        r1bio_t *r1_bio;
+        int size = offsetof(r1bio_t, bios[pi->raid_disks]);
+
+        /* allocate a r1bio with room for raid_disks entries in the bios array */
+        r1_bio = kmalloc(size, gfp_flags);
+        if (r1_bio)
+                memset(r1_bio, 0, size);
+        else
+                unplug_slaves(pi->mddev);
+
+        return r1_bio;
+}
+
+static void r1bio_pool_free(void *r1_bio, void *data)
+{
+        kfree(r1_bio);
+}
+
+#define RESYNC_BLOCK_SIZE (64*1024)
+//#define RESYNC_BLOCK_SIZE PAGE_SIZE
+#define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
+#define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
+#define RESYNC_WINDOW (2048*1024)
+
+static void * r1buf_pool_alloc(unsigned int __nocast gfp_flags, void *data)
+{
+        struct pool_info *pi = data;
+        struct page *page;
+        r1bio_t *r1_bio;
+        struct bio *bio;
+        int i, j;
+
+        r1_bio = r1bio_pool_alloc(gfp_flags, pi);
+        if (!r1_bio) {
+                unplug_slaves(pi->mddev);
+                return NULL;
+        }
+
+        /*
+         * Allocate bios : 1 for reading, n-1 for writing
+         */
+        for (j = pi->raid_disks ; j-- ; ) {
+                bio = bio_alloc(gfp_flags, RESYNC_PAGES);
+                if (!bio)
+                        goto out_free_bio;
+                r1_bio->bios[j] = bio;
+        }
+        /*
+         * Allocate RESYNC_PAGES data pages and attach them to
+         * the first bio;
+         */
+        bio = r1_bio->bios[0];
+        for (i = 0; i < RESYNC_PAGES; i++) {
+                page = alloc_page(gfp_flags);
+                if (unlikely(!page))
+                        goto out_free_pages;
+
+                bio->bi_io_vec[i].bv_page = page;
+        }
+
+        r1_bio->master_bio = NULL;
+
+        return r1_bio;
+
+out_free_pages:
+        for ( ; i > 0 ; i--)
+                __free_page(bio->bi_io_vec[i-1].bv_page);
+out_free_bio:
+        while ( ++j < pi->raid_disks )
+                bio_put(r1_bio->bios[j]);
+        r1bio_pool_free(r1_bio, data);
+        return NULL;
+}
+
+static void r1buf_pool_free(void *__r1_bio, void *data)
+{
+        struct pool_info *pi = data;
+        int i;
+        r1bio_t *r1bio = __r1_bio;
+        struct bio *bio = r1bio->bios[0];
+
+        for (i = 0; i < RESYNC_PAGES; i++) {
+                __free_page(bio->bi_io_vec[i].bv_page);
+                bio->bi_io_vec[i].bv_page = NULL;
+        }
+        for (i=0 ; i < pi->raid_disks; i++)
+                bio_put(r1bio->bios[i]);
+
+        r1bio_pool_free(r1bio, data);
+}
+
+static void put_all_bios(conf_t *conf, r1bio_t *r1_bio)
+{
+        int i;
+
+        for (i = 0; i < conf->raid_disks; i++) {
+                struct bio **bio = r1_bio->bios + i;
+                if (*bio)
+                        bio_put(*bio);
+                *bio = NULL;
+        }
+}
+
+static inline void free_r1bio(r1bio_t *r1_bio)
+{
+        unsigned long flags;
+
+        conf_t *conf = mddev_to_conf(r1_bio->mddev);
+
+        /*
+         * Wake up any possible resync thread that waits for the device
+         * to go idle.
+         */
+        spin_lock_irqsave(&conf->resync_lock, flags);
+        if (!--conf->nr_pending) {
+                wake_up(&conf->wait_idle);
+                wake_up(&conf->wait_resume);
+        }
+        spin_unlock_irqrestore(&conf->resync_lock, flags);
+
+        put_all_bios(conf, r1_bio);
+        mempool_free(r1_bio, conf->r1bio_pool);
+}
+
+static inline void put_buf(r1bio_t *r1_bio)
+{
+        conf_t *conf = mddev_to_conf(r1_bio->mddev);
+        unsigned long flags;
+
+        mempool_free(r1_bio, conf->r1buf_pool);
+
+        spin_lock_irqsave(&conf->resync_lock, flags);
+        if (!conf->barrier)
+                BUG();
+        --conf->barrier;
+        wake_up(&conf->wait_resume);
+        wake_up(&conf->wait_idle);
+
+        if (!--conf->nr_pending) {
+                wake_up(&conf->wait_idle);
+                wake_up(&conf->wait_resume);
+        }
+        spin_unlock_irqrestore(&conf->resync_lock, flags);
+}
+
+static void reschedule_retry(r1bio_t *r1_bio)
+{
+        unsigned long flags;
+        mddev_t *mddev = r1_bio->mddev;
+        conf_t *conf = mddev_to_conf(mddev);
+
+        spin_lock_irqsave(&conf->device_lock, flags);
+        list_add(&r1_bio->retry_list, &conf->retry_list);
+        spin_unlock_irqrestore(&conf->device_lock, flags);
+
+        md_wakeup_thread(mddev->thread);
+}
+
+/*
+ * raid_end_bio_io() is called when we have finished servicing a mirrored
+ * operation and are ready to return a success/failure code to the buffer
+ * cache layer.
+ */
+static void raid_end_bio_io(r1bio_t *r1_bio)
+{
+        struct bio *bio = r1_bio->master_bio;
+
+        bio_endio(bio, bio->bi_size,
+                test_bit(R1BIO_Uptodate, &r1_bio->state) ? 0 : -EIO);
+        free_r1bio(r1_bio);
+}
+
+/*
+ * Update disk head position estimator based on IRQ completion info.
+ */
+static inline void update_head_pos(int disk, r1bio_t *r1_bio)
+{
+        conf_t *conf = mddev_to_conf(r1_bio->mddev);
+
+        conf->mirrors[disk].head_position =
+                r1_bio->sector + (r1_bio->sectors);
+}
+
+static int raid1_end_read_request(struct bio *bio, unsigned int bytes_done, int error)
+{
+        int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
+        r1bio_t * r1_bio = (r1bio_t *)(bio->bi_private);
+        int mirror;
+        conf_t *conf = mddev_to_conf(r1_bio->mddev);
+
+        if (bio->bi_size)
+                return 1;
+        
+        mirror = r1_bio->read_disk;
+        /*
+         * this branch is our 'one mirror IO has finished' event handler:
+         */
+        if (!uptodate)
+                md_error(r1_bio->mddev, conf->mirrors[mirror].rdev);
+        else
+                /*
+                 * Set R1BIO_Uptodate in our master bio, so that
+                 * we will return a good error code for to the higher
+                 * levels even if IO on some other mirrored buffer fails.
+                 *
+                 * The 'master' represents the composite IO operation to
+                 * user-side. So if something waits for IO, then it will
+                 * wait for the 'master' bio.
+                 */
+                set_bit(R1BIO_Uptodate, &r1_bio->state);
+
+        update_head_pos(mirror, r1_bio);
+
+        /*
+         * we have only one bio on the read side
+         */
+        if (uptodate)
+                raid_end_bio_io(r1_bio);
+        else {
+                /*
+                 * oops, read error:
+                 */
+                char b[BDEVNAME_SIZE];
+                if (printk_ratelimit())
+                        printk(KERN_ERR "raid1: %s: rescheduling sector %llu\n",
+                               bdevname(conf->mirrors[mirror].rdev->bdev,b), (unsigned long long)r1_bio->sector);
+                reschedule_retry(r1_bio);
+        }
+
+        rdev_dec_pending(conf->mirrors[mirror].rdev, conf->mddev);
+        return 0;
+}
+
+static int raid1_end_write_request(struct bio *bio, unsigned int bytes_done, int error)
+{
+        int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
+        r1bio_t * r1_bio = (r1bio_t *)(bio->bi_private);
+        int mirror;
+        conf_t *conf = mddev_to_conf(r1_bio->mddev);
+
+        if (bio->bi_size)
+                return 1;
+
+        for (mirror = 0; mirror < conf->raid_disks; mirror++)
+                if (r1_bio->bios[mirror] == bio)
+                        break;
+
+        /*
+         * this branch is our 'one mirror IO has finished' event handler:
+         */
+        if (!uptodate)
+                md_error(r1_bio->mddev, conf->mirrors[mirror].rdev);
+        else
+                /*
+                 * Set R1BIO_Uptodate in our master bio, so that
+                 * we will return a good error code for to the higher
+                 * levels even if IO on some other mirrored buffer fails.
+                 *
+                 * The 'master' represents the composite IO operation to
+                 * user-side. So if something waits for IO, then it will
+                 * wait for the 'master' bio.
+                 */
+                set_bit(R1BIO_Uptodate, &r1_bio->state);
+
+        update_head_pos(mirror, r1_bio);
+
+        /*
+         *
+         * Let's see if all mirrored write operations have finished
+         * already.
+         */
+        if (atomic_dec_and_test(&r1_bio->remaining)) {
+                md_write_end(r1_bio->mddev);
+                raid_end_bio_io(r1_bio);
+        }
+
+        rdev_dec_pending(conf->mirrors[mirror].rdev, conf->mddev);
+        return 0;
+}
+
+
+/*
+ * This routine returns the disk from which the requested read should
+ * be done. There is a per-array 'next expected sequential IO' sector
+ * number - if this matches on the next IO then we use the last disk.
+ * There is also a per-disk 'last know head position' sector that is
+ * maintained from IRQ contexts, both the normal and the resync IO
+ * completion handlers update this position correctly. If there is no
+ * perfect sequential match then we pick the disk whose head is closest.
+ *
+ * If there are 2 mirrors in the same 2 devices, performance degrades
+ * because position is mirror, not device based.
+ *
+ * The rdev for the device selected will have nr_pending incremented.
+ */
+static int read_balance(conf_t *conf, r1bio_t *r1_bio)
+{
+        const unsigned long this_sector = r1_bio->sector;
+        int new_disk = conf->last_used, disk = new_disk;
+        const int sectors = r1_bio->sectors;
+        sector_t new_distance, current_distance;
+        mdk_rdev_t *new_rdev, *rdev;
+
+        rcu_read_lock();
+        /*
+         * Check if it if we can balance. We can balance on the whole
+         * device if no resync is going on, or below the resync window.
+         * We take the first readable disk when above the resync window.
+         */
+ retry:
+        if (conf->mddev->recovery_cp < MaxSector &&
+            (this_sector + sectors >= conf->next_resync)) {
+                /* Choose the first operation device, for consistancy */
+                new_disk = 0;
+
+                while ((new_rdev=conf->mirrors[new_disk].rdev) == NULL ||
+                       !new_rdev->in_sync) {
+                        new_disk++;
+                        if (new_disk == conf->raid_disks) {
+                                new_disk = -1;
+                                break;
+                        }
+                }
+                goto rb_out;
+        }
+
+
+        /* make sure the disk is operational */
+        while ((new_rdev=conf->mirrors[new_disk].rdev) == NULL ||
+               !new_rdev->in_sync) {
+                if (new_disk <= 0)
+                        new_disk = conf->raid_disks;
+                new_disk--;
+                if (new_disk == disk) {
+                        new_disk = -1;
+                        goto rb_out;
+                }
+        }
+        disk = new_disk;
+        /* now disk == new_disk == starting point for search */
+
+        /*
+         * Don't change to another disk for sequential reads:
+         */
+        if (conf->next_seq_sect == this_sector)
+                goto rb_out;
+        if (this_sector == conf->mirrors[new_disk].head_position)
+                goto rb_out;
+
+        current_distance = abs(this_sector - conf->mirrors[disk].head_position);
+
+        /* Find the disk whose head is closest */
+
+        do {
+                if (disk <= 0)
+                        disk = conf->raid_disks;
+                disk--;
+
+                if ((rdev=conf->mirrors[disk].rdev) == NULL ||
+                    !rdev->in_sync)
+                        continue;
+
+                if (!atomic_read(&rdev->nr_pending)) {
+                        new_disk = disk;
+                        new_rdev = rdev;
+                        break;
+                }
+                new_distance = abs(this_sector - conf->mirrors[disk].head_position);
+                if (new_distance < current_distance) {
+                        current_distance = new_distance;
+                        new_disk = disk;
+                        new_rdev = rdev;
+                }
+        } while (disk != conf->last_used);
+
+rb_out:
+
+
+        if (new_disk >= 0) {
+                conf->next_seq_sect = this_sector + sectors;
+                conf->last_used = new_disk;
+                atomic_inc(&new_rdev->nr_pending);
+                if (!new_rdev->in_sync) {
+                        /* cannot risk returning a device that failed
+                         * before we inc'ed nr_pending
+                         */
+                        atomic_dec(&new_rdev->nr_pending);
+                        goto retry;
+                }
+        }
+        rcu_read_unlock();
+
+        return new_disk;
+}
+
+static void unplug_slaves(mddev_t *mddev)
+{
+        conf_t *conf = mddev_to_conf(mddev);
+        int i;
+
+        rcu_read_lock();
+        for (i=0; i<mddev->raid_disks; i++) {
+                mdk_rdev_t *rdev = conf->mirrors[i].rdev;
+                if (rdev && !rdev->faulty && atomic_read(&rdev->nr_pending)) {
+                        request_queue_t *r_queue = bdev_get_queue(rdev->bdev);
+
+                        atomic_inc(&rdev->nr_pending);
+                        rcu_read_unlock();
+
+                        if (r_queue->unplug_fn)
+                                r_queue->unplug_fn(r_queue);
+
+                        rdev_dec_pending(rdev, mddev);
+                        rcu_read_lock();
+                }
+        }
+        rcu_read_unlock();
+}
+
+static void raid1_unplug(request_queue_t *q)
+{
+        unplug_slaves(q->queuedata);
+}
+
+static int raid1_issue_flush(request_queue_t *q, struct gendisk *disk,
+                             sector_t *error_sector)
+{
+        mddev_t *mddev = q->queuedata;
+        conf_t *conf = mddev_to_conf(mddev);
+        int i, ret = 0;
+
+        rcu_read_lock();
+        for (i=0; i<mddev->raid_disks && ret == 0; i++) {
+                mdk_rdev_t *rdev = conf->mirrors[i].rdev;
+                if (rdev && !rdev->faulty) {
+                        struct block_device *bdev = rdev->bdev;
+                        request_queue_t *r_queue = bdev_get_queue(bdev);
+
+                        if (!r_queue->issue_flush_fn)
+                                ret = -EOPNOTSUPP;
+                        else {
+                                atomic_inc(&rdev->nr_pending);
+                                rcu_read_unlock();
+                                ret = r_queue->issue_flush_fn(r_queue, bdev->bd_disk,
+                                                              error_sector);
+                                rdev_dec_pending(rdev, mddev);
+                                rcu_read_lock();
+                        }
+                }
+        }
+        rcu_read_unlock();
+        return ret;
+}
+
+/*
+ * Throttle resync depth, so that we can both get proper overlapping of
+ * requests, but are still able to handle normal requests quickly.
+ */
+#define RESYNC_DEPTH 32
+
+static void device_barrier(conf_t *conf, sector_t sect)
+{
+        spin_lock_irq(&conf->resync_lock);
+        wait_event_lock_irq(conf->wait_idle, !waitqueue_active(&conf->wait_resume),
+                            conf->resync_lock, unplug_slaves(conf->mddev));
+        
+        if (!conf->barrier++) {
+                wait_event_lock_irq(conf->wait_idle, !conf->nr_pending,
+                                    conf->resync_lock, unplug_slaves(conf->mddev));
+                if (conf->nr_pending)
+                        BUG();
+        }
+        wait_event_lock_irq(conf->wait_resume, conf->barrier < RESYNC_DEPTH,
+                            conf->resync_lock, unplug_slaves(conf->mddev));
+        conf->next_resync = sect;
+        spin_unlock_irq(&conf->resync_lock);
+}
+
+static int make_request(request_queue_t *q, struct bio * bio)
+{
+        mddev_t *mddev = q->queuedata;
+        conf_t *conf = mddev_to_conf(mddev);
+        mirror_info_t *mirror;
+        r1bio_t *r1_bio;
+        struct bio *read_bio;
+        int i, disks;
+        mdk_rdev_t *rdev;
+
+        /*
+         * Register the new request and wait if the reconstruction
+         * thread has put up a bar for new requests.
+         * Continue immediately if no resync is active currently.
+         */
+        spin_lock_irq(&conf->resync_lock);
+        wait_event_lock_irq(conf->wait_resume, !conf->barrier, conf->resync_lock, );
+        conf->nr_pending++;
+        spin_unlock_irq(&conf->resync_lock);
+
+        if (bio_data_dir(bio)==WRITE) {
+                disk_stat_inc(mddev->gendisk, writes);
+                disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bio));
+        } else {
+                disk_stat_inc(mddev->gendisk, reads);
+                disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bio));
+        }
+
+        /*
+         * make_request() can abort the operation when READA is being
+         * used and no empty request is available.
+         *
+         */
+        r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO);
+
+        r1_bio->master_bio = bio;
+        r1_bio->sectors = bio->bi_size >> 9;
+
+        r1_bio->mddev = mddev;
+        r1_bio->sector = bio->bi_sector;
+
+        r1_bio->state = 0;
+
+        if (bio_data_dir(bio) == READ) {
+                /*
+                 * read balancing logic:
+                 */
+                int rdisk = read_balance(conf, r1_bio);
+
+                if (rdisk < 0) {
+                        /* couldn't find anywhere to read from */
+                        raid_end_bio_io(r1_bio);
+                        return 0;
+                }
+                mirror = conf->mirrors + rdisk;
+
+                r1_bio->read_disk = rdisk;
+
+                read_bio = bio_clone(bio, GFP_NOIO);
+
+                r1_bio->bios[rdisk] = read_bio;
+
+                read_bio->bi_sector = r1_bio->sector + mirror->rdev->data_offset;
+                read_bio->bi_bdev = mirror->rdev->bdev;
+                read_bio->bi_end_io = raid1_end_read_request;
+                read_bio->bi_rw = READ;
+                read_bio->bi_private = r1_bio;
+
+                generic_make_request(read_bio);
+                return 0;
+        }
+
+        /*
+         * WRITE:
+         */
+        /* first select target devices under spinlock and
+         * inc refcount on their rdev.  Record them by setting
+         * bios[x] to bio
+         */
+        disks = conf->raid_disks;
+        rcu_read_lock();
+        for (i = 0;  i < disks; i++) {
+                if ((rdev=conf->mirrors[i].rdev) != NULL &&
+                    !rdev->faulty) {
+                        atomic_inc(&rdev->nr_pending);
+                        if (rdev->faulty) {
+                                atomic_dec(&rdev->nr_pending);
+                                r1_bio->bios[i] = NULL;
+                        } else
+                                r1_bio->bios[i] = bio;
+                } else
+                        r1_bio->bios[i] = NULL;
+        }
+        rcu_read_unlock();
+
+        atomic_set(&r1_bio->remaining, 1);
+        md_write_start(mddev);
+        for (i = 0; i < disks; i++) {
+                struct bio *mbio;
+                if (!r1_bio->bios[i])
+                        continue;
+
+                mbio = bio_clone(bio, GFP_NOIO);
+                r1_bio->bios[i] = mbio;
+
+                mbio->bi_sector = r1_bio->sector + conf->mirrors[i].rdev->data_offset;
+                mbio->bi_bdev = conf->mirrors[i].rdev->bdev;
+                mbio->bi_end_io = raid1_end_write_request;
+                mbio->bi_rw = WRITE;
+                mbio->bi_private = r1_bio;
+
+                atomic_inc(&r1_bio->remaining);
+                generic_make_request(mbio);
+        }
+
+        if (atomic_dec_and_test(&r1_bio->remaining)) {
+                md_write_end(mddev);
+                raid_end_bio_io(r1_bio);
+        }
+
+        return 0;
+}
+
+static void status(struct seq_file *seq, mddev_t *mddev)
+{
+        conf_t *conf = mddev_to_conf(mddev);
+        int i;
+
+        seq_printf(seq, " [%d/%d] [", conf->raid_disks,
+                                                conf->working_disks);
+        for (i = 0; i < conf->raid_disks; i++)
+                seq_printf(seq, "%s",
+                              conf->mirrors[i].rdev &&
+                              conf->mirrors[i].rdev->in_sync ? "U" : "_");
+        seq_printf(seq, "]");
+}
+
+
+static void error(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+        char b[BDEVNAME_SIZE];
+        conf_t *conf = mddev_to_conf(mddev);
+
+        /*
+         * If it is not operational, then we have already marked it as dead
+         * else if it is the last working disks, ignore the error, let the
+         * next level up know.
+         * else mark the drive as failed
+         */
+        if (rdev->in_sync
+            && conf->working_disks == 1)
+                /*
+                 * Don't fail the drive, act as though we were just a
+                 * normal single drive
+                 */
+                return;
+        if (rdev->in_sync) {
+                mddev->degraded++;
+                conf->working_disks--;
+                /*
+                 * if recovery is running, make sure it aborts.
+                 */
+                set_bit(MD_RECOVERY_ERR, &mddev->recovery);
+        }
+        rdev->in_sync = 0;
+        rdev->faulty = 1;
+        mddev->sb_dirty = 1;
+        printk(KERN_ALERT "raid1: Disk failure on %s, disabling device. \n"
+                "       Operation continuing on %d devices\n",
+                bdevname(rdev->bdev,b), conf->working_disks);
+}
+
+static void print_conf(conf_t *conf)
+{
+        int i;
+        mirror_info_t *tmp;
+
+        printk("RAID1 conf printout:\n");
+        if (!conf) {
+                printk("(!conf)\n");
+                return;
+        }
+        printk(" --- wd:%d rd:%d\n", conf->working_disks,
+                conf->raid_disks);
+
+        for (i = 0; i < conf->raid_disks; i++) {
+                char b[BDEVNAME_SIZE];
+                tmp = conf->mirrors + i;
+                if (tmp->rdev)
+                        printk(" disk %d, wo:%d, o:%d, dev:%s\n",
+                                i, !tmp->rdev->in_sync, !tmp->rdev->faulty,
+                                bdevname(tmp->rdev->bdev,b));
+        }
+}
+
+static void close_sync(conf_t *conf)
+{
+        spin_lock_irq(&conf->resync_lock);
+        wait_event_lock_irq(conf->wait_resume, !conf->barrier,
+                            conf->resync_lock,  unplug_slaves(conf->mddev));
+        spin_unlock_irq(&conf->resync_lock);
+
+        if (conf->barrier) BUG();
+        if (waitqueue_active(&conf->wait_idle)) BUG();
+
+        mempool_destroy(conf->r1buf_pool);
+        conf->r1buf_pool = NULL;
+}
+
+static int raid1_spare_active(mddev_t *mddev)
+{
+        int i;
+        conf_t *conf = mddev->private;
+        mirror_info_t *tmp;
+
+        /*
+         * Find all failed disks within the RAID1 configuration 
+         * and mark them readable
+         */
+        for (i = 0; i < conf->raid_disks; i++) {
+                tmp = conf->mirrors + i;
+                if (tmp->rdev 
+                    && !tmp->rdev->faulty
+                    && !tmp->rdev->in_sync) {
+                        conf->working_disks++;
+                        mddev->degraded--;
+                        tmp->rdev->in_sync = 1;
+                }
+        }
+
+        print_conf(conf);
+        return 0;
+}
+
+
+static int raid1_add_disk(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+        conf_t *conf = mddev->private;
+        int found = 0;
+        int mirror;
+        mirror_info_t *p;
+
+        for (mirror=0; mirror < mddev->raid_disks; mirror++)
+                if ( !(p=conf->mirrors+mirror)->rdev) {
+
+                        blk_queue_stack_limits(mddev->queue,
+                                               rdev->bdev->bd_disk->queue);
+                        /* as we don't honour merge_bvec_fn, we must never risk
+                         * violating it, so limit ->max_sector to one PAGE, as
+                         * a one page request is never in violation.
+                         */
+                        if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
+                            mddev->queue->max_sectors > (PAGE_SIZE>>9))
+                                blk_queue_max_sectors(mddev->queue, PAGE_SIZE>>9);
+
+                        p->head_position = 0;
+                        rdev->raid_disk = mirror;
+                        found = 1;
+                        p->rdev = rdev;
+                        break;
+                }
+
+        print_conf(conf);
+        return found;
+}
+
+static int raid1_remove_disk(mddev_t *mddev, int number)
+{
+        conf_t *conf = mddev->private;
+        int err = 0;
+        mdk_rdev_t *rdev;
+        mirror_info_t *p = conf->mirrors+ number;
+
+        print_conf(conf);
+        rdev = p->rdev;
+        if (rdev) {
+                if (rdev->in_sync ||
+                    atomic_read(&rdev->nr_pending)) {
+                        err = -EBUSY;
+                        goto abort;
+                }
+                p->rdev = NULL;
+                synchronize_kernel();
+                if (atomic_read(&rdev->nr_pending)) {
+                        /* lost the race, try later */
+                        err = -EBUSY;
+                        p->rdev = rdev;
+                }
+        }
+abort:
+
+        print_conf(conf);
+        return err;
+}
+
+
+static int end_sync_read(struct bio *bio, unsigned int bytes_done, int error)
+{
+        int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
+        r1bio_t * r1_bio = (r1bio_t *)(bio->bi_private);
+        conf_t *conf = mddev_to_conf(r1_bio->mddev);
+
+        if (bio->bi_size)
+                return 1;
+
+        if (r1_bio->bios[r1_bio->read_disk] != bio)
+                BUG();
+        update_head_pos(r1_bio->read_disk, r1_bio);
+        /*
+         * we have read a block, now it needs to be re-written,
+         * or re-read if the read failed.
+         * We don't do much here, just schedule handling by raid1d
+         */
+        if (!uptodate)
+                md_error(r1_bio->mddev,
+                         conf->mirrors[r1_bio->read_disk].rdev);
+        else
+                set_bit(R1BIO_Uptodate, &r1_bio->state);
+        rdev_dec_pending(conf->mirrors[r1_bio->read_disk].rdev, conf->mddev);
+        reschedule_retry(r1_bio);
+        return 0;
+}
+
+static int end_sync_write(struct bio *bio, unsigned int bytes_done, int error)
+{
+        int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
+        r1bio_t * r1_bio = (r1bio_t *)(bio->bi_private);
+        mddev_t *mddev = r1_bio->mddev;
+        conf_t *conf = mddev_to_conf(mddev);
+        int i;
+        int mirror=0;
+
+        if (bio->bi_size)
+                return 1;
+
+        for (i = 0; i < conf->raid_disks; i++)
+                if (r1_bio->bios[i] == bio) {
+                        mirror = i;
+                        break;
+                }
+        if (!uptodate)
+                md_error(mddev, conf->mirrors[mirror].rdev);
+        update_head_pos(mirror, r1_bio);
+
+        if (atomic_dec_and_test(&r1_bio->remaining)) {
+                md_done_sync(mddev, r1_bio->sectors, uptodate);
+                put_buf(r1_bio);
+        }
+        rdev_dec_pending(conf->mirrors[mirror].rdev, mddev);
+        return 0;
+}
+
+static void sync_request_write(mddev_t *mddev, r1bio_t *r1_bio)
+{
+        conf_t *conf = mddev_to_conf(mddev);
+        int i;
+        int disks = conf->raid_disks;
+        struct bio *bio, *wbio;
+
+        bio = r1_bio->bios[r1_bio->read_disk];
+
+        /*
+         * schedule writes
+         */
+        if (!test_bit(R1BIO_Uptodate, &r1_bio->state)) {
+                /*
+                 * There is no point trying a read-for-reconstruct as
+                 * reconstruct is about to be aborted
+                 */
+                char b[BDEVNAME_SIZE];
+                printk(KERN_ALERT "raid1: %s: unrecoverable I/O read error"
+                        " for block %llu\n",
+                        bdevname(bio->bi_bdev,b), 
+                        (unsigned long long)r1_bio->sector);
+                md_done_sync(mddev, r1_bio->sectors, 0);
+                put_buf(r1_bio);
+                return;
+        }
+
+        atomic_set(&r1_bio->remaining, 1);
+        for (i = 0; i < disks ; i++) {
+                wbio = r1_bio->bios[i];
+                if (wbio->bi_end_io != end_sync_write)
+                        continue;
+
+                atomic_inc(&conf->mirrors[i].rdev->nr_pending);
+                atomic_inc(&r1_bio->remaining);
+                md_sync_acct(conf->mirrors[i].rdev->bdev, wbio->bi_size >> 9);
+                generic_make_request(wbio);
+        }
+
+        if (atomic_dec_and_test(&r1_bio->remaining)) {
+                md_done_sync(mddev, r1_bio->sectors, 1);
+                put_buf(r1_bio);
+        }
+}
+
+/*
+ * This is a kernel thread which:
+ *
+ *      1.      Retries failed read operations on working mirrors.
+ *      2.      Updates the raid superblock when problems encounter.
+ *      3.      Performs writes following reads for array syncronising.
+ */
+
+static void raid1d(mddev_t *mddev)
+{
+        r1bio_t *r1_bio;
+        struct bio *bio;
+        unsigned long flags;
+        conf_t *conf = mddev_to_conf(mddev);
+        struct list_head *head = &conf->retry_list;
+        int unplug=0;
+        mdk_rdev_t *rdev;
+
+        md_check_recovery(mddev);
+        md_handle_safemode(mddev);
+        
+        for (;;) {
+                char b[BDEVNAME_SIZE];
+                spin_lock_irqsave(&conf->device_lock, flags);
+                if (list_empty(head))
+                        break;
+                r1_bio = list_entry(head->prev, r1bio_t, retry_list);
+                list_del(head->prev);
+                spin_unlock_irqrestore(&conf->device_lock, flags);
+
+                mddev = r1_bio->mddev;
+                conf = mddev_to_conf(mddev);
+                if (test_bit(R1BIO_IsSync, &r1_bio->state)) {
+                        sync_request_write(mddev, r1_bio);
+                        unplug = 1;
+                } else {
+                        int disk;
+                        bio = r1_bio->bios[r1_bio->read_disk];
+                        if ((disk=read_balance(conf, r1_bio)) == -1) {
+                                printk(KERN_ALERT "raid1: %s: unrecoverable I/O"
+                                       " read error for block %llu\n",
+                                       bdevname(bio->bi_bdev,b),
+                                       (unsigned long long)r1_bio->sector);
+                                raid_end_bio_io(r1_bio);
+                        } else {
+                                r1_bio->bios[r1_bio->read_disk] = NULL;
+                                r1_bio->read_disk = disk;
+                                bio_put(bio);
+                                bio = bio_clone(r1_bio->master_bio, GFP_NOIO);
+                                r1_bio->bios[r1_bio->read_disk] = bio;
+                                rdev = conf->mirrors[disk].rdev;
+                                if (printk_ratelimit())
+                                        printk(KERN_ERR "raid1: %s: redirecting sector %llu to"
+                                               " another mirror\n",
+                                               bdevname(rdev->bdev,b),
+                                               (unsigned long long)r1_bio->sector);
+                                bio->bi_sector = r1_bio->sector + rdev->data_offset;
+                                bio->bi_bdev = rdev->bdev;
+                                bio->bi_end_io = raid1_end_read_request;
+                                bio->bi_rw = READ;
+                                bio->bi_private = r1_bio;
+                                unplug = 1;
+                                generic_make_request(bio);
+                        }
+                }
+        }
+        spin_unlock_irqrestore(&conf->device_lock, flags);
+        if (unplug)
+                unplug_slaves(mddev);
+}
+
+
+static int init_resync(conf_t *conf)
+{
+        int buffs;
+
+        buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
+        if (conf->r1buf_pool)
+                BUG();
+        conf->r1buf_pool = mempool_create(buffs, r1buf_pool_alloc, r1buf_pool_free,
+                                          conf->poolinfo);
+        if (!conf->r1buf_pool)
+                return -ENOMEM;
+        conf->next_resync = 0;
+        return 0;
+}
+
+/*
+ * perform a "sync" on one "block"
+ *
+ * We need to make sure that no normal I/O request - particularly write
+ * requests - conflict with active sync requests.
+ *
+ * This is achieved by tracking pending requests and a 'barrier' concept
+ * that can be installed to exclude normal IO requests.
+ */
+
+static int sync_request(mddev_t *mddev, sector_t sector_nr, int go_faster)
+{
+        conf_t *conf = mddev_to_conf(mddev);
+        mirror_info_t *mirror;
+        r1bio_t *r1_bio;
+        struct bio *bio;
+        sector_t max_sector, nr_sectors;
+        int disk;
+        int i;
+        int write_targets = 0;
+
+        if (!conf->r1buf_pool)
+                if (init_resync(conf))
+                        return -ENOMEM;
+
+        max_sector = mddev->size << 1;
+        if (sector_nr >= max_sector) {
+                close_sync(conf);
+                return 0;
+        }
+
+        /*
+         * If there is non-resync activity waiting for us then
+         * put in a delay to throttle resync.
+         */
+        if (!go_faster && waitqueue_active(&conf->wait_resume))
+                msleep_interruptible(1000);
+        device_barrier(conf, sector_nr + RESYNC_SECTORS);
+
+        /*
+         * If reconstructing, and >1 working disc,
+         * could dedicate one to rebuild and others to
+         * service read requests ..
+         */
+        disk = conf->last_used;
+        /* make sure disk is operational */
+
+        while (conf->mirrors[disk].rdev == NULL ||
+               !conf->mirrors[disk].rdev->in_sync) {
+                if (disk <= 0)
+                        disk = conf->raid_disks;
+                disk--;
+                if (disk == conf->last_used)
+                        break;
+        }
+        conf->last_used = disk;
+        atomic_inc(&conf->mirrors[disk].rdev->nr_pending);
+
+
+        mirror = conf->mirrors + disk;
+
+        r1_bio = mempool_alloc(conf->r1buf_pool, GFP_NOIO);
+
+        spin_lock_irq(&conf->resync_lock);
+        conf->nr_pending++;
+        spin_unlock_irq(&conf->resync_lock);
+
+        r1_bio->mddev = mddev;
+        r1_bio->sector = sector_nr;
+        set_bit(R1BIO_IsSync, &r1_bio->state);
+        r1_bio->read_disk = disk;
+
+        for (i=0; i < conf->raid_disks; i++) {
+                bio = r1_bio->bios[i];
+
+                /* take from bio_init */
+                bio->bi_next = NULL;
+                bio->bi_flags |= 1 << BIO_UPTODATE;
+                bio->bi_rw = 0;
+                bio->bi_vcnt = 0;
+                bio->bi_idx = 0;
+                bio->bi_phys_segments = 0;
+                bio->bi_hw_segments = 0;
+                bio->bi_size = 0;
+                bio->bi_end_io = NULL;
+                bio->bi_private = NULL;
+
+                if (i == disk) {
+                        bio->bi_rw = READ;
+                        bio->bi_end_io = end_sync_read;
+                } else if (conf->mirrors[i].rdev &&
+                           !conf->mirrors[i].rdev->faulty &&
+                           (!conf->mirrors[i].rdev->in_sync ||
+                            sector_nr + RESYNC_SECTORS > mddev->recovery_cp)) {
+                        bio->bi_rw = WRITE;
+                        bio->bi_end_io = end_sync_write;
+                        write_targets ++;
+                } else
+                        continue;
+                bio->bi_sector = sector_nr + conf->mirrors[i].rdev->data_offset;
+                bio->bi_bdev = conf->mirrors[i].rdev->bdev;
+                bio->bi_private = r1_bio;
+        }
+        if (write_targets == 0) {
+                /* There is nowhere to write, so all non-sync
+                 * drives must be failed - so we are finished
+                 */
+                int rv = max_sector - sector_nr;
+                md_done_sync(mddev, rv, 1);
+                put_buf(r1_bio);
+                rdev_dec_pending(conf->mirrors[disk].rdev, mddev);
+                return rv;
+        }
+
+        nr_sectors = 0;
+        do {
+                struct page *page;
+                int len = PAGE_SIZE;
+                if (sector_nr + (len>>9) > max_sector)
+                        len = (max_sector - sector_nr) << 9;
+                if (len == 0)
+                        break;
+                for (i=0 ; i < conf->raid_disks; i++) {
+                        bio = r1_bio->bios[i];
+                        if (bio->bi_end_io) {
+                                page = r1_bio->bios[0]->bi_io_vec[bio->bi_vcnt].bv_page;
+                                if (bio_add_page(bio, page, len, 0) == 0) {
+                                        /* stop here */
+                                        r1_bio->bios[0]->bi_io_vec[bio->bi_vcnt].bv_page = page;
+                                        while (i > 0) {
+                                                i--;
+                                                bio = r1_bio->bios[i];
+                                                if (bio->bi_end_io==NULL) continue;
+                                                /* remove last page from this bio */
+                                                bio->bi_vcnt--;
+                                                bio->bi_size -= len;
+                                                bio->bi_flags &= ~(1<< BIO_SEG_VALID);
+                                        }
+                                        goto bio_full;
+                                }
+                        }
+                }
+                nr_sectors += len>>9;
+                sector_nr += len>>9;
+        } while (r1_bio->bios[disk]->bi_vcnt < RESYNC_PAGES);
+ bio_full:
+        bio = r1_bio->bios[disk];
+        r1_bio->sectors = nr_sectors;
+
+        md_sync_acct(mirror->rdev->bdev, nr_sectors);
+
+        generic_make_request(bio);
+
+        return nr_sectors;
+}
+
+static int run(mddev_t *mddev)
+{
+        conf_t *conf;
+        int i, j, disk_idx;
+        mirror_info_t *disk;
+        mdk_rdev_t *rdev;
+        struct list_head *tmp;
+
+        if (mddev->level != 1) {
+                printk("raid1: %s: raid level not set to mirroring (%d)\n",
+                       mdname(mddev), mddev->level);
+                goto out;
+        }
+        /*
+         * copy the already verified devices into our private RAID1
+         * bookkeeping area. [whatever we allocate in run(),
+         * should be freed in stop()]
+         */
+        conf = kmalloc(sizeof(conf_t), GFP_KERNEL);
+        mddev->private = conf;
+        if (!conf)
+                goto out_no_mem;
+
+        memset(conf, 0, sizeof(*conf));
+        conf->mirrors = kmalloc(sizeof(struct mirror_info)*mddev->raid_disks, 
+                                 GFP_KERNEL);
+        if (!conf->mirrors)
+                goto out_no_mem;
+
+        memset(conf->mirrors, 0, sizeof(struct mirror_info)*mddev->raid_disks);
+
+        conf->poolinfo = kmalloc(sizeof(*conf->poolinfo), GFP_KERNEL);
+        if (!conf->poolinfo)
+                goto out_no_mem;
+        conf->poolinfo->mddev = mddev;
+        conf->poolinfo->raid_disks = mddev->raid_disks;
+        conf->r1bio_pool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc,
+                                          r1bio_pool_free,
+                                          conf->poolinfo);
+        if (!conf->r1bio_pool)
+                goto out_no_mem;
+
+        mddev->queue->unplug_fn = raid1_unplug;
+
+        mddev->queue->issue_flush_fn = raid1_issue_flush;
+
+        ITERATE_RDEV(mddev, rdev, tmp) {
+                disk_idx = rdev->raid_disk;
+                if (disk_idx >= mddev->raid_disks
+                    || disk_idx < 0)
+                        continue;
+                disk = conf->mirrors + disk_idx;
+
+                disk->rdev = rdev;
+
+                blk_queue_stack_limits(mddev->queue,
+                                       rdev->bdev->bd_disk->queue);
+                /* as we don't honour merge_bvec_fn, we must never risk
+                 * violating it, so limit ->max_sector to one PAGE, as
+                 * a one page request is never in violation.
+                 */
+                if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
+                    mddev->queue->max_sectors > (PAGE_SIZE>>9))
+                        blk_queue_max_sectors(mddev->queue, PAGE_SIZE>>9);
+
+                disk->head_position = 0;
+                if (!rdev->faulty && rdev->in_sync)
+                        conf->working_disks++;
+        }
+        conf->raid_disks = mddev->raid_disks;
+        conf->mddev = mddev;
+        spin_lock_init(&conf->device_lock);
+        INIT_LIST_HEAD(&conf->retry_list);
+        if (conf->working_disks == 1)
+                mddev->recovery_cp = MaxSector;
+
+        spin_lock_init(&conf->resync_lock);
+        init_waitqueue_head(&conf->wait_idle);
+        init_waitqueue_head(&conf->wait_resume);
+
+        if (!conf->working_disks) {
+                printk(KERN_ERR "raid1: no operational mirrors for %s\n",
+                        mdname(mddev));
+                goto out_free_conf;
+        }
+
+        mddev->degraded = 0;
+        for (i = 0; i < conf->raid_disks; i++) {
+
+                disk = conf->mirrors + i;
+
+                if (!disk->rdev) {
+                        disk->head_position = 0;
+                        mddev->degraded++;
+                }
+        }
+
+        /*
+         * find the first working one and use it as a starting point
+         * to read balancing.
+         */
+        for (j = 0; j < conf->raid_disks &&
+                     (!conf->mirrors[j].rdev ||
+                      !conf->mirrors[j].rdev->in_sync) ; j++)
+                /* nothing */;
+        conf->last_used = j;
+
+
+
+        {
+                mddev->thread = md_register_thread(raid1d, mddev, "%s_raid1");
+                if (!mddev->thread) {
+                        printk(KERN_ERR 
+                                "raid1: couldn't allocate thread for %s\n", 
+                                mdname(mddev));
+                        goto out_free_conf;
+                }
+        }
+        printk(KERN_INFO 
+                "raid1: raid set %s active with %d out of %d mirrors\n",
+                mdname(mddev), mddev->raid_disks - mddev->degraded, 
+                mddev->raid_disks);
+        /*
+         * Ok, everything is just fine now
+         */
+        mddev->array_size = mddev->size;
+
+        return 0;
+
+out_no_mem:
+        printk(KERN_ERR "raid1: couldn't allocate memory for %s\n",
+               mdname(mddev));
+
+out_free_conf:
+        if (conf) {
+                if (conf->r1bio_pool)
+                        mempool_destroy(conf->r1bio_pool);
+                if (conf->mirrors)
+                        kfree(conf->mirrors);
+                if (conf->poolinfo)
+                        kfree(conf->poolinfo);
+                kfree(conf);
+                mddev->private = NULL;
+        }
+out:
+        return -EIO;
+}
+
+static int stop(mddev_t *mddev)
+{
+        conf_t *conf = mddev_to_conf(mddev);
+
+        md_unregister_thread(mddev->thread);
+        mddev->thread = NULL;
+        blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
+        if (conf->r1bio_pool)
+                mempool_destroy(conf->r1bio_pool);
+        if (conf->mirrors)
+                kfree(conf->mirrors);
+        if (conf->poolinfo)
+                kfree(conf->poolinfo);
+        kfree(conf);
+        mddev->private = NULL;
+        return 0;
+}
+
+static int raid1_resize(mddev_t *mddev, sector_t sectors)
+{
+        /* no resync is happening, and there is enough space
+         * on all devices, so we can resize.
+         * We need to make sure resync covers any new space.
+         * If the array is shrinking we should possibly wait until
+         * any io in the removed space completes, but it hardly seems
+         * worth it.
+         */
+        mddev->array_size = sectors>>1;
+        set_capacity(mddev->gendisk, mddev->array_size << 1);
+        mddev->changed = 1;
+        if (mddev->array_size > mddev->size && mddev->recovery_cp == MaxSector) {
+                mddev->recovery_cp = mddev->size << 1;
+                set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+        }
+        mddev->size = mddev->array_size;
+        return 0;
+}
+
+static int raid1_reshape(mddev_t *mddev, int raid_disks)
+{
+        /* We need to:
+         * 1/ resize the r1bio_pool
+         * 2/ resize conf->mirrors
+         *
+         * We allocate a new r1bio_pool if we can.
+         * Then raise a device barrier and wait until all IO stops.
+         * Then resize conf->mirrors and swap in the new r1bio pool.
+         */
+        mempool_t *newpool, *oldpool;
+        struct pool_info *newpoolinfo;
+        mirror_info_t *newmirrors;
+        conf_t *conf = mddev_to_conf(mddev);
+
+        int d;
+
+        for (d= raid_disks; d < conf->raid_disks; d++)
+                if (conf->mirrors[d].rdev)
+                        return -EBUSY;
+
+        newpoolinfo = kmalloc(sizeof(*newpoolinfo), GFP_KERNEL);
+        if (!newpoolinfo)
+                return -ENOMEM;
+        newpoolinfo->mddev = mddev;
+        newpoolinfo->raid_disks = raid_disks;
+
+        newpool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc,
+                                 r1bio_pool_free, newpoolinfo);
+        if (!newpool) {
+                kfree(newpoolinfo);
+                return -ENOMEM;
+        }
+        newmirrors = kmalloc(sizeof(struct mirror_info) * raid_disks, GFP_KERNEL);
+        if (!newmirrors) {
+                kfree(newpoolinfo);
+                mempool_destroy(newpool);
+                return -ENOMEM;
+        }
+        memset(newmirrors, 0, sizeof(struct mirror_info)*raid_disks);
+
+        spin_lock_irq(&conf->resync_lock);
+        conf->barrier++;
+        wait_event_lock_irq(conf->wait_idle, !conf->nr_pending,
+                            conf->resync_lock, unplug_slaves(mddev));
+        spin_unlock_irq(&conf->resync_lock);
+
+        /* ok, everything is stopped */
+        oldpool = conf->r1bio_pool;
+        conf->r1bio_pool = newpool;
+        for (d=0; d < raid_disks && d < conf->raid_disks; d++)
+                newmirrors[d] = conf->mirrors[d];
+        kfree(conf->mirrors);
+        conf->mirrors = newmirrors;
+        kfree(conf->poolinfo);
+        conf->poolinfo = newpoolinfo;
+
+        mddev->degraded += (raid_disks - conf->raid_disks);
+        conf->raid_disks = mddev->raid_disks = raid_disks;
+
+        spin_lock_irq(&conf->resync_lock);
+        conf->barrier--;
+        spin_unlock_irq(&conf->resync_lock);
+        wake_up(&conf->wait_resume);
+        wake_up(&conf->wait_idle);
+
+
+        set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+        md_wakeup_thread(mddev->thread);
+
+        mempool_destroy(oldpool);
+        return 0;
+}
+
+
+static mdk_personality_t raid1_personality =
+{
+        .name           = "raid1",
+        .owner          = THIS_MODULE,
+        .make_request   = make_request,
+        .run            = run,
+        .stop           = stop,
+        .status         = status,
+        .error_handler  = error,
+        .hot_add_disk   = raid1_add_disk,
+        .hot_remove_disk= raid1_remove_disk,
+        .spare_active   = raid1_spare_active,
+        .sync_request   = sync_request,
+        .resize         = raid1_resize,
+        .reshape        = raid1_reshape,
+};
+
+static int __init raid_init(void)
+{
+        return register_md_personality(RAID1, &raid1_personality);
+}
+
+static void raid_exit(void)
+{
+        unregister_md_personality(RAID1);
+}
+
+module_init(raid_init);
+module_exit(raid_exit);
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("md-personality-3"); /* RAID1 */
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
new file mode 100644
index 000000000000..b100bfe4fdca
--- /dev/null
+++ b/drivers/md/raid10.c
@@ -0,0 +1,1787 @@
+/*
+ * raid10.c : Multiple Devices driver for Linux
+ *
+ * Copyright (C) 2000-2004 Neil Brown
+ *
+ * RAID-10 support for md.
+ *
+ * Base on code in raid1.c.  See raid1.c for futher copyright information.
+ *
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * (for example /usr/src/linux/COPYING); if not, write to the Free
+ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/raid/raid10.h>
+
+/*
+ * RAID10 provides a combination of RAID0 and RAID1 functionality.
+ * The layout of data is defined by
+ *    chunk_size
+ *    raid_disks
+ *    near_copies (stored in low byte of layout)
+ *    far_copies (stored in second byte of layout)
+ *
+ * The data to be stored is divided into chunks using chunksize.
+ * Each device is divided into far_copies sections.
+ * In each section, chunks are laid out in a style similar to raid0, but
+ * near_copies copies of each chunk is stored (each on a different drive).
+ * The starting device for each section is offset near_copies from the starting
+ * device of the previous section.
+ * Thus there are (near_copies*far_copies) of each chunk, and each is on a different
+ * drive.
+ * near_copies and far_copies must be at least one, and their product is at most
+ * raid_disks.
+ */
+
+/*
+ * Number of guaranteed r10bios in case of extreme VM load:
+ */
+#define NR_RAID10_BIOS 256
+
+static void unplug_slaves(mddev_t *mddev);
+
+static void * r10bio_pool_alloc(unsigned int __nocast gfp_flags, void *data)
+{
+        conf_t *conf = data;
+        r10bio_t *r10_bio;
+        int size = offsetof(struct r10bio_s, devs[conf->copies]);
+
+        /* allocate a r10bio with room for raid_disks entries in the bios array */
+        r10_bio = kmalloc(size, gfp_flags);
+        if (r10_bio)
+                memset(r10_bio, 0, size);
+        else
+                unplug_slaves(conf->mddev);
+
+        return r10_bio;
+}
+
+static void r10bio_pool_free(void *r10_bio, void *data)
+{
+        kfree(r10_bio);
+}
+
+#define RESYNC_BLOCK_SIZE (64*1024)
+//#define RESYNC_BLOCK_SIZE PAGE_SIZE
+#define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
+#define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
+#define RESYNC_WINDOW (2048*1024)
+
+/*
+ * When performing a resync, we need to read and compare, so
+ * we need as many pages are there are copies.
+ * When performing a recovery, we need 2 bios, one for read,
+ * one for write (we recover only one drive per r10buf)
+ *
+ */
+static void * r10buf_pool_alloc(unsigned int __nocast gfp_flags, void *data)
+{
+        conf_t *conf = data;
+        struct page *page;
+        r10bio_t *r10_bio;
+        struct bio *bio;
+        int i, j;
+        int nalloc;
+
+        r10_bio = r10bio_pool_alloc(gfp_flags, conf);
+        if (!r10_bio) {
+                unplug_slaves(conf->mddev);
+                return NULL;
+        }
+
+        if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery))
+                nalloc = conf->copies; /* resync */
+        else
+                nalloc = 2; /* recovery */
+
+        /*
+         * Allocate bios.
+         */
+        for (j = nalloc ; j-- ; ) {
+                bio = bio_alloc(gfp_flags, RESYNC_PAGES);
+                if (!bio)
+                        goto out_free_bio;
+                r10_bio->devs[j].bio = bio;
+        }
+        /*
+         * Allocate RESYNC_PAGES data pages and attach them
+         * where needed.
+         */
+        for (j = 0 ; j < nalloc; j++) {
+                bio = r10_bio->devs[j].bio;
+                for (i = 0; i < RESYNC_PAGES; i++) {
+                        page = alloc_page(gfp_flags);
+                        if (unlikely(!page))
+                                goto out_free_pages;
+
+                        bio->bi_io_vec[i].bv_page = page;
+                }
+        }
+
+        return r10_bio;
+
+out_free_pages:
+        for ( ; i > 0 ; i--)
+                __free_page(bio->bi_io_vec[i-1].bv_page);
+        while (j--)
+                for (i = 0; i < RESYNC_PAGES ; i++)
+                        __free_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page);
+        j = -1;
+out_free_bio:
+        while ( ++j < nalloc )
+                bio_put(r10_bio->devs[j].bio);
+        r10bio_pool_free(r10_bio, conf);
+        return NULL;
+}
+
+static void r10buf_pool_free(void *__r10_bio, void *data)
+{
+        int i;
+        conf_t *conf = data;
+        r10bio_t *r10bio = __r10_bio;
+        int j;
+
+        for (j=0; j < conf->copies; j++) {
+                struct bio *bio = r10bio->devs[j].bio;
+                if (bio) {
+                        for (i = 0; i < RESYNC_PAGES; i++) {
+                                __free_page(bio->bi_io_vec[i].bv_page);
+                                bio->bi_io_vec[i].bv_page = NULL;
+                        }
+                        bio_put(bio);
+                }
+        }
+        r10bio_pool_free(r10bio, conf);
+}
+
+static void put_all_bios(conf_t *conf, r10bio_t *r10_bio)
+{
+        int i;
+
+        for (i = 0; i < conf->copies; i++) {
+                struct bio **bio = & r10_bio->devs[i].bio;
+                if (*bio)
+                        bio_put(*bio);
+                *bio = NULL;
+        }
+}
+
+static inline void free_r10bio(r10bio_t *r10_bio)
+{
+        unsigned long flags;
+
+        conf_t *conf = mddev_to_conf(r10_bio->mddev);
+
+        /*
+         * Wake up any possible resync thread that waits for the device
+         * to go idle.
+         */
+        spin_lock_irqsave(&conf->resync_lock, flags);
+        if (!--conf->nr_pending) {
+                wake_up(&conf->wait_idle);
+                wake_up(&conf->wait_resume);
+        }
+        spin_unlock_irqrestore(&conf->resync_lock, flags);
+
+        put_all_bios(conf, r10_bio);
+        mempool_free(r10_bio, conf->r10bio_pool);
+}
+
+static inline void put_buf(r10bio_t *r10_bio)
+{
+        conf_t *conf = mddev_to_conf(r10_bio->mddev);
+        unsigned long flags;
+
+        mempool_free(r10_bio, conf->r10buf_pool);
+
+        spin_lock_irqsave(&conf->resync_lock, flags);
+        if (!conf->barrier)
+                BUG();
+        --conf->barrier;
+        wake_up(&conf->wait_resume);
+        wake_up(&conf->wait_idle);
+
+        if (!--conf->nr_pending) {
+                wake_up(&conf->wait_idle);
+                wake_up(&conf->wait_resume);
+        }
+        spin_unlock_irqrestore(&conf->resync_lock, flags);
+}
+
+static void reschedule_retry(r10bio_t *r10_bio)
+{
+        unsigned long flags;
+        mddev_t *mddev = r10_bio->mddev;
+        conf_t *conf = mddev_to_conf(mddev);
+
+        spin_lock_irqsave(&conf->device_lock, flags);
+        list_add(&r10_bio->retry_list, &conf->retry_list);
+        spin_unlock_irqrestore(&conf->device_lock, flags);
+
+        md_wakeup_thread(mddev->thread);
+}
+
+/*
+ * raid_end_bio_io() is called when we have finished servicing a mirrored
+ * operation and are ready to return a success/failure code to the buffer
+ * cache layer.
+ */
+static void raid_end_bio_io(r10bio_t *r10_bio)
+{
+        struct bio *bio = r10_bio->master_bio;
+
+        bio_endio(bio, bio->bi_size,
+                test_bit(R10BIO_Uptodate, &r10_bio->state) ? 0 : -EIO);
+        free_r10bio(r10_bio);
+}
+
+/*
+ * Update disk head position estimator based on IRQ completion info.
+ */
+static inline void update_head_pos(int slot, r10bio_t *r10_bio)
+{
+        conf_t *conf = mddev_to_conf(r10_bio->mddev);
+
+        conf->mirrors[r10_bio->devs[slot].devnum].head_position =
+                r10_bio->devs[slot].addr + (r10_bio->sectors);
+}
+
+static int raid10_end_read_request(struct bio *bio, unsigned int bytes_done, int error)
+{
+        int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
+        r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
+        int slot, dev;
+        conf_t *conf = mddev_to_conf(r10_bio->mddev);
+
+        if (bio->bi_size)
+                return 1;
+
+        slot = r10_bio->read_slot;
+        dev = r10_bio->devs[slot].devnum;
+        /*
+         * this branch is our 'one mirror IO has finished' event handler:
+         */
+        if (!uptodate)
+                md_error(r10_bio->mddev, conf->mirrors[dev].rdev);
+        else
+                /*
+                 * Set R10BIO_Uptodate in our master bio, so that
+                 * we will return a good error code to the higher
+                 * levels even if IO on some other mirrored buffer fails.
+                 *
+                 * The 'master' represents the composite IO operation to
+                 * user-side. So if something waits for IO, then it will
+                 * wait for the 'master' bio.
+                 */
+                set_bit(R10BIO_Uptodate, &r10_bio->state);
+
+        update_head_pos(slot, r10_bio);
+
+        /*
+         * we have only one bio on the read side
+         */
+        if (uptodate)
+                raid_end_bio_io(r10_bio);
+        else {
+                /*
+                 * oops, read error:
+                 */
+                char b[BDEVNAME_SIZE];
+                if (printk_ratelimit())
+                        printk(KERN_ERR "raid10: %s: rescheduling sector %llu\n",
+                               bdevname(conf->mirrors[dev].rdev->bdev,b), (unsigned long long)r10_bio->sector);
+                reschedule_retry(r10_bio);
+        }
+
+        rdev_dec_pending(conf->mirrors[dev].rdev, conf->mddev);
+        return 0;
+}
+
+static int raid10_end_write_request(struct bio *bio, unsigned int bytes_done, int error)
+{
+        int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
+        r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
+        int slot, dev;
+        conf_t *conf = mddev_to_conf(r10_bio->mddev);
+
+        if (bio->bi_size)
+                return 1;
+
+        for (slot = 0; slot < conf->copies; slot++)
+                if (r10_bio->devs[slot].bio == bio)
+                        break;
+        dev = r10_bio->devs[slot].devnum;
+
+        /*
+         * this branch is our 'one mirror IO has finished' event handler:
+         */
+        if (!uptodate)
+                md_error(r10_bio->mddev, conf->mirrors[dev].rdev);
+        else
+                /*
+                 * Set R10BIO_Uptodate in our master bio, so that
+                 * we will return a good error code for to the higher
+                 * levels even if IO on some other mirrored buffer fails.
+                 *
+                 * The 'master' represents the composite IO operation to
+                 * user-side. So if something waits for IO, then it will
+                 * wait for the 'master' bio.
+                 */
+                set_bit(R10BIO_Uptodate, &r10_bio->state);
+
+        update_head_pos(slot, r10_bio);
+
+        /*
+         *
+         * Let's see if all mirrored write operations have finished
+         * already.
+         */
+        if (atomic_dec_and_test(&r10_bio->remaining)) {
+                md_write_end(r10_bio->mddev);
+                raid_end_bio_io(r10_bio);
+        }
+
+        rdev_dec_pending(conf->mirrors[dev].rdev, conf->mddev);
+        return 0;
+}
+
+
+/*
+ * RAID10 layout manager
+ * Aswell as the chunksize and raid_disks count, there are two
+ * parameters: near_copies and far_copies.
+ * near_copies * far_copies must be <= raid_disks.
+ * Normally one of these will be 1.
+ * If both are 1, we get raid0.
+ * If near_copies == raid_disks, we get raid1.
+ *
+ * Chunks are layed out in raid0 style with near_copies copies of the
+ * first chunk, followed by near_copies copies of the next chunk and
+ * so on.
+ * If far_copies > 1, then after 1/far_copies of the array has been assigned
+ * as described above, we start again with a device offset of near_copies.
+ * So we effectively have another copy of the whole array further down all
+ * the drives, but with blocks on different drives.
+ * With this layout, and block is never stored twice on the one device.
+ *
+ * raid10_find_phys finds the sector offset of a given virtual sector
+ * on each device that it is on. If a block isn't on a device,
+ * that entry in the array is set to MaxSector.
+ *
+ * raid10_find_virt does the reverse mapping, from a device and a
+ * sector offset to a virtual address
+ */
+
+static void raid10_find_phys(conf_t *conf, r10bio_t *r10bio)
+{
+        int n,f;
+        sector_t sector;
+        sector_t chunk;
+        sector_t stripe;
+        int dev;
+
+        int slot = 0;
+
+        /* now calculate first sector/dev */
+        chunk = r10bio->sector >> conf->chunk_shift;
+        sector = r10bio->sector & conf->chunk_mask;
+
+        chunk *= conf->near_copies;
+        stripe = chunk;
+        dev = sector_div(stripe, conf->raid_disks);
+
+        sector += stripe << conf->chunk_shift;
+
+        /* and calculate all the others */
+        for (n=0; n < conf->near_copies; n++) {
+                int d = dev;
+                sector_t s = sector;
+                r10bio->devs[slot].addr = sector;
+                r10bio->devs[slot].devnum = d;
+                slot++;
+
+                for (f = 1; f < conf->far_copies; f++) {
+                        d += conf->near_copies;
+                        if (d >= conf->raid_disks)
+                                d -= conf->raid_disks;
+                        s += conf->stride;
+                        r10bio->devs[slot].devnum = d;
+                        r10bio->devs[slot].addr = s;
+                        slot++;
+                }
+                dev++;
+                if (dev >= conf->raid_disks) {
+                        dev = 0;
+                        sector += (conf->chunk_mask + 1);
+                }
+        }
+        BUG_ON(slot != conf->copies);
+}
+
+static sector_t raid10_find_virt(conf_t *conf, sector_t sector, int dev)
+{
+        sector_t offset, chunk, vchunk;
+
+        while (sector > conf->stride) {
+                sector -= conf->stride;
+                if (dev < conf->near_copies)
+                        dev += conf->raid_disks - conf->near_copies;
+                else
+                        dev -= conf->near_copies;
+        }
+
+        offset = sector & conf->chunk_mask;
+        chunk = sector >> conf->chunk_shift;
+        vchunk = chunk * conf->raid_disks + dev;
+        sector_div(vchunk, conf->near_copies);
+        return (vchunk << conf->chunk_shift) + offset;
+}
+
+/**
+ *      raid10_mergeable_bvec -- tell bio layer if a two requests can be merged
+ *      @q: request queue
+ *      @bio: the buffer head that's been built up so far
+ *      @biovec: the request that could be merged to it.
+ *
+ *      Return amount of bytes we can accept at this offset
+ *      If near_copies == raid_disk, there are no striping issues,
+ *      but in that case, the function isn't called at all.
+ */
+static int raid10_mergeable_bvec(request_queue_t *q, struct bio *bio,
+                                struct bio_vec *bio_vec)
+{
+        mddev_t *mddev = q->queuedata;
+        sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev);
+        int max;
+        unsigned int chunk_sectors = mddev->chunk_size >> 9;
+        unsigned int bio_sectors = bio->bi_size >> 9;
+
+        max =  (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
+        if (max < 0) max = 0; /* bio_add cannot handle a negative return */
+        if (max <= bio_vec->bv_len && bio_sectors == 0)
+                return bio_vec->bv_len;
+        else
+                return max;
+}
+
+/*
+ * This routine returns the disk from which the requested read should
+ * be done. There is a per-array 'next expected sequential IO' sector
+ * number - if this matches on the next IO then we use the last disk.
+ * There is also a per-disk 'last know head position' sector that is
+ * maintained from IRQ contexts, both the normal and the resync IO
+ * completion handlers update this position correctly. If there is no
+ * perfect sequential match then we pick the disk whose head is closest.
+ *
+ * If there are 2 mirrors in the same 2 devices, performance degrades
+ * because position is mirror, not device based.
+ *
+ * The rdev for the device selected will have nr_pending incremented.
+ */
+
+/*
+ * FIXME: possibly should rethink readbalancing and do it differently
+ * depending on near_copies / far_copies geometry.
+ */
+static int read_balance(conf_t *conf, r10bio_t *r10_bio)
+{
+        const unsigned long this_sector = r10_bio->sector;
+        int disk, slot, nslot;
+        const int sectors = r10_bio->sectors;
+        sector_t new_distance, current_distance;
+
+        raid10_find_phys(conf, r10_bio);
+        rcu_read_lock();
+        /*
+         * Check if we can balance. We can balance on the whole
+         * device if no resync is going on, or below the resync window.
+         * We take the first readable disk when above the resync window.
+         */
+        if (conf->mddev->recovery_cp < MaxSector
+            && (this_sector + sectors >= conf->next_resync)) {
+                /* make sure that disk is operational */
+                slot = 0;
+                disk = r10_bio->devs[slot].devnum;
+
+                while (!conf->mirrors[disk].rdev ||
+                       !conf->mirrors[disk].rdev->in_sync) {
+                        slot++;
+                        if (slot == conf->copies) {
+                                slot = 0;
+                                disk = -1;
+                                break;
+                        }
+                        disk = r10_bio->devs[slot].devnum;
+                }
+                goto rb_out;
+        }
+
+
+        /* make sure the disk is operational */
+        slot = 0;
+        disk = r10_bio->devs[slot].devnum;
+        while (!conf->mirrors[disk].rdev ||
+               !conf->mirrors[disk].rdev->in_sync) {
+                slot ++;
+                if (slot == conf->copies) {
+                        disk = -1;
+                        goto rb_out;
+                }
+                disk = r10_bio->devs[slot].devnum;
+        }
+
+
+        current_distance = abs(this_sector - conf->mirrors[disk].head_position);
+
+        /* Find the disk whose head is closest */
+
+        for (nslot = slot; nslot < conf->copies; nslot++) {
+                int ndisk = r10_bio->devs[nslot].devnum;
+
+
+                if (!conf->mirrors[ndisk].rdev ||
+                    !conf->mirrors[ndisk].rdev->in_sync)
+                        continue;
+
+                if (!atomic_read(&conf->mirrors[ndisk].rdev->nr_pending)) {
+                        disk = ndisk;
+                        slot = nslot;
+                        break;
+                }
+                new_distance = abs(r10_bio->devs[nslot].addr -
+                                   conf->mirrors[ndisk].head_position);
+                if (new_distance < current_distance) {
+                        current_distance = new_distance;
+                        disk = ndisk;
+                        slot = nslot;
+                }
+        }
+
+rb_out:
+        r10_bio->read_slot = slot;
+/*      conf->next_seq_sect = this_sector + sectors;*/
+
+        if (disk >= 0 && conf->mirrors[disk].rdev)
+                atomic_inc(&conf->mirrors[disk].rdev->nr_pending);
+        rcu_read_unlock();
+
+        return disk;
+}
+
+static void unplug_slaves(mddev_t *mddev)
+{
+        conf_t *conf = mddev_to_conf(mddev);
+        int i;
+
+        rcu_read_lock();
+        for (i=0; i<mddev->raid_disks; i++) {
+                mdk_rdev_t *rdev = conf->mirrors[i].rdev;
+                if (rdev && !rdev->faulty && atomic_read(&rdev->nr_pending)) {
+                        request_queue_t *r_queue = bdev_get_queue(rdev->bdev);
+
+                        atomic_inc(&rdev->nr_pending);
+                        rcu_read_unlock();
+
+                        if (r_queue->unplug_fn)
+                                r_queue->unplug_fn(r_queue);
+
+                        rdev_dec_pending(rdev, mddev);
+                        rcu_read_lock();
+                }
+        }
+        rcu_read_unlock();
+}
+
+static void raid10_unplug(request_queue_t *q)
+{
+        unplug_slaves(q->queuedata);
+}
+
+static int raid10_issue_flush(request_queue_t *q, struct gendisk *disk,
+                             sector_t *error_sector)
+{
+        mddev_t *mddev = q->queuedata;
+        conf_t *conf = mddev_to_conf(mddev);
+        int i, ret = 0;
+
+        rcu_read_lock();
+        for (i=0; i<mddev->raid_disks && ret == 0; i++) {
+                mdk_rdev_t *rdev = conf->mirrors[i].rdev;
+                if (rdev && !rdev->faulty) {
+                        struct block_device *bdev = rdev->bdev;
+                        request_queue_t *r_queue = bdev_get_queue(bdev);
+
+                        if (!r_queue->issue_flush_fn)
+                                ret = -EOPNOTSUPP;
+                        else {
+                                atomic_inc(&rdev->nr_pending);
+                                rcu_read_unlock();
+                                ret = r_queue->issue_flush_fn(r_queue, bdev->bd_disk,
+                                                              error_sector);
+                                rdev_dec_pending(rdev, mddev);
+                                rcu_read_lock();
+                        }
+                }
+        }
+        rcu_read_unlock();
+        return ret;
+}
+
+/*
+ * Throttle resync depth, so that we can both get proper overlapping of
+ * requests, but are still able to handle normal requests quickly.
+ */
+#define RESYNC_DEPTH 32
+
+static void device_barrier(conf_t *conf, sector_t sect)
+{
+        spin_lock_irq(&conf->resync_lock);
+        wait_event_lock_irq(conf->wait_idle, !waitqueue_active(&conf->wait_resume),
+                            conf->resync_lock, unplug_slaves(conf->mddev));
+
+        if (!conf->barrier++) {
+                wait_event_lock_irq(conf->wait_idle, !conf->nr_pending,
+                                    conf->resync_lock, unplug_slaves(conf->mddev));
+                if (conf->nr_pending)
+                        BUG();
+        }
+        wait_event_lock_irq(conf->wait_resume, conf->barrier < RESYNC_DEPTH,
+                            conf->resync_lock, unplug_slaves(conf->mddev));
+        conf->next_resync = sect;
+        spin_unlock_irq(&conf->resync_lock);
+}
+
+static int make_request(request_queue_t *q, struct bio * bio)
+{
+        mddev_t *mddev = q->queuedata;
+        conf_t *conf = mddev_to_conf(mddev);
+        mirror_info_t *mirror;
+        r10bio_t *r10_bio;
+        struct bio *read_bio;
+        int i;
+        int chunk_sects = conf->chunk_mask + 1;
+
+        /* If this request crosses a chunk boundary, we need to
+         * split it.  This will only happen for 1 PAGE (or less) requests.
+         */
+        if (unlikely( (bio->bi_sector & conf->chunk_mask) + (bio->bi_size >> 9)
+                      > chunk_sects &&
+                    conf->near_copies < conf->raid_disks)) {
+                struct bio_pair *bp;
+                /* Sanity check -- queue functions should prevent this happening */
+                if (bio->bi_vcnt != 1 ||
+                    bio->bi_idx != 0)
+                        goto bad_map;
+                /* This is a one page bio that upper layers
+                 * refuse to split for us, so we need to split it.
+                 */
+                bp = bio_split(bio, bio_split_pool,
+                               chunk_sects - (bio->bi_sector & (chunk_sects - 1)) );
+                if (make_request(q, &bp->bio1))
+                        generic_make_request(&bp->bio1);
+                if (make_request(q, &bp->bio2))
+                        generic_make_request(&bp->bio2);
+
+                bio_pair_release(bp);
+                return 0;
+        bad_map:
+                printk("raid10_make_request bug: can't convert block across chunks"
+                       " or bigger than %dk %llu %d\n", chunk_sects/2,
+                       (unsigned long long)bio->bi_sector, bio->bi_size >> 10);
+
+                bio_io_error(bio, bio->bi_size);
+                return 0;
+        }
+
+        /*
+         * Register the new request and wait if the reconstruction
+         * thread has put up a bar for new requests.
+         * Continue immediately if no resync is active currently.
+         */
+        spin_lock_irq(&conf->resync_lock);
+        wait_event_lock_irq(conf->wait_resume, !conf->barrier, conf->resync_lock, );
+        conf->nr_pending++;
+        spin_unlock_irq(&conf->resync_lock);
+
+        if (bio_data_dir(bio)==WRITE) {
+                disk_stat_inc(mddev->gendisk, writes);
+                disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bio));
+        } else {
+                disk_stat_inc(mddev->gendisk, reads);
+                disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bio));
+        }
+
+        r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);
+
+        r10_bio->master_bio = bio;
+        r10_bio->sectors = bio->bi_size >> 9;
+
+        r10_bio->mddev = mddev;
+        r10_bio->sector = bio->bi_sector;
+
+        if (bio_data_dir(bio) == READ) {
+                /*
+                 * read balancing logic:
+                 */
+                int disk = read_balance(conf, r10_bio);
+                int slot = r10_bio->read_slot;
+                if (disk < 0) {
+                        raid_end_bio_io(r10_bio);
+                        return 0;
+                }
+                mirror = conf->mirrors + disk;
+
+                read_bio = bio_clone(bio, GFP_NOIO);
+
+                r10_bio->devs[slot].bio = read_bio;
+
+                read_bio->bi_sector = r10_bio->devs[slot].addr +
+                        mirror->rdev->data_offset;
+                read_bio->bi_bdev = mirror->rdev->bdev;
+                read_bio->bi_end_io = raid10_end_read_request;
+                read_bio->bi_rw = READ;
+                read_bio->bi_private = r10_bio;
+
+                generic_make_request(read_bio);
+                return 0;
+        }
+
+        /*
+         * WRITE:
+         */
+        /* first select target devices under spinlock and
+         * inc refcount on their rdev.  Record them by setting
+         * bios[x] to bio
+         */
+        raid10_find_phys(conf, r10_bio);
+        rcu_read_lock();
+        for (i = 0;  i < conf->copies; i++) {
+                int d = r10_bio->devs[i].devnum;
+                if (conf->mirrors[d].rdev &&
+                    !conf->mirrors[d].rdev->faulty) {
+                        atomic_inc(&conf->mirrors[d].rdev->nr_pending);
+                        r10_bio->devs[i].bio = bio;
+                } else
+                        r10_bio->devs[i].bio = NULL;
+        }
+        rcu_read_unlock();
+
+        atomic_set(&r10_bio->remaining, 1);
+        md_write_start(mddev);
+        for (i = 0; i < conf->copies; i++) {
+                struct bio *mbio;
+                int d = r10_bio->devs[i].devnum;
+                if (!r10_bio->devs[i].bio)
+                        continue;
+
+                mbio = bio_clone(bio, GFP_NOIO);
+                r10_bio->devs[i].bio = mbio;
+
+                mbio->bi_sector = r10_bio->devs[i].addr+
+                        conf->mirrors[d].rdev->data_offset;
+                mbio->bi_bdev = conf->mirrors[d].rdev->bdev;
+                mbio->bi_end_io = raid10_end_write_request;
+                mbio->bi_rw = WRITE;
+                mbio->bi_private = r10_bio;
+
+                atomic_inc(&r10_bio->remaining);
+                generic_make_request(mbio);
+        }
+
+        if (atomic_dec_and_test(&r10_bio->remaining)) {
+                md_write_end(mddev);
+                raid_end_bio_io(r10_bio);
+        }
+
+        return 0;
+}
+
+static void status(struct seq_file *seq, mddev_t *mddev)
+{
+        conf_t *conf = mddev_to_conf(mddev);
+        int i;
+
+        if (conf->near_copies < conf->raid_disks)
+                seq_printf(seq, " %dK chunks", mddev->chunk_size/1024);
+        if (conf->near_copies > 1)
+                seq_printf(seq, " %d near-copies", conf->near_copies);
+        if (conf->far_copies > 1)
+                seq_printf(seq, " %d far-copies", conf->far_copies);
+
+        seq_printf(seq, " [%d/%d] [", conf->raid_disks,
+                                                conf->working_disks);
+        for (i = 0; i < conf->raid_disks; i++)
+                seq_printf(seq, "%s",
+                              conf->mirrors[i].rdev &&
+                              conf->mirrors[i].rdev->in_sync ? "U" : "_");
+        seq_printf(seq, "]");
+}
+
+static void error(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+        char b[BDEVNAME_SIZE];
+        conf_t *conf = mddev_to_conf(mddev);
+
+        /*
+         * If it is not operational, then we have already marked it as dead
+         * else if it is the last working disks, ignore the error, let the
+         * next level up know.
+         * else mark the drive as failed
+         */
+        if (rdev->in_sync
+            && conf->working_disks == 1)
+                /*
+                 * Don't fail the drive, just return an IO error.
+                 * The test should really be more sophisticated than
+                 * "working_disks == 1", but it isn't critical, and
+                 * can wait until we do more sophisticated "is the drive
+                 * really dead" tests...
+                 */
+                return;
+        if (rdev->in_sync) {
+                mddev->degraded++;
+                conf->working_disks--;
+                /*
+                 * if recovery is running, make sure it aborts.
+                 */
+                set_bit(MD_RECOVERY_ERR, &mddev->recovery);
+        }
+        rdev->in_sync = 0;
+        rdev->faulty = 1;
+        mddev->sb_dirty = 1;
+        printk(KERN_ALERT "raid10: Disk failure on %s, disabling device. \n"
+                "       Operation continuing on %d devices\n",
+                bdevname(rdev->bdev,b), conf->working_disks);
+}
+
+static void print_conf(conf_t *conf)
+{
+        int i;
+        mirror_info_t *tmp;
+
+        printk("RAID10 conf printout:\n");
+        if (!conf) {
+                printk("(!conf)\n");
+                return;
+        }
+        printk(" --- wd:%d rd:%d\n", conf->working_disks,
+                conf->raid_disks);
+
+        for (i = 0; i < conf->raid_disks; i++) {
+                char b[BDEVNAME_SIZE];
+                tmp = conf->mirrors + i;
+                if (tmp->rdev)
+                        printk(" disk %d, wo:%d, o:%d, dev:%s\n",
+                                i, !tmp->rdev->in_sync, !tmp->rdev->faulty,
+                                bdevname(tmp->rdev->bdev,b));
+        }
+}
+
+static void close_sync(conf_t *conf)
+{
+        spin_lock_irq(&conf->resync_lock);
+        wait_event_lock_irq(conf->wait_resume, !conf->barrier,
+                            conf->resync_lock,  unplug_slaves(conf->mddev));
+        spin_unlock_irq(&conf->resync_lock);
+
+        if (conf->barrier) BUG();
+        if (waitqueue_active(&conf->wait_idle)) BUG();
+
+        mempool_destroy(conf->r10buf_pool);
+        conf->r10buf_pool = NULL;
+}
+
+static int raid10_spare_active(mddev_t *mddev)
+{
+        int i;
+        conf_t *conf = mddev->private;
+        mirror_info_t *tmp;
+
+        /*
+         * Find all non-in_sync disks within the RAID10 configuration
+         * and mark them in_sync
+         */
+        for (i = 0; i < conf->raid_disks; i++) {
+                tmp = conf->mirrors + i;
+                if (tmp->rdev
+                    && !tmp->rdev->faulty
+                    && !tmp->rdev->in_sync) {
+                        conf->working_disks++;
+                        mddev->degraded--;
+                        tmp->rdev->in_sync = 1;
+                }
+        }
+
+        print_conf(conf);
+        return 0;
+}
+
+
+static int raid10_add_disk(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+        conf_t *conf = mddev->private;
+        int found = 0;
+        int mirror;
+        mirror_info_t *p;
+
+        if (mddev->recovery_cp < MaxSector)
+                /* only hot-add to in-sync arrays, as recovery is
+                 * very different from resync
+                 */
+                return 0;
+
+        for (mirror=0; mirror < mddev->raid_disks; mirror++)
+                if ( !(p=conf->mirrors+mirror)->rdev) {
+
+                        blk_queue_stack_limits(mddev->queue,
+                                               rdev->bdev->bd_disk->queue);
+                        /* as we don't honour merge_bvec_fn, we must never risk
+                         * violating it, so limit ->max_sector to one PAGE, as
+                         * a one page request is never in violation.
+                         */
+                        if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
+                            mddev->queue->max_sectors > (PAGE_SIZE>>9))
+                                mddev->queue->max_sectors = (PAGE_SIZE>>9);
+
+                        p->head_position = 0;
+                        rdev->raid_disk = mirror;
+                        found = 1;
+                        p->rdev = rdev;
+                        break;
+                }
+
+        print_conf(conf);
+        return found;
+}
+
+static int raid10_remove_disk(mddev_t *mddev, int number)
+{
+        conf_t *conf = mddev->private;
+        int err = 0;
+        mdk_rdev_t *rdev;
+        mirror_info_t *p = conf->mirrors+ number;
+
+        print_conf(conf);
+        rdev = p->rdev;
+        if (rdev) {
+                if (rdev->in_sync ||
+                    atomic_read(&rdev->nr_pending)) {
+                        err = -EBUSY;
+                        goto abort;
+                }
+                p->rdev = NULL;
+                synchronize_kernel();
+                if (atomic_read(&rdev->nr_pending)) {
+                        /* lost the race, try later */
+                        err = -EBUSY;
+                        p->rdev = rdev;
+                }
+        }
+abort:
+
+        print_conf(conf);
+        return err;
+}
+
+
+static int end_sync_read(struct bio *bio, unsigned int bytes_done, int error)
+{
+        int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
+        r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
+        conf_t *conf = mddev_to_conf(r10_bio->mddev);
+        int i,d;
+
+        if (bio->bi_size)
+                return 1;
+
+        for (i=0; i<conf->copies; i++)
+                if (r10_bio->devs[i].bio == bio)
+                        break;
+        if (i == conf->copies)
+                BUG();
+        update_head_pos(i, r10_bio);
+        d = r10_bio->devs[i].devnum;
+        if (!uptodate)
+                md_error(r10_bio->mddev,
+                         conf->mirrors[d].rdev);
+
+        /* for reconstruct, we always reschedule after a read.
+         * for resync, only after all reads
+         */
+        if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
+            atomic_dec_and_test(&r10_bio->remaining)) {
+                /* we have read all the blocks,
+                 * do the comparison in process context in raid10d
+                 */
+                reschedule_retry(r10_bio);
+        }
+        rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev);
+        return 0;
+}
+
+static int end_sync_write(struct bio *bio, unsigned int bytes_done, int error)
+{
+        int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
+        r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
+        mddev_t *mddev = r10_bio->mddev;
+        conf_t *conf = mddev_to_conf(mddev);
+        int i,d;
+
+        if (bio->bi_size)
+                return 1;
+
+        for (i = 0; i < conf->copies; i++)
+                if (r10_bio->devs[i].bio == bio)
+                        break;
+        d = r10_bio->devs[i].devnum;
+
+        if (!uptodate)
+                md_error(mddev, conf->mirrors[d].rdev);
+        update_head_pos(i, r10_bio);
+
+        while (atomic_dec_and_test(&r10_bio->remaining)) {
+                if (r10_bio->master_bio == NULL) {
+                        /* the primary of several recovery bios */
+                        md_done_sync(mddev, r10_bio->sectors, 1);
+                        put_buf(r10_bio);
+                        break;
+                } else {
+                        r10bio_t *r10_bio2 = (r10bio_t *)r10_bio->master_bio;
+                        put_buf(r10_bio);
+                        r10_bio = r10_bio2;
+                }
+        }
+        rdev_dec_pending(conf->mirrors[d].rdev, mddev);
+        return 0;
+}
+
+/*
+ * Note: sync and recover and handled very differently for raid10
+ * This code is for resync.
+ * For resync, we read through virtual addresses and read all blocks.
+ * If there is any error, we schedule a write.  The lowest numbered
+ * drive is authoritative.
+ * However requests come for physical address, so we need to map.
+ * For every physical address there are raid_disks/copies virtual addresses,
+ * which is always are least one, but is not necessarly an integer.
+ * This means that a physical address can span multiple chunks, so we may
+ * have to submit multiple io requests for a single sync request.
+ */
+/*
+ * We check if all blocks are in-sync and only write to blocks that
+ * aren't in sync
+ */
+static void sync_request_write(mddev_t *mddev, r10bio_t *r10_bio)
+{
+        conf_t *conf = mddev_to_conf(mddev);
+        int i, first;
+        struct bio *tbio, *fbio;
+
+        atomic_set(&r10_bio->remaining, 1);
+
+        /* find the first device with a block */
+        for (i=0; i<conf->copies; i++)
+                if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags))
+                        break;
+
+        if (i == conf->copies)
+                goto done;
+
+        first = i;
+        fbio = r10_bio->devs[i].bio;
+
+        /* now find blocks with errors */
+        for (i=first+1 ; i < conf->copies ; i++) {
+                int vcnt, j, d;
+
+                if (!test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags))
+                        continue;
+                /* We know that the bi_io_vec layout is the same for
+                 * both 'first' and 'i', so we just compare them.
+                 * All vec entries are PAGE_SIZE;
+                 */
+                tbio = r10_bio->devs[i].bio;
+                vcnt = r10_bio->sectors >> (PAGE_SHIFT-9);
+                for (j = 0; j < vcnt; j++)
+                        if (memcmp(page_address(fbio->bi_io_vec[j].bv_page),
+                                   page_address(tbio->bi_io_vec[j].bv_page),
+                                   PAGE_SIZE))
+                                break;
+                if (j == vcnt)
+                        continue;
+                /* Ok, we need to write this bio
+                 * First we need to fixup bv_offset, bv_len and
+                 * bi_vecs, as the read request might have corrupted these
+                 */
+                tbio->bi_vcnt = vcnt;
+                tbio->bi_size = r10_bio->sectors << 9;
+                tbio->bi_idx = 0;
+                tbio->bi_phys_segments = 0;
+                tbio->bi_hw_segments = 0;
+                tbio->bi_hw_front_size = 0;
+                tbio->bi_hw_back_size = 0;
+                tbio->bi_flags &= ~(BIO_POOL_MASK - 1);
+                tbio->bi_flags |= 1 << BIO_UPTODATE;
+                tbio->bi_next = NULL;
+                tbio->bi_rw = WRITE;
+                tbio->bi_private = r10_bio;
+                tbio->bi_sector = r10_bio->devs[i].addr;
+
+                for (j=0; j < vcnt ; j++) {
+                        tbio->bi_io_vec[j].bv_offset = 0;
+                        tbio->bi_io_vec[j].bv_len = PAGE_SIZE;
+
+                        memcpy(page_address(tbio->bi_io_vec[j].bv_page),
+                               page_address(fbio->bi_io_vec[j].bv_page),
+                               PAGE_SIZE);
+                }
+                tbio->bi_end_io = end_sync_write;
+
+                d = r10_bio->devs[i].devnum;
+                atomic_inc(&conf->mirrors[d].rdev->nr_pending);
+                atomic_inc(&r10_bio->remaining);
+                md_sync_acct(conf->mirrors[d].rdev->bdev, tbio->bi_size >> 9);
+
+                tbio->bi_sector += conf->mirrors[d].rdev->data_offset;
+                tbio->bi_bdev = conf->mirrors[d].rdev->bdev;
+                generic_make_request(tbio);
+        }
+
+done:
+        if (atomic_dec_and_test(&r10_bio->remaining)) {
+                md_done_sync(mddev, r10_bio->sectors, 1);
+                put_buf(r10_bio);
+        }
+}
+
+/*
+ * Now for the recovery code.
+ * Recovery happens across physical sectors.
+ * We recover all non-is_sync drives by finding the virtual address of
+ * each, and then choose a working drive that also has that virt address.
+ * There is a separate r10_bio for each non-in_sync drive.
+ * Only the first two slots are in use. The first for reading,
+ * The second for writing.
+ *
+ */
+
+static void recovery_request_write(mddev_t *mddev, r10bio_t *r10_bio)
+{
+        conf_t *conf = mddev_to_conf(mddev);
+        int i, d;
+        struct bio *bio, *wbio;
+
+
+        /* move the pages across to the second bio
+         * and submit the write request
+         */
+        bio = r10_bio->devs[0].bio;
+        wbio = r10_bio->devs[1].bio;
+        for (i=0; i < wbio->bi_vcnt; i++) {
+                struct page *p = bio->bi_io_vec[i].bv_page;
+                bio->bi_io_vec[i].bv_page = wbio->bi_io_vec[i].bv_page;
+                wbio->bi_io_vec[i].bv_page = p;
+        }
+        d = r10_bio->devs[1].devnum;
+
+        atomic_inc(&conf->mirrors[d].rdev->nr_pending);
+        md_sync_acct(conf->mirrors[d].rdev->bdev, wbio->bi_size >> 9);
+        generic_make_request(wbio);
+}
+
+
+/*
+ * This is a kernel thread which:
+ *
+ *      1.      Retries failed read operations on working mirrors.
+ *      2.      Updates the raid superblock when problems encounter.
+ *      3.      Performs writes following reads for array syncronising.
+ */
+
+static void raid10d(mddev_t *mddev)
+{
+        r10bio_t *r10_bio;
+        struct bio *bio;
+        unsigned long flags;
+        conf_t *conf = mddev_to_conf(mddev);
+        struct list_head *head = &conf->retry_list;
+        int unplug=0;
+        mdk_rdev_t *rdev;
+
+        md_check_recovery(mddev);
+        md_handle_safemode(mddev);
+
+        for (;;) {
+                char b[BDEVNAME_SIZE];
+                spin_lock_irqsave(&conf->device_lock, flags);
+                if (list_empty(head))
+                        break;
+                r10_bio = list_entry(head->prev, r10bio_t, retry_list);
+                list_del(head->prev);
+                spin_unlock_irqrestore(&conf->device_lock, flags);
+
+                mddev = r10_bio->mddev;
+                conf = mddev_to_conf(mddev);
+                if (test_bit(R10BIO_IsSync, &r10_bio->state)) {
+                        sync_request_write(mddev, r10_bio);
+                        unplug = 1;
+                } else  if (test_bit(R10BIO_IsRecover, &r10_bio->state)) {
+                        recovery_request_write(mddev, r10_bio);
+                        unplug = 1;
+                } else {
+                        int mirror;
+                        bio = r10_bio->devs[r10_bio->read_slot].bio;
+                        r10_bio->devs[r10_bio->read_slot].bio = NULL;
+                        bio_put(bio);
+                        mirror = read_balance(conf, r10_bio);
+                        if (mirror == -1) {
+                                printk(KERN_ALERT "raid10: %s: unrecoverable I/O"
+                                       " read error for block %llu\n",
+                                       bdevname(bio->bi_bdev,b),
+                                       (unsigned long long)r10_bio->sector);
+                                raid_end_bio_io(r10_bio);
+                        } else {
+                                rdev = conf->mirrors[mirror].rdev;
+                                if (printk_ratelimit())
+                                        printk(KERN_ERR "raid10: %s: redirecting sector %llu to"
+                                               " another mirror\n",
+                                               bdevname(rdev->bdev,b),
+                                               (unsigned long long)r10_bio->sector);
+                                bio = bio_clone(r10_bio->master_bio, GFP_NOIO);
+                                r10_bio->devs[r10_bio->read_slot].bio = bio;
+                                bio->bi_sector = r10_bio->devs[r10_bio->read_slot].addr
+                                        + rdev->data_offset;
+                                bio->bi_bdev = rdev->bdev;
+                                bio->bi_rw = READ;
+                                bio->bi_private = r10_bio;
+                                bio->bi_end_io = raid10_end_read_request;
+                                unplug = 1;
+                                generic_make_request(bio);
+                        }
+                }
+        }
+        spin_unlock_irqrestore(&conf->device_lock, flags);
+        if (unplug)
+                unplug_slaves(mddev);
+}
+
+
+static int init_resync(conf_t *conf)
+{
+        int buffs;
+
+        buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
+        if (conf->r10buf_pool)
+                BUG();
+        conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf);
+        if (!conf->r10buf_pool)
+                return -ENOMEM;
+        conf->next_resync = 0;
+        return 0;
+}
+
+/*
+ * perform a "sync" on one "block"
+ *
+ * We need to make sure that no normal I/O request - particularly write
+ * requests - conflict with active sync requests.
+ *
+ * This is achieved by tracking pending requests and a 'barrier' concept
+ * that can be installed to exclude normal IO requests.
+ *
+ * Resync and recovery are handled very differently.
+ * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
+ *
+ * For resync, we iterate over virtual addresses, read all copies,
+ * and update if there are differences.  If only one copy is live,
+ * skip it.
+ * For recovery, we iterate over physical addresses, read a good
+ * value for each non-in_sync drive, and over-write.
+ *
+ * So, for recovery we may have several outstanding complex requests for a
+ * given address, one for each out-of-sync device.  We model this by allocating
+ * a number of r10_bio structures, one for each out-of-sync device.
+ * As we setup these structures, we collect all bio's together into a list
+ * which we then process collectively to add pages, and then process again
+ * to pass to generic_make_request.
+ *
+ * The r10_bio structures are linked using a borrowed master_bio pointer.
+ * This link is counted in ->remaining.  When the r10_bio that points to NULL
+ * has its remaining count decremented to 0, the whole complex operation
+ * is complete.
+ *
+ */
+
+static int sync_request(mddev_t *mddev, sector_t sector_nr, int go_faster)
+{
+        conf_t *conf = mddev_to_conf(mddev);
+        r10bio_t *r10_bio;
+        struct bio *biolist = NULL, *bio;
+        sector_t max_sector, nr_sectors;
+        int disk;
+        int i;
+
+        sector_t sectors_skipped = 0;
+        int chunks_skipped = 0;
+
+        if (!conf->r10buf_pool)
+                if (init_resync(conf))
+                        return -ENOMEM;
+
+ skipped:
+        max_sector = mddev->size << 1;
+        if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
+                max_sector = mddev->resync_max_sectors;
+        if (sector_nr >= max_sector) {
+                close_sync(conf);
+                return sectors_skipped;
+        }
+        if (chunks_skipped >= conf->raid_disks) {
+                /* if there has been nothing to do on any drive,
+                 * then there is nothing to do at all..
+                 */
+                sector_t sec = max_sector - sector_nr;
+                md_done_sync(mddev, sec, 1);
+                return sec + sectors_skipped;
+        }
+
+        /* make sure whole request will fit in a chunk - if chunks
+         * are meaningful
+         */
+        if (conf->near_copies < conf->raid_disks &&
+            max_sector > (sector_nr | conf->chunk_mask))
+                max_sector = (sector_nr | conf->chunk_mask) + 1;
+        /*
+         * If there is non-resync activity waiting for us then
+         * put in a delay to throttle resync.
+         */
+        if (!go_faster && waitqueue_active(&conf->wait_resume))
+                msleep_interruptible(1000);
+        device_barrier(conf, sector_nr + RESYNC_SECTORS);
+
+        /* Again, very different code for resync and recovery.
+         * Both must result in an r10bio with a list of bios that
+         * have bi_end_io, bi_sector, bi_bdev set,
+         * and bi_private set to the r10bio.
+         * For recovery, we may actually create several r10bios
+         * with 2 bios in each, that correspond to the bios in the main one.
+         * In this case, the subordinate r10bios link back through a
+         * borrowed master_bio pointer, and the counter in the master
+         * includes a ref from each subordinate.
+         */
+        /* First, we decide what to do and set ->bi_end_io
+         * To end_sync_read if we want to read, and
+         * end_sync_write if we will want to write.
+         */
+
+        if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
+                /* recovery... the complicated one */
+                int i, j, k;
+                r10_bio = NULL;
+
+                for (i=0 ; i<conf->raid_disks; i++)
+                        if (conf->mirrors[i].rdev &&
+                            !conf->mirrors[i].rdev->in_sync) {
+                                /* want to reconstruct this device */
+                                r10bio_t *rb2 = r10_bio;
+
+                                r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
+                                spin_lock_irq(&conf->resync_lock);
+                                conf->nr_pending++;
+                                if (rb2) conf->barrier++;
+                                spin_unlock_irq(&conf->resync_lock);
+                                atomic_set(&r10_bio->remaining, 0);
+
+                                r10_bio->master_bio = (struct bio*)rb2;
+                                if (rb2)
+                                        atomic_inc(&rb2->remaining);
+                                r10_bio->mddev = mddev;
+                                set_bit(R10BIO_IsRecover, &r10_bio->state);
+                                r10_bio->sector = raid10_find_virt(conf, sector_nr, i);
+                                raid10_find_phys(conf, r10_bio);
+                                for (j=0; j<conf->copies;j++) {
+                                        int d = r10_bio->devs[j].devnum;
+                                        if (conf->mirrors[d].rdev &&
+                                            conf->mirrors[d].rdev->in_sync) {
+                                                /* This is where we read from */
+                                                bio = r10_bio->devs[0].bio;
+                                                bio->bi_next = biolist;
+                                                biolist = bio;
+                                                bio->bi_private = r10_bio;
+                                                bio->bi_end_io = end_sync_read;
+                                                bio->bi_rw = 0;
+                                                bio->bi_sector = r10_bio->devs[j].addr +
+                                                        conf->mirrors[d].rdev->data_offset;
+                                                bio->bi_bdev = conf->mirrors[d].rdev->bdev;
+                                                atomic_inc(&conf->mirrors[d].rdev->nr_pending);
+                                                atomic_inc(&r10_bio->remaining);
+                                                /* and we write to 'i' */
+
+                                                for (k=0; k<conf->copies; k++)
+                                                        if (r10_bio->devs[k].devnum == i)
+                                                                break;
+                                                bio = r10_bio->devs[1].bio;
+                                                bio->bi_next = biolist;
+                                                biolist = bio;
+                                                bio->bi_private = r10_bio;
+                                                bio->bi_end_io = end_sync_write;
+                                                bio->bi_rw = 1;
+                                                bio->bi_sector = r10_bio->devs[k].addr +
+                                                        conf->mirrors[i].rdev->data_offset;
+                                                bio->bi_bdev = conf->mirrors[i].rdev->bdev;
+
+                                                r10_bio->devs[0].devnum = d;
+                                                r10_bio->devs[1].devnum = i;
+
+                                                break;
+                                        }
+                                }
+                                if (j == conf->copies) {
+                                        BUG();
+                                }
+                        }
+                if (biolist == NULL) {
+                        while (r10_bio) {
+                                r10bio_t *rb2 = r10_bio;
+                                r10_bio = (r10bio_t*) rb2->master_bio;
+                                rb2->master_bio = NULL;
+                                put_buf(rb2);
+                        }
+                        goto giveup;
+                }
+        } else {
+                /* resync. Schedule a read for every block at this virt offset */
+                int count = 0;
+                r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
+
+                spin_lock_irq(&conf->resync_lock);
+                conf->nr_pending++;
+                spin_unlock_irq(&conf->resync_lock);
+
+                r10_bio->mddev = mddev;
+                atomic_set(&r10_bio->remaining, 0);
+
+                r10_bio->master_bio = NULL;
+                r10_bio->sector = sector_nr;
+                set_bit(R10BIO_IsSync, &r10_bio->state);
+                raid10_find_phys(conf, r10_bio);
+                r10_bio->sectors = (sector_nr | conf->chunk_mask) - sector_nr +1;
+
+                for (i=0; i<conf->copies; i++) {
+                        int d = r10_bio->devs[i].devnum;
+                        bio = r10_bio->devs[i].bio;
+                        bio->bi_end_io = NULL;
+                        if (conf->mirrors[d].rdev == NULL ||
+                            conf->mirrors[d].rdev->faulty)
+                                continue;
+                        atomic_inc(&conf->mirrors[d].rdev->nr_pending);
+                        atomic_inc(&r10_bio->remaining);
+                        bio->bi_next = biolist;
+                        biolist = bio;
+                        bio->bi_private = r10_bio;
+                        bio->bi_end_io = end_sync_read;
+                        bio->bi_rw = 0;
+                        bio->bi_sector = r10_bio->devs[i].addr +
+                                conf->mirrors[d].rdev->data_offset;
+                        bio->bi_bdev = conf->mirrors[d].rdev->bdev;
+                        count++;
+                }
+
+                if (count < 2) {
+                        for (i=0; i<conf->copies; i++) {
+                                int d = r10_bio->devs[i].devnum;
+                                if (r10_bio->devs[i].bio->bi_end_io)
+                                        rdev_dec_pending(conf->mirrors[d].rdev, mddev);
+                        }
+                        put_buf(r10_bio);
+                        biolist = NULL;
+                        goto giveup;
+                }
+        }
+
+        for (bio = biolist; bio ; bio=bio->bi_next) {
+
+                bio->bi_flags &= ~(BIO_POOL_MASK - 1);
+                if (bio->bi_end_io)
+                        bio->bi_flags |= 1 << BIO_UPTODATE;
+                bio->bi_vcnt = 0;
+                bio->bi_idx = 0;
+                bio->bi_phys_segments = 0;
+                bio->bi_hw_segments = 0;
+                bio->bi_size = 0;
+        }
+
+        nr_sectors = 0;
+        do {
+                struct page *page;
+                int len = PAGE_SIZE;
+                disk = 0;
+                if (sector_nr + (len>>9) > max_sector)
+                        len = (max_sector - sector_nr) << 9;
+                if (len == 0)
+                        break;
+                for (bio= biolist ; bio ; bio=bio->bi_next) {
+                        page = bio->bi_io_vec[bio->bi_vcnt].bv_page;
+                        if (bio_add_page(bio, page, len, 0) == 0) {
+                                /* stop here */
+                                struct bio *bio2;
+                                bio->bi_io_vec[bio->bi_vcnt].bv_page = page;
+                                for (bio2 = biolist; bio2 && bio2 != bio; bio2 = bio2->bi_next) {
+                                        /* remove last page from this bio */
+                                        bio2->bi_vcnt--;
+                                        bio2->bi_size -= len;
+                                        bio2->bi_flags &= ~(1<< BIO_SEG_VALID);
+                                }
+                                goto bio_full;
+                        }
+                        disk = i;
+                }
+                nr_sectors += len>>9;
+                sector_nr += len>>9;
+        } while (biolist->bi_vcnt < RESYNC_PAGES);
+ bio_full:
+        r10_bio->sectors = nr_sectors;
+
+        while (biolist) {
+                bio = biolist;
+                biolist = biolist->bi_next;
+
+                bio->bi_next = NULL;
+                r10_bio = bio->bi_private;
+                r10_bio->sectors = nr_sectors;
+
+                if (bio->bi_end_io == end_sync_read) {
+                        md_sync_acct(bio->bi_bdev, nr_sectors);
+                        generic_make_request(bio);
+                }
+        }
+
+        return sectors_skipped + nr_sectors;
+ giveup:
+        /* There is nowhere to write, so all non-sync
+         * drives must be failed, so try the next chunk...
+         */
+        {
+        int sec = max_sector - sector_nr;
+        sectors_skipped += sec;
+        chunks_skipped ++;
+        sector_nr = max_sector;
+        md_done_sync(mddev, sec, 1);
+        goto skipped;
+        }
+}
+
+static int run(mddev_t *mddev)
+{
+        conf_t *conf;
+        int i, disk_idx;
+        mirror_info_t *disk;
+        mdk_rdev_t *rdev;
+        struct list_head *tmp;
+        int nc, fc;
+        sector_t stride, size;
+
+        if (mddev->level != 10) {
+                printk(KERN_ERR "raid10: %s: raid level not set correctly... (%d)\n",
+                       mdname(mddev), mddev->level);
+                goto out;
+        }
+        nc = mddev->layout & 255;
+        fc = (mddev->layout >> 8) & 255;
+        if ((nc*fc) <2 || (nc*fc) > mddev->raid_disks ||
+            (mddev->layout >> 16)) {
+                printk(KERN_ERR "raid10: %s: unsupported raid10 layout: 0x%8x\n",
+                       mdname(mddev), mddev->layout);
+                goto out;
+        }
+        /*
+         * copy the already verified devices into our private RAID10
+         * bookkeeping area. [whatever we allocate in run(),
+         * should be freed in stop()]
+         */
+        conf = kmalloc(sizeof(conf_t), GFP_KERNEL);
+        mddev->private = conf;
+        if (!conf) {
+                printk(KERN_ERR "raid10: couldn't allocate memory for %s\n",
+                        mdname(mddev));
+                goto out;
+        }
+        memset(conf, 0, sizeof(*conf));
+        conf->mirrors = kmalloc(sizeof(struct mirror_info)*mddev->raid_disks,
+                                 GFP_KERNEL);
+        if (!conf->mirrors) {
+                printk(KERN_ERR "raid10: couldn't allocate memory for %s\n",
+                       mdname(mddev));
+                goto out_free_conf;
+        }
+        memset(conf->mirrors, 0, sizeof(struct mirror_info)*mddev->raid_disks);
+
+        conf->near_copies = nc;
+        conf->far_copies = fc;
+        conf->copies = nc*fc;
+        conf->chunk_mask = (sector_t)(mddev->chunk_size>>9)-1;
+        conf->chunk_shift = ffz(~mddev->chunk_size) - 9;
+        stride = mddev->size >> (conf->chunk_shift-1);
+        sector_div(stride, fc);
+        conf->stride = stride << conf->chunk_shift;
+
+        conf->r10bio_pool = mempool_create(NR_RAID10_BIOS, r10bio_pool_alloc,
+                                                r10bio_pool_free, conf);
+        if (!conf->r10bio_pool) {
+                printk(KERN_ERR "raid10: couldn't allocate memory for %s\n",
+                        mdname(mddev));
+                goto out_free_conf;
+        }
+        mddev->queue->unplug_fn = raid10_unplug;
+
+        mddev->queue->issue_flush_fn = raid10_issue_flush;
+
+        ITERATE_RDEV(mddev, rdev, tmp) {
+                disk_idx = rdev->raid_disk;
+                if (disk_idx >= mddev->raid_disks
+                    || disk_idx < 0)
+                        continue;
+                disk = conf->mirrors + disk_idx;
+
+                disk->rdev = rdev;
+
+                blk_queue_stack_limits(mddev->queue,
+                                       rdev->bdev->bd_disk->queue);
+                /* as we don't honour merge_bvec_fn, we must never risk
+                 * violating it, so limit ->max_sector to one PAGE, as
+                 * a one page request is never in violation.
+                 */
+                if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
+                    mddev->queue->max_sectors > (PAGE_SIZE>>9))
+                        mddev->queue->max_sectors = (PAGE_SIZE>>9);
+
+                disk->head_position = 0;
+                if (!rdev->faulty && rdev->in_sync)
+                        conf->working_disks++;
+        }
+        conf->raid_disks = mddev->raid_disks;
+        conf->mddev = mddev;
+        spin_lock_init(&conf->device_lock);
+        INIT_LIST_HEAD(&conf->retry_list);
+
+        spin_lock_init(&conf->resync_lock);
+        init_waitqueue_head(&conf->wait_idle);
+        init_waitqueue_head(&conf->wait_resume);
+
+        if (!conf->working_disks) {
+                printk(KERN_ERR "raid10: no operational mirrors for %s\n",
+                        mdname(mddev));
+                goto out_free_conf;
+        }
+
+        mddev->degraded = 0;
+        for (i = 0; i < conf->raid_disks; i++) {
+
+                disk = conf->mirrors + i;
+
+                if (!disk->rdev) {
+                        disk->head_position = 0;
+                        mddev->degraded++;
+                }
+        }
+
+
+        mddev->thread = md_register_thread(raid10d, mddev, "%s_raid10");
+        if (!mddev->thread) {
+                printk(KERN_ERR
+                       "raid10: couldn't allocate thread for %s\n",
+                       mdname(mddev));
+                goto out_free_conf;
+        }
+
+        printk(KERN_INFO
+                "raid10: raid set %s active with %d out of %d devices\n",
+                mdname(mddev), mddev->raid_disks - mddev->degraded,
+                mddev->raid_disks);
+        /*
+         * Ok, everything is just fine now
+         */
+        size = conf->stride * conf->raid_disks;
+        sector_div(size, conf->near_copies);
+        mddev->array_size = size/2;
+        mddev->resync_max_sectors = size;
+
+        /* Calculate max read-ahead size.
+         * We need to readahead at least twice a whole stripe....
+         * maybe...
+         */
+        {
+                int stripe = conf->raid_disks * mddev->chunk_size / PAGE_CACHE_SIZE;
+                stripe /= conf->near_copies;
+                if (mddev->queue->backing_dev_info.ra_pages < 2* stripe)
+                        mddev->queue->backing_dev_info.ra_pages = 2* stripe;
+        }
+
+        if (conf->near_copies < mddev->raid_disks)
+                blk_queue_merge_bvec(mddev->queue, raid10_mergeable_bvec);
+        return 0;
+
+out_free_conf:
+        if (conf->r10bio_pool)
+                mempool_destroy(conf->r10bio_pool);
+        if (conf->mirrors)
+                kfree(conf->mirrors);
+        kfree(conf);
+        mddev->private = NULL;
+out:
+        return -EIO;
+}
+
+static int stop(mddev_t *mddev)
+{
+        conf_t *conf = mddev_to_conf(mddev);
+
+        md_unregister_thread(mddev->thread);
+        mddev->thread = NULL;
+        blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
+        if (conf->r10bio_pool)
+                mempool_destroy(conf->r10bio_pool);
+        if (conf->mirrors)
+                kfree(conf->mirrors);
+        kfree(conf);
+        mddev->private = NULL;
+        return 0;
+}
+
+
+static mdk_personality_t raid10_personality =
+{
+        .name           = "raid10",
+        .owner          = THIS_MODULE,
+        .make_request   = make_request,
+        .run            = run,
+        .stop           = stop,
+        .status         = status,
+        .error_handler  = error,
+        .hot_add_disk   = raid10_add_disk,
+        .hot_remove_disk= raid10_remove_disk,
+        .spare_active   = raid10_spare_active,
+        .sync_request   = sync_request,
+};
+
+static int __init raid_init(void)
+{
+        return register_md_personality(RAID10, &raid10_personality);
+}
+
+static void raid_exit(void)
+{
+        unregister_md_personality(RAID10);
+}
+
+module_init(raid_init);
+module_exit(raid_exit);
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("md-personality-9"); /* RAID10 */
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
new file mode 100644
index 000000000000..52c3a81c4aa7
--- /dev/null
+++ b/drivers/md/raid5.c
@@ -0,0 +1,1965 @@
+/*
+ * raid5.c : Multiple Devices driver for Linux
+ *         Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
+ *         Copyright (C) 1999, 2000 Ingo Molnar
+ *
+ * RAID-5 management functions.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * (for example /usr/src/linux/COPYING); if not, write to the Free
+ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+
+#include <linux/config.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/raid/raid5.h>
+#include <linux/highmem.h>
+#include <linux/bitops.h>
+#include <asm/atomic.h>
+
+/*
+ * Stripe cache
+ */
+
+#define NR_STRIPES              256
+#define STRIPE_SIZE             PAGE_SIZE
+#define STRIPE_SHIFT            (PAGE_SHIFT - 9)
+#define STRIPE_SECTORS          (STRIPE_SIZE>>9)
+#define IO_THRESHOLD            1
+#define HASH_PAGES              1
+#define HASH_PAGES_ORDER        0
+#define NR_HASH                 (HASH_PAGES * PAGE_SIZE / sizeof(struct stripe_head *))
+#define HASH_MASK               (NR_HASH - 1)
+
+#define stripe_hash(conf, sect) ((conf)->stripe_hashtbl[((sect) >> STRIPE_SHIFT) & HASH_MASK])
+
+/* bio's attached to a stripe+device for I/O are linked together in bi_sector
+ * order without overlap.  There may be several bio's per stripe+device, and
+ * a bio could span several devices.
+ * When walking this list for a particular stripe+device, we must never proceed
+ * beyond a bio that extends past this device, as the next bio might no longer
+ * be valid.
+ * This macro is used to determine the 'next' bio in the list, given the sector
+ * of the current stripe+device
+ */
+#define r5_next_bio(bio, sect) ( ( (bio)->bi_sector + ((bio)->bi_size>>9) < sect + STRIPE_SECTORS) ? (bio)->bi_next : NULL)
+/*
+ * The following can be used to debug the driver
+ */
+#define RAID5_DEBUG     0
+#define RAID5_PARANOIA  1
+#if RAID5_PARANOIA && defined(CONFIG_SMP)
+# define CHECK_DEVLOCK() assert_spin_locked(&conf->device_lock)
+#else
+# define CHECK_DEVLOCK()
+#endif
+
+#define PRINTK(x...) ((void)(RAID5_DEBUG && printk(x)))
+#if RAID5_DEBUG
+#define inline
+#define __inline__
+#endif
+
+static void print_raid5_conf (raid5_conf_t *conf);
+
+static inline void __release_stripe(raid5_conf_t *conf, struct stripe_head *sh)
+{
+        if (atomic_dec_and_test(&sh->count)) {
+                if (!list_empty(&sh->lru))
+                        BUG();
+                if (atomic_read(&conf->active_stripes)==0)
+                        BUG();
+                if (test_bit(STRIPE_HANDLE, &sh->state)) {
+                        if (test_bit(STRIPE_DELAYED, &sh->state))
+                                list_add_tail(&sh->lru, &conf->delayed_list);
+                        else
+                                list_add_tail(&sh->lru, &conf->handle_list);
+                        md_wakeup_thread(conf->mddev->thread);
+                } else {
+                        if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
+                                atomic_dec(&conf->preread_active_stripes);
+                                if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD)
+                                        md_wakeup_thread(conf->mddev->thread);
+                        }
+                        list_add_tail(&sh->lru, &conf->inactive_list);
+                        atomic_dec(&conf->active_stripes);
+                        if (!conf->inactive_blocked ||
+                            atomic_read(&conf->active_stripes) < (NR_STRIPES*3/4))
+                                wake_up(&conf->wait_for_stripe);
+                }
+        }
+}
+static void release_stripe(struct stripe_head *sh)
+{
+        raid5_conf_t *conf = sh->raid_conf;
+        unsigned long flags;
+        
+        spin_lock_irqsave(&conf->device_lock, flags);
+        __release_stripe(conf, sh);
+        spin_unlock_irqrestore(&conf->device_lock, flags);
+}
+
+static void remove_hash(struct stripe_head *sh)
+{
+        PRINTK("remove_hash(), stripe %llu\n", (unsigned long long)sh->sector);
+
+        if (sh->hash_pprev) {
+                if (sh->hash_next)
+                        sh->hash_next->hash_pprev = sh->hash_pprev;
+                *sh->hash_pprev = sh->hash_next;
+                sh->hash_pprev = NULL;
+        }
+}
+
+static __inline__ void insert_hash(raid5_conf_t *conf, struct stripe_head *sh)
+{
+        struct stripe_head **shp = &stripe_hash(conf, sh->sector);
+
+        PRINTK("insert_hash(), stripe %llu\n", (unsigned long long)sh->sector);
+
+        CHECK_DEVLOCK();
+        if ((sh->hash_next = *shp) != NULL)
+                (*shp)->hash_pprev = &sh->hash_next;
+        *shp = sh;
+        sh->hash_pprev = shp;
+}
+
+
+/* find an idle stripe, make sure it is unhashed, and return it. */
+static struct stripe_head *get_free_stripe(raid5_conf_t *conf)
+{
+        struct stripe_head *sh = NULL;
+        struct list_head *first;
+
+        CHECK_DEVLOCK();
+        if (list_empty(&conf->inactive_list))
+                goto out;
+        first = conf->inactive_list.next;
+        sh = list_entry(first, struct stripe_head, lru);
+        list_del_init(first);
+        remove_hash(sh);
+        atomic_inc(&conf->active_stripes);
+out:
+        return sh;
+}
+
+static void shrink_buffers(struct stripe_head *sh, int num)
+{
+        struct page *p;
+        int i;
+
+        for (i=0; i<num ; i++) {
+                p = sh->dev[i].page;
+                if (!p)
+                        continue;
+                sh->dev[i].page = NULL;
+                page_cache_release(p);
+        }
+}
+
+static int grow_buffers(struct stripe_head *sh, int num)
+{
+        int i;
+
+        for (i=0; i<num; i++) {
+                struct page *page;
+
+                if (!(page = alloc_page(GFP_KERNEL))) {
+                        return 1;
+                }
+                sh->dev[i].page = page;
+        }
+        return 0;
+}
+
+static void raid5_build_block (struct stripe_head *sh, int i);
+
+static inline void init_stripe(struct stripe_head *sh, sector_t sector, int pd_idx)
+{
+        raid5_conf_t *conf = sh->raid_conf;
+        int disks = conf->raid_disks, i;
+
+        if (atomic_read(&sh->count) != 0)
+                BUG();
+        if (test_bit(STRIPE_HANDLE, &sh->state))
+                BUG();
+        
+        CHECK_DEVLOCK();
+        PRINTK("init_stripe called, stripe %llu\n", 
+                (unsigned long long)sh->sector);
+
+        remove_hash(sh);
+        
+        sh->sector = sector;
+        sh->pd_idx = pd_idx;
+        sh->state = 0;
+
+        for (i=disks; i--; ) {
+                struct r5dev *dev = &sh->dev[i];
+
+                if (dev->toread || dev->towrite || dev->written ||
+                    test_bit(R5_LOCKED, &dev->flags)) {
+                        printk("sector=%llx i=%d %p %p %p %d\n",
+                               (unsigned long long)sh->sector, i, dev->toread,
+                               dev->towrite, dev->written,
+                               test_bit(R5_LOCKED, &dev->flags));
+                        BUG();
+                }
+                dev->flags = 0;
+                raid5_build_block(sh, i);
+        }
+        insert_hash(conf, sh);
+}
+
+static struct stripe_head *__find_stripe(raid5_conf_t *conf, sector_t sector)
+{
+        struct stripe_head *sh;
+
+        CHECK_DEVLOCK();
+        PRINTK("__find_stripe, sector %llu\n", (unsigned long long)sector);
+        for (sh = stripe_hash(conf, sector); sh; sh = sh->hash_next)
+                if (sh->sector == sector)
+                        return sh;
+        PRINTK("__stripe %llu not in cache\n", (unsigned long long)sector);
+        return NULL;
+}
+
+static void unplug_slaves(mddev_t *mddev);
+static void raid5_unplug_device(request_queue_t *q);
+
+static struct stripe_head *get_active_stripe(raid5_conf_t *conf, sector_t sector,
+                                             int pd_idx, int noblock) 
+{
+        struct stripe_head *sh;
+
+        PRINTK("get_stripe, sector %llu\n", (unsigned long long)sector);
+
+        spin_lock_irq(&conf->device_lock);
+
+        do {
+                sh = __find_stripe(conf, sector);
+                if (!sh) {
+                        if (!conf->inactive_blocked)
+                                sh = get_free_stripe(conf);
+                        if (noblock && sh == NULL)
+                                break;
+                        if (!sh) {
+                                conf->inactive_blocked = 1;
+                                wait_event_lock_irq(conf->wait_for_stripe,
+                                                    !list_empty(&conf->inactive_list) &&
+                                                    (atomic_read(&conf->active_stripes) < (NR_STRIPES *3/4)
+                                                     || !conf->inactive_blocked),
+                                                    conf->device_lock,
+                                                    unplug_slaves(conf->mddev);
+                                        );
+                                conf->inactive_blocked = 0;
+                        } else
+                                init_stripe(sh, sector, pd_idx);
+                } else {
+                        if (atomic_read(&sh->count)) {
+                                if (!list_empty(&sh->lru))
+                                        BUG();
+                        } else {
+                                if (!test_bit(STRIPE_HANDLE, &sh->state))
+                                        atomic_inc(&conf->active_stripes);
+                                if (list_empty(&sh->lru))
+                                        BUG();
+                                list_del_init(&sh->lru);
+                        }
+                }
+        } while (sh == NULL);
+
+        if (sh)
+                atomic_inc(&sh->count);
+
+        spin_unlock_irq(&conf->device_lock);
+        return sh;
+}
+
+static int grow_stripes(raid5_conf_t *conf, int num)
+{
+        struct stripe_head *sh;
+        kmem_cache_t *sc;
+        int devs = conf->raid_disks;
+
+        sprintf(conf->cache_name, "raid5/%s", mdname(conf->mddev));
+
+        sc = kmem_cache_create(conf->cache_name, 
+                               sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
+                               0, 0, NULL, NULL);
+        if (!sc)
+                return 1;
+        conf->slab_cache = sc;
+        while (num--) {
+                sh = kmem_cache_alloc(sc, GFP_KERNEL);
+                if (!sh)
+                        return 1;
+                memset(sh, 0, sizeof(*sh) + (devs-1)*sizeof(struct r5dev));
+                sh->raid_conf = conf;
+                spin_lock_init(&sh->lock);
+
+                if (grow_buffers(sh, conf->raid_disks)) {
+                        shrink_buffers(sh, conf->raid_disks);
+                        kmem_cache_free(sc, sh);
+                        return 1;
+                }
+                /* we just created an active stripe so... */
+                atomic_set(&sh->count, 1);
+                atomic_inc(&conf->active_stripes);
+                INIT_LIST_HEAD(&sh->lru);
+                release_stripe(sh);
+        }
+        return 0;
+}
+
+static void shrink_stripes(raid5_conf_t *conf)
+{
+        struct stripe_head *sh;
+
+        while (1) {
+                spin_lock_irq(&conf->device_lock);
+                sh = get_free_stripe(conf);
+                spin_unlock_irq(&conf->device_lock);
+                if (!sh)
+                        break;
+                if (atomic_read(&sh->count))
+                        BUG();
+                shrink_buffers(sh, conf->raid_disks);
+                kmem_cache_free(conf->slab_cache, sh);
+                atomic_dec(&conf->active_stripes);
+        }
+        kmem_cache_destroy(conf->slab_cache);
+        conf->slab_cache = NULL;
+}
+
+static int raid5_end_read_request (struct bio * bi, unsigned int bytes_done,
+                                   int error)
+{
+        struct stripe_head *sh = bi->bi_private;
+        raid5_conf_t *conf = sh->raid_conf;
+        int disks = conf->raid_disks, i;
+        int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
+
+        if (bi->bi_size)
+                return 1;
+
+        for (i=0 ; i<disks; i++)
+                if (bi == &sh->dev[i].req)
+                        break;
+
+        PRINTK("end_read_request %llu/%d, count: %d, uptodate %d.\n", 
+                (unsigned long long)sh->sector, i, atomic_read(&sh->count), 
+                uptodate);
+        if (i == disks) {
+                BUG();
+                return 0;
+        }
+
+        if (uptodate) {
+#if 0
+                struct bio *bio;
+                unsigned long flags;
+                spin_lock_irqsave(&conf->device_lock, flags);
+                /* we can return a buffer if we bypassed the cache or
+                 * if the top buffer is not in highmem.  If there are
+                 * multiple buffers, leave the extra work to
+                 * handle_stripe
+                 */
+                buffer = sh->bh_read[i];
+                if (buffer &&
+                    (!PageHighMem(buffer->b_page)
+                     || buffer->b_page == bh->b_page )
+                        ) {
+                        sh->bh_read[i] = buffer->b_reqnext;
+                        buffer->b_reqnext = NULL;
+                } else
+                        buffer = NULL;
+                spin_unlock_irqrestore(&conf->device_lock, flags);
+                if (sh->bh_page[i]==bh->b_page)
+                        set_buffer_uptodate(bh);
+                if (buffer) {
+                        if (buffer->b_page != bh->b_page)
+                                memcpy(buffer->b_data, bh->b_data, bh->b_size);
+                        buffer->b_end_io(buffer, 1);
+                }
+#else
+                set_bit(R5_UPTODATE, &sh->dev[i].flags);
+#endif          
+        } else {
+                md_error(conf->mddev, conf->disks[i].rdev);
+                clear_bit(R5_UPTODATE, &sh->dev[i].flags);
+        }
+        rdev_dec_pending(conf->disks[i].rdev, conf->mddev);
+#if 0
+        /* must restore b_page before unlocking buffer... */
+        if (sh->bh_page[i] != bh->b_page) {
+                bh->b_page = sh->bh_page[i];
+                bh->b_data = page_address(bh->b_page);
+                clear_buffer_uptodate(bh);
+        }
+#endif
+        clear_bit(R5_LOCKED, &sh->dev[i].flags);
+        set_bit(STRIPE_HANDLE, &sh->state);
+        release_stripe(sh);
+        return 0;
+}
+
+static int raid5_end_write_request (struct bio *bi, unsigned int bytes_done,
+                                    int error)
+{
+        struct stripe_head *sh = bi->bi_private;
+        raid5_conf_t *conf = sh->raid_conf;
+        int disks = conf->raid_disks, i;
+        unsigned long flags;
+        int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
+
+        if (bi->bi_size)
+                return 1;
+
+        for (i=0 ; i<disks; i++)
+                if (bi == &sh->dev[i].req)
+                        break;
+
+        PRINTK("end_write_request %llu/%d, count %d, uptodate: %d.\n", 
+                (unsigned long long)sh->sector, i, atomic_read(&sh->count),
+                uptodate);
+        if (i == disks) {
+                BUG();
+                return 0;
+        }
+
+        spin_lock_irqsave(&conf->device_lock, flags);
+        if (!uptodate)
+                md_error(conf->mddev, conf->disks[i].rdev);
+
+        rdev_dec_pending(conf->disks[i].rdev, conf->mddev);
+        
+        clear_bit(R5_LOCKED, &sh->dev[i].flags);
+        set_bit(STRIPE_HANDLE, &sh->state);
+        __release_stripe(conf, sh);
+        spin_unlock_irqrestore(&conf->device_lock, flags);
+        return 0;
+}
+
+
+static sector_t compute_blocknr(struct stripe_head *sh, int i);
+        
+static void raid5_build_block (struct stripe_head *sh, int i)
+{
+        struct r5dev *dev = &sh->dev[i];
+
+        bio_init(&dev->req);
+        dev->req.bi_io_vec = &dev->vec;
+        dev->req.bi_vcnt++;
+        dev->req.bi_max_vecs++;
+        dev->vec.bv_page = dev->page;
+        dev->vec.bv_len = STRIPE_SIZE;
+        dev->vec.bv_offset = 0;
+
+        dev->req.bi_sector = sh->sector;
+        dev->req.bi_private = sh;
+
+        dev->flags = 0;
+        if (i != sh->pd_idx)
+                dev->sector = compute_blocknr(sh, i);
+}
+
+static void error(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+        char b[BDEVNAME_SIZE];
+        raid5_conf_t *conf = (raid5_conf_t *) mddev->private;
+        PRINTK("raid5: error called\n");
+
+        if (!rdev->faulty) {
+                mddev->sb_dirty = 1;
+                if (rdev->in_sync) {
+                        conf->working_disks--;
+                        mddev->degraded++;
+                        conf->failed_disks++;
+                        rdev->in_sync = 0;
+                        /*
+                         * if recovery was running, make sure it aborts.
+                         */
+                        set_bit(MD_RECOVERY_ERR, &mddev->recovery);
+                }
+                rdev->faulty = 1;
+                printk (KERN_ALERT
+                        "raid5: Disk failure on %s, disabling device."
+                        " Operation continuing on %d devices\n",
+                        bdevname(rdev->bdev,b), conf->working_disks);
+        }
+}       
+
+/*
+ * Input: a 'big' sector number,
+ * Output: index of the data and parity disk, and the sector # in them.
+ */
+static sector_t raid5_compute_sector(sector_t r_sector, unsigned int raid_disks,
+                        unsigned int data_disks, unsigned int * dd_idx,
+                        unsigned int * pd_idx, raid5_conf_t *conf)
+{
+        long stripe;
+        unsigned long chunk_number;
+        unsigned int chunk_offset;
+        sector_t new_sector;
+        int sectors_per_chunk = conf->chunk_size >> 9;
+
+        /* First compute the information on this sector */
+
+        /*
+         * Compute the chunk number and the sector offset inside the chunk
+         */
+        chunk_offset = sector_div(r_sector, sectors_per_chunk);
+        chunk_number = r_sector;
+        BUG_ON(r_sector != chunk_number);
+
+        /*
+         * Compute the stripe number
+         */
+        stripe = chunk_number / data_disks;
+
+        /*
+         * Compute the data disk and parity disk indexes inside the stripe
+         */
+        *dd_idx = chunk_number % data_disks;
+
+        /*
+         * Select the parity disk based on the user selected algorithm.
+         */
+        if (conf->level == 4)
+                *pd_idx = data_disks;
+        else switch (conf->algorithm) {
+                case ALGORITHM_LEFT_ASYMMETRIC:
+                        *pd_idx = data_disks - stripe % raid_disks;
+                        if (*dd_idx >= *pd_idx)
+                                (*dd_idx)++;
+                        break;
+                case ALGORITHM_RIGHT_ASYMMETRIC:
+                        *pd_idx = stripe % raid_disks;
+                        if (*dd_idx >= *pd_idx)
+                                (*dd_idx)++;
+                        break;
+                case ALGORITHM_LEFT_SYMMETRIC:
+                        *pd_idx = data_disks - stripe % raid_disks;
+                        *dd_idx = (*pd_idx + 1 + *dd_idx) % raid_disks;
+                        break;
+                case ALGORITHM_RIGHT_SYMMETRIC:
+                        *pd_idx = stripe % raid_disks;
+                        *dd_idx = (*pd_idx + 1 + *dd_idx) % raid_disks;
+                        break;
+                default:
+                        printk("raid5: unsupported algorithm %d\n",
+                                conf->algorithm);
+        }
+
+        /*
+         * Finally, compute the new sector number
+         */
+        new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
+        return new_sector;
+}
+
+
+static sector_t compute_blocknr(struct stripe_head *sh, int i)
+{
+        raid5_conf_t *conf = sh->raid_conf;
+        int raid_disks = conf->raid_disks, data_disks = raid_disks - 1;
+        sector_t new_sector = sh->sector, check;
+        int sectors_per_chunk = conf->chunk_size >> 9;
+        sector_t stripe;
+        int chunk_offset;
+        int chunk_number, dummy1, dummy2, dd_idx = i;
+        sector_t r_sector;
+
+        chunk_offset = sector_div(new_sector, sectors_per_chunk);
+        stripe = new_sector;
+        BUG_ON(new_sector != stripe);
+
+        
+        switch (conf->algorithm) {
+                case ALGORITHM_LEFT_ASYMMETRIC:
+                case ALGORITHM_RIGHT_ASYMMETRIC:
+                        if (i > sh->pd_idx)
+                                i--;
+                        break;
+                case ALGORITHM_LEFT_SYMMETRIC:
+                case ALGORITHM_RIGHT_SYMMETRIC:
+                        if (i < sh->pd_idx)
+                                i += raid_disks;
+                        i -= (sh->pd_idx + 1);
+                        break;
+                default:
+                        printk("raid5: unsupported algorithm %d\n",
+                                conf->algorithm);
+        }
+
+        chunk_number = stripe * data_disks + i;
+        r_sector = (sector_t)chunk_number * sectors_per_chunk + chunk_offset;
+
+        check = raid5_compute_sector (r_sector, raid_disks, data_disks, &dummy1, &dummy2, conf);
+        if (check != sh->sector || dummy1 != dd_idx || dummy2 != sh->pd_idx) {
+                printk("compute_blocknr: map not correct\n");
+                return 0;
+        }
+        return r_sector;
+}
+
+
+
+/*
+ * Copy data between a page in the stripe cache, and a bio.
+ * There are no alignment or size guarantees between the page or the
+ * bio except that there is some overlap.
+ * All iovecs in the bio must be considered.
+ */
+static void copy_data(int frombio, struct bio *bio,
+                     struct page *page,
+                     sector_t sector)
+{
+        char *pa = page_address(page);
+        struct bio_vec *bvl;
+        int i;
+        int page_offset;
+
+        if (bio->bi_sector >= sector)
+                page_offset = (signed)(bio->bi_sector - sector) * 512;
+        else
+                page_offset = (signed)(sector - bio->bi_sector) * -512;
+        bio_for_each_segment(bvl, bio, i) {
+                int len = bio_iovec_idx(bio,i)->bv_len;
+                int clen;
+                int b_offset = 0;
+
+                if (page_offset < 0) {
+                        b_offset = -page_offset;
+                        page_offset += b_offset;
+                        len -= b_offset;
+                }
+
+                if (len > 0 && page_offset + len > STRIPE_SIZE)
+                        clen = STRIPE_SIZE - page_offset;
+                else clen = len;
+                        
+                if (clen > 0) {
+                        char *ba = __bio_kmap_atomic(bio, i, KM_USER0);
+                        if (frombio)
+                                memcpy(pa+page_offset, ba+b_offset, clen);
+                        else
+                                memcpy(ba+b_offset, pa+page_offset, clen);
+                        __bio_kunmap_atomic(ba, KM_USER0);
+                }
+                if (clen < len) /* hit end of page */
+                        break;
+                page_offset +=  len;
+        }
+}
+
+#define check_xor()     do {                                            \
+                           if (count == MAX_XOR_BLOCKS) {               \
+                                xor_block(count, STRIPE_SIZE, ptr);     \
+                                count = 1;                              \
+                           }                                            \
+                        } while(0)
+
+
+static void compute_block(struct stripe_head *sh, int dd_idx)
+{
+        raid5_conf_t *conf = sh->raid_conf;
+        int i, count, disks = conf->raid_disks;
+        void *ptr[MAX_XOR_BLOCKS], *p;
+
+        PRINTK("compute_block, stripe %llu, idx %d\n", 
+                (unsigned long long)sh->sector, dd_idx);
+
+        ptr[0] = page_address(sh->dev[dd_idx].page);
+        memset(ptr[0], 0, STRIPE_SIZE);
+        count = 1;
+        for (i = disks ; i--; ) {
+                if (i == dd_idx)
+                        continue;
+                p = page_address(sh->dev[i].page);
+                if (test_bit(R5_UPTODATE, &sh->dev[i].flags))
+                        ptr[count++] = p;
+                else
+                        printk("compute_block() %d, stripe %llu, %d"
+                                " not present\n", dd_idx,
+                                (unsigned long long)sh->sector, i);
+
+                check_xor();
+        }
+        if (count != 1)
+                xor_block(count, STRIPE_SIZE, ptr);
+        set_bit(R5_UPTODATE, &sh->dev[dd_idx].flags);
+}
+
+static void compute_parity(struct stripe_head *sh, int method)
+{
+        raid5_conf_t *conf = sh->raid_conf;
+        int i, pd_idx = sh->pd_idx, disks = conf->raid_disks, count;
+        void *ptr[MAX_XOR_BLOCKS];
+        struct bio *chosen;
+
+        PRINTK("compute_parity, stripe %llu, method %d\n",
+                (unsigned long long)sh->sector, method);
+
+        count = 1;
+        ptr[0] = page_address(sh->dev[pd_idx].page);
+        switch(method) {
+        case READ_MODIFY_WRITE:
+                if (!test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags))
+                        BUG();
+                for (i=disks ; i-- ;) {
+                        if (i==pd_idx)
+                                continue;
+                        if (sh->dev[i].towrite &&
+                            test_bit(R5_UPTODATE, &sh->dev[i].flags)) {
+                                ptr[count++] = page_address(sh->dev[i].page);
+                                chosen = sh->dev[i].towrite;
+                                sh->dev[i].towrite = NULL;
+
+                                if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
+                                        wake_up(&conf->wait_for_overlap);
+
+                                if (sh->dev[i].written) BUG();
+                                sh->dev[i].written = chosen;
+                                check_xor();
+                        }
+                }
+                break;
+        case RECONSTRUCT_WRITE:
+                memset(ptr[0], 0, STRIPE_SIZE);
+                for (i= disks; i-- ;)
+                        if (i!=pd_idx && sh->dev[i].towrite) {
+                                chosen = sh->dev[i].towrite;
+                                sh->dev[i].towrite = NULL;
+
+                                if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
+                                        wake_up(&conf->wait_for_overlap);
+
+                                if (sh->dev[i].written) BUG();
+                                sh->dev[i].written = chosen;
+                        }
+                break;
+        case CHECK_PARITY:
+                break;
+        }
+        if (count>1) {
+                xor_block(count, STRIPE_SIZE, ptr);
+                count = 1;
+        }
+        
+        for (i = disks; i--;)
+                if (sh->dev[i].written) {
+                        sector_t sector = sh->dev[i].sector;
+                        struct bio *wbi = sh->dev[i].written;
+                        while (wbi && wbi->bi_sector < sector + STRIPE_SECTORS) {
+                                copy_data(1, wbi, sh->dev[i].page, sector);
+                                wbi = r5_next_bio(wbi, sector);
+                        }
+
+                        set_bit(R5_LOCKED, &sh->dev[i].flags);
+                        set_bit(R5_UPTODATE, &sh->dev[i].flags);
+                }
+
+        switch(method) {
+        case RECONSTRUCT_WRITE:
+        case CHECK_PARITY:
+                for (i=disks; i--;)
+                        if (i != pd_idx) {
+                                ptr[count++] = page_address(sh->dev[i].page);
+                                check_xor();
+                        }
+                break;
+        case READ_MODIFY_WRITE:
+                for (i = disks; i--;)
+                        if (sh->dev[i].written) {
+                                ptr[count++] = page_address(sh->dev[i].page);
+                                check_xor();
+                        }
+        }
+        if (count != 1)
+                xor_block(count, STRIPE_SIZE, ptr);
+        
+        if (method != CHECK_PARITY) {
+                set_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
+                set_bit(R5_LOCKED,   &sh->dev[pd_idx].flags);
+        } else
+                clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
+}
+
+/*
+ * Each stripe/dev can have one or more bion attached.
+ * toread/towrite point to the first in a chain. 
+ * The bi_next chain must be in order.
+ */
+static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite)
+{
+        struct bio **bip;
+        raid5_conf_t *conf = sh->raid_conf;
+
+        PRINTK("adding bh b#%llu to stripe s#%llu\n",
+                (unsigned long long)bi->bi_sector,
+                (unsigned long long)sh->sector);
+
+
+        spin_lock(&sh->lock);
+        spin_lock_irq(&conf->device_lock);
+        if (forwrite)
+                bip = &sh->dev[dd_idx].towrite;
+        else
+                bip = &sh->dev[dd_idx].toread;
+        while (*bip && (*bip)->bi_sector < bi->bi_sector) {
+                if ((*bip)->bi_sector + ((*bip)->bi_size >> 9) > bi->bi_sector)
+                        goto overlap;
+                bip = & (*bip)->bi_next;
+        }
+        if (*bip && (*bip)->bi_sector < bi->bi_sector + ((bi->bi_size)>>9))
+                goto overlap;
+
+        if (*bip && bi->bi_next && (*bip) != bi->bi_next)
+                BUG();
+        if (*bip)
+                bi->bi_next = *bip;
+        *bip = bi;
+        bi->bi_phys_segments ++;
+        spin_unlock_irq(&conf->device_lock);
+        spin_unlock(&sh->lock);
+
+        PRINTK("added bi b#%llu to stripe s#%llu, disk %d.\n",
+                (unsigned long long)bi->bi_sector,
+                (unsigned long long)sh->sector, dd_idx);
+
+        if (forwrite) {
+                /* check if page is covered */
+                sector_t sector = sh->dev[dd_idx].sector;
+                for (bi=sh->dev[dd_idx].towrite;
+                     sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
+                             bi && bi->bi_sector <= sector;
+                     bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
+                        if (bi->bi_sector + (bi->bi_size>>9) >= sector)
+                                sector = bi->bi_sector + (bi->bi_size>>9);
+                }
+                if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
+                        set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags);
+        }
+        return 1;
+
+ overlap:
+        set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
+        spin_unlock_irq(&conf->device_lock);
+        spin_unlock(&sh->lock);
+        return 0;
+}
+
+
+/*
+ * handle_stripe - do things to a stripe.
+ *
+ * We lock the stripe and then examine the state of various bits
+ * to see what needs to be done.
+ * Possible results:
+ *    return some read request which now have data
+ *    return some write requests which are safely on disc
+ *    schedule a read on some buffers
+ *    schedule a write of some buffers
+ *    return confirmation of parity correctness
+ *
+ * Parity calculations are done inside the stripe lock
+ * buffers are taken off read_list or write_list, and bh_cache buffers
+ * get BH_Lock set before the stripe lock is released.
+ *
+ */
+ 
+static void handle_stripe(struct stripe_head *sh)
+{
+        raid5_conf_t *conf = sh->raid_conf;
+        int disks = conf->raid_disks;
+        struct bio *return_bi= NULL;
+        struct bio *bi;
+        int i;
+        int syncing;
+        int locked=0, uptodate=0, to_read=0, to_write=0, failed=0, written=0;
+        int non_overwrite = 0;
+        int failed_num=0;
+        struct r5dev *dev;
+
+        PRINTK("handling stripe %llu, cnt=%d, pd_idx=%d\n",
+                (unsigned long long)sh->sector, atomic_read(&sh->count),
+                sh->pd_idx);
+
+        spin_lock(&sh->lock);
+        clear_bit(STRIPE_HANDLE, &sh->state);
+        clear_bit(STRIPE_DELAYED, &sh->state);
+
+        syncing = test_bit(STRIPE_SYNCING, &sh->state);
+        /* Now to look around and see what can be done */
+
+        for (i=disks; i--; ) {
+                mdk_rdev_t *rdev;
+                dev = &sh->dev[i];
+                clear_bit(R5_Insync, &dev->flags);
+                clear_bit(R5_Syncio, &dev->flags);
+
+                PRINTK("check %d: state 0x%lx read %p write %p written %p\n",
+                        i, dev->flags, dev->toread, dev->towrite, dev->written);
+                /* maybe we can reply to a read */
+                if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread) {
+                        struct bio *rbi, *rbi2;
+                        PRINTK("Return read for disc %d\n", i);
+                        spin_lock_irq(&conf->device_lock);
+                        rbi = dev->toread;
+                        dev->toread = NULL;
+                        if (test_and_clear_bit(R5_Overlap, &dev->flags))
+                                wake_up(&conf->wait_for_overlap);
+                        spin_unlock_irq(&conf->device_lock);
+                        while (rbi && rbi->bi_sector < dev->sector + STRIPE_SECTORS) {
+                                copy_data(0, rbi, dev->page, dev->sector);
+                                rbi2 = r5_next_bio(rbi, dev->sector);
+                                spin_lock_irq(&conf->device_lock);
+                                if (--rbi->bi_phys_segments == 0) {
+                                        rbi->bi_next = return_bi;
+                                        return_bi = rbi;
+                                }
+                                spin_unlock_irq(&conf->device_lock);
+                                rbi = rbi2;
+                        }
+                }
+
+                /* now count some things */
+                if (test_bit(R5_LOCKED, &dev->flags)) locked++;
+                if (test_bit(R5_UPTODATE, &dev->flags)) uptodate++;
+
+                
+                if (dev->toread) to_read++;
+                if (dev->towrite) {
+                        to_write++;
+                        if (!test_bit(R5_OVERWRITE, &dev->flags))
+                                non_overwrite++;
+                }
+                if (dev->written) written++;
+                rdev = conf->disks[i].rdev; /* FIXME, should I be looking rdev */
+                if (!rdev || !rdev->in_sync) {
+                        failed++;
+                        failed_num = i;
+                } else
+                        set_bit(R5_Insync, &dev->flags);
+        }
+        PRINTK("locked=%d uptodate=%d to_read=%d"
+                " to_write=%d failed=%d failed_num=%d\n",
+                locked, uptodate, to_read, to_write, failed, failed_num);
+        /* check if the array has lost two devices and, if so, some requests might
+         * need to be failed
+         */
+        if (failed > 1 && to_read+to_write+written) {
+                spin_lock_irq(&conf->device_lock);
+                for (i=disks; i--; ) {
+                        /* fail all writes first */
+                        bi = sh->dev[i].towrite;
+                        sh->dev[i].towrite = NULL;
+                        if (bi) to_write--;
+
+                        if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
+                                wake_up(&conf->wait_for_overlap);
+
+                        while (bi && bi->bi_sector < sh->dev[i].sector + STRIPE_SECTORS){
+                                struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
+                                clear_bit(BIO_UPTODATE, &bi->bi_flags);
+                                if (--bi->bi_phys_segments == 0) {
+                                        md_write_end(conf->mddev);
+                                        bi->bi_next = return_bi;
+                                        return_bi = bi;
+                                }
+                                bi = nextbi;
+                        }
+                        /* and fail all 'written' */
+                        bi = sh->dev[i].written;
+                        sh->dev[i].written = NULL;
+                        while (bi && bi->bi_sector < sh->dev[i].sector + STRIPE_SECTORS) {
+                                struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
+                                clear_bit(BIO_UPTODATE, &bi->bi_flags);
+                                if (--bi->bi_phys_segments == 0) {
+                                        md_write_end(conf->mddev);
+                                        bi->bi_next = return_bi;
+                                        return_bi = bi;
+                                }
+                                bi = bi2;
+                        }
+
+                        /* fail any reads if this device is non-operational */
+                        if (!test_bit(R5_Insync, &sh->dev[i].flags)) {
+                                bi = sh->dev[i].toread;
+                                sh->dev[i].toread = NULL;
+                                if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
+                                        wake_up(&conf->wait_for_overlap);
+                                if (bi) to_read--;
+                                while (bi && bi->bi_sector < sh->dev[i].sector + STRIPE_SECTORS){
+                                        struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
+                                        clear_bit(BIO_UPTODATE, &bi->bi_flags);
+                                        if (--bi->bi_phys_segments == 0) {
+                                                bi->bi_next = return_bi;
+                                                return_bi = bi;
+                                        }
+                                        bi = nextbi;
+                                }
+                        }
+                }
+                spin_unlock_irq(&conf->device_lock);
+        }
+        if (failed > 1 && syncing) {
+                md_done_sync(conf->mddev, STRIPE_SECTORS,0);
+                clear_bit(STRIPE_SYNCING, &sh->state);
+                syncing = 0;
+        }
+
+        /* might be able to return some write requests if the parity block
+         * is safe, or on a failed drive
+         */
+        dev = &sh->dev[sh->pd_idx];
+        if ( written &&
+             ( (test_bit(R5_Insync, &dev->flags) && !test_bit(R5_LOCKED, &dev->flags) &&
+                test_bit(R5_UPTODATE, &dev->flags))
+               || (failed == 1 && failed_num == sh->pd_idx))
+            ) {
+            /* any written block on an uptodate or failed drive can be returned.
+             * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but 
+             * never LOCKED, so we don't need to test 'failed' directly.
+             */
+            for (i=disks; i--; )
+                if (sh->dev[i].written) {
+                    dev = &sh->dev[i];
+                    if (!test_bit(R5_LOCKED, &dev->flags) &&
+                         test_bit(R5_UPTODATE, &dev->flags) ) {
+                        /* We can return any write requests */
+                            struct bio *wbi, *wbi2;
+                            PRINTK("Return write for disc %d\n", i);
+                            spin_lock_irq(&conf->device_lock);
+                            wbi = dev->written;
+                            dev->written = NULL;
+                            while (wbi && wbi->bi_sector < dev->sector + STRIPE_SECTORS) {
+                                    wbi2 = r5_next_bio(wbi, dev->sector);
+                                    if (--wbi->bi_phys_segments == 0) {
+                                            md_write_end(conf->mddev);
+                                            wbi->bi_next = return_bi;
+                                            return_bi = wbi;
+                                    }
+                                    wbi = wbi2;
+                            }
+                            spin_unlock_irq(&conf->device_lock);
+                    }
+                }
+        }
+
+        /* Now we might consider reading some blocks, either to check/generate
+         * parity, or to satisfy requests
+         * or to load a block that is being partially written.
+         */
+        if (to_read || non_overwrite || (syncing && (uptodate < disks))) {
+                for (i=disks; i--;) {
+                        dev = &sh->dev[i];
+                        if (!test_bit(R5_LOCKED, &dev->flags) && !test_bit(R5_UPTODATE, &dev->flags) &&
+                            (dev->toread ||
+                             (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) ||
+                             syncing ||
+                             (failed && (sh->dev[failed_num].toread ||
+                                         (sh->dev[failed_num].towrite && !test_bit(R5_OVERWRITE, &sh->dev[failed_num].flags))))
+                                    )
+                                ) {
+                                /* we would like to get this block, possibly
+                                 * by computing it, but we might not be able to
+                                 */
+                                if (uptodate == disks-1) {
+                                        PRINTK("Computing block %d\n", i);
+                                        compute_block(sh, i);
+                                        uptodate++;
+                                } else if (test_bit(R5_Insync, &dev->flags)) {
+                                        set_bit(R5_LOCKED, &dev->flags);
+                                        set_bit(R5_Wantread, &dev->flags);
+#if 0
+                                        /* if I am just reading this block and we don't have
+                                           a failed drive, or any pending writes then sidestep the cache */
+                                        if (sh->bh_read[i] && !sh->bh_read[i]->b_reqnext &&
+                                            ! syncing && !failed && !to_write) {
+                                                sh->bh_cache[i]->b_page =  sh->bh_read[i]->b_page;
+                                                sh->bh_cache[i]->b_data =  sh->bh_read[i]->b_data;
+                                        }
+#endif
+                                        locked++;
+                                        PRINTK("Reading block %d (sync=%d)\n", 
+                                                i, syncing);
+                                        if (syncing)
+                                                md_sync_acct(conf->disks[i].rdev->bdev,
+                                                             STRIPE_SECTORS);
+                                }
+                        }
+                }
+                set_bit(STRIPE_HANDLE, &sh->state);
+        }
+
+        /* now to consider writing and what else, if anything should be read */
+        if (to_write) {
+                int rmw=0, rcw=0;
+                for (i=disks ; i--;) {
+                        /* would I have to read this buffer for read_modify_write */
+                        dev = &sh->dev[i];
+                        if ((dev->towrite || i == sh->pd_idx) &&
+                            (!test_bit(R5_LOCKED, &dev->flags) 
+#if 0
+|| sh->bh_page[i]!=bh->b_page
+#endif
+                                    ) &&
+                            !test_bit(R5_UPTODATE, &dev->flags)) {
+                                if (test_bit(R5_Insync, &dev->flags)
+/*                                  && !(!mddev->insync && i == sh->pd_idx) */
+                                        )
+                                        rmw++;
+                                else rmw += 2*disks;  /* cannot read it */
+                        }
+                        /* Would I have to read this buffer for reconstruct_write */
+                        if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx &&
+                            (!test_bit(R5_LOCKED, &dev->flags) 
+#if 0
+|| sh->bh_page[i] != bh->b_page
+#endif
+                                    ) &&
+                            !test_bit(R5_UPTODATE, &dev->flags)) {
+                                if (test_bit(R5_Insync, &dev->flags)) rcw++;
+                                else rcw += 2*disks;
+                        }
+                }
+                PRINTK("for sector %llu, rmw=%d rcw=%d\n", 
+                        (unsigned long long)sh->sector, rmw, rcw);
+                set_bit(STRIPE_HANDLE, &sh->state);
+                if (rmw < rcw && rmw > 0)
+                        /* prefer read-modify-write, but need to get some data */
+                        for (i=disks; i--;) {
+                                dev = &sh->dev[i];
+                                if ((dev->towrite || i == sh->pd_idx) &&
+                                    !test_bit(R5_LOCKED, &dev->flags) && !test_bit(R5_UPTODATE, &dev->flags) &&
+                                    test_bit(R5_Insync, &dev->flags)) {
+                                        if (test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
+                                        {
+                                                PRINTK("Read_old block %d for r-m-w\n", i);
+                                                set_bit(R5_LOCKED, &dev->flags);
+                                                set_bit(R5_Wantread, &dev->flags);
+                                                locked++;
+                                        } else {
+                                                set_bit(STRIPE_DELAYED, &sh->state);
+                                                set_bit(STRIPE_HANDLE, &sh->state);
+                                        }
+                                }
+                        }
+                if (rcw <= rmw && rcw > 0)
+                        /* want reconstruct write, but need to get some data */
+                        for (i=disks; i--;) {
+                                dev = &sh->dev[i];
+                                if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx &&
+                                    !test_bit(R5_LOCKED, &dev->flags) && !test_bit(R5_UPTODATE, &dev->flags) &&
+                                    test_bit(R5_Insync, &dev->flags)) {
+                                        if (test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
+                                        {
+                                                PRINTK("Read_old block %d for Reconstruct\n", i);
+                                                set_bit(R5_LOCKED, &dev->flags);
+                                                set_bit(R5_Wantread, &dev->flags);
+                                                locked++;
+                                        } else {
+                                                set_bit(STRIPE_DELAYED, &sh->state);
+                                                set_bit(STRIPE_HANDLE, &sh->state);
+                                        }
+                                }
+                        }
+                /* now if nothing is locked, and if we have enough data, we can start a write request */
+                if (locked == 0 && (rcw == 0 ||rmw == 0)) {
+                        PRINTK("Computing parity...\n");
+                        compute_parity(sh, rcw==0 ? RECONSTRUCT_WRITE : READ_MODIFY_WRITE);
+                        /* now every locked buffer is ready to be written */
+                        for (i=disks; i--;)
+                                if (test_bit(R5_LOCKED, &sh->dev[i].flags)) {
+                                        PRINTK("Writing block %d\n", i);
+                                        locked++;
+                                        set_bit(R5_Wantwrite, &sh->dev[i].flags);
+                                        if (!test_bit(R5_Insync, &sh->dev[i].flags)
+                                            || (i==sh->pd_idx && failed == 0))
+                                                set_bit(STRIPE_INSYNC, &sh->state);
+                                }
+                        if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
+                                atomic_dec(&conf->preread_active_stripes);
+                                if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD)
+                                        md_wakeup_thread(conf->mddev->thread);
+                        }
+                }
+        }
+
+        /* maybe we need to check and possibly fix the parity for this stripe
+         * Any reads will already have been scheduled, so we just see if enough data
+         * is available
+         */
+        if (syncing && locked == 0 &&
+            !test_bit(STRIPE_INSYNC, &sh->state) && failed <= 1) {
+                set_bit(STRIPE_HANDLE, &sh->state);
+                if (failed == 0) {
+                        char *pagea;
+                        if (uptodate != disks)
+                                BUG();
+                        compute_parity(sh, CHECK_PARITY);
+                        uptodate--;
+                        pagea = page_address(sh->dev[sh->pd_idx].page);
+                        if ((*(u32*)pagea) == 0 &&
+                            !memcmp(pagea, pagea+4, STRIPE_SIZE-4)) {
+                                /* parity is correct (on disc, not in buffer any more) */
+                                set_bit(STRIPE_INSYNC, &sh->state);
+                        }
+                }
+                if (!test_bit(STRIPE_INSYNC, &sh->state)) {
+                        if (failed==0)
+                                failed_num = sh->pd_idx;
+                        /* should be able to compute the missing block and write it to spare */
+                        if (!test_bit(R5_UPTODATE, &sh->dev[failed_num].flags)) {
+                                if (uptodate+1 != disks)
+                                        BUG();
+                                compute_block(sh, failed_num);
+                                uptodate++;
+                        }
+                        if (uptodate != disks)
+                                BUG();
+                        dev = &sh->dev[failed_num];
+                        set_bit(R5_LOCKED, &dev->flags);
+                        set_bit(R5_Wantwrite, &dev->flags);
+                        locked++;
+                        set_bit(STRIPE_INSYNC, &sh->state);
+                        set_bit(R5_Syncio, &dev->flags);
+                }
+        }
+        if (syncing && locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) {
+                md_done_sync(conf->mddev, STRIPE_SECTORS,1);
+                clear_bit(STRIPE_SYNCING, &sh->state);
+        }
+        
+        spin_unlock(&sh->lock);
+
+        while ((bi=return_bi)) {
+                int bytes = bi->bi_size;
+
+                return_bi = bi->bi_next;
+                bi->bi_next = NULL;
+                bi->bi_size = 0;
+                bi->bi_end_io(bi, bytes, 0);
+        }
+        for (i=disks; i-- ;) {
+                int rw;
+                struct bio *bi;
+                mdk_rdev_t *rdev;
+                if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags))
+                        rw = 1;
+                else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
+                        rw = 0;
+                else
+                        continue;
+ 
+                bi = &sh->dev[i].req;
+ 
+                bi->bi_rw = rw;
+                if (rw)
+                        bi->bi_end_io = raid5_end_write_request;
+                else
+                        bi->bi_end_io = raid5_end_read_request;
+ 
+                rcu_read_lock();
+                rdev = conf->disks[i].rdev;
+                if (rdev && rdev->faulty)
+                        rdev = NULL;
+                if (rdev)
+                        atomic_inc(&rdev->nr_pending);
+                rcu_read_unlock();
+ 
+                if (rdev) {
+                        if (test_bit(R5_Syncio, &sh->dev[i].flags))
+                                md_sync_acct(rdev->bdev, STRIPE_SECTORS);
+
+                        bi->bi_bdev = rdev->bdev;
+                        PRINTK("for %llu schedule op %ld on disc %d\n",
+                                (unsigned long long)sh->sector, bi->bi_rw, i);
+                        atomic_inc(&sh->count);
+                        bi->bi_sector = sh->sector + rdev->data_offset;
+                        bi->bi_flags = 1 << BIO_UPTODATE;
+                        bi->bi_vcnt = 1;        
+                        bi->bi_max_vecs = 1;
+                        bi->bi_idx = 0;
+                        bi->bi_io_vec = &sh->dev[i].vec;
+                        bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
+                        bi->bi_io_vec[0].bv_offset = 0;
+                        bi->bi_size = STRIPE_SIZE;
+                        bi->bi_next = NULL;
+                        generic_make_request(bi);
+                } else {
+                        PRINTK("skip op %ld on disc %d for sector %llu\n",
+                                bi->bi_rw, i, (unsigned long long)sh->sector);
+                        clear_bit(R5_LOCKED, &sh->dev[i].flags);
+                        set_bit(STRIPE_HANDLE, &sh->state);
+                }
+        }
+}
+
+static inline void raid5_activate_delayed(raid5_conf_t *conf)
+{
+        if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
+                while (!list_empty(&conf->delayed_list)) {
+                        struct list_head *l = conf->delayed_list.next;
+                        struct stripe_head *sh;
+                        sh = list_entry(l, struct stripe_head, lru);
+                        list_del_init(l);
+                        clear_bit(STRIPE_DELAYED, &sh->state);
+                        if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
+                                atomic_inc(&conf->preread_active_stripes);
+                        list_add_tail(&sh->lru, &conf->handle_list);
+                }
+        }
+}
+
+static void unplug_slaves(mddev_t *mddev)
+{
+        raid5_conf_t *conf = mddev_to_conf(mddev);
+        int i;
+
+        rcu_read_lock();
+        for (i=0; i<mddev->raid_disks; i++) {
+                mdk_rdev_t *rdev = conf->disks[i].rdev;
+                if (rdev && !rdev->faulty && atomic_read(&rdev->nr_pending)) {
+                        request_queue_t *r_queue = bdev_get_queue(rdev->bdev);
+
+                        atomic_inc(&rdev->nr_pending);
+                        rcu_read_unlock();
+
+                        if (r_queue->unplug_fn)
+                                r_queue->unplug_fn(r_queue);
+
+                        rdev_dec_pending(rdev, mddev);
+                        rcu_read_lock();
+                }
+        }
+        rcu_read_unlock();
+}
+
+static void raid5_unplug_device(request_queue_t *q)
+{
+        mddev_t *mddev = q->queuedata;
+        raid5_conf_t *conf = mddev_to_conf(mddev);
+        unsigned long flags;
+
+        spin_lock_irqsave(&conf->device_lock, flags);
+
+        if (blk_remove_plug(q))
+                raid5_activate_delayed(conf);
+        md_wakeup_thread(mddev->thread);
+
+        spin_unlock_irqrestore(&conf->device_lock, flags);
+
+        unplug_slaves(mddev);
+}
+
+static int raid5_issue_flush(request_queue_t *q, struct gendisk *disk,
+                             sector_t *error_sector)
+{
+        mddev_t *mddev = q->queuedata;
+        raid5_conf_t *conf = mddev_to_conf(mddev);
+        int i, ret = 0;
+
+        rcu_read_lock();
+        for (i=0; i<mddev->raid_disks && ret == 0; i++) {
+                mdk_rdev_t *rdev = conf->disks[i].rdev;
+                if (rdev && !rdev->faulty) {
+                        struct block_device *bdev = rdev->bdev;
+                        request_queue_t *r_queue = bdev_get_queue(bdev);
+
+                        if (!r_queue->issue_flush_fn)
+                                ret = -EOPNOTSUPP;
+                        else {
+                                atomic_inc(&rdev->nr_pending);
+                                rcu_read_unlock();
+                                ret = r_queue->issue_flush_fn(r_queue, bdev->bd_disk,
+                                                              error_sector);
+                                rdev_dec_pending(rdev, mddev);
+                                rcu_read_lock();
+                        }
+                }
+        }
+        rcu_read_unlock();
+        return ret;
+}
+
+static inline void raid5_plug_device(raid5_conf_t *conf)
+{
+        spin_lock_irq(&conf->device_lock);
+        blk_plug_device(conf->mddev->queue);
+        spin_unlock_irq(&conf->device_lock);
+}
+
+static int make_request (request_queue_t *q, struct bio * bi)
+{
+        mddev_t *mddev = q->queuedata;
+        raid5_conf_t *conf = mddev_to_conf(mddev);
+        const unsigned int raid_disks = conf->raid_disks;
+        const unsigned int data_disks = raid_disks - 1;
+        unsigned int dd_idx, pd_idx;
+        sector_t new_sector;
+        sector_t logical_sector, last_sector;
+        struct stripe_head *sh;
+
+        if (bio_data_dir(bi)==WRITE) {
+                disk_stat_inc(mddev->gendisk, writes);
+                disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bi));
+        } else {
+                disk_stat_inc(mddev->gendisk, reads);
+                disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bi));
+        }
+
+        logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
+        last_sector = bi->bi_sector + (bi->bi_size>>9);
+        bi->bi_next = NULL;
+        bi->bi_phys_segments = 1;       /* over-loaded to count active stripes */
+        if ( bio_data_dir(bi) == WRITE )
+                md_write_start(mddev);
+        for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
+                DEFINE_WAIT(w);
+                
+                new_sector = raid5_compute_sector(logical_sector,
+                                                  raid_disks, data_disks, &dd_idx, &pd_idx, conf);
+
+                PRINTK("raid5: make_request, sector %llu logical %llu\n",
+                        (unsigned long long)new_sector, 
+                        (unsigned long long)logical_sector);
+
+        retry:
+                prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
+                sh = get_active_stripe(conf, new_sector, pd_idx, (bi->bi_rw&RWA_MASK));
+                if (sh) {
+                        if (!add_stripe_bio(sh, bi, dd_idx, (bi->bi_rw&RW_MASK))) {
+                                /* Add failed due to overlap.  Flush everything
+                                 * and wait a while
+                                 */
+                                raid5_unplug_device(mddev->queue);
+                                release_stripe(sh);
+                                schedule();
+                                goto retry;
+                        }
+                        finish_wait(&conf->wait_for_overlap, &w);
+                        raid5_plug_device(conf);
+                        handle_stripe(sh);
+                        release_stripe(sh);
+
+                } else {
+                        /* cannot get stripe for read-ahead, just give-up */
+                        clear_bit(BIO_UPTODATE, &bi->bi_flags);
+                        finish_wait(&conf->wait_for_overlap, &w);
+                        break;
+                }
+                        
+        }
+        spin_lock_irq(&conf->device_lock);
+        if (--bi->bi_phys_segments == 0) {
+                int bytes = bi->bi_size;
+
+                if ( bio_data_dir(bi) == WRITE )
+                        md_write_end(mddev);
+                bi->bi_size = 0;
+                bi->bi_end_io(bi, bytes, 0);
+        }
+        spin_unlock_irq(&conf->device_lock);
+        return 0;
+}
+
+/* FIXME go_faster isn't used */
+static int sync_request (mddev_t *mddev, sector_t sector_nr, int go_faster)
+{
+        raid5_conf_t *conf = (raid5_conf_t *) mddev->private;
+        struct stripe_head *sh;
+        int sectors_per_chunk = conf->chunk_size >> 9;
+        sector_t x;
+        unsigned long stripe;
+        int chunk_offset;
+        int dd_idx, pd_idx;
+        sector_t first_sector;
+        int raid_disks = conf->raid_disks;
+        int data_disks = raid_disks-1;
+
+        if (sector_nr >= mddev->size <<1) {
+                /* just being told to finish up .. nothing much to do */
+                unplug_slaves(mddev);
+                return 0;
+        }
+        /* if there is 1 or more failed drives and we are trying
+         * to resync, then assert that we are finished, because there is
+         * nothing we can do.
+         */
+        if (mddev->degraded >= 1 && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
+                int rv = (mddev->size << 1) - sector_nr;
+                md_done_sync(mddev, rv, 1);
+                return rv;
+        }
+
+        x = sector_nr;
+        chunk_offset = sector_div(x, sectors_per_chunk);
+        stripe = x;
+        BUG_ON(x != stripe);
+
+        first_sector = raid5_compute_sector((sector_t)stripe*data_disks*sectors_per_chunk
+                + chunk_offset, raid_disks, data_disks, &dd_idx, &pd_idx, conf);
+        sh = get_active_stripe(conf, sector_nr, pd_idx, 1);
+        if (sh == NULL) {
+                sh = get_active_stripe(conf, sector_nr, pd_idx, 0);
+                /* make sure we don't swamp the stripe cache if someone else
+                 * is trying to get access 
+                 */
+                set_current_state(TASK_UNINTERRUPTIBLE);
+                schedule_timeout(1);
+        }
+        spin_lock(&sh->lock);   
+        set_bit(STRIPE_SYNCING, &sh->state);
+        clear_bit(STRIPE_INSYNC, &sh->state);
+        spin_unlock(&sh->lock);
+
+        handle_stripe(sh);
+        release_stripe(sh);
+
+        return STRIPE_SECTORS;
+}
+
+/*
+ * This is our raid5 kernel thread.
+ *
+ * We scan the hash table for stripes which can be handled now.
+ * During the scan, completed stripes are saved for us by the interrupt
+ * handler, so that they will not have to wait for our next wakeup.
+ */
+static void raid5d (mddev_t *mddev)
+{
+        struct stripe_head *sh;
+        raid5_conf_t *conf = mddev_to_conf(mddev);
+        int handled;
+
+        PRINTK("+++ raid5d active\n");
+
+        md_check_recovery(mddev);
+        md_handle_safemode(mddev);
+
+        handled = 0;
+        spin_lock_irq(&conf->device_lock);
+        while (1) {
+                struct list_head *first;
+
+                if (list_empty(&conf->handle_list) &&
+                    atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD &&
+                    !blk_queue_plugged(mddev->queue) &&
+                    !list_empty(&conf->delayed_list))
+                        raid5_activate_delayed(conf);
+
+                if (list_empty(&conf->handle_list))
+                        break;
+
+                first = conf->handle_list.next;
+                sh = list_entry(first, struct stripe_head, lru);
+
+                list_del_init(first);
+                atomic_inc(&sh->count);
+                if (atomic_read(&sh->count)!= 1)
+                        BUG();
+                spin_unlock_irq(&conf->device_lock);
+                
+                handled++;
+                handle_stripe(sh);
+                release_stripe(sh);
+
+                spin_lock_irq(&conf->device_lock);
+        }
+        PRINTK("%d stripes handled\n", handled);
+
+        spin_unlock_irq(&conf->device_lock);
+
+        unplug_slaves(mddev);
+
+        PRINTK("--- raid5d inactive\n");
+}
+
+static int run (mddev_t *mddev)
+{
+        raid5_conf_t *conf;
+        int raid_disk, memory;
+        mdk_rdev_t *rdev;
+        struct disk_info *disk;
+        struct list_head *tmp;
+
+        if (mddev->level != 5 && mddev->level != 4) {
+                printk("raid5: %s: raid level not set to 4/5 (%d)\n", mdname(mddev), mddev->level);
+                return -EIO;
+        }
+
+        mddev->private = kmalloc (sizeof (raid5_conf_t)
+                                  + mddev->raid_disks * sizeof(struct disk_info),
+                                  GFP_KERNEL);
+        if ((conf = mddev->private) == NULL)
+                goto abort;
+        memset (conf, 0, sizeof (*conf) + mddev->raid_disks * sizeof(struct disk_info) );
+        conf->mddev = mddev;
+
+        if ((conf->stripe_hashtbl = (struct stripe_head **) __get_free_pages(GFP_ATOMIC, HASH_PAGES_ORDER)) == NULL)
+                goto abort;
+        memset(conf->stripe_hashtbl, 0, HASH_PAGES * PAGE_SIZE);
+
+        spin_lock_init(&conf->device_lock);
+        init_waitqueue_head(&conf->wait_for_stripe);
+        init_waitqueue_head(&conf->wait_for_overlap);
+        INIT_LIST_HEAD(&conf->handle_list);
+        INIT_LIST_HEAD(&conf->delayed_list);
+        INIT_LIST_HEAD(&conf->inactive_list);
+        atomic_set(&conf->active_stripes, 0);
+        atomic_set(&conf->preread_active_stripes, 0);
+
+        mddev->queue->unplug_fn = raid5_unplug_device;
+        mddev->queue->issue_flush_fn = raid5_issue_flush;
+
+        PRINTK("raid5: run(%s) called.\n", mdname(mddev));
+
+        ITERATE_RDEV(mddev,rdev,tmp) {
+                raid_disk = rdev->raid_disk;
+                if (raid_disk >= mddev->raid_disks
+                    || raid_disk < 0)
+                        continue;
+                disk = conf->disks + raid_disk;
+
+                disk->rdev = rdev;
+
+                if (rdev->in_sync) {
+                        char b[BDEVNAME_SIZE];
+                        printk(KERN_INFO "raid5: device %s operational as raid"
+                                " disk %d\n", bdevname(rdev->bdev,b),
+                                raid_disk);
+                        conf->working_disks++;
+                }
+        }
+
+        conf->raid_disks = mddev->raid_disks;
+        /*
+         * 0 for a fully functional array, 1 for a degraded array.
+         */
+        mddev->degraded = conf->failed_disks = conf->raid_disks - conf->working_disks;
+        conf->mddev = mddev;
+        conf->chunk_size = mddev->chunk_size;
+        conf->level = mddev->level;
+        conf->algorithm = mddev->layout;
+        conf->max_nr_stripes = NR_STRIPES;
+
+        /* device size must be a multiple of chunk size */
+        mddev->size &= ~(mddev->chunk_size/1024 -1);
+
+        if (!conf->chunk_size || conf->chunk_size % 4) {
+                printk(KERN_ERR "raid5: invalid chunk size %d for %s\n",
+                        conf->chunk_size, mdname(mddev));
+                goto abort;
+        }
+        if (conf->algorithm > ALGORITHM_RIGHT_SYMMETRIC) {
+                printk(KERN_ERR 
+                        "raid5: unsupported parity algorithm %d for %s\n",
+                        conf->algorithm, mdname(mddev));
+                goto abort;
+        }
+        if (mddev->degraded > 1) {
+                printk(KERN_ERR "raid5: not enough operational devices for %s"
+                        " (%d/%d failed)\n",
+                        mdname(mddev), conf->failed_disks, conf->raid_disks);
+                goto abort;
+        }
+
+        if (mddev->degraded == 1 &&
+            mddev->recovery_cp != MaxSector) {
+                printk(KERN_ERR 
+                        "raid5: cannot start dirty degraded array for %s\n",
+                        mdname(mddev));
+                goto abort;
+        }
+
+        {
+                mddev->thread = md_register_thread(raid5d, mddev, "%s_raid5");
+                if (!mddev->thread) {
+                        printk(KERN_ERR 
+                                "raid5: couldn't allocate thread for %s\n",
+                                mdname(mddev));
+                        goto abort;
+                }
+        }
+memory = conf->max_nr_stripes * (sizeof(struct stripe_head) +
+                 conf->raid_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
+        if (grow_stripes(conf, conf->max_nr_stripes)) {
+                printk(KERN_ERR 
+                        "raid5: couldn't allocate %dkB for buffers\n", memory);
+                shrink_stripes(conf);
+                md_unregister_thread(mddev->thread);
+                goto abort;
+        } else
+                printk(KERN_INFO "raid5: allocated %dkB for %s\n",
+                        memory, mdname(mddev));
+
+        if (mddev->degraded == 0)
+                printk("raid5: raid level %d set %s active with %d out of %d"
+                        " devices, algorithm %d\n", conf->level, mdname(mddev), 
+                        mddev->raid_disks-mddev->degraded, mddev->raid_disks,
+                        conf->algorithm);
+        else
+                printk(KERN_ALERT "raid5: raid level %d set %s active with %d"
+                        " out of %d devices, algorithm %d\n", conf->level,
+                        mdname(mddev), mddev->raid_disks - mddev->degraded,
+                        mddev->raid_disks, conf->algorithm);
+
+        print_raid5_conf(conf);
+
+        /* read-ahead size must cover two whole stripes, which is
+         * 2 * (n-1) * chunksize where 'n' is the number of raid devices
+         */
+        {
+                int stripe = (mddev->raid_disks-1) * mddev->chunk_size
+                        / PAGE_CACHE_SIZE;
+                if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
+                        mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
+        }
+
+        /* Ok, everything is just fine now */
+        mddev->array_size =  mddev->size * (mddev->raid_disks - 1);
+        return 0;
+abort:
+        if (conf) {
+                print_raid5_conf(conf);
+                if (conf->stripe_hashtbl)
+                        free_pages((unsigned long) conf->stripe_hashtbl,
+                                                        HASH_PAGES_ORDER);
+                kfree(conf);
+        }
+        mddev->private = NULL;
+        printk(KERN_ALERT "raid5: failed to run raid set %s\n", mdname(mddev));
+        return -EIO;
+}
+
+
+
+static int stop (mddev_t *mddev)
+{
+        raid5_conf_t *conf = (raid5_conf_t *) mddev->private;
+
+        md_unregister_thread(mddev->thread);
+        mddev->thread = NULL;
+        shrink_stripes(conf);
+        free_pages((unsigned long) conf->stripe_hashtbl, HASH_PAGES_ORDER);
+        blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
+        kfree(conf);
+        mddev->private = NULL;
+        return 0;
+}
+
+#if RAID5_DEBUG
+static void print_sh (struct stripe_head *sh)
+{
+        int i;
+
+        printk("sh %llu, pd_idx %d, state %ld.\n",
+                (unsigned long long)sh->sector, sh->pd_idx, sh->state);
+        printk("sh %llu,  count %d.\n",
+                (unsigned long long)sh->sector, atomic_read(&sh->count));
+        printk("sh %llu, ", (unsigned long long)sh->sector);
+        for (i = 0; i < sh->raid_conf->raid_disks; i++) {
+                printk("(cache%d: %p %ld) ", 
+                        i, sh->dev[i].page, sh->dev[i].flags);
+        }
+        printk("\n");
+}
+
+static void printall (raid5_conf_t *conf)
+{
+        struct stripe_head *sh;
+        int i;
+
+        spin_lock_irq(&conf->device_lock);
+        for (i = 0; i < NR_HASH; i++) {
+                sh = conf->stripe_hashtbl[i];
+                for (; sh; sh = sh->hash_next) {
+                        if (sh->raid_conf != conf)
+                                continue;
+                        print_sh(sh);
+                }
+        }
+        spin_unlock_irq(&conf->device_lock);
+}
+#endif
+
+static void status (struct seq_file *seq, mddev_t *mddev)
+{
+        raid5_conf_t *conf = (raid5_conf_t *) mddev->private;
+        int i;
+
+        seq_printf (seq, " level %d, %dk chunk, algorithm %d", mddev->level, mddev->chunk_size >> 10, mddev->layout);
+        seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->working_disks);
+        for (i = 0; i < conf->raid_disks; i++)
+                seq_printf (seq, "%s",
+                               conf->disks[i].rdev &&
+                               conf->disks[i].rdev->in_sync ? "U" : "_");
+        seq_printf (seq, "]");
+#if RAID5_DEBUG
+#define D(x) \
+        seq_printf (seq, "<"#x":%d>", atomic_read(&conf->x))
+        printall(conf);
+#endif
+}
+
+static void print_raid5_conf (raid5_conf_t *conf)
+{
+        int i;
+        struct disk_info *tmp;
+
+        printk("RAID5 conf printout:\n");
+        if (!conf) {
+                printk("(conf==NULL)\n");
+                return;
+        }
+        printk(" --- rd:%d wd:%d fd:%d\n", conf->raid_disks,
+                 conf->working_disks, conf->failed_disks);
+
+        for (i = 0; i < conf->raid_disks; i++) {
+                char b[BDEVNAME_SIZE];
+                tmp = conf->disks + i;
+                if (tmp->rdev)
+                printk(" disk %d, o:%d, dev:%s\n",
+                        i, !tmp->rdev->faulty,
+                        bdevname(tmp->rdev->bdev,b));
+        }
+}
+
+static int raid5_spare_active(mddev_t *mddev)
+{
+        int i;
+        raid5_conf_t *conf = mddev->private;
+        struct disk_info *tmp;
+
+        for (i = 0; i < conf->raid_disks; i++) {
+                tmp = conf->disks + i;
+                if (tmp->rdev
+                    && !tmp->rdev->faulty
+                    && !tmp->rdev->in_sync) {
+                        mddev->degraded--;
+                        conf->failed_disks--;
+                        conf->working_disks++;
+                        tmp->rdev->in_sync = 1;
+                }
+        }
+        print_raid5_conf(conf);
+        return 0;
+}
+
+static int raid5_remove_disk(mddev_t *mddev, int number)
+{
+        raid5_conf_t *conf = mddev->private;
+        int err = 0;
+        mdk_rdev_t *rdev;
+        struct disk_info *p = conf->disks + number;
+
+        print_raid5_conf(conf);
+        rdev = p->rdev;
+        if (rdev) {
+                if (rdev->in_sync ||
+                    atomic_read(&rdev->nr_pending)) {
+                        err = -EBUSY;
+                        goto abort;
+                }
+                p->rdev = NULL;
+                synchronize_kernel();
+                if (atomic_read(&rdev->nr_pending)) {
+                        /* lost the race, try later */
+                        err = -EBUSY;
+                        p->rdev = rdev;
+                }
+        }
+abort:
+
+        print_raid5_conf(conf);
+        return err;
+}
+
+static int raid5_add_disk(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+        raid5_conf_t *conf = mddev->private;
+        int found = 0;
+        int disk;
+        struct disk_info *p;
+
+        if (mddev->degraded > 1)
+                /* no point adding a device */
+                return 0;
+
+        /*
+         * find the disk ...
+         */
+        for (disk=0; disk < mddev->raid_disks; disk++)
+                if ((p=conf->disks + disk)->rdev == NULL) {
+                        rdev->in_sync = 0;
+                        rdev->raid_disk = disk;
+                        found = 1;
+                        p->rdev = rdev;
+                        break;
+                }
+        print_raid5_conf(conf);
+        return found;
+}
+
+static int raid5_resize(mddev_t *mddev, sector_t sectors)
+{
+        /* no resync is happening, and there is enough space
+         * on all devices, so we can resize.
+         * We need to make sure resync covers any new space.
+         * If the array is shrinking we should possibly wait until
+         * any io in the removed space completes, but it hardly seems
+         * worth it.
+         */
+        sectors &= ~((sector_t)mddev->chunk_size/512 - 1);
+        mddev->array_size = (sectors * (mddev->raid_disks-1))>>1;
+        set_capacity(mddev->gendisk, mddev->array_size << 1);
+        mddev->changed = 1;
+        if (sectors/2  > mddev->size && mddev->recovery_cp == MaxSector) {
+                mddev->recovery_cp = mddev->size << 1;
+                set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+        }
+        mddev->size = sectors /2;
+        return 0;
+}
+
+static mdk_personality_t raid5_personality=
+{
+        .name           = "raid5",
+        .owner          = THIS_MODULE,
+        .make_request   = make_request,
+        .run            = run,
+        .stop           = stop,
+        .status         = status,
+        .error_handler  = error,
+        .hot_add_disk   = raid5_add_disk,
+        .hot_remove_disk= raid5_remove_disk,
+        .spare_active   = raid5_spare_active,
+        .sync_request   = sync_request,
+        .resize         = raid5_resize,
+};
+
+static int __init raid5_init (void)
+{
+        return register_md_personality (RAID5, &raid5_personality);
+}
+
+static void raid5_exit (void)
+{
+        unregister_md_personality (RAID5);
+}
+
+module_init(raid5_init);
+module_exit(raid5_exit);
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("md-personality-4"); /* RAID5 */
diff --git a/drivers/md/raid6.h b/drivers/md/raid6.h
new file mode 100644
index 000000000000..f80ee6350edf
--- /dev/null
+++ b/drivers/md/raid6.h
@@ -0,0 +1,135 @@
+/* -*- linux-c -*- ------------------------------------------------------- *
+ *
+ *   Copyright 2003 H. Peter Anvin - All Rights Reserved
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+ *   Bostom MA 02111-1307, USA; either version 2 of the License, or
+ *   (at your option) any later version; incorporated herein by reference.
+ *
+ * ----------------------------------------------------------------------- */
+
+#ifndef LINUX_RAID_RAID6_H
+#define LINUX_RAID_RAID6_H
+
+#ifdef __KERNEL__
+
+/* Set to 1 to use kernel-wide empty_zero_page */
+#define RAID6_USE_EMPTY_ZERO_PAGE 0
+
+#include <linux/module.h>
+#include <linux/stddef.h>
+#include <linux/compiler.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/mempool.h>
+#include <linux/list.h>
+#include <linux/vmalloc.h>
+#include <linux/raid/md.h>
+#include <linux/raid/raid5.h>
+
+typedef raid5_conf_t raid6_conf_t; /* Same configuration */
+
+/* Additional compute_parity mode -- updates the parity w/o LOCKING */
+#define UPDATE_PARITY   4
+
+/* We need a pre-zeroed page... if we don't want to use the kernel-provided
+   one define it here */
+#if RAID6_USE_EMPTY_ZERO_PAGE
+# define raid6_empty_zero_page empty_zero_page
+#else
+extern const char raid6_empty_zero_page[PAGE_SIZE];
+#endif
+
+#else /* ! __KERNEL__ */
+/* Used for testing in user space */
+
+#include <errno.h>
+#include <inttypes.h>
+#include <limits.h>
+#include <stddef.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+
+/* Not standard, but glibc defines it */
+#define BITS_PER_LONG __WORDSIZE
+
+typedef uint8_t  u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+typedef uint64_t u64;
+
+#ifndef PAGE_SIZE
+# define PAGE_SIZE 4096
+#endif
+extern const char raid6_empty_zero_page[PAGE_SIZE];
+
+#define __init
+#define __exit
+#define __attribute_const__ __attribute__((const))
+
+#define preempt_enable()
+#define preempt_disable()
+
+#endif /* __KERNEL__ */
+
+/* Routine choices */
+struct raid6_calls {
+        void (*gen_syndrome)(int, size_t, void **);
+        int  (*valid)(void);    /* Returns 1 if this routine set is usable */
+        const char *name;       /* Name of this routine set */
+        int prefer;             /* Has special performance attribute */
+};
+
+/* Selected algorithm */
+extern struct raid6_calls raid6_call;
+
+/* Algorithm list */
+extern const struct raid6_calls * const raid6_algos[];
+int raid6_select_algo(void);
+
+/* Return values from chk_syndrome */
+#define RAID6_OK        0
+#define RAID6_P_BAD     1
+#define RAID6_Q_BAD     2
+#define RAID6_PQ_BAD    3
+
+/* Galois field tables */
+extern const u8 raid6_gfmul[256][256] __attribute__((aligned(256)));
+extern const u8 raid6_gfexp[256]      __attribute__((aligned(256)));
+extern const u8 raid6_gfinv[256]      __attribute__((aligned(256)));
+extern const u8 raid6_gfexi[256]      __attribute__((aligned(256)));
+
+/* Recovery routines */
+void raid6_2data_recov(int disks, size_t bytes, int faila, int failb, void **ptrs);
+void raid6_datap_recov(int disks, size_t bytes, int faila, void **ptrs);
+void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, void **ptrs);
+
+/* Some definitions to allow code to be compiled for testing in userspace */
+#ifndef __KERNEL__
+
+# define jiffies        raid6_jiffies()
+# define printk         printf
+# define GFP_KERNEL     0
+# define __get_free_pages(x,y)  ((unsigned long)mmap(NULL, PAGE_SIZE << (y), PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, 0, 0))
+# define free_pages(x,y)        munmap((void *)(x), (y)*PAGE_SIZE)
+
+static inline void cpu_relax(void)
+{
+        /* Nothing */
+}
+
+#undef  HZ
+#define HZ 1000
+static inline uint32_t raid6_jiffies(void)
+{
+        struct timeval tv;
+        gettimeofday(&tv, NULL);
+        return tv.tv_sec*1000 + tv.tv_usec/1000;
+}
+
+#endif /* ! __KERNEL__ */
+
+#endif /* LINUX_RAID_RAID6_H */
diff --git a/drivers/md/raid6algos.c b/drivers/md/raid6algos.c
new file mode 100644
index 000000000000..acf386fc4b4f
--- /dev/null
+++ b/drivers/md/raid6algos.c
@@ -0,0 +1,153 @@
+/* -*- linux-c -*- ------------------------------------------------------- *
+ *
+ *   Copyright 2002 H. Peter Anvin - All Rights Reserved
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+ *   Bostom MA 02111-1307, USA; either version 2 of the License, or
+ *   (at your option) any later version; incorporated herein by reference.
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * raid6algos.c
+ *
+ * Algorithm list and algorithm selection for RAID-6
+ */
+
+#include "raid6.h"
+#ifndef __KERNEL__
+#include <sys/mman.h>
+#endif
+
+struct raid6_calls raid6_call;
+
+/* Various routine sets */
+extern const struct raid6_calls raid6_intx1;
+extern const struct raid6_calls raid6_intx2;
+extern const struct raid6_calls raid6_intx4;
+extern const struct raid6_calls raid6_intx8;
+extern const struct raid6_calls raid6_intx16;
+extern const struct raid6_calls raid6_intx32;
+extern const struct raid6_calls raid6_mmxx1;
+extern const struct raid6_calls raid6_mmxx2;
+extern const struct raid6_calls raid6_sse1x1;
+extern const struct raid6_calls raid6_sse1x2;
+extern const struct raid6_calls raid6_sse2x1;
+extern const struct raid6_calls raid6_sse2x2;
+extern const struct raid6_calls raid6_sse2x4;
+extern const struct raid6_calls raid6_altivec1;
+extern const struct raid6_calls raid6_altivec2;
+extern const struct raid6_calls raid6_altivec4;
+extern const struct raid6_calls raid6_altivec8;
+
+const struct raid6_calls * const raid6_algos[] = {
+        &raid6_intx1,
+        &raid6_intx2,
+        &raid6_intx4,
+        &raid6_intx8,
+#if defined(__ia64__)
+        &raid6_intx16,
+        &raid6_intx32,
+#endif
+#if defined(__i386__)
+        &raid6_mmxx1,
+        &raid6_mmxx2,
+        &raid6_sse1x1,
+        &raid6_sse1x2,
+        &raid6_sse2x1,
+        &raid6_sse2x2,
+#endif
+#if defined(__x86_64__)
+        &raid6_sse2x1,
+        &raid6_sse2x2,
+        &raid6_sse2x4,
+#endif
+#ifdef CONFIG_ALTIVEC
+        &raid6_altivec1,
+        &raid6_altivec2,
+        &raid6_altivec4,
+        &raid6_altivec8,
+#endif
+        NULL
+};
+
+#ifdef __KERNEL__
+#define RAID6_TIME_JIFFIES_LG2  4
+#else
+/* Need more time to be stable in userspace */
+#define RAID6_TIME_JIFFIES_LG2  9
+#endif
+
+/* Try to pick the best algorithm */
+/* This code uses the gfmul table as convenient data set to abuse */
+
+int __init raid6_select_algo(void)
+{
+        const struct raid6_calls * const * algo;
+        const struct raid6_calls * best;
+        char *syndromes;
+        void *dptrs[(65536/PAGE_SIZE)+2];
+        int i, disks;
+        unsigned long perf, bestperf;
+        int bestprefer;
+        unsigned long j0, j1;
+
+        disks = (65536/PAGE_SIZE)+2;
+        for ( i = 0 ; i < disks-2 ; i++ ) {
+                dptrs[i] = ((char *)raid6_gfmul) + PAGE_SIZE*i;
+        }
+
+        /* Normal code - use a 2-page allocation to avoid D$ conflict */
+        syndromes = (void *) __get_free_pages(GFP_KERNEL, 1);
+
+        if ( !syndromes ) {
+                printk("raid6: Yikes!  No memory available.\n");
+                return -ENOMEM;
+        }
+
+        dptrs[disks-2] = syndromes;
+        dptrs[disks-1] = syndromes + PAGE_SIZE;
+
+        bestperf = 0;  bestprefer = 0;  best = NULL;
+
+        for ( algo = raid6_algos ; *algo ; algo++ ) {
+                if ( !(*algo)->valid || (*algo)->valid() ) {
+                        perf = 0;
+
+                        preempt_disable();
+                        j0 = jiffies;
+                        while ( (j1 = jiffies) == j0 )
+                                cpu_relax();
+                        while ( (jiffies-j1) < (1 << RAID6_TIME_JIFFIES_LG2) ) {
+                                (*algo)->gen_syndrome(disks, PAGE_SIZE, dptrs);
+                                perf++;
+                        }
+                        preempt_enable();
+
+                        if ( (*algo)->prefer > bestprefer ||
+                             ((*algo)->prefer == bestprefer &&
+                              perf > bestperf) ) {
+                                best = *algo;
+                                bestprefer = best->prefer;
+                                bestperf = perf;
+                        }
+                        printk("raid6: %-8s %5ld MB/s\n", (*algo)->name,
+                               (perf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2));
+                }
+        }
+
+        if ( best )
+                printk("raid6: using algorithm %s (%ld MB/s)\n",
+                       best->name,
+                       (bestperf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2));
+        else
+                printk("raid6: Yikes!  No algorithm found!\n");
+
+        raid6_call = *best;
+
+        free_pages((unsigned long)syndromes, 1);
+
+        return best ? 0 : -EINVAL;
+}
diff --git a/drivers/md/raid6altivec.uc b/drivers/md/raid6altivec.uc
new file mode 100644
index 000000000000..1de8f030eee0
--- /dev/null
+++ b/drivers/md/raid6altivec.uc
@@ -0,0 +1,122 @@
+/* -*- linux-c -*- ------------------------------------------------------- *
+ *
+ *   Copyright 2002-2004 H. Peter Anvin - All Rights Reserved
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+ *   Bostom MA 02111-1307, USA; either version 2 of the License, or
+ *   (at your option) any later version; incorporated herein by reference.
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * raid6altivec$#.c
+ *
+ * $#-way unrolled portable integer math RAID-6 instruction set
+ *
+ * This file is postprocessed using unroll.pl
+ *
+ * <benh> hpa: in process,
+ * you can just "steal" the vec unit with enable_kernel_altivec() (but
+ * bracked this with preempt_disable/enable or in a lock)
+ */
+
+#include "raid6.h"
+
+#ifdef CONFIG_ALTIVEC
+
+#include <altivec.h>
+#include <asm/system.h>
+#include <asm/cputable.h>
+
+/*
+ * This is the C data type to use
+ */
+
+typedef vector unsigned char unative_t;
+
+#define NBYTES(x) ((vector unsigned char) {x,x,x,x, x,x,x,x, x,x,x,x, x,x,x,x})
+#define NSIZE   sizeof(unative_t)
+
+/*
+ * The SHLBYTE() operation shifts each byte left by 1, *not*
+ * rolling over into the next byte
+ */
+static inline __attribute_const__ unative_t SHLBYTE(unative_t v)
+{
+        return vec_add(v,v);
+}
+
+/*
+ * The MASK() operation returns 0xFF in any byte for which the high
+ * bit is 1, 0x00 for any byte for which the high bit is 0.
+ */
+static inline __attribute_const__ unative_t MASK(unative_t v)
+{
+        unative_t zv = NBYTES(0);
+
+        /* vec_cmpgt returns a vector bool char; thus the need for the cast */
+        return (unative_t)vec_cmpgt(zv, v);
+}
+
+
+/* This is noinline to make damned sure that gcc doesn't move any of the
+   Altivec code around the enable/disable code */
+static void noinline
+raid6_altivec$#_gen_syndrome_real(int disks, size_t bytes, void **ptrs)
+{
+        u8 **dptr = (u8 **)ptrs;
+        u8 *p, *q;
+        int d, z, z0;
+
+        unative_t wd$$, wq$$, wp$$, w1$$, w2$$;
+        unative_t x1d = NBYTES(0x1d);
+
+        z0 = disks - 3;         /* Highest data disk */
+        p = dptr[z0+1];         /* XOR parity */
+        q = dptr[z0+2];         /* RS syndrome */
+
+        for ( d = 0 ; d < bytes ; d += NSIZE*$# ) {
+                wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE];
+                for ( z = z0-1 ; z >= 0 ; z-- ) {
+                        wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
+                        wp$$ = vec_xor(wp$$, wd$$);
+                        w2$$ = MASK(wq$$);
+                        w1$$ = SHLBYTE(wq$$);
+                        w2$$ = vec_and(w2$$, x1d);
+                        w1$$ = vec_xor(w1$$, w2$$);
+                        wq$$ = vec_xor(w1$$, wd$$);
+                }
+                *(unative_t *)&p[d+NSIZE*$$] = wp$$;
+                *(unative_t *)&q[d+NSIZE*$$] = wq$$;
+        }
+}
+
+static void raid6_altivec$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+        preempt_disable();
+        enable_kernel_altivec();
+
+        raid6_altivec$#_gen_syndrome_real(disks, bytes, ptrs);
+
+        preempt_enable();
+}
+
+int raid6_have_altivec(void);
+#if $# == 1
+int raid6_have_altivec(void)
+{
+        /* This assumes either all CPUs have Altivec or none does */
+        return cpu_has_feature(CPU_FTR_ALTIVEC);
+}
+#endif
+
+const struct raid6_calls raid6_altivec$# = {
+        raid6_altivec$#_gen_syndrome,
+        raid6_have_altivec,
+        "altivecx$#",
+        0
+};
+
+#endif /* CONFIG_ALTIVEC */
diff --git a/drivers/md/raid6int.uc b/drivers/md/raid6int.uc
new file mode 100644
index 000000000000..ad004cee0e26
--- /dev/null
+++ b/drivers/md/raid6int.uc
@@ -0,0 +1,117 @@
+/* -*- linux-c -*- ------------------------------------------------------- *
+ *
+ *   Copyright 2002-2004 H. Peter Anvin - All Rights Reserved
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+ *   Bostom MA 02111-1307, USA; either version 2 of the License, or
+ *   (at your option) any later version; incorporated herein by reference.
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * raid6int$#.c
+ *
+ * $#-way unrolled portable integer math RAID-6 instruction set
+ *
+ * This file is postprocessed using unroll.pl
+ */
+
+#include "raid6.h"
+
+/*
+ * This is the C data type to use
+ */
+
+/* Change this from BITS_PER_LONG if there is something better... */
+#if BITS_PER_LONG == 64
+# define NBYTES(x) ((x) * 0x0101010101010101UL)
+# define NSIZE  8
+# define NSHIFT 3
+# define NSTRING "64"
+typedef u64 unative_t;
+#else
+# define NBYTES(x) ((x) * 0x01010101U)
+# define NSIZE  4
+# define NSHIFT 2
+# define NSTRING "32"
+typedef u32 unative_t;
+#endif
+
+
+
+/*
+ * IA-64 wants insane amounts of unrolling.  On other architectures that
+ * is just a waste of space.
+ */
+#if ($# <= 8) || defined(__ia64__)
+
+
+/*
+ * These sub-operations are separate inlines since they can sometimes be
+ * specially optimized using architecture-specific hacks.
+ */
+
+/*
+ * The SHLBYTE() operation shifts each byte left by 1, *not*
+ * rolling over into the next byte
+ */
+static inline __attribute_const__ unative_t SHLBYTE(unative_t v)
+{
+        unative_t vv;
+
+        vv = (v << 1) & NBYTES(0xfe);
+        return vv;
+}
+
+/*
+ * The MASK() operation returns 0xFF in any byte for which the high
+ * bit is 1, 0x00 for any byte for which the high bit is 0.
+ */
+static inline __attribute_const__ unative_t MASK(unative_t v)
+{
+        unative_t vv;
+
+        vv = v & NBYTES(0x80);
+        vv = (vv << 1) - (vv >> 7); /* Overflow on the top bit is OK */
+        return vv;
+}
+
+
+static void raid6_int$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+        u8 **dptr = (u8 **)ptrs;
+        u8 *p, *q;
+        int d, z, z0;
+
+        unative_t wd$$, wq$$, wp$$, w1$$, w2$$;
+
+        z0 = disks - 3;         /* Highest data disk */
+        p = dptr[z0+1];         /* XOR parity */
+        q = dptr[z0+2];         /* RS syndrome */
+
+        for ( d = 0 ; d < bytes ; d += NSIZE*$# ) {
+                wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE];
+                for ( z = z0-1 ; z >= 0 ; z-- ) {
+                        wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
+                        wp$$ ^= wd$$;
+                        w2$$ = MASK(wq$$);
+                        w1$$ = SHLBYTE(wq$$);
+                        w2$$ &= NBYTES(0x1d);
+                        w1$$ ^= w2$$;
+                        wq$$ = w1$$ ^ wd$$;
+                }
+                *(unative_t *)&p[d+NSIZE*$$] = wp$$;
+                *(unative_t *)&q[d+NSIZE*$$] = wq$$;
+        }
+}
+
+const struct raid6_calls raid6_intx$# = {
+        raid6_int$#_gen_syndrome,
+        NULL,           /* always valid */
+        "int" NSTRING "x$#",
+        0
+};
+
+#endif
diff --git a/drivers/md/raid6main.c b/drivers/md/raid6main.c
new file mode 100644
index 000000000000..7e30ab29691a
--- /dev/null
+++ b/drivers/md/raid6main.c
@@ -0,0 +1,2136 @@
+/*
+ * raid6main.c : Multiple Devices driver for Linux
+ *         Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
+ *         Copyright (C) 1999, 2000 Ingo Molnar
+ *         Copyright (C) 2002, 2003 H. Peter Anvin
+ *
+ * RAID-6 management functions.  This code is derived from raid5.c.
+ * Last merge from raid5.c bkcvs version 1.79 (kernel 2.6.1).
+ *
+ * Thanks to Penguin Computing for making the RAID-6 development possible
+ * by donating a test server!
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * (for example /usr/src/linux/COPYING); if not, write to the Free
+ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+
+#include <linux/config.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/highmem.h>
+#include <linux/bitops.h>
+#include <asm/atomic.h>
+#include "raid6.h"
+
+/*
+ * Stripe cache
+ */
+
+#define NR_STRIPES              256
+#define STRIPE_SIZE             PAGE_SIZE
+#define STRIPE_SHIFT            (PAGE_SHIFT - 9)
+#define STRIPE_SECTORS          (STRIPE_SIZE>>9)
+#define IO_THRESHOLD            1
+#define HASH_PAGES              1
+#define HASH_PAGES_ORDER        0
+#define NR_HASH                 (HASH_PAGES * PAGE_SIZE / sizeof(struct stripe_head *))
+#define HASH_MASK               (NR_HASH - 1)
+
+#define stripe_hash(conf, sect) ((conf)->stripe_hashtbl[((sect) >> STRIPE_SHIFT) & HASH_MASK])
+
+/* bio's attached to a stripe+device for I/O are linked together in bi_sector
+ * order without overlap.  There may be several bio's per stripe+device, and
+ * a bio could span several devices.
+ * When walking this list for a particular stripe+device, we must never proceed
+ * beyond a bio that extends past this device, as the next bio might no longer
+ * be valid.
+ * This macro is used to determine the 'next' bio in the list, given the sector
+ * of the current stripe+device
+ */
+#define r5_next_bio(bio, sect) ( ( (bio)->bi_sector + ((bio)->bi_size>>9) < sect + STRIPE_SECTORS) ? (bio)->bi_next : NULL)
+/*
+ * The following can be used to debug the driver
+ */
+#define RAID6_DEBUG     0       /* Extremely verbose printk */
+#define RAID6_PARANOIA  1       /* Check spinlocks */
+#define RAID6_DUMPSTATE 0       /* Include stripe cache state in /proc/mdstat */
+#if RAID6_PARANOIA && defined(CONFIG_SMP)
+# define CHECK_DEVLOCK() assert_spin_locked(&conf->device_lock)
+#else
+# define CHECK_DEVLOCK()
+#endif
+
+#define PRINTK(x...) ((void)(RAID6_DEBUG && printk(KERN_DEBUG x)))
+#if RAID6_DEBUG
+#undef inline
+#undef __inline__
+#define inline
+#define __inline__
+#endif
+
+#if !RAID6_USE_EMPTY_ZERO_PAGE
+/* In .bss so it's zeroed */
+const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(256)));
+#endif
+
+static inline int raid6_next_disk(int disk, int raid_disks)
+{
+        disk++;
+        return (disk < raid_disks) ? disk : 0;
+}
+
+static void print_raid6_conf (raid6_conf_t *conf);
+
+static inline void __release_stripe(raid6_conf_t *conf, struct stripe_head *sh)
+{
+        if (atomic_dec_and_test(&sh->count)) {
+                if (!list_empty(&sh->lru))
+                        BUG();
+                if (atomic_read(&conf->active_stripes)==0)
+                        BUG();
+                if (test_bit(STRIPE_HANDLE, &sh->state)) {
+                        if (test_bit(STRIPE_DELAYED, &sh->state))
+                                list_add_tail(&sh->lru, &conf->delayed_list);
+                        else
+                                list_add_tail(&sh->lru, &conf->handle_list);
+                        md_wakeup_thread(conf->mddev->thread);
+                } else {
+                        if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
+                                atomic_dec(&conf->preread_active_stripes);
+                                if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD)
+                                        md_wakeup_thread(conf->mddev->thread);
+                        }
+                        list_add_tail(&sh->lru, &conf->inactive_list);
+                        atomic_dec(&conf->active_stripes);
+                        if (!conf->inactive_blocked ||
+                            atomic_read(&conf->active_stripes) < (NR_STRIPES*3/4))
+                                wake_up(&conf->wait_for_stripe);
+                }
+        }
+}
+static void release_stripe(struct stripe_head *sh)
+{
+        raid6_conf_t *conf = sh->raid_conf;
+        unsigned long flags;
+
+        spin_lock_irqsave(&conf->device_lock, flags);
+        __release_stripe(conf, sh);
+        spin_unlock_irqrestore(&conf->device_lock, flags);
+}
+
+static void remove_hash(struct stripe_head *sh)
+{
+        PRINTK("remove_hash(), stripe %llu\n", (unsigned long long)sh->sector);
+
+        if (sh->hash_pprev) {
+                if (sh->hash_next)
+                        sh->hash_next->hash_pprev = sh->hash_pprev;
+                *sh->hash_pprev = sh->hash_next;
+                sh->hash_pprev = NULL;
+        }
+}
+
+static __inline__ void insert_hash(raid6_conf_t *conf, struct stripe_head *sh)
+{
+        struct stripe_head **shp = &stripe_hash(conf, sh->sector);
+
+        PRINTK("insert_hash(), stripe %llu\n", (unsigned long long)sh->sector);
+
+        CHECK_DEVLOCK();
+        if ((sh->hash_next = *shp) != NULL)
+                (*shp)->hash_pprev = &sh->hash_next;
+        *shp = sh;
+        sh->hash_pprev = shp;
+}
+
+
+/* find an idle stripe, make sure it is unhashed, and return it. */
+static struct stripe_head *get_free_stripe(raid6_conf_t *conf)
+{
+        struct stripe_head *sh = NULL;
+        struct list_head *first;
+
+        CHECK_DEVLOCK();
+        if (list_empty(&conf->inactive_list))
+                goto out;
+        first = conf->inactive_list.next;
+        sh = list_entry(first, struct stripe_head, lru);
+        list_del_init(first);
+        remove_hash(sh);
+        atomic_inc(&conf->active_stripes);
+out:
+        return sh;
+}
+
+static void shrink_buffers(struct stripe_head *sh, int num)
+{
+        struct page *p;
+        int i;
+
+        for (i=0; i<num ; i++) {
+                p = sh->dev[i].page;
+                if (!p)
+                        continue;
+                sh->dev[i].page = NULL;
+                page_cache_release(p);
+        }
+}
+
+static int grow_buffers(struct stripe_head *sh, int num)
+{
+        int i;
+
+        for (i=0; i<num; i++) {
+                struct page *page;
+
+                if (!(page = alloc_page(GFP_KERNEL))) {
+                        return 1;
+                }
+                sh->dev[i].page = page;
+        }
+        return 0;
+}
+
+static void raid6_build_block (struct stripe_head *sh, int i);
+
+static inline void init_stripe(struct stripe_head *sh, sector_t sector, int pd_idx)
+{
+        raid6_conf_t *conf = sh->raid_conf;
+        int disks = conf->raid_disks, i;
+
+        if (atomic_read(&sh->count) != 0)
+                BUG();
+        if (test_bit(STRIPE_HANDLE, &sh->state))
+                BUG();
+
+        CHECK_DEVLOCK();
+        PRINTK("init_stripe called, stripe %llu\n",
+                (unsigned long long)sh->sector);
+
+        remove_hash(sh);
+
+        sh->sector = sector;
+        sh->pd_idx = pd_idx;
+        sh->state = 0;
+
+        for (i=disks; i--; ) {
+                struct r5dev *dev = &sh->dev[i];
+
+                if (dev->toread || dev->towrite || dev->written ||
+                    test_bit(R5_LOCKED, &dev->flags)) {
+                        PRINTK("sector=%llx i=%d %p %p %p %d\n",
+                               (unsigned long long)sh->sector, i, dev->toread,
+                               dev->towrite, dev->written,
+                               test_bit(R5_LOCKED, &dev->flags));
+                        BUG();
+                }
+                dev->flags = 0;
+                raid6_build_block(sh, i);
+        }
+        insert_hash(conf, sh);
+}
+
+static struct stripe_head *__find_stripe(raid6_conf_t *conf, sector_t sector)
+{
+        struct stripe_head *sh;
+
+        CHECK_DEVLOCK();
+        PRINTK("__find_stripe, sector %llu\n", (unsigned long long)sector);
+        for (sh = stripe_hash(conf, sector); sh; sh = sh->hash_next)
+                if (sh->sector == sector)
+                        return sh;
+        PRINTK("__stripe %llu not in cache\n", (unsigned long long)sector);
+        return NULL;
+}
+
+static void unplug_slaves(mddev_t *mddev);
+
+static struct stripe_head *get_active_stripe(raid6_conf_t *conf, sector_t sector,
+                                             int pd_idx, int noblock)
+{
+        struct stripe_head *sh;
+
+        PRINTK("get_stripe, sector %llu\n", (unsigned long long)sector);
+
+        spin_lock_irq(&conf->device_lock);
+
+        do {
+                sh = __find_stripe(conf, sector);
+                if (!sh) {
+                        if (!conf->inactive_blocked)
+                                sh = get_free_stripe(conf);
+                        if (noblock && sh == NULL)
+                                break;
+                        if (!sh) {
+                                conf->inactive_blocked = 1;
+                                wait_event_lock_irq(conf->wait_for_stripe,
+                                                    !list_empty(&conf->inactive_list) &&
+                                                    (atomic_read(&conf->active_stripes) < (NR_STRIPES *3/4)
+                                                     || !conf->inactive_blocked),
+                                                    conf->device_lock,
+                                                    unplug_slaves(conf->mddev);
+                                        );
+                                conf->inactive_blocked = 0;
+                        } else
+                                init_stripe(sh, sector, pd_idx);
+                } else {
+                        if (atomic_read(&sh->count)) {
+                                if (!list_empty(&sh->lru))
+                                        BUG();
+                        } else {
+                                if (!test_bit(STRIPE_HANDLE, &sh->state))
+                                        atomic_inc(&conf->active_stripes);
+                                if (list_empty(&sh->lru))
+                                        BUG();
+                                list_del_init(&sh->lru);
+                        }
+                }
+        } while (sh == NULL);
+
+        if (sh)
+                atomic_inc(&sh->count);
+
+        spin_unlock_irq(&conf->device_lock);
+        return sh;
+}
+
+static int grow_stripes(raid6_conf_t *conf, int num)
+{
+        struct stripe_head *sh;
+        kmem_cache_t *sc;
+        int devs = conf->raid_disks;
+
+        sprintf(conf->cache_name, "raid6/%s", mdname(conf->mddev));
+
+        sc = kmem_cache_create(conf->cache_name,
+                               sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
+                               0, 0, NULL, NULL);
+        if (!sc)
+                return 1;
+        conf->slab_cache = sc;
+        while (num--) {
+                sh = kmem_cache_alloc(sc, GFP_KERNEL);
+                if (!sh)
+                        return 1;
+                memset(sh, 0, sizeof(*sh) + (devs-1)*sizeof(struct r5dev));
+                sh->raid_conf = conf;
+                spin_lock_init(&sh->lock);
+
+                if (grow_buffers(sh, conf->raid_disks)) {
+                        shrink_buffers(sh, conf->raid_disks);
+                        kmem_cache_free(sc, sh);
+                        return 1;
+                }
+                /* we just created an active stripe so... */
+                atomic_set(&sh->count, 1);
+                atomic_inc(&conf->active_stripes);
+                INIT_LIST_HEAD(&sh->lru);
+                release_stripe(sh);
+        }
+        return 0;
+}
+
+static void shrink_stripes(raid6_conf_t *conf)
+{
+        struct stripe_head *sh;
+
+        while (1) {
+                spin_lock_irq(&conf->device_lock);
+                sh = get_free_stripe(conf);
+                spin_unlock_irq(&conf->device_lock);
+                if (!sh)
+                        break;
+                if (atomic_read(&sh->count))
+                        BUG();
+                shrink_buffers(sh, conf->raid_disks);
+                kmem_cache_free(conf->slab_cache, sh);
+                atomic_dec(&conf->active_stripes);
+        }
+        kmem_cache_destroy(conf->slab_cache);
+        conf->slab_cache = NULL;
+}
+
+static int raid6_end_read_request (struct bio * bi, unsigned int bytes_done,
+                                   int error)
+{
+        struct stripe_head *sh = bi->bi_private;
+        raid6_conf_t *conf = sh->raid_conf;
+        int disks = conf->raid_disks, i;
+        int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
+
+        if (bi->bi_size)
+                return 1;
+
+        for (i=0 ; i<disks; i++)
+                if (bi == &sh->dev[i].req)
+                        break;
+
+        PRINTK("end_read_request %llu/%d, count: %d, uptodate %d.\n",
+                (unsigned long long)sh->sector, i, atomic_read(&sh->count),
+                uptodate);
+        if (i == disks) {
+                BUG();
+                return 0;
+        }
+
+        if (uptodate) {
+#if 0
+                struct bio *bio;
+                unsigned long flags;
+                spin_lock_irqsave(&conf->device_lock, flags);
+                /* we can return a buffer if we bypassed the cache or
+                 * if the top buffer is not in highmem.  If there are
+                 * multiple buffers, leave the extra work to
+                 * handle_stripe
+                 */
+                buffer = sh->bh_read[i];
+                if (buffer &&
+                    (!PageHighMem(buffer->b_page)
+                     || buffer->b_page == bh->b_page )
+                        ) {
+                        sh->bh_read[i] = buffer->b_reqnext;
+                        buffer->b_reqnext = NULL;
+                } else
+                        buffer = NULL;
+                spin_unlock_irqrestore(&conf->device_lock, flags);
+                if (sh->bh_page[i]==bh->b_page)
+                        set_buffer_uptodate(bh);
+                if (buffer) {
+                        if (buffer->b_page != bh->b_page)
+                                memcpy(buffer->b_data, bh->b_data, bh->b_size);
+                        buffer->b_end_io(buffer, 1);
+                }
+#else
+                set_bit(R5_UPTODATE, &sh->dev[i].flags);
+#endif
+        } else {
+                md_error(conf->mddev, conf->disks[i].rdev);
+                clear_bit(R5_UPTODATE, &sh->dev[i].flags);
+        }
+        rdev_dec_pending(conf->disks[i].rdev, conf->mddev);
+#if 0
+        /* must restore b_page before unlocking buffer... */
+        if (sh->bh_page[i] != bh->b_page) {
+                bh->b_page = sh->bh_page[i];
+                bh->b_data = page_address(bh->b_page);
+                clear_buffer_uptodate(bh);
+        }
+#endif
+        clear_bit(R5_LOCKED, &sh->dev[i].flags);
+        set_bit(STRIPE_HANDLE, &sh->state);
+        release_stripe(sh);
+        return 0;
+}
+
+static int raid6_end_write_request (struct bio *bi, unsigned int bytes_done,
+                                    int error)
+{
+        struct stripe_head *sh = bi->bi_private;
+        raid6_conf_t *conf = sh->raid_conf;
+        int disks = conf->raid_disks, i;
+        unsigned long flags;
+        int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
+
+        if (bi->bi_size)
+                return 1;
+
+        for (i=0 ; i<disks; i++)
+                if (bi == &sh->dev[i].req)
+                        break;
+
+        PRINTK("end_write_request %llu/%d, count %d, uptodate: %d.\n",
+                (unsigned long long)sh->sector, i, atomic_read(&sh->count),
+                uptodate);
+        if (i == disks) {
+                BUG();
+                return 0;
+        }
+
+        spin_lock_irqsave(&conf->device_lock, flags);
+        if (!uptodate)
+                md_error(conf->mddev, conf->disks[i].rdev);
+
+        rdev_dec_pending(conf->disks[i].rdev, conf->mddev);
+
+        clear_bit(R5_LOCKED, &sh->dev[i].flags);
+        set_bit(STRIPE_HANDLE, &sh->state);
+        __release_stripe(conf, sh);
+        spin_unlock_irqrestore(&conf->device_lock, flags);
+        return 0;
+}
+
+
+static sector_t compute_blocknr(struct stripe_head *sh, int i);
+
+static void raid6_build_block (struct stripe_head *sh, int i)
+{
+        struct r5dev *dev = &sh->dev[i];
+        int pd_idx = sh->pd_idx;
+        int qd_idx = raid6_next_disk(pd_idx, sh->raid_conf->raid_disks);
+
+        bio_init(&dev->req);
+        dev->req.bi_io_vec = &dev->vec;
+        dev->req.bi_vcnt++;
+        dev->req.bi_max_vecs++;
+        dev->vec.bv_page = dev->page;
+        dev->vec.bv_len = STRIPE_SIZE;
+        dev->vec.bv_offset = 0;
+
+        dev->req.bi_sector = sh->sector;
+        dev->req.bi_private = sh;
+
+        dev->flags = 0;
+        if (i != pd_idx && i != qd_idx)
+                dev->sector = compute_blocknr(sh, i);
+}
+
+static void error(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+        char b[BDEVNAME_SIZE];
+        raid6_conf_t *conf = (raid6_conf_t *) mddev->private;
+        PRINTK("raid6: error called\n");
+
+        if (!rdev->faulty) {
+                mddev->sb_dirty = 1;
+                if (rdev->in_sync) {
+                        conf->working_disks--;
+                        mddev->degraded++;
+                        conf->failed_disks++;
+                        rdev->in_sync = 0;
+                        /*
+                         * if recovery was running, make sure it aborts.
+                         */
+                        set_bit(MD_RECOVERY_ERR, &mddev->recovery);
+                }
+                rdev->faulty = 1;
+                printk (KERN_ALERT
+                        "raid6: Disk failure on %s, disabling device."
+                        " Operation continuing on %d devices\n",
+                        bdevname(rdev->bdev,b), conf->working_disks);
+        }
+}
+
+/*
+ * Input: a 'big' sector number,
+ * Output: index of the data and parity disk, and the sector # in them.
+ */
+static sector_t raid6_compute_sector(sector_t r_sector, unsigned int raid_disks,
+                        unsigned int data_disks, unsigned int * dd_idx,
+                        unsigned int * pd_idx, raid6_conf_t *conf)
+{
+        long stripe;
+        unsigned long chunk_number;
+        unsigned int chunk_offset;
+        sector_t new_sector;
+        int sectors_per_chunk = conf->chunk_size >> 9;
+
+        /* First compute the information on this sector */
+
+        /*
+         * Compute the chunk number and the sector offset inside the chunk
+         */
+        chunk_offset = sector_div(r_sector, sectors_per_chunk);
+        chunk_number = r_sector;
+        if ( r_sector != chunk_number ) {
+                printk(KERN_CRIT "raid6: ERROR: r_sector = %llu, chunk_number = %lu\n",
+                       (unsigned long long)r_sector, (unsigned long)chunk_number);
+                BUG();
+        }
+
+        /*
+         * Compute the stripe number
+         */
+        stripe = chunk_number / data_disks;
+
+        /*
+         * Compute the data disk and parity disk indexes inside the stripe
+         */
+        *dd_idx = chunk_number % data_disks;
+
+        /*
+         * Select the parity disk based on the user selected algorithm.
+         */
+
+        /**** FIX THIS ****/
+        switch (conf->algorithm) {
+        case ALGORITHM_LEFT_ASYMMETRIC:
+                *pd_idx = raid_disks - 1 - (stripe % raid_disks);
+                if (*pd_idx == raid_disks-1)
+                        (*dd_idx)++;    /* Q D D D P */
+                else if (*dd_idx >= *pd_idx)
+                        (*dd_idx) += 2; /* D D P Q D */
+                break;
+        case ALGORITHM_RIGHT_ASYMMETRIC:
+                *pd_idx = stripe % raid_disks;
+                if (*pd_idx == raid_disks-1)
+                        (*dd_idx)++;    /* Q D D D P */
+                else if (*dd_idx >= *pd_idx)
+                        (*dd_idx) += 2; /* D D P Q D */
+                break;
+        case ALGORITHM_LEFT_SYMMETRIC:
+                *pd_idx = raid_disks - 1 - (stripe % raid_disks);
+                *dd_idx = (*pd_idx + 2 + *dd_idx) % raid_disks;
+                break;
+        case ALGORITHM_RIGHT_SYMMETRIC:
+                *pd_idx = stripe % raid_disks;
+                *dd_idx = (*pd_idx + 2 + *dd_idx) % raid_disks;
+                break;
+        default:
+                printk (KERN_CRIT "raid6: unsupported algorithm %d\n",
+                        conf->algorithm);
+        }
+
+        PRINTK("raid6: chunk_number = %lu, pd_idx = %u, dd_idx = %u\n",
+               chunk_number, *pd_idx, *dd_idx);
+
+        /*
+         * Finally, compute the new sector number
+         */
+        new_sector = (sector_t) stripe * sectors_per_chunk + chunk_offset;
+        return new_sector;
+}
+
+
+static sector_t compute_blocknr(struct stripe_head *sh, int i)
+{
+        raid6_conf_t *conf = sh->raid_conf;
+        int raid_disks = conf->raid_disks, data_disks = raid_disks - 2;
+        sector_t new_sector = sh->sector, check;
+        int sectors_per_chunk = conf->chunk_size >> 9;
+        sector_t stripe;
+        int chunk_offset;
+        int chunk_number, dummy1, dummy2, dd_idx = i;
+        sector_t r_sector;
+        int i0 = i;
+
+        chunk_offset = sector_div(new_sector, sectors_per_chunk);
+        stripe = new_sector;
+        if ( new_sector != stripe ) {
+                printk(KERN_CRIT "raid6: ERROR: new_sector = %llu, stripe = %lu\n",
+                       (unsigned long long)new_sector, (unsigned long)stripe);
+                BUG();
+        }
+
+        switch (conf->algorithm) {
+                case ALGORITHM_LEFT_ASYMMETRIC:
+                case ALGORITHM_RIGHT_ASYMMETRIC:
+                        if (sh->pd_idx == raid_disks-1)
+                                i--;    /* Q D D D P */
+                        else if (i > sh->pd_idx)
+                                i -= 2; /* D D P Q D */
+                        break;
+                case ALGORITHM_LEFT_SYMMETRIC:
+                case ALGORITHM_RIGHT_SYMMETRIC:
+                        if (sh->pd_idx == raid_disks-1)
+                                i--; /* Q D D D P */
+                        else {
+                                /* D D P Q D */
+                                if (i < sh->pd_idx)
+                                        i += raid_disks;
+                                i -= (sh->pd_idx + 2);
+                        }
+                        break;
+                default:
+                        printk (KERN_CRIT "raid6: unsupported algorithm %d\n",
+                                conf->algorithm);
+        }
+
+        PRINTK("raid6: compute_blocknr: pd_idx = %u, i0 = %u, i = %u\n", sh->pd_idx, i0, i);
+
+        chunk_number = stripe * data_disks + i;
+        r_sector = (sector_t)chunk_number * sectors_per_chunk + chunk_offset;
+
+        check = raid6_compute_sector (r_sector, raid_disks, data_disks, &dummy1, &dummy2, conf);
+        if (check != sh->sector || dummy1 != dd_idx || dummy2 != sh->pd_idx) {
+                printk(KERN_CRIT "raid6: compute_blocknr: map not correct\n");
+                return 0;
+        }
+        return r_sector;
+}
+
+
+
+/*
+ * Copy data between a page in the stripe cache, and one or more bion
+ * The page could align with the middle of the bio, or there could be
+ * several bion, each with several bio_vecs, which cover part of the page
+ * Multiple bion are linked together on bi_next.  There may be extras
+ * at the end of this list.  We ignore them.
+ */
+static void copy_data(int frombio, struct bio *bio,
+                     struct page *page,
+                     sector_t sector)
+{
+        char *pa = page_address(page);
+        struct bio_vec *bvl;
+        int i;
+        int page_offset;
+
+        if (bio->bi_sector >= sector)
+                page_offset = (signed)(bio->bi_sector - sector) * 512;
+        else
+                page_offset = (signed)(sector - bio->bi_sector) * -512;
+        bio_for_each_segment(bvl, bio, i) {
+                int len = bio_iovec_idx(bio,i)->bv_len;
+                int clen;
+                int b_offset = 0;
+
+                if (page_offset < 0) {
+                        b_offset = -page_offset;
+                        page_offset += b_offset;
+                        len -= b_offset;
+                }
+
+                if (len > 0 && page_offset + len > STRIPE_SIZE)
+                        clen = STRIPE_SIZE - page_offset;
+                else clen = len;
+
+                if (clen > 0) {
+                        char *ba = __bio_kmap_atomic(bio, i, KM_USER0);
+                        if (frombio)
+                                memcpy(pa+page_offset, ba+b_offset, clen);
+                        else
+                                memcpy(ba+b_offset, pa+page_offset, clen);
+                        __bio_kunmap_atomic(ba, KM_USER0);
+                }
+                if (clen < len) /* hit end of page */
+                        break;
+                page_offset +=  len;
+        }
+}
+
+#define check_xor()     do {                                            \
+                           if (count == MAX_XOR_BLOCKS) {               \
+                                xor_block(count, STRIPE_SIZE, ptr);     \
+                                count = 1;                              \
+                           }                                            \
+                        } while(0)
+
+/* Compute P and Q syndromes */
+static void compute_parity(struct stripe_head *sh, int method)
+{
+        raid6_conf_t *conf = sh->raid_conf;
+        int i, pd_idx = sh->pd_idx, qd_idx, d0_idx, disks = conf->raid_disks, count;
+        struct bio *chosen;
+        /**** FIX THIS: This could be very bad if disks is close to 256 ****/
+        void *ptrs[disks];
+
+        qd_idx = raid6_next_disk(pd_idx, disks);
+        d0_idx = raid6_next_disk(qd_idx, disks);
+
+        PRINTK("compute_parity, stripe %llu, method %d\n",
+                (unsigned long long)sh->sector, method);
+
+        switch(method) {
+        case READ_MODIFY_WRITE:
+                BUG();          /* READ_MODIFY_WRITE N/A for RAID-6 */
+        case RECONSTRUCT_WRITE:
+                for (i= disks; i-- ;)
+                        if ( i != pd_idx && i != qd_idx && sh->dev[i].towrite ) {
+                                chosen = sh->dev[i].towrite;
+                                sh->dev[i].towrite = NULL;
+
+                                if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
+                                        wake_up(&conf->wait_for_overlap);
+
+                                if (sh->dev[i].written) BUG();
+                                sh->dev[i].written = chosen;
+                        }
+                break;
+        case CHECK_PARITY:
+                BUG();          /* Not implemented yet */
+        }
+
+        for (i = disks; i--;)
+                if (sh->dev[i].written) {
+                        sector_t sector = sh->dev[i].sector;
+                        struct bio *wbi = sh->dev[i].written;
+                        while (wbi && wbi->bi_sector < sector + STRIPE_SECTORS) {
+                                copy_data(1, wbi, sh->dev[i].page, sector);
+                                wbi = r5_next_bio(wbi, sector);
+                        }
+
+                        set_bit(R5_LOCKED, &sh->dev[i].flags);
+                        set_bit(R5_UPTODATE, &sh->dev[i].flags);
+                }
+
+//      switch(method) {
+//      case RECONSTRUCT_WRITE:
+//      case CHECK_PARITY:
+//      case UPDATE_PARITY:
+                /* Note that unlike RAID-5, the ordering of the disks matters greatly. */
+                /* FIX: Is this ordering of drives even remotely optimal? */
+                count = 0;
+                i = d0_idx;
+                do {
+                        ptrs[count++] = page_address(sh->dev[i].page);
+                        if (count <= disks-2 && !test_bit(R5_UPTODATE, &sh->dev[i].flags))
+                                printk("block %d/%d not uptodate on parity calc\n", i,count);
+                        i = raid6_next_disk(i, disks);
+                } while ( i != d0_idx );
+//              break;
+//      }
+
+        raid6_call.gen_syndrome(disks, STRIPE_SIZE, ptrs);
+
+        switch(method) {
+        case RECONSTRUCT_WRITE:
+                set_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
+                set_bit(R5_UPTODATE, &sh->dev[qd_idx].flags);
+                set_bit(R5_LOCKED,   &sh->dev[pd_idx].flags);
+                set_bit(R5_LOCKED,   &sh->dev[qd_idx].flags);
+                break;
+        case UPDATE_PARITY:
+                set_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
+                set_bit(R5_UPTODATE, &sh->dev[qd_idx].flags);
+                break;
+        }
+}
+
+/* Compute one missing block */
+static void compute_block_1(struct stripe_head *sh, int dd_idx)
+{
+        raid6_conf_t *conf = sh->raid_conf;
+        int i, count, disks = conf->raid_disks;
+        void *ptr[MAX_XOR_BLOCKS], *p;
+        int pd_idx = sh->pd_idx;
+        int qd_idx = raid6_next_disk(pd_idx, disks);
+
+        PRINTK("compute_block_1, stripe %llu, idx %d\n",
+                (unsigned long long)sh->sector, dd_idx);
+
+        if ( dd_idx == qd_idx ) {
+                /* We're actually computing the Q drive */
+                compute_parity(sh, UPDATE_PARITY);
+        } else {
+                ptr[0] = page_address(sh->dev[dd_idx].page);
+                memset(ptr[0], 0, STRIPE_SIZE);
+                count = 1;
+                for (i = disks ; i--; ) {
+                        if (i == dd_idx || i == qd_idx)
+                                continue;
+                        p = page_address(sh->dev[i].page);
+                        if (test_bit(R5_UPTODATE, &sh->dev[i].flags))
+                                ptr[count++] = p;
+                        else
+                                printk("compute_block() %d, stripe %llu, %d"
+                                       " not present\n", dd_idx,
+                                       (unsigned long long)sh->sector, i);
+
+                        check_xor();
+                }
+                if (count != 1)
+                        xor_block(count, STRIPE_SIZE, ptr);
+                set_bit(R5_UPTODATE, &sh->dev[dd_idx].flags);
+        }
+}
+
+/* Compute two missing blocks */
+static void compute_block_2(struct stripe_head *sh, int dd_idx1, int dd_idx2)
+{
+        raid6_conf_t *conf = sh->raid_conf;
+        int i, count, disks = conf->raid_disks;
+        int pd_idx = sh->pd_idx;
+        int qd_idx = raid6_next_disk(pd_idx, disks);
+        int d0_idx = raid6_next_disk(qd_idx, disks);
+        int faila, failb;
+
+        /* faila and failb are disk numbers relative to d0_idx */
+        /* pd_idx become disks-2 and qd_idx become disks-1 */
+        faila = (dd_idx1 < d0_idx) ? dd_idx1+(disks-d0_idx) : dd_idx1-d0_idx;
+        failb = (dd_idx2 < d0_idx) ? dd_idx2+(disks-d0_idx) : dd_idx2-d0_idx;
+
+        BUG_ON(faila == failb);
+        if ( failb < faila ) { int tmp = faila; faila = failb; failb = tmp; }
+
+        PRINTK("compute_block_2, stripe %llu, idx %d,%d (%d,%d)\n",
+               (unsigned long long)sh->sector, dd_idx1, dd_idx2, faila, failb);
+
+        if ( failb == disks-1 ) {
+                /* Q disk is one of the missing disks */
+                if ( faila == disks-2 ) {
+                        /* Missing P+Q, just recompute */
+                        compute_parity(sh, UPDATE_PARITY);
+                        return;
+                } else {
+                        /* We're missing D+Q; recompute D from P */
+                        compute_block_1(sh, (dd_idx1 == qd_idx) ? dd_idx2 : dd_idx1);
+                        compute_parity(sh, UPDATE_PARITY); /* Is this necessary? */
+                        return;
+                }
+        }
+
+        /* We're missing D+P or D+D; build pointer table */
+        {
+                /**** FIX THIS: This could be very bad if disks is close to 256 ****/
+                void *ptrs[disks];
+
+                count = 0;
+                i = d0_idx;
+                do {
+                        ptrs[count++] = page_address(sh->dev[i].page);
+                        i = raid6_next_disk(i, disks);
+                        if (i != dd_idx1 && i != dd_idx2 &&
+                            !test_bit(R5_UPTODATE, &sh->dev[i].flags))
+                                printk("compute_2 with missing block %d/%d\n", count, i);
+                } while ( i != d0_idx );
+
+                if ( failb == disks-2 ) {
+                        /* We're missing D+P. */
+                        raid6_datap_recov(disks, STRIPE_SIZE, faila, ptrs);
+                } else {
+                        /* We're missing D+D. */
+                        raid6_2data_recov(disks, STRIPE_SIZE, faila, failb, ptrs);
+                }
+
+                /* Both the above update both missing blocks */
+                set_bit(R5_UPTODATE, &sh->dev[dd_idx1].flags);
+                set_bit(R5_UPTODATE, &sh->dev[dd_idx2].flags);
+        }
+}
+
+
+/*
+ * Each stripe/dev can have one or more bion attached.
+ * toread/towrite point to the first in a chain.
+ * The bi_next chain must be in order.
+ */
+static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite)
+{
+        struct bio **bip;
+        raid6_conf_t *conf = sh->raid_conf;
+
+        PRINTK("adding bh b#%llu to stripe s#%llu\n",
+                (unsigned long long)bi->bi_sector,
+                (unsigned long long)sh->sector);
+
+
+        spin_lock(&sh->lock);
+        spin_lock_irq(&conf->device_lock);
+        if (forwrite)
+                bip = &sh->dev[dd_idx].towrite;
+        else
+                bip = &sh->dev[dd_idx].toread;
+        while (*bip && (*bip)->bi_sector < bi->bi_sector) {
+                if ((*bip)->bi_sector + ((*bip)->bi_size >> 9) > bi->bi_sector)
+                        goto overlap;
+                bip = &(*bip)->bi_next;
+        }
+        if (*bip && (*bip)->bi_sector < bi->bi_sector + ((bi->bi_size)>>9))
+                goto overlap;
+
+        if (*bip && bi->bi_next && (*bip) != bi->bi_next)
+                BUG();
+        if (*bip)
+                bi->bi_next = *bip;
+        *bip = bi;
+        bi->bi_phys_segments ++;
+        spin_unlock_irq(&conf->device_lock);
+        spin_unlock(&sh->lock);
+
+        PRINTK("added bi b#%llu to stripe s#%llu, disk %d.\n",
+                (unsigned long long)bi->bi_sector,
+                (unsigned long long)sh->sector, dd_idx);
+
+        if (forwrite) {
+                /* check if page is covered */
+                sector_t sector = sh->dev[dd_idx].sector;
+                for (bi=sh->dev[dd_idx].towrite;
+                     sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
+                             bi && bi->bi_sector <= sector;
+                     bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
+                        if (bi->bi_sector + (bi->bi_size>>9) >= sector)
+                                sector = bi->bi_sector + (bi->bi_size>>9);
+                }
+                if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
+                        set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags);
+        }
+        return 1;
+
+ overlap:
+        set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
+        spin_unlock_irq(&conf->device_lock);
+        spin_unlock(&sh->lock);
+        return 0;
+}
+
+
+/*
+ * handle_stripe - do things to a stripe.
+ *
+ * We lock the stripe and then examine the state of various bits
+ * to see what needs to be done.
+ * Possible results:
+ *    return some read request which now have data
+ *    return some write requests which are safely on disc
+ *    schedule a read on some buffers
+ *    schedule a write of some buffers
+ *    return confirmation of parity correctness
+ *
+ * Parity calculations are done inside the stripe lock
+ * buffers are taken off read_list or write_list, and bh_cache buffers
+ * get BH_Lock set before the stripe lock is released.
+ *
+ */
+
+static void handle_stripe(struct stripe_head *sh)
+{
+        raid6_conf_t *conf = sh->raid_conf;
+        int disks = conf->raid_disks;
+        struct bio *return_bi= NULL;
+        struct bio *bi;
+        int i;
+        int syncing;
+        int locked=0, uptodate=0, to_read=0, to_write=0, failed=0, written=0;
+        int non_overwrite = 0;
+        int failed_num[2] = {0, 0};
+        struct r5dev *dev, *pdev, *qdev;
+        int pd_idx = sh->pd_idx;
+        int qd_idx = raid6_next_disk(pd_idx, disks);
+        int p_failed, q_failed;
+
+        PRINTK("handling stripe %llu, state=%#lx cnt=%d, pd_idx=%d, qd_idx=%d\n",
+               (unsigned long long)sh->sector, sh->state, atomic_read(&sh->count),
+               pd_idx, qd_idx);
+
+        spin_lock(&sh->lock);
+        clear_bit(STRIPE_HANDLE, &sh->state);
+        clear_bit(STRIPE_DELAYED, &sh->state);
+
+        syncing = test_bit(STRIPE_SYNCING, &sh->state);
+        /* Now to look around and see what can be done */
+
+        for (i=disks; i--; ) {
+                mdk_rdev_t *rdev;
+                dev = &sh->dev[i];
+                clear_bit(R5_Insync, &dev->flags);
+                clear_bit(R5_Syncio, &dev->flags);
+
+                PRINTK("check %d: state 0x%lx read %p write %p written %p\n",
+                        i, dev->flags, dev->toread, dev->towrite, dev->written);
+                /* maybe we can reply to a read */
+                if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread) {
+                        struct bio *rbi, *rbi2;
+                        PRINTK("Return read for disc %d\n", i);
+                        spin_lock_irq(&conf->device_lock);
+                        rbi = dev->toread;
+                        dev->toread = NULL;
+                        if (test_and_clear_bit(R5_Overlap, &dev->flags))
+                                wake_up(&conf->wait_for_overlap);
+                        spin_unlock_irq(&conf->device_lock);
+                        while (rbi && rbi->bi_sector < dev->sector + STRIPE_SECTORS) {
+                                copy_data(0, rbi, dev->page, dev->sector);
+                                rbi2 = r5_next_bio(rbi, dev->sector);
+                                spin_lock_irq(&conf->device_lock);
+                                if (--rbi->bi_phys_segments == 0) {
+                                        rbi->bi_next = return_bi;
+                                        return_bi = rbi;
+                                }
+                                spin_unlock_irq(&conf->device_lock);
+                                rbi = rbi2;
+                        }
+                }
+
+                /* now count some things */
+                if (test_bit(R5_LOCKED, &dev->flags)) locked++;
+                if (test_bit(R5_UPTODATE, &dev->flags)) uptodate++;
+
+
+                if (dev->toread) to_read++;
+                if (dev->towrite) {
+                        to_write++;
+                        if (!test_bit(R5_OVERWRITE, &dev->flags))
+                                non_overwrite++;
+                }
+                if (dev->written) written++;
+                rdev = conf->disks[i].rdev; /* FIXME, should I be looking rdev */
+                if (!rdev || !rdev->in_sync) {
+                        if ( failed < 2 )
+                                failed_num[failed] = i;
+                        failed++;
+                } else
+                        set_bit(R5_Insync, &dev->flags);
+        }
+        PRINTK("locked=%d uptodate=%d to_read=%d"
+               " to_write=%d failed=%d failed_num=%d,%d\n",
+               locked, uptodate, to_read, to_write, failed,
+               failed_num[0], failed_num[1]);
+        /* check if the array has lost >2 devices and, if so, some requests might
+         * need to be failed
+         */
+        if (failed > 2 && to_read+to_write+written) {
+                spin_lock_irq(&conf->device_lock);
+                for (i=disks; i--; ) {
+                        /* fail all writes first */
+                        bi = sh->dev[i].towrite;
+                        sh->dev[i].towrite = NULL;
+                        if (bi) to_write--;
+
+                        if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
+                                wake_up(&conf->wait_for_overlap);
+
+                        while (bi && bi->bi_sector < sh->dev[i].sector + STRIPE_SECTORS){
+                                struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
+                                clear_bit(BIO_UPTODATE, &bi->bi_flags);
+                                if (--bi->bi_phys_segments == 0) {
+                                        md_write_end(conf->mddev);
+                                        bi->bi_next = return_bi;
+                                        return_bi = bi;
+                                }
+                                bi = nextbi;
+                        }
+                        /* and fail all 'written' */
+                        bi = sh->dev[i].written;
+                        sh->dev[i].written = NULL;
+                        while (bi && bi->bi_sector < sh->dev[i].sector + STRIPE_SECTORS) {
+                                struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
+                                clear_bit(BIO_UPTODATE, &bi->bi_flags);
+                                if (--bi->bi_phys_segments == 0) {
+                                        md_write_end(conf->mddev);
+                                        bi->bi_next = return_bi;
+                                        return_bi = bi;
+                                }
+                                bi = bi2;
+                        }
+
+                        /* fail any reads if this device is non-operational */
+                        if (!test_bit(R5_Insync, &sh->dev[i].flags)) {
+                                bi = sh->dev[i].toread;
+                                sh->dev[i].toread = NULL;
+                                if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
+                                        wake_up(&conf->wait_for_overlap);
+                                if (bi) to_read--;
+                                while (bi && bi->bi_sector < sh->dev[i].sector + STRIPE_SECTORS){
+                                        struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
+                                        clear_bit(BIO_UPTODATE, &bi->bi_flags);
+                                        if (--bi->bi_phys_segments == 0) {
+                                                bi->bi_next = return_bi;
+                                                return_bi = bi;
+                                        }
+                                        bi = nextbi;
+                                }
+                        }
+                }
+                spin_unlock_irq(&conf->device_lock);
+        }
+        if (failed > 2 && syncing) {
+                md_done_sync(conf->mddev, STRIPE_SECTORS,0);
+                clear_bit(STRIPE_SYNCING, &sh->state);
+                syncing = 0;
+        }
+
+        /*
+         * might be able to return some write requests if the parity blocks
+         * are safe, or on a failed drive
+         */
+        pdev = &sh->dev[pd_idx];
+        p_failed = (failed >= 1 && failed_num[0] == pd_idx)
+                || (failed >= 2 && failed_num[1] == pd_idx);
+        qdev = &sh->dev[qd_idx];
+        q_failed = (failed >= 1 && failed_num[0] == qd_idx)
+                || (failed >= 2 && failed_num[1] == qd_idx);
+
+        if ( written &&
+             ( p_failed || ((test_bit(R5_Insync, &pdev->flags)
+                             && !test_bit(R5_LOCKED, &pdev->flags)
+                             && test_bit(R5_UPTODATE, &pdev->flags))) ) &&
+             ( q_failed || ((test_bit(R5_Insync, &qdev->flags)
+                             && !test_bit(R5_LOCKED, &qdev->flags)
+                             && test_bit(R5_UPTODATE, &qdev->flags))) ) ) {
+                /* any written block on an uptodate or failed drive can be
+                 * returned.  Note that if we 'wrote' to a failed drive,
+                 * it will be UPTODATE, but never LOCKED, so we don't need
+                 * to test 'failed' directly.
+                 */
+                for (i=disks; i--; )
+                        if (sh->dev[i].written) {
+                                dev = &sh->dev[i];
+                                if (!test_bit(R5_LOCKED, &dev->flags) &&
+                                    test_bit(R5_UPTODATE, &dev->flags) ) {
+                                        /* We can return any write requests */
+                                        struct bio *wbi, *wbi2;
+                                        PRINTK("Return write for stripe %llu disc %d\n",
+                                               (unsigned long long)sh->sector, i);
+                                        spin_lock_irq(&conf->device_lock);
+                                        wbi = dev->written;
+                                        dev->written = NULL;
+                                        while (wbi && wbi->bi_sector < dev->sector + STRIPE_SECTORS) {
+                                                wbi2 = r5_next_bio(wbi, dev->sector);
+                                                if (--wbi->bi_phys_segments == 0) {
+                                                        md_write_end(conf->mddev);
+                                                        wbi->bi_next = return_bi;
+                                                        return_bi = wbi;
+                                                }
+                                                wbi = wbi2;
+                                        }
+                                        spin_unlock_irq(&conf->device_lock);
+                                }
+                        }
+        }
+
+        /* Now we might consider reading some blocks, either to check/generate
+         * parity, or to satisfy requests
+         * or to load a block that is being partially written.
+         */
+        if (to_read || non_overwrite || (to_write && failed) || (syncing && (uptodate < disks))) {
+                for (i=disks; i--;) {
+                        dev = &sh->dev[i];
+                        if (!test_bit(R5_LOCKED, &dev->flags) && !test_bit(R5_UPTODATE, &dev->flags) &&
+                            (dev->toread ||
+                             (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) ||
+                             syncing ||
+                             (failed >= 1 && (sh->dev[failed_num[0]].toread || to_write)) ||
+                             (failed >= 2 && (sh->dev[failed_num[1]].toread || to_write))
+                                    )
+                                ) {
+                                /* we would like to get this block, possibly
+                                 * by computing it, but we might not be able to
+                                 */
+                                if (uptodate == disks-1) {
+                                        PRINTK("Computing stripe %llu block %d\n",
+                                               (unsigned long long)sh->sector, i);
+                                        compute_block_1(sh, i);
+                                        uptodate++;
+                                } else if ( uptodate == disks-2 && failed >= 2 ) {
+                                        /* Computing 2-failure is *very* expensive; only do it if failed >= 2 */
+                                        int other;
+                                        for (other=disks; other--;) {
+                                                if ( other == i )
+                                                        continue;
+                                                if ( !test_bit(R5_UPTODATE, &sh->dev[other].flags) )
+                                                        break;
+                                        }
+                                        BUG_ON(other < 0);
+                                        PRINTK("Computing stripe %llu blocks %d,%d\n",
+                                               (unsigned long long)sh->sector, i, other);
+                                        compute_block_2(sh, i, other);
+                                        uptodate += 2;
+                                } else if (test_bit(R5_Insync, &dev->flags)) {
+                                        set_bit(R5_LOCKED, &dev->flags);
+                                        set_bit(R5_Wantread, &dev->flags);
+#if 0
+                                        /* if I am just reading this block and we don't have
+                                           a failed drive, or any pending writes then sidestep the cache */
+                                        if (sh->bh_read[i] && !sh->bh_read[i]->b_reqnext &&
+                                            ! syncing && !failed && !to_write) {
+                                                sh->bh_cache[i]->b_page =  sh->bh_read[i]->b_page;
+                                                sh->bh_cache[i]->b_data =  sh->bh_read[i]->b_data;
+                                        }
+#endif
+                                        locked++;
+                                        PRINTK("Reading block %d (sync=%d)\n",
+                                                i, syncing);
+                                        if (syncing)
+                                                md_sync_acct(conf->disks[i].rdev->bdev,
+                                                             STRIPE_SECTORS);
+                                }
+                        }
+                }
+                set_bit(STRIPE_HANDLE, &sh->state);
+        }
+
+        /* now to consider writing and what else, if anything should be read */
+        if (to_write) {
+                int rcw=0, must_compute=0;
+                for (i=disks ; i--;) {
+                        dev = &sh->dev[i];
+                        /* Would I have to read this buffer for reconstruct_write */
+                        if (!test_bit(R5_OVERWRITE, &dev->flags)
+                            && i != pd_idx && i != qd_idx
+                            && (!test_bit(R5_LOCKED, &dev->flags)
+#if 0
+                                || sh->bh_page[i] != bh->b_page
+#endif
+                                    ) &&
+                            !test_bit(R5_UPTODATE, &dev->flags)) {
+                                if (test_bit(R5_Insync, &dev->flags)) rcw++;
+                                else {
+                                        PRINTK("raid6: must_compute: disk %d flags=%#lx\n", i, dev->flags);
+                                        must_compute++;
+                                }
+                        }
+                }
+                PRINTK("for sector %llu, rcw=%d, must_compute=%d\n",
+                       (unsigned long long)sh->sector, rcw, must_compute);
+                set_bit(STRIPE_HANDLE, &sh->state);
+
+                if (rcw > 0)
+                        /* want reconstruct write, but need to get some data */
+                        for (i=disks; i--;) {
+                                dev = &sh->dev[i];
+                                if (!test_bit(R5_OVERWRITE, &dev->flags)
+                                    && !(failed == 0 && (i == pd_idx || i == qd_idx))
+                                    && !test_bit(R5_LOCKED, &dev->flags) && !test_bit(R5_UPTODATE, &dev->flags) &&
+                                    test_bit(R5_Insync, &dev->flags)) {
+                                        if (test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
+                                        {
+                                                PRINTK("Read_old stripe %llu block %d for Reconstruct\n",
+                                                       (unsigned long long)sh->sector, i);
+                                                set_bit(R5_LOCKED, &dev->flags);
+                                                set_bit(R5_Wantread, &dev->flags);
+                                                locked++;
+                                        } else {
+                                                PRINTK("Request delayed stripe %llu block %d for Reconstruct\n",
+                                                       (unsigned long long)sh->sector, i);
+                                                set_bit(STRIPE_DELAYED, &sh->state);
+                                                set_bit(STRIPE_HANDLE, &sh->state);
+                                        }
+                                }
+                        }
+                /* now if nothing is locked, and if we have enough data, we can start a write request */
+                if (locked == 0 && rcw == 0) {
+                        if ( must_compute > 0 ) {
+                                /* We have failed blocks and need to compute them */
+                                switch ( failed ) {
+                                case 0: BUG();
+                                case 1: compute_block_1(sh, failed_num[0]); break;
+                                case 2: compute_block_2(sh, failed_num[0], failed_num[1]); break;
+                                default: BUG(); /* This request should have been failed? */
+                                }
+                        }
+
+                        PRINTK("Computing parity for stripe %llu\n", (unsigned long long)sh->sector);
+                        compute_parity(sh, RECONSTRUCT_WRITE);
+                        /* now every locked buffer is ready to be written */
+                        for (i=disks; i--;)
+                                if (test_bit(R5_LOCKED, &sh->dev[i].flags)) {
+                                        PRINTK("Writing stripe %llu block %d\n",
+                                               (unsigned long long)sh->sector, i);
+                                        locked++;
+                                        set_bit(R5_Wantwrite, &sh->dev[i].flags);
+#if 0 /**** FIX: I don't understand the logic here... ****/
+                                        if (!test_bit(R5_Insync, &sh->dev[i].flags)
+                                            || ((i==pd_idx || i==qd_idx) && failed == 0)) /* FIX? */
+                                                set_bit(STRIPE_INSYNC, &sh->state);
+#endif
+                                }
+                        if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
+                                atomic_dec(&conf->preread_active_stripes);
+                                if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD)
+                                        md_wakeup_thread(conf->mddev->thread);
+                        }
+                }
+        }
+
+        /* maybe we need to check and possibly fix the parity for this stripe
+         * Any reads will already have been scheduled, so we just see if enough data
+         * is available
+         */
+        if (syncing && locked == 0 &&
+            !test_bit(STRIPE_INSYNC, &sh->state) && failed <= 2) {
+                set_bit(STRIPE_HANDLE, &sh->state);
+#if 0 /* RAID-6: Don't support CHECK PARITY yet */
+                if (failed == 0) {
+                        char *pagea;
+                        if (uptodate != disks)
+                                BUG();
+                        compute_parity(sh, CHECK_PARITY);
+                        uptodate--;
+                        pagea = page_address(sh->dev[pd_idx].page);
+                        if ((*(u32*)pagea) == 0 &&
+                            !memcmp(pagea, pagea+4, STRIPE_SIZE-4)) {
+                                /* parity is correct (on disc, not in buffer any more) */
+                                set_bit(STRIPE_INSYNC, &sh->state);
+                        }
+                }
+#endif
+                if (!test_bit(STRIPE_INSYNC, &sh->state)) {
+                        int failed_needupdate[2];
+                        struct r5dev *adev, *bdev;
+
+                        if ( failed < 1 )
+                                failed_num[0] = pd_idx;
+                        if ( failed < 2 )
+                                failed_num[1] = (failed_num[0] == qd_idx) ? pd_idx : qd_idx;
+
+                        failed_needupdate[0] = !test_bit(R5_UPTODATE, &sh->dev[failed_num[0]].flags);
+                        failed_needupdate[1] = !test_bit(R5_UPTODATE, &sh->dev[failed_num[1]].flags);
+
+                        PRINTK("sync: failed=%d num=%d,%d fnu=%u%u\n",
+                               failed, failed_num[0], failed_num[1], failed_needupdate[0], failed_needupdate[1]);
+
+#if 0  /* RAID-6: This code seems to require that CHECK_PARITY destroys the uptodateness of the parity */
+                        /* should be able to compute the missing block(s) and write to spare */
+                        if ( failed_needupdate[0] ^ failed_needupdate[1] ) {
+                                if (uptodate+1 != disks)
+                                        BUG();
+                                compute_block_1(sh, failed_needupdate[0] ? failed_num[0] : failed_num[1]);
+                                uptodate++;
+                        } else if ( failed_needupdate[0] & failed_needupdate[1] ) {
+                                if (uptodate+2 != disks)
+                                        BUG();
+                                compute_block_2(sh, failed_num[0], failed_num[1]);
+                                uptodate += 2;
+                        }
+#else
+                        compute_block_2(sh, failed_num[0], failed_num[1]);
+                        uptodate += failed_needupdate[0] + failed_needupdate[1];
+#endif
+
+                        if (uptodate != disks)
+                                BUG();
+
+                        PRINTK("Marking for sync stripe %llu blocks %d,%d\n",
+                               (unsigned long long)sh->sector, failed_num[0], failed_num[1]);
+
+                        /**** FIX: Should we really do both of these unconditionally? ****/
+                        adev = &sh->dev[failed_num[0]];
+                        locked += !test_bit(R5_LOCKED, &adev->flags);
+                        set_bit(R5_LOCKED, &adev->flags);
+                        set_bit(R5_Wantwrite, &adev->flags);
+                        bdev = &sh->dev[failed_num[1]];
+                        locked += !test_bit(R5_LOCKED, &bdev->flags);
+                        set_bit(R5_LOCKED, &bdev->flags);
+                        set_bit(R5_Wantwrite, &bdev->flags);
+
+                        set_bit(STRIPE_INSYNC, &sh->state);
+                        set_bit(R5_Syncio, &adev->flags);
+                        set_bit(R5_Syncio, &bdev->flags);
+                }
+        }
+        if (syncing && locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) {
+                md_done_sync(conf->mddev, STRIPE_SECTORS,1);
+                clear_bit(STRIPE_SYNCING, &sh->state);
+        }
+
+        spin_unlock(&sh->lock);
+
+        while ((bi=return_bi)) {
+                int bytes = bi->bi_size;
+
+                return_bi = bi->bi_next;
+                bi->bi_next = NULL;
+                bi->bi_size = 0;
+                bi->bi_end_io(bi, bytes, 0);
+        }
+        for (i=disks; i-- ;) {
+                int rw;
+                struct bio *bi;
+                mdk_rdev_t *rdev;
+                if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags))
+                        rw = 1;
+                else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
+                        rw = 0;
+                else
+                        continue;
+
+                bi = &sh->dev[i].req;
+
+                bi->bi_rw = rw;
+                if (rw)
+                        bi->bi_end_io = raid6_end_write_request;
+                else
+                        bi->bi_end_io = raid6_end_read_request;
+
+                rcu_read_lock();
+                rdev = conf->disks[i].rdev;
+                if (rdev && rdev->faulty)
+                        rdev = NULL;
+                if (rdev)
+                        atomic_inc(&rdev->nr_pending);
+                rcu_read_unlock();
+
+                if (rdev) {
+                        if (test_bit(R5_Syncio, &sh->dev[i].flags))
+                                md_sync_acct(rdev->bdev, STRIPE_SECTORS);
+
+                        bi->bi_bdev = rdev->bdev;
+                        PRINTK("for %llu schedule op %ld on disc %d\n",
+                                (unsigned long long)sh->sector, bi->bi_rw, i);
+                        atomic_inc(&sh->count);
+                        bi->bi_sector = sh->sector + rdev->data_offset;
+                        bi->bi_flags = 1 << BIO_UPTODATE;
+                        bi->bi_vcnt = 1;
+                        bi->bi_max_vecs = 1;
+                        bi->bi_idx = 0;
+                        bi->bi_io_vec = &sh->dev[i].vec;
+                        bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
+                        bi->bi_io_vec[0].bv_offset = 0;
+                        bi->bi_size = STRIPE_SIZE;
+                        bi->bi_next = NULL;
+                        generic_make_request(bi);
+                } else {
+                        PRINTK("skip op %ld on disc %d for sector %llu\n",
+                                bi->bi_rw, i, (unsigned long long)sh->sector);
+                        clear_bit(R5_LOCKED, &sh->dev[i].flags);
+                        set_bit(STRIPE_HANDLE, &sh->state);
+                }
+        }
+}
+
+static inline void raid6_activate_delayed(raid6_conf_t *conf)
+{
+        if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
+                while (!list_empty(&conf->delayed_list)) {
+                        struct list_head *l = conf->delayed_list.next;
+                        struct stripe_head *sh;
+                        sh = list_entry(l, struct stripe_head, lru);
+                        list_del_init(l);
+                        clear_bit(STRIPE_DELAYED, &sh->state);
+                        if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
+                                atomic_inc(&conf->preread_active_stripes);
+                        list_add_tail(&sh->lru, &conf->handle_list);
+                }
+        }
+}
+
+static void unplug_slaves(mddev_t *mddev)
+{
+        raid6_conf_t *conf = mddev_to_conf(mddev);
+        int i;
+
+        rcu_read_lock();
+        for (i=0; i<mddev->raid_disks; i++) {
+                mdk_rdev_t *rdev = conf->disks[i].rdev;
+                if (rdev && !rdev->faulty && atomic_read(&rdev->nr_pending)) {
+                        request_queue_t *r_queue = bdev_get_queue(rdev->bdev);
+
+                        atomic_inc(&rdev->nr_pending);
+                        rcu_read_unlock();
+
+                        if (r_queue->unplug_fn)
+                                r_queue->unplug_fn(r_queue);
+
+                        rdev_dec_pending(rdev, mddev);
+                        rcu_read_lock();
+                }
+        }
+        rcu_read_unlock();
+}
+
+static void raid6_unplug_device(request_queue_t *q)
+{
+        mddev_t *mddev = q->queuedata;
+        raid6_conf_t *conf = mddev_to_conf(mddev);
+        unsigned long flags;
+
+        spin_lock_irqsave(&conf->device_lock, flags);
+
+        if (blk_remove_plug(q))
+                raid6_activate_delayed(conf);
+        md_wakeup_thread(mddev->thread);
+
+        spin_unlock_irqrestore(&conf->device_lock, flags);
+
+        unplug_slaves(mddev);
+}
+
+static int raid6_issue_flush(request_queue_t *q, struct gendisk *disk,
+                             sector_t *error_sector)
+{
+        mddev_t *mddev = q->queuedata;
+        raid6_conf_t *conf = mddev_to_conf(mddev);
+        int i, ret = 0;
+
+        rcu_read_lock();
+        for (i=0; i<mddev->raid_disks && ret == 0; i++) {
+                mdk_rdev_t *rdev = conf->disks[i].rdev;
+                if (rdev && !rdev->faulty) {
+                        struct block_device *bdev = rdev->bdev;
+                        request_queue_t *r_queue = bdev_get_queue(bdev);
+
+                        if (!r_queue->issue_flush_fn)
+                                ret = -EOPNOTSUPP;
+                        else {
+                                atomic_inc(&rdev->nr_pending);
+                                rcu_read_unlock();
+                                ret = r_queue->issue_flush_fn(r_queue, bdev->bd_disk,
+                                                              error_sector);
+                                rdev_dec_pending(rdev, mddev);
+                                rcu_read_lock();
+                        }
+                }
+        }
+        rcu_read_unlock();
+        return ret;
+}
+
+static inline void raid6_plug_device(raid6_conf_t *conf)
+{
+        spin_lock_irq(&conf->device_lock);
+        blk_plug_device(conf->mddev->queue);
+        spin_unlock_irq(&conf->device_lock);
+}
+
+static int make_request (request_queue_t *q, struct bio * bi)
+{
+        mddev_t *mddev = q->queuedata;
+        raid6_conf_t *conf = mddev_to_conf(mddev);
+        const unsigned int raid_disks = conf->raid_disks;
+        const unsigned int data_disks = raid_disks - 2;
+        unsigned int dd_idx, pd_idx;
+        sector_t new_sector;
+        sector_t logical_sector, last_sector;
+        struct stripe_head *sh;
+
+        if (bio_data_dir(bi)==WRITE) {
+                disk_stat_inc(mddev->gendisk, writes);
+                disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bi));
+        } else {
+                disk_stat_inc(mddev->gendisk, reads);
+                disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bi));
+        }
+
+        logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
+        last_sector = bi->bi_sector + (bi->bi_size>>9);
+
+        bi->bi_next = NULL;
+        bi->bi_phys_segments = 1;       /* over-loaded to count active stripes */
+        if ( bio_data_dir(bi) == WRITE )
+                md_write_start(mddev);
+        for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
+                DEFINE_WAIT(w);
+
+                new_sector = raid6_compute_sector(logical_sector,
+                                                  raid_disks, data_disks, &dd_idx, &pd_idx, conf);
+
+                PRINTK("raid6: make_request, sector %llu logical %llu\n",
+                       (unsigned long long)new_sector,
+                       (unsigned long long)logical_sector);
+
+        retry:
+                prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
+                sh = get_active_stripe(conf, new_sector, pd_idx, (bi->bi_rw&RWA_MASK));
+                if (sh) {
+                        if (!add_stripe_bio(sh, bi, dd_idx, (bi->bi_rw&RW_MASK))) {
+                                /* Add failed due to overlap.  Flush everything
+                                 * and wait a while
+                                 */
+                                raid6_unplug_device(mddev->queue);
+                                release_stripe(sh);
+                                schedule();
+                                goto retry;
+                        }
+                        finish_wait(&conf->wait_for_overlap, &w);
+                        raid6_plug_device(conf);
+                        handle_stripe(sh);
+                        release_stripe(sh);
+                } else {
+                        /* cannot get stripe for read-ahead, just give-up */
+                        clear_bit(BIO_UPTODATE, &bi->bi_flags);
+                        finish_wait(&conf->wait_for_overlap, &w);
+                        break;
+                }
+
+        }
+        spin_lock_irq(&conf->device_lock);
+        if (--bi->bi_phys_segments == 0) {
+                int bytes = bi->bi_size;
+
+                if ( bio_data_dir(bi) == WRITE )
+                        md_write_end(mddev);
+                bi->bi_size = 0;
+                bi->bi_end_io(bi, bytes, 0);
+        }
+        spin_unlock_irq(&conf->device_lock);
+        return 0;
+}
+
+/* FIXME go_faster isn't used */
+static int sync_request (mddev_t *mddev, sector_t sector_nr, int go_faster)
+{
+        raid6_conf_t *conf = (raid6_conf_t *) mddev->private;
+        struct stripe_head *sh;
+        int sectors_per_chunk = conf->chunk_size >> 9;
+        sector_t x;
+        unsigned long stripe;
+        int chunk_offset;
+        int dd_idx, pd_idx;
+        sector_t first_sector;
+        int raid_disks = conf->raid_disks;
+        int data_disks = raid_disks - 2;
+
+        if (sector_nr >= mddev->size <<1) {
+                /* just being told to finish up .. nothing much to do */
+                unplug_slaves(mddev);
+                return 0;
+        }
+        /* if there are 2 or more failed drives and we are trying
+         * to resync, then assert that we are finished, because there is
+         * nothing we can do.
+         */
+        if (mddev->degraded >= 2 && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
+                int rv = (mddev->size << 1) - sector_nr;
+                md_done_sync(mddev, rv, 1);
+                return rv;
+        }
+
+        x = sector_nr;
+        chunk_offset = sector_div(x, sectors_per_chunk);
+        stripe = x;
+        BUG_ON(x != stripe);
+
+        first_sector = raid6_compute_sector((sector_t)stripe*data_disks*sectors_per_chunk
+                + chunk_offset, raid_disks, data_disks, &dd_idx, &pd_idx, conf);
+        sh = get_active_stripe(conf, sector_nr, pd_idx, 1);
+        if (sh == NULL) {
+                sh = get_active_stripe(conf, sector_nr, pd_idx, 0);
+                /* make sure we don't swamp the stripe cache if someone else
+                 * is trying to get access
+                 */
+                set_current_state(TASK_UNINTERRUPTIBLE);
+                schedule_timeout(1);
+        }
+        spin_lock(&sh->lock);
+        set_bit(STRIPE_SYNCING, &sh->state);
+        clear_bit(STRIPE_INSYNC, &sh->state);
+        spin_unlock(&sh->lock);
+
+        handle_stripe(sh);
+        release_stripe(sh);
+
+        return STRIPE_SECTORS;
+}
+
+/*
+ * This is our raid6 kernel thread.
+ *
+ * We scan the hash table for stripes which can be handled now.
+ * During the scan, completed stripes are saved for us by the interrupt
+ * handler, so that they will not have to wait for our next wakeup.
+ */
+static void raid6d (mddev_t *mddev)
+{
+        struct stripe_head *sh;
+        raid6_conf_t *conf = mddev_to_conf(mddev);
+        int handled;
+
+        PRINTK("+++ raid6d active\n");
+
+        md_check_recovery(mddev);
+        md_handle_safemode(mddev);
+
+        handled = 0;
+        spin_lock_irq(&conf->device_lock);
+        while (1) {
+                struct list_head *first;
+
+                if (list_empty(&conf->handle_list) &&
+                    atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD &&
+                    !blk_queue_plugged(mddev->queue) &&
+                    !list_empty(&conf->delayed_list))
+                        raid6_activate_delayed(conf);
+
+                if (list_empty(&conf->handle_list))
+                        break;
+
+                first = conf->handle_list.next;
+                sh = list_entry(first, struct stripe_head, lru);
+
+                list_del_init(first);
+                atomic_inc(&sh->count);
+                if (atomic_read(&sh->count)!= 1)
+                        BUG();
+                spin_unlock_irq(&conf->device_lock);
+
+                handled++;
+                handle_stripe(sh);
+                release_stripe(sh);
+
+                spin_lock_irq(&conf->device_lock);
+        }
+        PRINTK("%d stripes handled\n", handled);
+
+        spin_unlock_irq(&conf->device_lock);
+
+        unplug_slaves(mddev);
+
+        PRINTK("--- raid6d inactive\n");
+}
+
+static int run (mddev_t *mddev)
+{
+        raid6_conf_t *conf;
+        int raid_disk, memory;
+        mdk_rdev_t *rdev;
+        struct disk_info *disk;
+        struct list_head *tmp;
+
+        if (mddev->level != 6) {
+                PRINTK("raid6: %s: raid level not set to 6 (%d)\n", mdname(mddev), mddev->level);
+                return -EIO;
+        }
+
+        mddev->private = kmalloc (sizeof (raid6_conf_t)
+                                  + mddev->raid_disks * sizeof(struct disk_info),
+                                  GFP_KERNEL);
+        if ((conf = mddev->private) == NULL)
+                goto abort;
+        memset (conf, 0, sizeof (*conf) + mddev->raid_disks * sizeof(struct disk_info) );
+        conf->mddev = mddev;
+
+        if ((conf->stripe_hashtbl = (struct stripe_head **) __get_free_pages(GFP_ATOMIC, HASH_PAGES_ORDER)) == NULL)
+                goto abort;
+        memset(conf->stripe_hashtbl, 0, HASH_PAGES * PAGE_SIZE);
+
+        spin_lock_init(&conf->device_lock);
+        init_waitqueue_head(&conf->wait_for_stripe);
+        init_waitqueue_head(&conf->wait_for_overlap);
+        INIT_LIST_HEAD(&conf->handle_list);
+        INIT_LIST_HEAD(&conf->delayed_list);
+        INIT_LIST_HEAD(&conf->inactive_list);
+        atomic_set(&conf->active_stripes, 0);
+        atomic_set(&conf->preread_active_stripes, 0);
+
+        mddev->queue->unplug_fn = raid6_unplug_device;
+        mddev->queue->issue_flush_fn = raid6_issue_flush;
+
+        PRINTK("raid6: run(%s) called.\n", mdname(mddev));
+
+        ITERATE_RDEV(mddev,rdev,tmp) {
+                raid_disk = rdev->raid_disk;
+                if (raid_disk >= mddev->raid_disks
+                    || raid_disk < 0)
+                        continue;
+                disk = conf->disks + raid_disk;
+
+                disk->rdev = rdev;
+
+                if (rdev->in_sync) {
+                        char b[BDEVNAME_SIZE];
+                        printk(KERN_INFO "raid6: device %s operational as raid"
+                               " disk %d\n", bdevname(rdev->bdev,b),
+                               raid_disk);
+                        conf->working_disks++;
+                }
+        }
+
+        conf->raid_disks = mddev->raid_disks;
+
+        /*
+         * 0 for a fully functional array, 1 or 2 for a degraded array.
+         */
+        mddev->degraded = conf->failed_disks = conf->raid_disks - conf->working_disks;
+        conf->mddev = mddev;
+        conf->chunk_size = mddev->chunk_size;
+        conf->level = mddev->level;
+        conf->algorithm = mddev->layout;
+        conf->max_nr_stripes = NR_STRIPES;
+
+        /* device size must be a multiple of chunk size */
+        mddev->size &= ~(mddev->chunk_size/1024 -1);
+
+        if (conf->raid_disks < 4) {
+                printk(KERN_ERR "raid6: not enough configured devices for %s (%d, minimum 4)\n",
+                       mdname(mddev), conf->raid_disks);
+                goto abort;
+        }
+        if (!conf->chunk_size || conf->chunk_size % 4) {
+                printk(KERN_ERR "raid6: invalid chunk size %d for %s\n",
+                       conf->chunk_size, mdname(mddev));
+                goto abort;
+        }
+        if (conf->algorithm > ALGORITHM_RIGHT_SYMMETRIC) {
+                printk(KERN_ERR
+                       "raid6: unsupported parity algorithm %d for %s\n",
+                       conf->algorithm, mdname(mddev));
+                goto abort;
+        }
+        if (mddev->degraded > 2) {
+                printk(KERN_ERR "raid6: not enough operational devices for %s"
+                       " (%d/%d failed)\n",
+                       mdname(mddev), conf->failed_disks, conf->raid_disks);
+                goto abort;
+        }
+
+#if 0                           /* FIX: For now */
+        if (mddev->degraded > 0 &&
+            mddev->recovery_cp != MaxSector) {
+                printk(KERN_ERR "raid6: cannot start dirty degraded array for %s\n", mdname(mddev));
+                goto abort;
+        }
+#endif
+
+        {
+                mddev->thread = md_register_thread(raid6d, mddev, "%s_raid6");
+                if (!mddev->thread) {
+                        printk(KERN_ERR
+                               "raid6: couldn't allocate thread for %s\n",
+                               mdname(mddev));
+                        goto abort;
+                }
+        }
+
+        memory = conf->max_nr_stripes * (sizeof(struct stripe_head) +
+                 conf->raid_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
+        if (grow_stripes(conf, conf->max_nr_stripes)) {
+                printk(KERN_ERR
+                       "raid6: couldn't allocate %dkB for buffers\n", memory);
+                shrink_stripes(conf);
+                md_unregister_thread(mddev->thread);
+                goto abort;
+        } else
+                printk(KERN_INFO "raid6: allocated %dkB for %s\n",
+                       memory, mdname(mddev));
+
+        if (mddev->degraded == 0)
+                printk(KERN_INFO "raid6: raid level %d set %s active with %d out of %d"
+                       " devices, algorithm %d\n", conf->level, mdname(mddev),
+                       mddev->raid_disks-mddev->degraded, mddev->raid_disks,
+                       conf->algorithm);
+        else
+                printk(KERN_ALERT "raid6: raid level %d set %s active with %d"
+                       " out of %d devices, algorithm %d\n", conf->level,
+                       mdname(mddev), mddev->raid_disks - mddev->degraded,
+                       mddev->raid_disks, conf->algorithm);
+
+        print_raid6_conf(conf);
+
+        /* read-ahead size must cover two whole stripes, which is
+         * 2 * (n-2) * chunksize where 'n' is the number of raid devices
+         */
+        {
+                int stripe = (mddev->raid_disks-2) * mddev->chunk_size
+                        / PAGE_CACHE_SIZE;
+                if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
+                        mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
+        }
+
+        /* Ok, everything is just fine now */
+        mddev->array_size =  mddev->size * (mddev->raid_disks - 2);
+        return 0;
+abort:
+        if (conf) {
+                print_raid6_conf(conf);
+                if (conf->stripe_hashtbl)
+                        free_pages((unsigned long) conf->stripe_hashtbl,
+                                                        HASH_PAGES_ORDER);
+                kfree(conf);
+        }
+        mddev->private = NULL;
+        printk(KERN_ALERT "raid6: failed to run raid set %s\n", mdname(mddev));
+        return -EIO;
+}
+
+
+
+static int stop (mddev_t *mddev)
+{
+        raid6_conf_t *conf = (raid6_conf_t *) mddev->private;
+
+        md_unregister_thread(mddev->thread);
+        mddev->thread = NULL;
+        shrink_stripes(conf);
+        free_pages((unsigned long) conf->stripe_hashtbl, HASH_PAGES_ORDER);
+        blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
+        kfree(conf);
+        mddev->private = NULL;
+        return 0;
+}
+
+#if RAID6_DUMPSTATE
+static void print_sh (struct seq_file *seq, struct stripe_head *sh)
+{
+        int i;
+
+        seq_printf(seq, "sh %llu, pd_idx %d, state %ld.\n",
+                   (unsigned long long)sh->sector, sh->pd_idx, sh->state);
+        seq_printf(seq, "sh %llu,  count %d.\n",
+                   (unsigned long long)sh->sector, atomic_read(&sh->count));
+        seq_printf(seq, "sh %llu, ", (unsigned long long)sh->sector);
+        for (i = 0; i < sh->raid_conf->raid_disks; i++) {
+                seq_printf(seq, "(cache%d: %p %ld) ",
+                           i, sh->dev[i].page, sh->dev[i].flags);
+        }
+        seq_printf(seq, "\n");
+}
+
+static void printall (struct seq_file *seq, raid6_conf_t *conf)
+{
+        struct stripe_head *sh;
+        int i;
+
+        spin_lock_irq(&conf->device_lock);
+        for (i = 0; i < NR_HASH; i++) {
+                sh = conf->stripe_hashtbl[i];
+                for (; sh; sh = sh->hash_next) {
+                        if (sh->raid_conf != conf)
+                                continue;
+                        print_sh(seq, sh);
+                }
+        }
+        spin_unlock_irq(&conf->device_lock);
+}
+#endif
+
+static void status (struct seq_file *seq, mddev_t *mddev)
+{
+        raid6_conf_t *conf = (raid6_conf_t *) mddev->private;
+        int i;
+
+        seq_printf (seq, " level %d, %dk chunk, algorithm %d", mddev->level, mddev->chunk_size >> 10, mddev->layout);
+        seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->working_disks);
+        for (i = 0; i < conf->raid_disks; i++)
+                seq_printf (seq, "%s",
+                            conf->disks[i].rdev &&
+                            conf->disks[i].rdev->in_sync ? "U" : "_");
+        seq_printf (seq, "]");
+#if RAID6_DUMPSTATE
+        seq_printf (seq, "\n");
+        printall(seq, conf);
+#endif
+}
+
+static void print_raid6_conf (raid6_conf_t *conf)
+{
+        int i;
+        struct disk_info *tmp;
+
+        printk("RAID6 conf printout:\n");
+        if (!conf) {
+                printk("(conf==NULL)\n");
+                return;
+        }
+        printk(" --- rd:%d wd:%d fd:%d\n", conf->raid_disks,
+                 conf->working_disks, conf->failed_disks);
+
+        for (i = 0; i < conf->raid_disks; i++) {
+                char b[BDEVNAME_SIZE];
+                tmp = conf->disks + i;
+                if (tmp->rdev)
+                printk(" disk %d, o:%d, dev:%s\n",
+                        i, !tmp->rdev->faulty,
+                        bdevname(tmp->rdev->bdev,b));
+        }
+}
+
+static int raid6_spare_active(mddev_t *mddev)
+{
+        int i;
+        raid6_conf_t *conf = mddev->private;
+        struct disk_info *tmp;
+
+        for (i = 0; i < conf->raid_disks; i++) {
+                tmp = conf->disks + i;
+                if (tmp->rdev
+                    && !tmp->rdev->faulty
+                    && !tmp->rdev->in_sync) {
+                        mddev->degraded--;
+                        conf->failed_disks--;
+                        conf->working_disks++;
+                        tmp->rdev->in_sync = 1;
+                }
+        }
+        print_raid6_conf(conf);
+        return 0;
+}
+
+static int raid6_remove_disk(mddev_t *mddev, int number)
+{
+        raid6_conf_t *conf = mddev->private;
+        int err = 0;
+        mdk_rdev_t *rdev;
+        struct disk_info *p = conf->disks + number;
+
+        print_raid6_conf(conf);
+        rdev = p->rdev;
+        if (rdev) {
+                if (rdev->in_sync ||
+                    atomic_read(&rdev->nr_pending)) {
+                        err = -EBUSY;
+                        goto abort;
+                }
+                p->rdev = NULL;
+                synchronize_kernel();
+                if (atomic_read(&rdev->nr_pending)) {
+                        /* lost the race, try later */
+                        err = -EBUSY;
+                        p->rdev = rdev;
+                }
+        }
+
+abort:
+
+        print_raid6_conf(conf);
+        return err;
+}
+
+static int raid6_add_disk(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+        raid6_conf_t *conf = mddev->private;
+        int found = 0;
+        int disk;
+        struct disk_info *p;
+
+        if (mddev->degraded > 2)
+                /* no point adding a device */
+                return 0;
+        /*
+         * find the disk ...
+         */
+        for (disk=0; disk < mddev->raid_disks; disk++)
+                if ((p=conf->disks + disk)->rdev == NULL) {
+                        rdev->in_sync = 0;
+                        rdev->raid_disk = disk;
+                        found = 1;
+                        p->rdev = rdev;
+                        break;
+                }
+        print_raid6_conf(conf);
+        return found;
+}
+
+static int raid6_resize(mddev_t *mddev, sector_t sectors)
+{
+        /* no resync is happening, and there is enough space
+         * on all devices, so we can resize.
+         * We need to make sure resync covers any new space.
+         * If the array is shrinking we should possibly wait until
+         * any io in the removed space completes, but it hardly seems
+         * worth it.
+         */
+        sectors &= ~((sector_t)mddev->chunk_size/512 - 1);
+        mddev->array_size = (sectors * (mddev->raid_disks-2))>>1;
+        set_capacity(mddev->gendisk, mddev->array_size << 1);
+        mddev->changed = 1;
+        if (sectors/2  > mddev->size && mddev->recovery_cp == MaxSector) {
+                mddev->recovery_cp = mddev->size << 1;
+                set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+        }
+        mddev->size = sectors /2;
+        return 0;
+}
+
+static mdk_personality_t raid6_personality=
+{
+        .name           = "raid6",
+        .owner          = THIS_MODULE,
+        .make_request   = make_request,
+        .run            = run,
+        .stop           = stop,
+        .status         = status,
+        .error_handler  = error,
+        .hot_add_disk   = raid6_add_disk,
+        .hot_remove_disk= raid6_remove_disk,
+        .spare_active   = raid6_spare_active,
+        .sync_request   = sync_request,
+        .resize         = raid6_resize,
+};
+
+static int __init raid6_init (void)
+{
+        int e;
+
+        e = raid6_select_algo();
+        if ( e )
+                return e;
+
+        return register_md_personality (RAID6, &raid6_personality);
+}
+
+static void raid6_exit (void)
+{
+        unregister_md_personality (RAID6);
+}
+
+module_init(raid6_init);
+module_exit(raid6_exit);
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("md-personality-8"); /* RAID6 */
diff --git a/drivers/md/raid6mmx.c b/drivers/md/raid6mmx.c
new file mode 100644
index 000000000000..359157aaf9e0
--- /dev/null
+++ b/drivers/md/raid6mmx.c
@@ -0,0 +1,150 @@
+/* -*- linux-c -*- ------------------------------------------------------- *
+ *
+ *   Copyright 2002 H. Peter Anvin - All Rights Reserved
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+ *   Bostom MA 02111-1307, USA; either version 2 of the License, or
+ *   (at your option) any later version; incorporated herein by reference.
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * raid6mmx.c
+ *
+ * MMX implementation of RAID-6 syndrome functions
+ */
+
+#if defined(__i386__)
+
+#include "raid6.h"
+#include "raid6x86.h"
+
+/* Shared with raid6sse1.c */
+const struct raid6_mmx_constants {
+        u64 x1d;
+} raid6_mmx_constants = {
+        0x1d1d1d1d1d1d1d1dULL,
+};
+
+static int raid6_have_mmx(void)
+{
+#ifdef __KERNEL__
+        /* Not really "boot_cpu" but "all_cpus" */
+        return boot_cpu_has(X86_FEATURE_MMX);
+#else
+        /* User space test code */
+        u32 features = cpuid_features();
+        return ( (features & (1<<23)) == (1<<23) );
+#endif
+}
+
+/*
+ * Plain MMX implementation
+ */
+static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+        u8 **dptr = (u8 **)ptrs;
+        u8 *p, *q;
+        int d, z, z0;
+        raid6_mmx_save_t sa;
+
+        z0 = disks - 3;         /* Highest data disk */
+        p = dptr[z0+1];         /* XOR parity */
+        q = dptr[z0+2];         /* RS syndrome */
+
+        raid6_before_mmx(&sa);
+
+        asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
+        asm volatile("pxor %mm5,%mm5"); /* Zero temp */
+
+        for ( d = 0 ; d < bytes ; d += 8 ) {
+                asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
+                asm volatile("movq %mm2,%mm4"); /* Q[0] */
+                for ( z = z0-1 ; z >= 0 ; z-- ) {
+                        asm volatile("movq %0,%%mm6" : : "m" (dptr[z][d]));
+                        asm volatile("pcmpgtb %mm4,%mm5");
+                        asm volatile("paddb %mm4,%mm4");
+                        asm volatile("pand %mm0,%mm5");
+                        asm volatile("pxor %mm5,%mm4");
+                        asm volatile("pxor %mm5,%mm5");
+                        asm volatile("pxor %mm6,%mm2");
+                        asm volatile("pxor %mm6,%mm4");
+                }
+                asm volatile("movq %%mm2,%0" : "=m" (p[d]));
+                asm volatile("pxor %mm2,%mm2");
+                asm volatile("movq %%mm4,%0" : "=m" (q[d]));
+                asm volatile("pxor %mm4,%mm4");
+        }
+
+        raid6_after_mmx(&sa);
+}
+
+const struct raid6_calls raid6_mmxx1 = {
+        raid6_mmx1_gen_syndrome,
+        raid6_have_mmx,
+        "mmxx1",
+        0
+};
+
+/*
+ * Unrolled-by-2 MMX implementation
+ */
+static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+        u8 **dptr = (u8 **)ptrs;
+        u8 *p, *q;
+        int d, z, z0;
+        raid6_mmx_save_t sa;
+
+        z0 = disks - 3;         /* Highest data disk */
+        p = dptr[z0+1];         /* XOR parity */
+        q = dptr[z0+2];         /* RS syndrome */
+
+        raid6_before_mmx(&sa);
+
+        asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
+        asm volatile("pxor %mm5,%mm5"); /* Zero temp */
+        asm volatile("pxor %mm7,%mm7"); /* Zero temp */
+
+        for ( d = 0 ; d < bytes ; d += 16 ) {
+                asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
+                asm volatile("movq %0,%%mm3" : : "m" (dptr[z0][d+8]));
+                asm volatile("movq %mm2,%mm4"); /* Q[0] */
+                asm volatile("movq %mm3,%mm6"); /* Q[1] */
+                for ( z = z0-1 ; z >= 0 ; z-- ) {
+                        asm volatile("pcmpgtb %mm4,%mm5");
+                        asm volatile("pcmpgtb %mm6,%mm7");
+                        asm volatile("paddb %mm4,%mm4");
+                        asm volatile("paddb %mm6,%mm6");
+                        asm volatile("pand %mm0,%mm5");
+                        asm volatile("pand %mm0,%mm7");
+                        asm volatile("pxor %mm5,%mm4");
+                        asm volatile("pxor %mm7,%mm6");
+                        asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d]));
+                        asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8]));
+                        asm volatile("pxor %mm5,%mm2");
+                        asm volatile("pxor %mm7,%mm3");
+                        asm volatile("pxor %mm5,%mm4");
+                        asm volatile("pxor %mm7,%mm6");
+                        asm volatile("pxor %mm5,%mm5");
+                        asm volatile("pxor %mm7,%mm7");
+                }
+                asm volatile("movq %%mm2,%0" : "=m" (p[d]));
+                asm volatile("movq %%mm3,%0" : "=m" (p[d+8]));
+                asm volatile("movq %%mm4,%0" : "=m" (q[d]));
+                asm volatile("movq %%mm6,%0" : "=m" (q[d+8]));
+        }
+
+        raid6_after_mmx(&sa);
+}
+
+const struct raid6_calls raid6_mmxx2 = {
+        raid6_mmx2_gen_syndrome,
+        raid6_have_mmx,
+        "mmxx2",
+        0
+};
+
+#endif
diff --git a/drivers/md/raid6recov.c b/drivers/md/raid6recov.c
new file mode 100644
index 000000000000..a8c4d9451bd9
--- /dev/null
+++ b/drivers/md/raid6recov.c
@@ -0,0 +1,133 @@
+/* -*- linux-c -*- ------------------------------------------------------- *
+ *
+ *   Copyright 2002 H. Peter Anvin - All Rights Reserved
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+ *   Bostom MA 02111-1307, USA; either version 2 of the License, or
+ *   (at your option) any later version; incorporated herein by reference.
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * raid6recov.c
+ *
+ * RAID-6 data recovery in dual failure mode.  In single failure mode,
+ * use the RAID-5 algorithm (or, in the case of Q failure, just reconstruct
+ * the syndrome.)
+ */
+
+#include "raid6.h"
+
+/* Recover two failed data blocks. */
+void raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
+                       void **ptrs)
+{
+        u8 *p, *q, *dp, *dq;
+        u8 px, qx, db;
+        const u8 *pbmul;        /* P multiplier table for B data */
+        const u8 *qmul;         /* Q multiplier table (for both) */
+
+        p = (u8 *)ptrs[disks-2];
+        q = (u8 *)ptrs[disks-1];
+
+        /* Compute syndrome with zero for the missing data pages
+           Use the dead data pages as temporary storage for
+           delta p and delta q */
+        dp = (u8 *)ptrs[faila];
+        ptrs[faila] = (void *)raid6_empty_zero_page;
+        ptrs[disks-2] = dp;
+        dq = (u8 *)ptrs[failb];
+        ptrs[failb] = (void *)raid6_empty_zero_page;
+        ptrs[disks-1] = dq;
+
+        raid6_call.gen_syndrome(disks, bytes, ptrs);
+
+        /* Restore pointer table */
+        ptrs[faila]   = dp;
+        ptrs[failb]   = dq;
+        ptrs[disks-2] = p;
+        ptrs[disks-1] = q;
+
+        /* Now, pick the proper data tables */
+        pbmul = raid6_gfmul[raid6_gfexi[failb-faila]];
+        qmul  = raid6_gfmul[raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]];
+
+        /* Now do it... */
+        while ( bytes-- ) {
+                px    = *p ^ *dp;
+                qx    = qmul[*q ^ *dq];
+                *dq++ = db = pbmul[px] ^ qx; /* Reconstructed B */
+                *dp++ = db ^ px; /* Reconstructed A */
+                p++; q++;
+        }
+}
+
+
+
+
+/* Recover failure of one data block plus the P block */
+void raid6_datap_recov(int disks, size_t bytes, int faila, void **ptrs)
+{
+        u8 *p, *q, *dq;
+        const u8 *qmul;         /* Q multiplier table */
+
+        p = (u8 *)ptrs[disks-2];
+        q = (u8 *)ptrs[disks-1];
+
+        /* Compute syndrome with zero for the missing data page
+           Use the dead data page as temporary storage for delta q */
+        dq = (u8 *)ptrs[faila];
+        ptrs[faila] = (void *)raid6_empty_zero_page;
+        ptrs[disks-1] = dq;
+
+        raid6_call.gen_syndrome(disks, bytes, ptrs);
+
+        /* Restore pointer table */
+        ptrs[faila]   = dq;
+        ptrs[disks-1] = q;
+
+        /* Now, pick the proper data tables */
+        qmul  = raid6_gfmul[raid6_gfinv[raid6_gfexp[faila]]];
+
+        /* Now do it... */
+        while ( bytes-- ) {
+                *p++ ^= *dq = qmul[*q ^ *dq];
+                q++; dq++;
+        }
+}
+
+
+#ifndef __KERNEL__              /* Testing only */
+
+/* Recover two failed blocks. */
+void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, void **ptrs)
+{
+        if ( faila > failb ) {
+                int tmp = faila;
+                faila = failb;
+                failb = tmp;
+        }
+
+        if ( failb == disks-1 ) {
+                if ( faila == disks-2 ) {
+                        /* P+Q failure.  Just rebuild the syndrome. */
+                        raid6_call.gen_syndrome(disks, bytes, ptrs);
+                } else {
+                        /* data+Q failure.  Reconstruct data from P,
+                           then rebuild syndrome. */
+                        /* NOT IMPLEMENTED - equivalent to RAID-5 */
+                }
+        } else {
+                if ( failb == disks-2 ) {
+                        /* data+P failure. */
+                        raid6_datap_recov(disks, bytes, faila, ptrs);
+                } else {
+                        /* data+data failure. */
+                        raid6_2data_recov(disks, bytes, faila, failb, ptrs);
+                }
+        }
+}
+
+#endif
diff --git a/drivers/md/raid6sse1.c b/drivers/md/raid6sse1.c
new file mode 100644
index 000000000000..f7e7859f71aa
--- /dev/null
+++ b/drivers/md/raid6sse1.c
@@ -0,0 +1,171 @@
+/* -*- linux-c -*- ------------------------------------------------------- *
+ *
+ *   Copyright 2002 H. Peter Anvin - All Rights Reserved
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+ *   Bostom MA 02111-1307, USA; either version 2 of the License, or
+ *   (at your option) any later version; incorporated herein by reference.
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * raid6sse1.c
+ *
+ * SSE-1/MMXEXT implementation of RAID-6 syndrome functions
+ *
+ * This is really an MMX implementation, but it requires SSE-1 or
+ * AMD MMXEXT for prefetch support and a few other features.  The
+ * support for nontemporal memory accesses is enough to make this
+ * worthwhile as a separate implementation.
+ */
+
+#if defined(__i386__)
+
+#include "raid6.h"
+#include "raid6x86.h"
+
+/* Defined in raid6mmx.c */
+extern const struct raid6_mmx_constants {
+        u64 x1d;
+} raid6_mmx_constants;
+
+static int raid6_have_sse1_or_mmxext(void)
+{
+#ifdef __KERNEL__
+        /* Not really boot_cpu but "all_cpus" */
+        return boot_cpu_has(X86_FEATURE_MMX) &&
+                (boot_cpu_has(X86_FEATURE_XMM) ||
+                 boot_cpu_has(X86_FEATURE_MMXEXT));
+#else
+        /* User space test code - this incorrectly breaks on some Athlons */
+        u32 features = cpuid_features();
+        return ( (features & (5<<23)) == (5<<23) );
+#endif
+}
+
+/*
+ * Plain SSE1 implementation
+ */
+static void raid6_sse11_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+        u8 **dptr = (u8 **)ptrs;
+        u8 *p, *q;
+        int d, z, z0;
+        raid6_mmx_save_t sa;
+
+        z0 = disks - 3;         /* Highest data disk */
+        p = dptr[z0+1];         /* XOR parity */
+        q = dptr[z0+2];         /* RS syndrome */
+
+        /* This is really MMX code, not SSE */
+        raid6_before_mmx(&sa);
+
+        asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
+        asm volatile("pxor %mm5,%mm5"); /* Zero temp */
+
+        for ( d = 0 ; d < bytes ; d += 8 ) {
+                asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
+                asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
+                asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));
+                asm volatile("movq %mm2,%mm4"); /* Q[0] */
+                asm volatile("movq %0,%%mm6" : : "m" (dptr[z0-1][d]));
+                for ( z = z0-2 ; z >= 0 ; z-- ) {
+                        asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
+                        asm volatile("pcmpgtb %mm4,%mm5");
+                        asm volatile("paddb %mm4,%mm4");
+                        asm volatile("pand %mm0,%mm5");
+                        asm volatile("pxor %mm5,%mm4");
+                        asm volatile("pxor %mm5,%mm5");
+                        asm volatile("pxor %mm6,%mm2");
+                        asm volatile("pxor %mm6,%mm4");
+                        asm volatile("movq %0,%%mm6" : : "m" (dptr[z][d]));
+                }
+                asm volatile("pcmpgtb %mm4,%mm5");
+                asm volatile("paddb %mm4,%mm4");
+                asm volatile("pand %mm0,%mm5");
+                asm volatile("pxor %mm5,%mm4");
+                asm volatile("pxor %mm5,%mm5");
+                asm volatile("pxor %mm6,%mm2");
+                asm volatile("pxor %mm6,%mm4");
+
+                asm volatile("movntq %%mm2,%0" : "=m" (p[d]));
+                asm volatile("movntq %%mm4,%0" : "=m" (q[d]));
+        }
+
+        raid6_after_mmx(&sa);
+        asm volatile("sfence" : : : "memory");
+}
+
+const struct raid6_calls raid6_sse1x1 = {
+        raid6_sse11_gen_syndrome,
+        raid6_have_sse1_or_mmxext,
+        "sse1x1",
+        1                       /* Has cache hints */
+};
+
+/*
+ * Unrolled-by-2 SSE1 implementation
+ */
+static void raid6_sse12_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+        u8 **dptr = (u8 **)ptrs;
+        u8 *p, *q;
+        int d, z, z0;
+        raid6_mmx_save_t sa;
+
+        z0 = disks - 3;         /* Highest data disk */
+        p = dptr[z0+1];         /* XOR parity */
+        q = dptr[z0+2];         /* RS syndrome */
+
+        raid6_before_mmx(&sa);
+
+        asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
+        asm volatile("pxor %mm5,%mm5"); /* Zero temp */
+        asm volatile("pxor %mm7,%mm7"); /* Zero temp */
+
+        /* We uniformly assume a single prefetch covers at least 16 bytes */
+        for ( d = 0 ; d < bytes ; d += 16 ) {
+                asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
+                asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
+                asm volatile("movq %0,%%mm3" : : "m" (dptr[z0][d+8])); /* P[1] */
+                asm volatile("movq %mm2,%mm4"); /* Q[0] */
+                asm volatile("movq %mm3,%mm6"); /* Q[1] */
+                for ( z = z0-1 ; z >= 0 ; z-- ) {
+                        asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
+                        asm volatile("pcmpgtb %mm4,%mm5");
+                        asm volatile("pcmpgtb %mm6,%mm7");
+                        asm volatile("paddb %mm4,%mm4");
+                        asm volatile("paddb %mm6,%mm6");
+                        asm volatile("pand %mm0,%mm5");
+                        asm volatile("pand %mm0,%mm7");
+                        asm volatile("pxor %mm5,%mm4");
+                        asm volatile("pxor %mm7,%mm6");
+                        asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d]));
+                        asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8]));
+                        asm volatile("pxor %mm5,%mm2");
+                        asm volatile("pxor %mm7,%mm3");
+                        asm volatile("pxor %mm5,%mm4");
+                        asm volatile("pxor %mm7,%mm6");
+                        asm volatile("pxor %mm5,%mm5");
+                        asm volatile("pxor %mm7,%mm7");
+                }
+                asm volatile("movntq %%mm2,%0" : "=m" (p[d]));
+                asm volatile("movntq %%mm3,%0" : "=m" (p[d+8]));
+                asm volatile("movntq %%mm4,%0" : "=m" (q[d]));
+                asm volatile("movntq %%mm6,%0" : "=m" (q[d+8]));
+        }
+
+        raid6_after_mmx(&sa);
+        asm volatile("sfence" : :: "memory");
+}
+
+const struct raid6_calls raid6_sse1x2 = {
+        raid6_sse12_gen_syndrome,
+        raid6_have_sse1_or_mmxext,
+        "sse1x2",
+        1                       /* Has cache hints */
+};
+
+#endif
diff --git a/drivers/md/raid6sse2.c b/drivers/md/raid6sse2.c
new file mode 100644
index 000000000000..b3aa7fe0877e
--- /dev/null
+++ b/drivers/md/raid6sse2.c
@@ -0,0 +1,270 @@
+/* -*- linux-c -*- ------------------------------------------------------- *
+ *
+ *   Copyright 2002 H. Peter Anvin - All Rights Reserved
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+ *   Bostom MA 02111-1307, USA; either version 2 of the License, or
+ *   (at your option) any later version; incorporated herein by reference.
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * raid6sse2.c
+ *
+ * SSE-2 implementation of RAID-6 syndrome functions
+ *
+ */
+
+#if defined(__i386__) || defined(__x86_64__)
+
+#include "raid6.h"
+#include "raid6x86.h"
+
+static const struct raid6_sse_constants {
+        u64 x1d[2];
+} raid6_sse_constants  __attribute__((aligned(16))) = {
+        { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL },
+};
+
+static int raid6_have_sse2(void)
+{
+#ifdef __KERNEL__
+        /* Not really boot_cpu but "all_cpus" */
+        return boot_cpu_has(X86_FEATURE_MMX) &&
+                boot_cpu_has(X86_FEATURE_FXSR) &&
+                boot_cpu_has(X86_FEATURE_XMM) &&
+                boot_cpu_has(X86_FEATURE_XMM2);
+#else
+        /* User space test code */
+        u32 features = cpuid_features();
+        return ( (features & (15<<23)) == (15<<23) );
+#endif
+}
+
+/*
+ * Plain SSE2 implementation
+ */
+static void raid6_sse21_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+        u8 **dptr = (u8 **)ptrs;
+        u8 *p, *q;
+        int d, z, z0;
+        raid6_sse_save_t sa;
+
+        z0 = disks - 3;         /* Highest data disk */
+        p = dptr[z0+1];         /* XOR parity */
+        q = dptr[z0+2];         /* RS syndrome */
+
+        raid6_before_sse2(&sa);
+
+        asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
+        asm volatile("pxor %xmm5,%xmm5");       /* Zero temp */
+
+        for ( d = 0 ; d < bytes ; d += 16 ) {
+                asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
+                asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d])); /* P[0] */
+                asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));
+                asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */
+                asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z0-1][d]));
+                for ( z = z0-2 ; z >= 0 ; z-- ) {
+                        asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
+                        asm volatile("pcmpgtb %xmm4,%xmm5");
+                        asm volatile("paddb %xmm4,%xmm4");
+                        asm volatile("pand %xmm0,%xmm5");
+                        asm volatile("pxor %xmm5,%xmm4");
+                        asm volatile("pxor %xmm5,%xmm5");
+                        asm volatile("pxor %xmm6,%xmm2");
+                        asm volatile("pxor %xmm6,%xmm4");
+                        asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z][d]));
+                }
+                asm volatile("pcmpgtb %xmm4,%xmm5");
+                asm volatile("paddb %xmm4,%xmm4");
+                asm volatile("pand %xmm0,%xmm5");
+                asm volatile("pxor %xmm5,%xmm4");
+                asm volatile("pxor %xmm5,%xmm5");
+                asm volatile("pxor %xmm6,%xmm2");
+                asm volatile("pxor %xmm6,%xmm4");
+
+                asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
+                asm volatile("pxor %xmm2,%xmm2");
+                asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
+                asm volatile("pxor %xmm4,%xmm4");
+        }
+
+        raid6_after_sse2(&sa);
+        asm volatile("sfence" : : : "memory");
+}
+
+const struct raid6_calls raid6_sse2x1 = {
+        raid6_sse21_gen_syndrome,
+        raid6_have_sse2,
+        "sse2x1",
+        1                       /* Has cache hints */
+};
+
+/*
+ * Unrolled-by-2 SSE2 implementation
+ */
+static void raid6_sse22_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+        u8 **dptr = (u8 **)ptrs;
+        u8 *p, *q;
+        int d, z, z0;
+        raid6_sse_save_t sa;
+
+        z0 = disks - 3;         /* Highest data disk */
+        p = dptr[z0+1];         /* XOR parity */
+        q = dptr[z0+2];         /* RS syndrome */
+
+        raid6_before_sse2(&sa);
+
+        asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
+        asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */
+        asm volatile("pxor %xmm7,%xmm7"); /* Zero temp */
+
+        /* We uniformly assume a single prefetch covers at least 32 bytes */
+        for ( d = 0 ; d < bytes ; d += 32 ) {
+                asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
+                asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d]));    /* P[0] */
+                asm volatile("movdqa %0,%%xmm3" : : "m" (dptr[z0][d+16])); /* P[1] */
+                asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */
+                asm volatile("movdqa %xmm3,%xmm6"); /* Q[1] */
+                for ( z = z0-1 ; z >= 0 ; z-- ) {
+                        asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
+                        asm volatile("pcmpgtb %xmm4,%xmm5");
+                        asm volatile("pcmpgtb %xmm6,%xmm7");
+                        asm volatile("paddb %xmm4,%xmm4");
+                        asm volatile("paddb %xmm6,%xmm6");
+                        asm volatile("pand %xmm0,%xmm5");
+                        asm volatile("pand %xmm0,%xmm7");
+                        asm volatile("pxor %xmm5,%xmm4");
+                        asm volatile("pxor %xmm7,%xmm6");
+                        asm volatile("movdqa %0,%%xmm5" : : "m" (dptr[z][d]));
+                        asm volatile("movdqa %0,%%xmm7" : : "m" (dptr[z][d+16]));
+                        asm volatile("pxor %xmm5,%xmm2");
+                        asm volatile("pxor %xmm7,%xmm3");
+                        asm volatile("pxor %xmm5,%xmm4");
+                        asm volatile("pxor %xmm7,%xmm6");
+                        asm volatile("pxor %xmm5,%xmm5");
+                        asm volatile("pxor %xmm7,%xmm7");
+                }
+                asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
+                asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
+                asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
+                asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
+        }
+
+        raid6_after_sse2(&sa);
+        asm volatile("sfence" : : : "memory");
+}
+
+const struct raid6_calls raid6_sse2x2 = {
+        raid6_sse22_gen_syndrome,
+        raid6_have_sse2,
+        "sse2x2",
+        1                       /* Has cache hints */
+};
+
+#endif
+
+#ifdef __x86_64__
+
+/*
+ * Unrolled-by-4 SSE2 implementation
+ */
+static void raid6_sse24_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+        u8 **dptr = (u8 **)ptrs;
+        u8 *p, *q;
+        int d, z, z0;
+        raid6_sse16_save_t sa;
+
+        z0 = disks - 3;         /* Highest data disk */
+        p = dptr[z0+1];         /* XOR parity */
+        q = dptr[z0+2];         /* RS syndrome */
+
+        raid6_before_sse16(&sa);
+
+        asm volatile("movdqa %0,%%xmm0" :: "m" (raid6_sse_constants.x1d[0]));
+        asm volatile("pxor %xmm2,%xmm2");       /* P[0] */
+        asm volatile("pxor %xmm3,%xmm3");       /* P[1] */
+        asm volatile("pxor %xmm4,%xmm4");       /* Q[0] */
+        asm volatile("pxor %xmm5,%xmm5");       /* Zero temp */
+        asm volatile("pxor %xmm6,%xmm6");       /* Q[1] */
+        asm volatile("pxor %xmm7,%xmm7");       /* Zero temp */
+        asm volatile("pxor %xmm10,%xmm10");     /* P[2] */
+        asm volatile("pxor %xmm11,%xmm11");     /* P[3] */
+        asm volatile("pxor %xmm12,%xmm12");     /* Q[2] */
+        asm volatile("pxor %xmm13,%xmm13");     /* Zero temp */
+        asm volatile("pxor %xmm14,%xmm14");     /* Q[3] */
+        asm volatile("pxor %xmm15,%xmm15");     /* Zero temp */
+
+        for ( d = 0 ; d < bytes ; d += 64 ) {
+                for ( z = z0 ; z >= 0 ; z-- ) {
+                        /* The second prefetch seems to improve performance... */
+                        asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
+                        asm volatile("prefetchnta %0" :: "m" (dptr[z][d+32]));
+                        asm volatile("pcmpgtb %xmm4,%xmm5");
+                        asm volatile("pcmpgtb %xmm6,%xmm7");
+                        asm volatile("pcmpgtb %xmm12,%xmm13");
+                        asm volatile("pcmpgtb %xmm14,%xmm15");
+                        asm volatile("paddb %xmm4,%xmm4");
+                        asm volatile("paddb %xmm6,%xmm6");
+                        asm volatile("paddb %xmm12,%xmm12");
+                        asm volatile("paddb %xmm14,%xmm14");
+                        asm volatile("pand %xmm0,%xmm5");
+                        asm volatile("pand %xmm0,%xmm7");
+                        asm volatile("pand %xmm0,%xmm13");
+                        asm volatile("pand %xmm0,%xmm15");
+                        asm volatile("pxor %xmm5,%xmm4");
+                        asm volatile("pxor %xmm7,%xmm6");
+                        asm volatile("pxor %xmm13,%xmm12");
+                        asm volatile("pxor %xmm15,%xmm14");
+                        asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
+                        asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
+                        asm volatile("movdqa %0,%%xmm13" :: "m" (dptr[z][d+32]));
+                        asm volatile("movdqa %0,%%xmm15" :: "m" (dptr[z][d+48]));
+                        asm volatile("pxor %xmm5,%xmm2");
+                        asm volatile("pxor %xmm7,%xmm3");
+                        asm volatile("pxor %xmm13,%xmm10");
+                        asm volatile("pxor %xmm15,%xmm11");
+                        asm volatile("pxor %xmm5,%xmm4");
+                        asm volatile("pxor %xmm7,%xmm6");
+                        asm volatile("pxor %xmm13,%xmm12");
+                        asm volatile("pxor %xmm15,%xmm14");
+                        asm volatile("pxor %xmm5,%xmm5");
+                        asm volatile("pxor %xmm7,%xmm7");
+                        asm volatile("pxor %xmm13,%xmm13");
+                        asm volatile("pxor %xmm15,%xmm15");
+                }
+                asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
+                asm volatile("pxor %xmm2,%xmm2");
+                asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
+                asm volatile("pxor %xmm3,%xmm3");
+                asm volatile("movntdq %%xmm10,%0" : "=m" (p[d+32]));
+                asm volatile("pxor %xmm10,%xmm10");
+                asm volatile("movntdq %%xmm11,%0" : "=m" (p[d+48]));
+                asm volatile("pxor %xmm11,%xmm11");
+                asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
+                asm volatile("pxor %xmm4,%xmm4");
+                asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
+                asm volatile("pxor %xmm6,%xmm6");
+                asm volatile("movntdq %%xmm12,%0" : "=m" (q[d+32]));
+                asm volatile("pxor %xmm12,%xmm12");
+                asm volatile("movntdq %%xmm14,%0" : "=m" (q[d+48]));
+                asm volatile("pxor %xmm14,%xmm14");
+        }
+        asm volatile("sfence" : : : "memory");
+        raid6_after_sse16(&sa);
+}
+
+const struct raid6_calls raid6_sse2x4 = {
+        raid6_sse24_gen_syndrome,
+        raid6_have_sse2,
+        "sse2x4",
+        1                       /* Has cache hints */
+};
+
+#endif
diff --git a/drivers/md/raid6test/Makefile b/drivers/md/raid6test/Makefile
new file mode 100644
index 000000000000..557806728609
--- /dev/null
+++ b/drivers/md/raid6test/Makefile
@@ -0,0 +1,58 @@
+#
+# This is a simple Makefile to test some of the RAID-6 code
+# from userspace.
+#
+
+CC       = gcc
+OPTFLAGS = -O2                  # Adjust as desired
+CFLAGS   = -I.. -g $(OPTFLAGS)
+LD       = ld
+PERL     = perl
+
+.c.o:
+        $(CC) $(CFLAGS) -c -o $@ $<
+
+%.c: ../%.c
+        cp -f $< $@
+
+%.uc: ../%.uc
+        cp -f $< $@
+
+all:    raid6.o raid6test
+
+raid6.o: raid6int1.o raid6int2.o raid6int4.o raid6int8.o raid6int16.o \
+         raid6int32.o \
+         raid6mmx.o raid6sse1.o raid6sse2.o \
+         raid6recov.o raid6algos.o \
+         raid6tables.o
+        $(LD) -r -o $@ $^
+
+raid6test: raid6.o test.c
+        $(CC) $(CFLAGS) -o raid6test $^
+
+raid6int1.c: raid6int.uc ../unroll.pl
+        $(PERL) ../unroll.pl 1 < raid6int.uc > $@
+
+raid6int2.c: raid6int.uc ../unroll.pl
+        $(PERL) ../unroll.pl 2 < raid6int.uc > $@
+
+raid6int4.c: raid6int.uc ../unroll.pl
+        $(PERL) ../unroll.pl 4 < raid6int.uc > $@
+
+raid6int8.c: raid6int.uc ../unroll.pl
+        $(PERL) ../unroll.pl 8 < raid6int.uc > $@
+
+raid6int16.c: raid6int.uc ../unroll.pl
+        $(PERL) ../unroll.pl 16 < raid6int.uc > $@
+
+raid6int32.c: raid6int.uc ../unroll.pl
+        $(PERL) ../unroll.pl 32 < raid6int.uc > $@
+
+raid6tables.c: mktables
+        ./mktables > raid6tables.c
+
+clean:
+        rm -f *.o mktables mktables.c raid6int.uc raid6*.c raid6test
+
+spotless: clean
+        rm -f *~
diff --git a/drivers/md/raid6test/test.c b/drivers/md/raid6test/test.c
new file mode 100644
index 000000000000..0d5cd57accd7
--- /dev/null
+++ b/drivers/md/raid6test/test.c
@@ -0,0 +1,103 @@
+/* -*- linux-c -*- ------------------------------------------------------- *
+ *
+ *   Copyright 2002 H. Peter Anvin - All Rights Reserved
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+ *   Bostom MA 02111-1307, USA; either version 2 of the License, or
+ *   (at your option) any later version; incorporated herein by reference.
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * raid6test.c
+ *
+ * Test RAID-6 recovery with various algorithms
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include "raid6.h"
+
+#define NDISKS          16      /* Including P and Q */
+
+const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(256)));
+struct raid6_calls raid6_call;
+
+char *dataptrs[NDISKS];
+char data[NDISKS][PAGE_SIZE];
+char recovi[PAGE_SIZE], recovj[PAGE_SIZE];
+
+void makedata(void)
+{
+        int i, j;
+
+        for (  i = 0 ; i < NDISKS ; i++ ) {
+                for ( j = 0 ; j < PAGE_SIZE ; j++ ) {
+                        data[i][j] = rand();
+                }
+                dataptrs[i] = data[i];
+        }
+}
+
+int main(int argc, char *argv[])
+{
+        const struct raid6_calls * const * algo;
+        int i, j;
+        int erra, errb;
+
+        makedata();
+
+        for ( algo = raid6_algos ; *algo ; algo++ ) {
+                if ( !(*algo)->valid || (*algo)->valid() ) {
+                        raid6_call = **algo;
+
+                        /* Nuke syndromes */
+                        memset(data[NDISKS-2], 0xee, 2*PAGE_SIZE);
+
+                        /* Generate assumed good syndrome */
+                        raid6_call.gen_syndrome(NDISKS, PAGE_SIZE, (void **)&dataptrs);
+
+                        for ( i = 0 ; i < NDISKS-1 ; i++ ) {
+                                for ( j = i+1 ; j < NDISKS ; j++ ) {
+                                        memset(recovi, 0xf0, PAGE_SIZE);
+                                        memset(recovj, 0xba, PAGE_SIZE);
+
+                                        dataptrs[i] = recovi;
+                                        dataptrs[j] = recovj;
+
+                                        raid6_dual_recov(NDISKS, PAGE_SIZE, i, j, (void **)&dataptrs);
+
+                                        erra = memcmp(data[i], recovi, PAGE_SIZE);
+                                        errb = memcmp(data[j], recovj, PAGE_SIZE);
+
+                                        if ( i < NDISKS-2 && j == NDISKS-1 ) {
+                                                /* We don't implement the DQ failure scenario, since it's
+                                                   equivalent to a RAID-5 failure (XOR, then recompute Q) */
+                                        } else {
+                                                printf("algo=%-8s  faila=%3d(%c)  failb=%3d(%c)  %s\n",
+                                                       raid6_call.name,
+                                                       i, (i==NDISKS-2)?'P':'D',
+                                                       j, (j==NDISKS-1)?'Q':(j==NDISKS-2)?'P':'D',
+                                                       (!erra && !errb) ? "OK" :
+                                                       !erra ? "ERRB" :
+                                                       !errb ? "ERRA" :
+                                                       "ERRAB");
+                                        }
+
+                                        dataptrs[i] = data[i];
+                                        dataptrs[j] = data[j];
+                                }
+                        }
+                }
+                printf("\n");
+        }
+
+        printf("\n");
+        /* Pick the best algorithm test */
+        raid6_select_algo();
+
+        return 0;
+}
diff --git a/drivers/md/raid6x86.h b/drivers/md/raid6x86.h
new file mode 100644
index 000000000000..4cf20534fe44
--- /dev/null
+++ b/drivers/md/raid6x86.h
@@ -0,0 +1,245 @@
+/* ----------------------------------------------------------------------- *
+ *
+ *   Copyright 2002-2004 H. Peter Anvin - All Rights Reserved
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+ *   Bostom MA 02111-1307, USA; either version 2 of the License, or
+ *   (at your option) any later version; incorporated herein by reference.
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * raid6x86.h
+ *
+ * Definitions common to x86 and x86-64 RAID-6 code only
+ */
+
+#ifndef LINUX_RAID_RAID6X86_H
+#define LINUX_RAID_RAID6X86_H
+
+#if defined(__i386__) || defined(__x86_64__)
+
+#ifdef __x86_64__
+
+typedef struct {
+        unsigned int fsave[27];
+        unsigned long cr0;
+} raid6_mmx_save_t __attribute__((aligned(16)));
+
+/* N.B.: For SSE we only save %xmm0-%xmm7 even for x86-64, since
+   the code doesn't know about the additional x86-64 registers */
+typedef struct {
+        unsigned int sarea[8*4+2];
+        unsigned long cr0;
+} raid6_sse_save_t __attribute__((aligned(16)));
+
+/* This is for x86-64-specific code which uses all 16 XMM registers */
+typedef struct {
+        unsigned int sarea[16*4+2];
+        unsigned long cr0;
+} raid6_sse16_save_t __attribute__((aligned(16)));
+
+/* On x86-64 the stack *SHOULD* be 16-byte aligned, but currently this
+   is buggy in the kernel and it's only 8-byte aligned in places, so
+   we need to do this anyway.  Sigh. */
+#define SAREA(x) ((unsigned int *)((((unsigned long)&(x)->sarea)+15) & ~15))
+
+#else /* __i386__ */
+
+typedef struct {
+        unsigned int fsave[27];
+        unsigned long cr0;
+} raid6_mmx_save_t;
+
+/* On i386, the stack is only 8-byte aligned, but SSE requires 16-byte
+   alignment.  The +3 is so we have the slack space to manually align
+   a properly-sized area correctly.  */
+typedef struct {
+        unsigned int sarea[8*4+3];
+        unsigned long cr0;
+} raid6_sse_save_t;
+
+/* Find the 16-byte aligned save area */
+#define SAREA(x) ((unsigned int *)((((unsigned long)&(x)->sarea)+15) & ~15))
+
+#endif
+
+#ifdef __KERNEL__ /* Real code */
+
+/* Note: %cr0 is 32 bits on i386 and 64 bits on x86-64 */
+
+static inline unsigned long raid6_get_fpu(void)
+{
+        unsigned long cr0;
+
+        preempt_disable();
+        asm volatile("mov %%cr0,%0 ; clts" : "=r" (cr0));
+        return cr0;
+}
+
+static inline void raid6_put_fpu(unsigned long cr0)
+{
+        asm volatile("mov %0,%%cr0" : : "r" (cr0));
+        preempt_enable();
+}
+
+#else /* Dummy code for user space testing */
+
+static inline unsigned long raid6_get_fpu(void)
+{
+        return 0xf00ba6;
+}
+
+static inline void raid6_put_fpu(unsigned long cr0)
+{
+        (void)cr0;
+}
+
+#endif
+
+static inline void raid6_before_mmx(raid6_mmx_save_t *s)
+{
+        s->cr0 = raid6_get_fpu();
+        asm volatile("fsave %0 ; fwait" : "=m" (s->fsave[0]));
+}
+
+static inline void raid6_after_mmx(raid6_mmx_save_t *s)
+{
+        asm volatile("frstor %0" : : "m" (s->fsave[0]));
+        raid6_put_fpu(s->cr0);
+}
+
+static inline void raid6_before_sse(raid6_sse_save_t *s)
+{
+        unsigned int *rsa = SAREA(s);
+
+        s->cr0 = raid6_get_fpu();
+
+        asm volatile("movaps %%xmm0,%0" : "=m" (rsa[0]));
+        asm volatile("movaps %%xmm1,%0" : "=m" (rsa[4]));
+        asm volatile("movaps %%xmm2,%0" : "=m" (rsa[8]));
+        asm volatile("movaps %%xmm3,%0" : "=m" (rsa[12]));
+        asm volatile("movaps %%xmm4,%0" : "=m" (rsa[16]));
+        asm volatile("movaps %%xmm5,%0" : "=m" (rsa[20]));
+        asm volatile("movaps %%xmm6,%0" : "=m" (rsa[24]));
+        asm volatile("movaps %%xmm7,%0" : "=m" (rsa[28]));
+}
+
+static inline void raid6_after_sse(raid6_sse_save_t *s)
+{
+        unsigned int *rsa = SAREA(s);
+
+        asm volatile("movaps %0,%%xmm0" : : "m" (rsa[0]));
+        asm volatile("movaps %0,%%xmm1" : : "m" (rsa[4]));
+        asm volatile("movaps %0,%%xmm2" : : "m" (rsa[8]));
+        asm volatile("movaps %0,%%xmm3" : : "m" (rsa[12]));
+        asm volatile("movaps %0,%%xmm4" : : "m" (rsa[16]));
+        asm volatile("movaps %0,%%xmm5" : : "m" (rsa[20]));
+        asm volatile("movaps %0,%%xmm6" : : "m" (rsa[24]));
+        asm volatile("movaps %0,%%xmm7" : : "m" (rsa[28]));
+
+        raid6_put_fpu(s->cr0);
+}
+
+static inline void raid6_before_sse2(raid6_sse_save_t *s)
+{
+        unsigned int *rsa = SAREA(s);
+
+        s->cr0 = raid6_get_fpu();
+
+        asm volatile("movdqa %%xmm0,%0" : "=m" (rsa[0]));
+        asm volatile("movdqa %%xmm1,%0" : "=m" (rsa[4]));
+        asm volatile("movdqa %%xmm2,%0" : "=m" (rsa[8]));
+        asm volatile("movdqa %%xmm3,%0" : "=m" (rsa[12]));
+        asm volatile("movdqa %%xmm4,%0" : "=m" (rsa[16]));
+        asm volatile("movdqa %%xmm5,%0" : "=m" (rsa[20]));
+        asm volatile("movdqa %%xmm6,%0" : "=m" (rsa[24]));
+        asm volatile("movdqa %%xmm7,%0" : "=m" (rsa[28]));
+}
+
+static inline void raid6_after_sse2(raid6_sse_save_t *s)
+{
+        unsigned int *rsa = SAREA(s);
+
+        asm volatile("movdqa %0,%%xmm0" : : "m" (rsa[0]));
+        asm volatile("movdqa %0,%%xmm1" : : "m" (rsa[4]));
+        asm volatile("movdqa %0,%%xmm2" : : "m" (rsa[8]));
+        asm volatile("movdqa %0,%%xmm3" : : "m" (rsa[12]));
+        asm volatile("movdqa %0,%%xmm4" : : "m" (rsa[16]));
+        asm volatile("movdqa %0,%%xmm5" : : "m" (rsa[20]));
+        asm volatile("movdqa %0,%%xmm6" : : "m" (rsa[24]));
+        asm volatile("movdqa %0,%%xmm7" : : "m" (rsa[28]));
+
+        raid6_put_fpu(s->cr0);
+}
+
+#ifdef __x86_64__
+
+static inline void raid6_before_sse16(raid6_sse16_save_t *s)
+{
+        unsigned int *rsa = SAREA(s);
+
+        s->cr0 = raid6_get_fpu();
+
+        asm volatile("movdqa %%xmm0,%0" : "=m" (rsa[0]));
+        asm volatile("movdqa %%xmm1,%0" : "=m" (rsa[4]));
+        asm volatile("movdqa %%xmm2,%0" : "=m" (rsa[8]));
+        asm volatile("movdqa %%xmm3,%0" : "=m" (rsa[12]));
+        asm volatile("movdqa %%xmm4,%0" : "=m" (rsa[16]));
+        asm volatile("movdqa %%xmm5,%0" : "=m" (rsa[20]));
+        asm volatile("movdqa %%xmm6,%0" : "=m" (rsa[24]));
+        asm volatile("movdqa %%xmm7,%0" : "=m" (rsa[28]));
+        asm volatile("movdqa %%xmm8,%0" : "=m" (rsa[32]));
+        asm volatile("movdqa %%xmm9,%0" : "=m" (rsa[36]));
+        asm volatile("movdqa %%xmm10,%0" : "=m" (rsa[40]));
+        asm volatile("movdqa %%xmm11,%0" : "=m" (rsa[44]));
+        asm volatile("movdqa %%xmm12,%0" : "=m" (rsa[48]));
+        asm volatile("movdqa %%xmm13,%0" : "=m" (rsa[52]));
+        asm volatile("movdqa %%xmm14,%0" : "=m" (rsa[56]));
+        asm volatile("movdqa %%xmm15,%0" : "=m" (rsa[60]));
+}
+
+static inline void raid6_after_sse16(raid6_sse16_save_t *s)
+{
+        unsigned int *rsa = SAREA(s);
+
+        asm volatile("movdqa %0,%%xmm0" : : "m" (rsa[0]));
+        asm volatile("movdqa %0,%%xmm1" : : "m" (rsa[4]));
+        asm volatile("movdqa %0,%%xmm2" : : "m" (rsa[8]));
+        asm volatile("movdqa %0,%%xmm3" : : "m" (rsa[12]));
+        asm volatile("movdqa %0,%%xmm4" : : "m" (rsa[16]));
+        asm volatile("movdqa %0,%%xmm5" : : "m" (rsa[20]));
+        asm volatile("movdqa %0,%%xmm6" : : "m" (rsa[24]));
+        asm volatile("movdqa %0,%%xmm7" : : "m" (rsa[28]));
+        asm volatile("movdqa %0,%%xmm8" : : "m" (rsa[32]));
+        asm volatile("movdqa %0,%%xmm9" : : "m" (rsa[36]));
+        asm volatile("movdqa %0,%%xmm10" : : "m" (rsa[40]));
+        asm volatile("movdqa %0,%%xmm11" : : "m" (rsa[44]));
+        asm volatile("movdqa %0,%%xmm12" : : "m" (rsa[48]));
+        asm volatile("movdqa %0,%%xmm13" : : "m" (rsa[52]));
+        asm volatile("movdqa %0,%%xmm14" : : "m" (rsa[56]));
+        asm volatile("movdqa %0,%%xmm15" : : "m" (rsa[60]));
+
+        raid6_put_fpu(s->cr0);
+}
+
+#endif /* __x86_64__ */
+
+/* User space test hack */
+#ifndef __KERNEL__
+static inline int cpuid_features(void)
+{
+        u32 eax = 1;
+        u32 ebx, ecx, edx;
+
+        asm volatile("cpuid" :
+                     "+a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx));
+
+        return edx;
+}
+#endif /* ndef __KERNEL__ */
+
+#endif
+#endif
diff --git a/drivers/md/unroll.pl b/drivers/md/unroll.pl
new file mode 100644
index 000000000000..3acc710a20ea
--- /dev/null
+++ b/drivers/md/unroll.pl
@@ -0,0 +1,24 @@
+#!/usr/bin/perl
+#
+# Take a piece of C code and for each line which contains the sequence $$
+# repeat n times with $ replaced by 0...n-1; the sequence $# is replaced
+# by the unrolling factor, and $* with a single $
+#
+
+($n) = @ARGV;
+$n += 0;
+
+while ( defined($line = <STDIN>) ) {
+    if ( $line =~ /\$\$/ ) {
+        $rep = $n;
+    } else {
+        $rep = 1;
+    }
+    for ( $i = 0 ; $i < $rep ; $i++ ) {
+        $tmp = $line;
+        $tmp =~ s/\$\$/$i/g;
+        $tmp =~ s/\$\#/$n/g;
+        $tmp =~ s/\$\*/\$/g;
+        print $tmp;
+    }
+}
diff --git a/drivers/md/xor.c b/drivers/md/xor.c
new file mode 100644
index 000000000000..324897c4be4e
--- /dev/null
+++ b/drivers/md/xor.c
@@ -0,0 +1,154 @@
+/*
+ * xor.c : Multiple Devices driver for Linux
+ *
+ * Copyright (C) 1996, 1997, 1998, 1999, 2000,
+ * Ingo Molnar, Matti Aarnio, Jakub Jelinek, Richard Henderson.
+ *
+ * Dispatch optimized RAID-5 checksumming functions.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * (for example /usr/src/linux/COPYING); if not, write to the Free
+ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#define BH_TRACE 0
+#include <linux/module.h>
+#include <linux/raid/md.h>
+#include <linux/raid/xor.h>
+#include <asm/xor.h>
+
+/* The xor routines to use.  */
+static struct xor_block_template *active_template;
+
+void
+xor_block(unsigned int count, unsigned int bytes, void **ptr)
+{
+        unsigned long *p0, *p1, *p2, *p3, *p4;
+
+        p0 = (unsigned long *) ptr[0];
+        p1 = (unsigned long *) ptr[1];
+        if (count == 2) {
+                active_template->do_2(bytes, p0, p1);
+                return;
+        }
+
+        p2 = (unsigned long *) ptr[2];
+        if (count == 3) {
+                active_template->do_3(bytes, p0, p1, p2);
+                return;
+        }
+
+        p3 = (unsigned long *) ptr[3];
+        if (count == 4) {
+                active_template->do_4(bytes, p0, p1, p2, p3);
+                return;
+        }
+
+        p4 = (unsigned long *) ptr[4];
+        active_template->do_5(bytes, p0, p1, p2, p3, p4);
+}
+
+/* Set of all registered templates.  */
+static struct xor_block_template *template_list;
+
+#define BENCH_SIZE (PAGE_SIZE)
+
+static void
+do_xor_speed(struct xor_block_template *tmpl, void *b1, void *b2)
+{
+        int speed;
+        unsigned long now;
+        int i, count, max;
+
+        tmpl->next = template_list;
+        template_list = tmpl;
+
+        /*
+         * Count the number of XORs done during a whole jiffy, and use
+         * this to calculate the speed of checksumming.  We use a 2-page
+         * allocation to have guaranteed color L1-cache layout.
+         */
+        max = 0;
+        for (i = 0; i < 5; i++) {
+                now = jiffies;
+                count = 0;
+                while (jiffies == now) {
+                        mb();
+                        tmpl->do_2(BENCH_SIZE, b1, b2);
+                        mb();
+                        count++;
+                        mb();
+                }
+                if (count > max)
+                        max = count;
+        }
+
+        speed = max * (HZ * BENCH_SIZE / 1024);
+        tmpl->speed = speed;
+
+        printk("   %-10s: %5d.%03d MB/sec\n", tmpl->name,
+               speed / 1000, speed % 1000);
+}
+
+static int
+calibrate_xor_block(void)
+{
+        void *b1, *b2;
+        struct xor_block_template *f, *fastest;
+
+        b1 = (void *) __get_free_pages(GFP_KERNEL, 2);
+        if (! b1) {
+                printk("raid5: Yikes!  No memory available.\n");
+                return -ENOMEM;
+        }
+        b2 = b1 + 2*PAGE_SIZE + BENCH_SIZE;
+
+        /*
+         * If this arch/cpu has a short-circuited selection, don't loop through all
+         * the possible functions, just test the best one
+         */
+
+        fastest = NULL;
+
+#ifdef XOR_SELECT_TEMPLATE
+                fastest = XOR_SELECT_TEMPLATE(fastest);
+#endif
+
+#define xor_speed(templ)        do_xor_speed((templ), b1, b2)
+
+        if (fastest) {
+                printk(KERN_INFO "raid5: automatically using best checksumming function: %s\n",
+                        fastest->name);
+                xor_speed(fastest);
+        } else {
+                printk(KERN_INFO "raid5: measuring checksumming speed\n");
+                XOR_TRY_TEMPLATES;
+                fastest = template_list;
+                for (f = fastest; f; f = f->next)
+                        if (f->speed > fastest->speed)
+                                fastest = f;
+        }
+
+        printk("raid5: using function: %s (%d.%03d MB/sec)\n",
+               fastest->name, fastest->speed / 1000, fastest->speed % 1000);
+
+#undef xor_speed
+
+        free_pages((unsigned long)b1, 2);
+
+        active_template = fastest;
+        return 0;
+}
+
+static __exit void xor_exit(void) { }
+
+EXPORT_SYMBOL(xor_block);
+MODULE_LICENSE("GPL");
+
+module_init(calibrate_xor_block);
+module_exit(xor_exit);