6 files changed, 361 insertions, 28 deletions

diff --git a/fs/exofs/common.h b/fs/exofs/common.h
index ce1c71692599..b1b178e61718 100644
--- a/fs/exofs/common.h
+++ b/fs/exofs/common.h
@@ -49,6 +49,7 @@
 #define EXOFS_MIN_PID   0x10000 /* Smallest partition ID */
 #define EXOFS_OBJ_OFF   0x10000 /* offset for objects */
 #define EXOFS_SUPER_ID  0x10000 /* object ID for on-disk superblock */
+#define EXOFS_DEVTABLE_ID 0x10001 /* object ID for on-disk device table */
 #define EXOFS_ROOT_ID   0x10002 /* object ID for root directory */
 
 /* exofs Application specific page/attribute */
@@ -78,17 +79,67 @@ enum {
 #define EXOFS_SUPER_MAGIC       0x5DF5
 
 /*
- * The file system control block - stored in an object's data (mainly, the one
- * with ID EXOFS_SUPER_ID).  This is where the in-memory superblock is stored
- * on disk.  Right now it just has a magic value, which is basically a sanity
- * check on our ability to communicate with the object store.
+ * The file system control block - stored in object EXOFS_SUPER_ID's data.
+ * This is where the in-memory superblock is stored on disk.
  */
+enum {EXOFS_FSCB_VER = 1, EXOFS_DT_VER = 1};
 struct exofs_fscb {
         __le64  s_nextid;       /* Highest object ID used */
-        __le32  s_numfiles;     /* Number of files on fs */
+        __le64  s_numfiles;     /* Number of files on fs */
+        __le32  s_version;      /* == EXOFS_FSCB_VER */
         __le16  s_magic;        /* Magic signature */
         __le16  s_newfs;        /* Non-zero if this is a new fs */
-};
+
+        /* From here on it's a static part, only written by mkexofs */
+        __le64  s_dev_table_oid;   /* Resurved, not used */
+        __le64  s_dev_table_count; /* == 0 means no dev_table */
+} __packed;
+
+/*
+ * Describes the raid used in the FS. It is part of the device table.
+ * This here is taken from the pNFS-objects definition. In exofs we
+ * use one raid policy through-out the filesystem. (NOTE: the funny
+ * alignment at begining. We take care of it at exofs_device_table.
+ */
+struct exofs_dt_data_map {
+        __le32  cb_num_comps;
+        __le64  cb_stripe_unit;
+        __le32  cb_group_width;
+        __le32  cb_group_depth;
+        __le32  cb_mirror_cnt;
+        __le32  cb_raid_algorithm;
+} __packed;
+
+/*
+ * This is an osd device information descriptor. It is a single entry in
+ * the exofs device table. It describes an osd target lun which
+ * contains data belonging to this FS. (Same partition_id on all devices)
+ */
+struct exofs_dt_device_info {
+        __le32  systemid_len;
+        u8      systemid[OSD_SYSTEMID_LEN];
+        __le64  long_name_offset;       /* If !0 then offset-in-file */
+        __le32  osdname_len;            /* */
+        u8      osdname[44];            /* Embbeded, Ususally an asci uuid */
+} __packed;
+
+/*
+ * The EXOFS device table - stored in object EXOFS_DEVTABLE_ID's data.
+ * It contains the raid used for this multy-device FS and an array of
+ * participating devices.
+ */
+struct exofs_device_table {
+        __le32                          dt_version;     /* == EXOFS_DT_VER */
+        struct exofs_dt_data_map        dt_data_map;    /* Raid policy to use */
+
+        /* Resurved space For future use. Total includeing this:
+         * (8 * sizeof(le64))
+         */
+        __le64                          __Resurved[4];
+
+        __le64                          dt_num_devices; /* Array size */
+        struct exofs_dt_device_info     dt_dev_table[]; /* Array of devices */
+} __packed;
 
 /****************************************************************************
  * inode-related things
diff --git a/fs/exofs/exofs.h b/fs/exofs/exofs.h
index 2e08859a89e8..c35fd4623986 100644
--- a/fs/exofs/exofs.h
+++ b/fs/exofs/exofs.h
@@ -37,6 +37,11 @@
 #include <linux/time.h>
 #include "common.h"
 
+/* FIXME: Remove once pnfs hits mainline
+ * #include <linux/exportfs/pnfs_osd_xdr.h>
+ */
+#include "pnfs.h"
+
 #define EXOFS_ERR(fmt, a...) printk(KERN_ERR "exofs: " fmt, ##a)
 
 #ifdef CONFIG_EXOFS_DEBUG
@@ -54,7 +59,6 @@
  * our extension to the in-memory superblock
  */
 struct exofs_sb_info {
-        struct osd_dev  *s_dev;                 /* returned by get_osd_dev    */
         struct exofs_fscb s_fscb;               /* Written often, pre-allocate*/
         osd_id          s_pid;                  /* partition ID of file system*/
         int             s_timeout;              /* timeout for OSD operations */
@@ -63,7 +67,11 @@ struct exofs_sb_info {
         spinlock_t      s_next_gen_lock;        /* spinlock for gen # update  */
         u32             s_next_generation;      /* next gen # to use          */
         atomic_t        s_curr_pending;         /* number of pending commands */
-        uint8_t         s_cred[OSD_CAP_LEN];    /* all-powerful credential    */
+        uint8_t         s_cred[OSD_CAP_LEN];    /* credential for the fscb    */
+
+        struct pnfs_osd_data_map data_map;      /* Default raid to use        */
+        unsigned        s_numdevs;              /* Num of devices in array    */
+        struct osd_dev  *s_ods[1];              /* Variable length, minimum 1 */
 };
 
 /*
diff --git a/fs/exofs/inode.c b/fs/exofs/inode.c
index 7578950fd135..698a8636d39c 100644
--- a/fs/exofs/inode.c
+++ b/fs/exofs/inode.c
@@ -62,7 +62,10 @@ static void _pcol_init(struct page_collect *pcol, unsigned expected_pages,
         struct exofs_sb_info *sbi = inode->i_sb->s_fs_info;
 
         pcol->sbi = sbi;
-        pcol->req_q = osd_request_queue(sbi->s_dev);
+        /* Create master bios on first Q, later on cloning, each clone will be
+         * allocated on it's destination Q
+         */
+        pcol->req_q = osd_request_queue(sbi->s_ods[0]);
         pcol->inode = inode;
         pcol->expected_pages = expected_pages;
 
diff --git a/fs/exofs/ios.c b/fs/exofs/ios.c
index bb2f9d341fdf..5bad01fa1f9f 100644
--- a/fs/exofs/ios.c
+++ b/fs/exofs/ios.c
@@ -71,7 +71,7 @@ int exofs_get_io_state(struct exofs_sb_info *sbi, struct exofs_io_state** pios)
         /*TODO: Maybe use kmem_cach per sbi of size
          * exofs_io_state_size(sbi->s_numdevs)
          */
-        ios = kzalloc(exofs_io_state_size(1), GFP_KERNEL);
+        ios = kzalloc(exofs_io_state_size(sbi->s_numdevs), GFP_KERNEL);
         if (unlikely(!ios)) {
                 *pios = NULL;
                 return -ENOMEM;
@@ -209,10 +209,10 @@ int exofs_sbi_create(struct exofs_io_state *ios)
 {
         int i, ret;
 
-        for (i = 0; i < 1; i++) {
+        for (i = 0; i < ios->sbi->s_numdevs; i++) {
                 struct osd_request *or;
 
-                or = osd_start_request(ios->sbi->s_dev, GFP_KERNEL);
+                or = osd_start_request(ios->sbi->s_ods[i], GFP_KERNEL);
                 if (unlikely(!or)) {
                         EXOFS_ERR("%s: osd_start_request failed\n", __func__);
                         ret = -ENOMEM;
@@ -233,10 +233,10 @@ int exofs_sbi_remove(struct exofs_io_state *ios)
 {
         int i, ret;
 
-        for (i = 0; i < 1; i++) {
+        for (i = 0; i < ios->sbi->s_numdevs; i++) {
                 struct osd_request *or;
 
-                or = osd_start_request(ios->sbi->s_dev, GFP_KERNEL);
+                or = osd_start_request(ios->sbi->s_ods[i], GFP_KERNEL);
                 if (unlikely(!or)) {
                         EXOFS_ERR("%s: osd_start_request failed\n", __func__);
                         ret = -ENOMEM;
@@ -257,10 +257,10 @@ int exofs_sbi_write(struct exofs_io_state *ios)
 {
         int i, ret;
 
-        for (i = 0; i < 1; i++) {
+        for (i = 0; i < ios->sbi->s_numdevs; i++) {
                 struct osd_request *or;
 
-                or = osd_start_request(ios->sbi->s_dev, GFP_KERNEL);
+                or = osd_start_request(ios->sbi->s_ods[i], GFP_KERNEL);
                 if (unlikely(!or)) {
                         EXOFS_ERR("%s: osd_start_request failed\n", __func__);
                         ret = -ENOMEM;
@@ -272,7 +272,21 @@ int exofs_sbi_write(struct exofs_io_state *ios)
                 if (ios->bio) {
                         struct bio *bio;
 
-                        bio = ios->bio;
+                        if (i != 0) {
+                                bio = bio_kmalloc(GFP_KERNEL,
+                                                  ios->bio->bi_max_vecs);
+                                if (unlikely(!bio)) {
+                                        ret = -ENOMEM;
+                                        goto out;
+                                }
+
+                                __bio_clone(bio, ios->bio);
+                                bio->bi_bdev = NULL;
+                                bio->bi_next = NULL;
+                                ios->per_dev[i].bio =  bio;
+                        } else {
+                                bio = ios->bio;
+                        }
 
                         osd_req_write(or, &ios->obj, ios->offset, bio,
                                       ios->length);
@@ -306,8 +320,10 @@ int exofs_sbi_read(struct exofs_io_state *ios)
 
         for (i = 0; i < 1; i++) {
                 struct osd_request *or;
+                unsigned first_dev = (unsigned)ios->obj.id;
 
-                or = osd_start_request(ios->sbi->s_dev, GFP_KERNEL);
+                first_dev %= ios->sbi->s_numdevs;
+                or = osd_start_request(ios->sbi->s_ods[first_dev], GFP_KERNEL);
                 if (unlikely(!or)) {
                         EXOFS_ERR("%s: osd_start_request failed\n", __func__);
                         ret = -ENOMEM;
@@ -382,10 +398,10 @@ int exofs_oi_truncate(struct exofs_i_info *oi, u64 size)
         attr = g_attr_logical_length;
         attr.val_ptr = &newsize;
 
-        for (i = 0; i < 1; i++) {
+        for (i = 0; i < sbi->s_numdevs; i++) {
                 struct osd_request *or;
 
-                or = osd_start_request(sbi->s_dev, GFP_KERNEL);
+                or = osd_start_request(sbi->s_ods[i], GFP_KERNEL);
                 if (unlikely(!or)) {
                         EXOFS_ERR("%s: osd_start_request failed\n", __func__);
                         ret = -ENOMEM;
diff --git a/fs/exofs/pnfs.h b/fs/exofs/pnfs.h
new file mode 100644
index 000000000000..423033addd1f
--- /dev/null
+++ b/fs/exofs/pnfs.h
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2008, 2009
+ * Boaz Harrosh <bharrosh@panasas.com>
+ *
+ * This file is part of exofs.
+ *
+ * exofs is free software; you can redistribute it and/or modify it under the
+ * terms of the GNU General Public License  version 2 as published by the Free
+ * Software Foundation.
+ *
+ */
+
+/* FIXME: Remove this file once pnfs hits mainline */
+
+#ifndef __EXOFS_PNFS_H__
+#define __EXOFS_PNFS_H__
+
+#if defined(CONFIG_PNFS)
+
+
+/* FIXME: move this file to: linux/exportfs/pnfs_osd_xdr.h */
+#include "../nfs/objlayout/pnfs_osd_xdr.h"
+
+#else /* defined(CONFIG_PNFS) */
+
+enum pnfs_iomode {
+        IOMODE_READ = 1,
+        IOMODE_RW = 2,
+        IOMODE_ANY = 3,
+};
+
+/* Layout Structure */
+enum pnfs_osd_raid_algorithm4 {
+        PNFS_OSD_RAID_0         = 1,
+        PNFS_OSD_RAID_4         = 2,
+        PNFS_OSD_RAID_5         = 3,
+        PNFS_OSD_RAID_PQ        = 4     /* Reed-Solomon P+Q */
+};
+
+struct pnfs_osd_data_map {
+        u32     odm_num_comps;
+        u64     odm_stripe_unit;
+        u32     odm_group_width;
+        u32     odm_group_depth;
+        u32     odm_mirror_cnt;
+        u32     odm_raid_algorithm;
+};
+
+#endif /* else defined(CONFIG_PNFS) */
+
+#endif /* __EXOFS_PNFS_H__ */
diff --git a/fs/exofs/super.c b/fs/exofs/super.c
index 4cd97f526d49..a1d1e77b12eb 100644
--- a/fs/exofs/super.c
+++ b/fs/exofs/super.c
@@ -214,12 +214,17 @@ int exofs_sync_fs(struct super_block *sb, int wait)
         if (ret)
                 goto out;
 
-        ios->length = sizeof(*fscb);
+        /* Note: We only write the changing part of the fscb. .i.e upto the
+         *       the fscb->s_dev_table_oid member. There is no read-modify-write
+         *       here.
+         */
+        ios->length = offsetof(struct exofs_fscb, s_dev_table_oid);
         memset(fscb, 0, ios->length);
         fscb->s_nextid = cpu_to_le64(sbi->s_nextid);
         fscb->s_numfiles = cpu_to_le32(sbi->s_numfiles);
         fscb->s_magic = cpu_to_le16(sb->s_magic);
         fscb->s_newfs = 0;
+        fscb->s_version = EXOFS_FSCB_VER;
 
         ios->obj.id = EXOFS_SUPER_ID;
         ios->offset = 0;
@@ -257,6 +262,20 @@ static void _exofs_print_device(const char *msg, const char *dev_path,
                 msg, dev_path ?: "", odi->osdname, _LLU(pid));
 }
 
+void exofs_free_sbi(struct exofs_sb_info *sbi)
+{
+        while (sbi->s_numdevs) {
+                int i = --sbi->s_numdevs;
+                struct osd_dev *od = sbi->s_ods[i];
+
+                if (od) {
+                        sbi->s_ods[i] = NULL;
+                        osduld_put_device(od);
+                }
+        }
+        kfree(sbi);
+}
+
 /*
  * This function is called when the vfs is freeing the superblock.  We just
  * need to free our own part.
@@ -279,12 +298,182 @@ static void exofs_put_super(struct super_block *sb)
                                   msecs_to_jiffies(100));
         }
 
-        _exofs_print_device("Unmounting", NULL, sbi->s_dev, sbi->s_pid);
-        osduld_put_device(sbi->s_dev);
-        kfree(sb->s_fs_info);
+        _exofs_print_device("Unmounting", NULL, sbi->s_ods[0], sbi->s_pid);
+
+        exofs_free_sbi(sbi);
         sb->s_fs_info = NULL;
 }
 
+static int _read_and_match_data_map(struct exofs_sb_info *sbi, unsigned numdevs,
+                                    struct exofs_device_table *dt)
+{
+        sbi->data_map.odm_num_comps   =
+                                le32_to_cpu(dt->dt_data_map.cb_num_comps);
+        sbi->data_map.odm_stripe_unit =
+                                le64_to_cpu(dt->dt_data_map.cb_stripe_unit);
+        sbi->data_map.odm_group_width =
+                                le32_to_cpu(dt->dt_data_map.cb_group_width);
+        sbi->data_map.odm_group_depth =
+                                le32_to_cpu(dt->dt_data_map.cb_group_depth);
+        sbi->data_map.odm_mirror_cnt  =
+                                le32_to_cpu(dt->dt_data_map.cb_mirror_cnt);
+        sbi->data_map.odm_raid_algorithm  =
+                                le32_to_cpu(dt->dt_data_map.cb_raid_algorithm);
+
+/* FIXME: Hard coded mirror only for now. if not so do not mount */
+        if ((sbi->data_map.odm_num_comps != numdevs) ||
+            (sbi->data_map.odm_stripe_unit != EXOFS_BLKSIZE) ||
+            (sbi->data_map.odm_raid_algorithm != PNFS_OSD_RAID_0) ||
+            (sbi->data_map.odm_mirror_cnt != (numdevs - 1)))
+                return -EINVAL;
+        else
+                return 0;
+}
+
+/* @odi is valid only as long as @fscb_dev is valid */
+static int exofs_devs_2_odi(struct exofs_dt_device_info *dt_dev,
+                             struct osd_dev_info *odi)
+{
+        odi->systemid_len = le32_to_cpu(dt_dev->systemid_len);
+        memcpy(odi->systemid, dt_dev->systemid, odi->systemid_len);
+
+        odi->osdname_len = le32_to_cpu(dt_dev->osdname_len);
+        odi->osdname = dt_dev->osdname;
+
+        /* FIXME support long names. Will need a _put function */
+        if (dt_dev->long_name_offset)
+                return -EINVAL;
+
+        /* Make sure osdname is printable!
+         * mkexofs should give us space for a null-terminator else the
+         * device-table is invalid.
+         */
+        if (unlikely(odi->osdname_len >= sizeof(dt_dev->osdname)))
+                odi->osdname_len = sizeof(dt_dev->osdname) - 1;
+        dt_dev->osdname[odi->osdname_len] = 0;
+
+        /* If it's all zeros something is bad we read past end-of-obj */
+        return !(odi->systemid_len || odi->osdname_len);
+}
+
+static int exofs_read_lookup_dev_table(struct exofs_sb_info **psbi,
+                                       unsigned table_count)
+{
+        struct exofs_sb_info *sbi = *psbi;
+        struct osd_dev *fscb_od;
+        struct osd_obj_id obj = {.partition = sbi->s_pid,
+                                 .id = EXOFS_DEVTABLE_ID};
+        struct exofs_device_table *dt;
+        unsigned table_bytes = table_count * sizeof(dt->dt_dev_table[0]) +
+                                             sizeof(*dt);
+        unsigned numdevs, i;
+        int ret;
+
+        dt = kmalloc(table_bytes, GFP_KERNEL);
+        if (unlikely(!dt)) {
+                EXOFS_ERR("ERROR: allocating %x bytes for device table\n",
+                          table_bytes);
+                return -ENOMEM;
+        }
+
+        fscb_od = sbi->s_ods[0];
+        sbi->s_ods[0] = NULL;
+        sbi->s_numdevs = 0;
+        ret = exofs_read_kern(fscb_od, sbi->s_cred, &obj, 0, dt, table_bytes);
+        if (unlikely(ret)) {
+                EXOFS_ERR("ERROR: reading device table\n");
+                goto out;
+        }
+
+        numdevs = le64_to_cpu(dt->dt_num_devices);
+        if (unlikely(!numdevs)) {
+                ret = -EINVAL;
+                goto out;
+        }
+        WARN_ON(table_count != numdevs);
+
+        ret = _read_and_match_data_map(sbi, numdevs, dt);
+        if (unlikely(ret))
+                goto out;
+
+        if (likely(numdevs > 1)) {
+                unsigned size = numdevs * sizeof(sbi->s_ods[0]);
+
+                sbi = krealloc(sbi, sizeof(*sbi) + size, GFP_KERNEL);
+                if (unlikely(!sbi)) {
+                        ret = -ENOMEM;
+                        goto out;
+                }
+                memset(&sbi->s_ods[1], 0, size - sizeof(sbi->s_ods[0]));
+                *psbi = sbi;
+        }
+
+        for (i = 0; i < numdevs; i++) {
+                struct exofs_fscb fscb;
+                struct osd_dev_info odi;
+                struct osd_dev *od;
+
+                if (exofs_devs_2_odi(&dt->dt_dev_table[i], &odi)) {
+                        EXOFS_ERR("ERROR: Read all-zeros device entry\n");
+                        ret = -EINVAL;
+                        goto out;
+                }
+
+                printk(KERN_NOTICE "Add device[%d]: osd_name-%s\n",
+                       i, odi.osdname);
+
+                /* On all devices the device table is identical. The user can
+                 * specify any one of the participating devices on the command
+                 * line. We always keep them in device-table order.
+                 */
+                if (fscb_od && osduld_device_same(fscb_od, &odi)) {
+                        sbi->s_ods[i] = fscb_od;
+                        ++sbi->s_numdevs;
+                        fscb_od = NULL;
+                        continue;
+                }
+
+                od = osduld_info_lookup(&odi);
+                if (unlikely(IS_ERR(od))) {
+                        ret = PTR_ERR(od);
+                        EXOFS_ERR("ERROR: device requested is not found "
+                                  "osd_name-%s =>%d\n", odi.osdname, ret);
+                        goto out;
+                }
+
+                sbi->s_ods[i] = od;
+                ++sbi->s_numdevs;
+
+                /* Read the fscb of the other devices to make sure the FS
+                 * partition is there.
+                 */
+                ret = exofs_read_kern(od, sbi->s_cred, &obj, 0, &fscb,
+                                      sizeof(fscb));
+                if (unlikely(ret)) {
+                        EXOFS_ERR("ERROR: Malformed participating device "
+                                  "error reading fscb osd_name-%s\n",
+                                  odi.osdname);
+                        goto out;
+                }
+
+                /* TODO: verify other information is correct and FS-uuid
+                 *       matches. Benny what did you say about device table
+                 *       generation and old devices?
+                 */
+        }
+
+out:
+        kfree(dt);
+        if (unlikely(!ret && fscb_od)) {
+                EXOFS_ERR(
+                      "ERROR: Bad device-table container device not present\n");
+                osduld_put_device(fscb_od);
+                ret = -EINVAL;
+        }
+
+        return ret;
+}
+
 /*
  * Read the superblock from the OSD and fill in the fields
  */
@@ -296,6 +485,7 @@ static int exofs_fill_super(struct super_block *sb, void *data, int silent)
         struct osd_dev *od;             /* Master device                 */
         struct exofs_fscb fscb;         /*on-disk superblock info        */
         struct osd_obj_id obj;
+        unsigned table_count;
         int ret;
 
         sbi = kzalloc(sizeof(*sbi), GFP_KERNEL);
@@ -309,7 +499,8 @@ static int exofs_fill_super(struct super_block *sb, void *data, int silent)
                 goto free_sbi;
         }
 
-        sbi->s_dev = od;
+        sbi->s_ods[0] = od;
+        sbi->s_numdevs = 1;
         sbi->s_pid = opts->pid;
         sbi->s_timeout = opts->timeout;
 
@@ -342,11 +533,24 @@ static int exofs_fill_super(struct super_block *sb, void *data, int silent)
                 ret = -EINVAL;
                 goto free_sbi;
         }
+        if (le32_to_cpu(fscb.s_version) != EXOFS_FSCB_VER) {
+                EXOFS_ERR("ERROR: Bad FSCB version expected-%d got-%d\n",
+                          EXOFS_FSCB_VER, le32_to_cpu(fscb.s_version));
+                ret = -EINVAL;
+                goto free_sbi;
+        }
 
         /* start generation numbers from a random point */
         get_random_bytes(&sbi->s_next_generation, sizeof(u32));
         spin_lock_init(&sbi->s_next_gen_lock);
 
+        table_count = le64_to_cpu(fscb.s_dev_table_count);
+        if (table_count) {
+                ret = exofs_read_lookup_dev_table(&sbi, table_count);
+                if (unlikely(ret))
+                        goto free_sbi;
+        }
+
         /* set up operation vectors */
         sb->s_fs_info = sbi;
         sb->s_op = &exofs_sops;
@@ -374,14 +578,14 @@ static int exofs_fill_super(struct super_block *sb, void *data, int silent)
                 goto free_sbi;
         }
 
-        _exofs_print_device("Mounting", opts->dev_name, sbi->s_dev, sbi->s_pid);
+        _exofs_print_device("Mounting", opts->dev_name, sbi->s_ods[0],
+                            sbi->s_pid);
         return 0;
 
 free_sbi:
         EXOFS_ERR("Unable to mount exofs on %s pid=0x%llx err=%d\n",
                   opts->dev_name, sbi->s_pid, ret);
-        osduld_put_device(sbi->s_dev); /* NULL safe */
-        kfree(sbi);
+        exofs_free_sbi(sbi);
         return ret;
 }