24 files changed, 2446 insertions, 2952 deletions

diff --git a/block/Kconfig.iosched b/block/Kconfig.iosched
index 3199b76f795d..421bef9c4c48 100644
--- a/block/Kconfig.iosched
+++ b/block/Kconfig.iosched
@@ -23,8 +23,6 @@ config IOSCHED_DEADLINE
 
 config IOSCHED_CFQ
         tristate "CFQ I/O scheduler"
-        # If BLK_CGROUP is a module, CFQ has to be built as module.
-        depends on (BLK_CGROUP=m && m) || !BLK_CGROUP || BLK_CGROUP=y
         default y
         ---help---
           The CFQ I/O scheduler tries to distribute bandwidth equally
@@ -34,8 +32,6 @@ config IOSCHED_CFQ
 
           This is the default I/O scheduler.
 
-          Note: If BLK_CGROUP=m, then CFQ can be built only as module.
-
 config CFQ_GROUP_IOSCHED
         bool "CFQ Group Scheduling support"
         depends on IOSCHED_CFQ && BLK_CGROUP
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 126c341955de..02cf6335e9bd 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -11,1570 +11,612 @@
  *                    Nauman Rafique <nauman@google.com>
  */
 #include <linux/ioprio.h>
-#include <linux/seq_file.h>
 #include <linux/kdev_t.h>
 #include <linux/module.h>
 #include <linux/err.h>
 #include <linux/blkdev.h>
 #include <linux/slab.h>
-#include "blk-cgroup.h"
 #include <linux/genhd.h>
+#include <linux/delay.h>
+#include <linux/atomic.h>
+#include "blk-cgroup.h"
+#include "blk.h"
 
 #define MAX_KEY_LEN 100
 
-static DEFINE_SPINLOCK(blkio_list_lock);
-static LIST_HEAD(blkio_list);
+static DEFINE_MUTEX(blkcg_pol_mutex);
 
-struct blkio_cgroup blkio_root_cgroup = { .weight = 2*BLKIO_WEIGHT_DEFAULT };
-EXPORT_SYMBOL_GPL(blkio_root_cgroup);
+struct blkcg blkcg_root = { .cfq_weight = 2 * CFQ_WEIGHT_DEFAULT };
+EXPORT_SYMBOL_GPL(blkcg_root);
 
-/* for encoding cft->private value on file */
-#define BLKIOFILE_PRIVATE(x, val)       (((x) << 16) | (val))
-/* What policy owns the file, proportional or throttle */
-#define BLKIOFILE_POLICY(val)           (((val) >> 16) & 0xffff)
-#define BLKIOFILE_ATTR(val)             ((val) & 0xffff)
+static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
 
-static inline void blkio_policy_insert_node(struct blkio_cgroup *blkcg,
-                                            struct blkio_policy_node *pn)
+struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup)
 {
-        list_add(&pn->node, &blkcg->policy_list);
+        return container_of(cgroup_subsys_state(cgroup, blkio_subsys_id),
+                            struct blkcg, css);
 }
+EXPORT_SYMBOL_GPL(cgroup_to_blkcg);
 
-static inline bool cftype_blkg_same_policy(struct cftype *cft,
-                        struct blkio_group *blkg)
+static struct blkcg *task_blkcg(struct task_struct *tsk)
 {
-        enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
-
-        if (blkg->plid == plid)
-                return 1;
-
-        return 0;
+        return container_of(task_subsys_state(tsk, blkio_subsys_id),
+                            struct blkcg, css);
 }
 
-/* Determines if policy node matches cgroup file being accessed */
-static inline bool pn_matches_cftype(struct cftype *cft,
-                        struct blkio_policy_node *pn)
+struct blkcg *bio_blkcg(struct bio *bio)
 {
-        enum blkio_policy_id plid = BLKIOFILE_POLICY(cft->private);
-        int fileid = BLKIOFILE_ATTR(cft->private);
-
-        return (plid == pn->plid && fileid == pn->fileid);
+        if (bio && bio->bi_css)
+                return container_of(bio->bi_css, struct blkcg, css);
+        return task_blkcg(current);
 }
+EXPORT_SYMBOL_GPL(bio_blkcg);
 
-/* Must be called with blkcg->lock held */
-static inline void blkio_policy_delete_node(struct blkio_policy_node *pn)
+static bool blkcg_policy_enabled(struct request_queue *q,
+                                 const struct blkcg_policy *pol)
 {
-        list_del(&pn->node);
+        return pol && test_bit(pol->plid, q->blkcg_pols);
 }
 
-/* Must be called with blkcg->lock held */
-static struct blkio_policy_node *
-blkio_policy_search_node(const struct blkio_cgroup *blkcg, dev_t dev,
-                enum blkio_policy_id plid, int fileid)
+/**
+ * blkg_free - free a blkg
+ * @blkg: blkg to free
+ *
+ * Free @blkg which may be partially allocated.
+ */
+static void blkg_free(struct blkcg_gq *blkg)
 {
-        struct blkio_policy_node *pn;
-
-        list_for_each_entry(pn, &blkcg->policy_list, node) {
-                if (pn->dev == dev && pn->plid == plid && pn->fileid == fileid)
-                        return pn;
-        }
+        int i;
 
-        return NULL;
-}
+        if (!blkg)
+                return;
 
-struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup)
-{
-        return container_of(cgroup_subsys_state(cgroup, blkio_subsys_id),
-                            struct blkio_cgroup, css);
-}
-EXPORT_SYMBOL_GPL(cgroup_to_blkio_cgroup);
+        for (i = 0; i < BLKCG_MAX_POLS; i++) {
+                struct blkcg_policy *pol = blkcg_policy[i];
+                struct blkg_policy_data *pd = blkg->pd[i];
 
-struct blkio_cgroup *task_blkio_cgroup(struct task_struct *tsk)
-{
-        return container_of(task_subsys_state(tsk, blkio_subsys_id),
-                            struct blkio_cgroup, css);
-}
-EXPORT_SYMBOL_GPL(task_blkio_cgroup);
+                if (!pd)
+                        continue;
 
-static inline void
-blkio_update_group_weight(struct blkio_group *blkg, unsigned int weight)
-{
-        struct blkio_policy_type *blkiop;
+                if (pol && pol->pd_exit_fn)
+                        pol->pd_exit_fn(blkg);
 
-        list_for_each_entry(blkiop, &blkio_list, list) {
-                /* If this policy does not own the blkg, do not send updates */
-                if (blkiop->plid != blkg->plid)
-                        continue;
-                if (blkiop->ops.blkio_update_group_weight_fn)
-                        blkiop->ops.blkio_update_group_weight_fn(blkg->key,
-                                                        blkg, weight);
+                kfree(pd);
         }
+
+        kfree(blkg);
 }
 
-static inline void blkio_update_group_bps(struct blkio_group *blkg, u64 bps,
-                                int fileid)
+/**
+ * blkg_alloc - allocate a blkg