aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/cgroup.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/cgroup.c')
-rw-r--r--kernel/cgroup.c904
1 files changed, 559 insertions, 345 deletions
diff --git a/kernel/cgroup.c b/kernel/cgroup.c
index a5d3b5325f77..2905977e0f33 100644
--- a/kernel/cgroup.c
+++ b/kernel/cgroup.c
@@ -63,6 +63,9 @@
63 63
64#include <linux/atomic.h> 64#include <linux/atomic.h>
65 65
66/* css deactivation bias, makes css->refcnt negative to deny new trygets */
67#define CSS_DEACT_BIAS INT_MIN
68
66/* 69/*
67 * cgroup_mutex is the master lock. Any modification to cgroup or its 70 * cgroup_mutex is the master lock. Any modification to cgroup or its
68 * hierarchy must be performed while holding it. 71 * hierarchy must be performed while holding it.
@@ -127,6 +130,9 @@ struct cgroupfs_root {
127 /* A list running through the active hierarchies */ 130 /* A list running through the active hierarchies */
128 struct list_head root_list; 131 struct list_head root_list;
129 132
133 /* All cgroups on this root, cgroup_mutex protected */
134 struct list_head allcg_list;
135
130 /* Hierarchy-specific flags */ 136 /* Hierarchy-specific flags */
131 unsigned long flags; 137 unsigned long flags;
132 138
@@ -145,6 +151,15 @@ struct cgroupfs_root {
145static struct cgroupfs_root rootnode; 151static struct cgroupfs_root rootnode;
146 152
147/* 153/*
154 * cgroupfs file entry, pointed to from leaf dentry->d_fsdata.
155 */
156struct cfent {
157 struct list_head node;
158 struct dentry *dentry;
159 struct cftype *type;
160};
161
162/*
148 * CSS ID -- ID per subsys's Cgroup Subsys State(CSS). used only when 163 * CSS ID -- ID per subsys's Cgroup Subsys State(CSS). used only when
149 * cgroup_subsys->use_id != 0. 164 * cgroup_subsys->use_id != 0.
150 */ 165 */
@@ -239,6 +254,14 @@ int cgroup_lock_is_held(void)
239 254
240EXPORT_SYMBOL_GPL(cgroup_lock_is_held); 255EXPORT_SYMBOL_GPL(cgroup_lock_is_held);
241 256
257/* the current nr of refs, always >= 0 whether @css is deactivated or not */
258static int css_refcnt(struct cgroup_subsys_state *css)
259{
260 int v = atomic_read(&css->refcnt);
261
262 return v >= 0 ? v : v - CSS_DEACT_BIAS;
263}
264
242/* convenient tests for these bits */ 265/* convenient tests for these bits */
243inline int cgroup_is_removed(const struct cgroup *cgrp) 266inline int cgroup_is_removed(const struct cgroup *cgrp)
244{ 267{
@@ -279,6 +302,21 @@ list_for_each_entry(_ss, &_root->subsys_list, sibling)
279#define for_each_active_root(_root) \ 302#define for_each_active_root(_root) \
280list_for_each_entry(_root, &roots, root_list) 303list_for_each_entry(_root, &roots, root_list)
281 304
305static inline struct cgroup *__d_cgrp(struct dentry *dentry)
306{
307 return dentry->d_fsdata;
308}
309
310static inline struct cfent *__d_cfe(struct dentry *dentry)
311{
312 return dentry->d_fsdata;
313}
314
315static inline struct cftype *__d_cft(struct dentry *dentry)
316{
317 return __d_cfe(dentry)->type;
318}
319
282/* the list of cgroups eligible for automatic release. Protected by 320/* the list of cgroups eligible for automatic release. Protected by
283 * release_list_lock */ 321 * release_list_lock */
284static LIST_HEAD(release_list); 322static LIST_HEAD(release_list);
@@ -816,12 +854,17 @@ static int cgroup_call_pre_destroy(struct cgroup *cgrp)
816 struct cgroup_subsys *ss; 854 struct cgroup_subsys *ss;
817 int ret = 0; 855 int ret = 0;
818 856
819 for_each_subsys(cgrp->root, ss) 857 for_each_subsys(cgrp->root, ss) {
820 if (ss->pre_destroy) { 858 if (!ss->pre_destroy)
821 ret = ss->pre_destroy(ss, cgrp); 859 continue;
822 if (ret) 860
823 break; 861 ret = ss->pre_destroy(cgrp);
862 if (ret) {
863 /* ->pre_destroy() failure is being deprecated */
864 WARN_ON_ONCE(!ss->__DEPRECATED_clear_css_refs);
865 break;
824 } 866 }
867 }
825 868
826 return ret; 869 return ret;
827} 870}
@@ -846,7 +889,7 @@ static void cgroup_diput(struct dentry *dentry, struct inode *inode)
846 * Release the subsystem state objects. 889 * Release the subsystem state objects.
847 */ 890 */
848 for_each_subsys(cgrp->root, ss) 891 for_each_subsys(cgrp->root, ss)
849 ss->destroy(ss, cgrp); 892 ss->destroy(cgrp);
850 893
851 cgrp->root->number_of_cgroups--; 894 cgrp->root->number_of_cgroups--;
852 mutex_unlock(&cgroup_mutex); 895 mutex_unlock(&cgroup_mutex);
@@ -864,6 +907,14 @@ static void cgroup_diput(struct dentry *dentry, struct inode *inode)
864 BUG_ON(!list_empty(&cgrp->pidlists)); 907 BUG_ON(!list_empty(&cgrp->pidlists));
865 908
866 kfree_rcu(cgrp, rcu_head); 909 kfree_rcu(cgrp, rcu_head);
910 } else {
911 struct cfent *cfe = __d_cfe(dentry);
912 struct cgroup *cgrp = dentry->d_parent->d_fsdata;
913
914 WARN_ONCE(!list_empty(&cfe->node) &&
915 cgrp != &cgrp->root->top_cgroup,
916 "cfe still linked for %s\n", cfe->type->name);
917 kfree(cfe);
867 } 918 }
868 iput(inode); 919 iput(inode);
869} 920}
@@ -882,34 +933,36 @@ static void remove_dir(struct dentry *d)
882 dput(parent); 933 dput(parent);
883} 934}
884 935
885static void cgroup_clear_directory(struct dentry *dentry) 936static int cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft)
886{ 937{
887 struct list_head *node; 938 struct cfent *cfe;
888 939
889 BUG_ON(!mutex_is_locked(&dentry->d_inode->i_mutex)); 940 lockdep_assert_held(&cgrp->dentry->d_inode->i_mutex);
890 spin_lock(&dentry->d_lock); 941 lockdep_assert_held(&cgroup_mutex);
891 node = dentry->d_subdirs.next; 942
892 while (node != &dentry->d_subdirs) { 943 list_for_each_entry(cfe, &cgrp->files, node) {
893 struct dentry *d = list_entry(node, struct dentry, d_u.d_child); 944 struct dentry *d = cfe->dentry;
894 945
895 spin_lock_nested(&d->d_lock, DENTRY_D_LOCK_NESTED); 946 if (cft && cfe->type != cft)
896 list_del_init(node); 947 continue;
897 if (d->d_inode) { 948
898 /* This should never be called on a cgroup 949 dget(d);
899 * directory with child cgroups */ 950 d_delete(d);
900 BUG_ON(d->d_inode->i_mode & S_IFDIR); 951 simple_unlink(d->d_inode, d);
901 dget_dlock(d); 952 list_del_init(&cfe->node);
902 spin_unlock(&d->d_lock); 953 dput(d);
903 spin_unlock(&dentry->d_lock); 954
904 d_delete(d); 955 return 0;
905 simple_unlink(dentry->d_inode, d);
906 dput(d);
907 spin_lock(&dentry->d_lock);
908 } else
909 spin_unlock(&d->d_lock);
910 node = dentry->d_subdirs.next;
911 } 956 }
912 spin_unlock(&dentry->d_lock); 957 return -ENOENT;
958}
959
960static void cgroup_clear_directory(struct dentry *dir)
961{
962 struct cgroup *cgrp = __d_cgrp(dir);
963
964 while (!list_empty(&cgrp->files))
965 cgroup_rm_file(cgrp, NULL);
913} 966}
914 967
915/* 968/*
@@ -1015,7 +1068,7 @@ static int rebind_subsystems(struct cgroupfs_root *root,
1015 list_move(&ss->sibling, &root->subsys_list); 1068 list_move(&ss->sibling, &root->subsys_list);
1016 ss->root = root; 1069 ss->root = root;
1017 if (ss->bind) 1070 if (ss->bind)
1018 ss->bind(ss, cgrp); 1071 ss->bind(cgrp);
1019 mutex_unlock(&ss->hierarchy_mutex); 1072 mutex_unlock(&ss->hierarchy_mutex);
1020 /* refcount was already taken, and we're keeping it */ 1073 /* refcount was already taken, and we're keeping it */
1021 } else if (bit & removed_bits) { 1074 } else if (bit & removed_bits) {
@@ -1025,7 +1078,7 @@ static int rebind_subsystems(struct cgroupfs_root *root,
1025 BUG_ON(cgrp->subsys[i]->cgroup != cgrp); 1078 BUG_ON(cgrp->subsys[i]->cgroup != cgrp);
1026 mutex_lock(&ss->hierarchy_mutex); 1079 mutex_lock(&ss->hierarchy_mutex);
1027 if (ss->bind) 1080 if (ss->bind)
1028 ss->bind(ss, dummytop); 1081 ss->bind(dummytop);
1029 dummytop->subsys[i]->cgroup = dummytop; 1082 dummytop->subsys[i]->cgroup = dummytop;
1030 cgrp->subsys[i] = NULL; 1083 cgrp->subsys[i] = NULL;
1031 subsys[i]->root = &rootnode; 1084 subsys[i]->root = &rootnode;
@@ -1294,6 +1347,11 @@ static int cgroup_remount(struct super_block *sb, int *flags, char *data)
1294 if (ret) 1347 if (ret)
1295 goto out_unlock; 1348 goto out_unlock;
1296 1349
1350 /* See feature-removal-schedule.txt */
1351 if (opts.subsys_bits != root->actual_subsys_bits || opts.release_agent)
1352 pr_warning("cgroup: option changes via remount are deprecated (pid=%d comm=%s)\n",
1353 task_tgid_nr(current), current->comm);
1354
1297 /* Don't allow flags or name to change at remount */ 1355 /* Don't allow flags or name to change at remount */
1298 if (opts.flags != root->flags || 1356 if (opts.flags != root->flags ||
1299 (opts.name && strcmp(opts.name, root->name))) { 1357 (opts.name && strcmp(opts.name, root->name))) {
@@ -1308,7 +1366,8 @@ static int cgroup_remount(struct super_block *sb, int *flags, char *data)
1308 goto out_unlock; 1366 goto out_unlock;
1309 } 1367 }
1310 1368
1311 /* (re)populate subsystem files */ 1369 /* clear out any existing files and repopulate subsystem files */
1370 cgroup_clear_directory(cgrp->dentry);
1312 cgroup_populate_dir(cgrp); 1371 cgroup_populate_dir(cgrp);
1313 1372
1314 if (opts.release_agent) 1373 if (opts.release_agent)
@@ -1333,6 +1392,7 @@ static void init_cgroup_housekeeping(struct cgroup *cgrp)
1333{ 1392{
1334 INIT_LIST_HEAD(&cgrp->sibling); 1393 INIT_LIST_HEAD(&cgrp->sibling);
1335 INIT_LIST_HEAD(&cgrp->children); 1394 INIT_LIST_HEAD(&cgrp->children);
1395 INIT_LIST_HEAD(&cgrp->files);
1336 INIT_LIST_HEAD(&cgrp->css_sets); 1396 INIT_LIST_HEAD(&cgrp->css_sets);
1337 INIT_LIST_HEAD(&cgrp->release_list); 1397 INIT_LIST_HEAD(&cgrp->release_list);
1338 INIT_LIST_HEAD(&cgrp->pidlists); 1398 INIT_LIST_HEAD(&cgrp->pidlists);
@@ -1344,11 +1404,14 @@ static void init_cgroup_housekeeping(struct cgroup *cgrp)
1344static void init_cgroup_root(struct cgroupfs_root *root) 1404static void init_cgroup_root(struct cgroupfs_root *root)
1345{ 1405{
1346 struct cgroup *cgrp = &root->top_cgroup; 1406 struct cgroup *cgrp = &root->top_cgroup;
1407
1347 INIT_LIST_HEAD(&root->subsys_list); 1408 INIT_LIST_HEAD(&root->subsys_list);
1348 INIT_LIST_HEAD(&root->root_list); 1409 INIT_LIST_HEAD(&root->root_list);
1410 INIT_LIST_HEAD(&root->allcg_list);
1349 root->number_of_cgroups = 1; 1411 root->number_of_cgroups = 1;
1350 cgrp->root = root; 1412 cgrp->root = root;
1351 cgrp->top_cgroup = cgrp; 1413 cgrp->top_cgroup = cgrp;
1414 list_add_tail(&cgrp->allcg_node, &root->allcg_list);
1352 init_cgroup_housekeeping(cgrp); 1415 init_cgroup_housekeeping(cgrp);
1353} 1416}
1354 1417
@@ -1472,7 +1535,6 @@ static int cgroup_get_rootdir(struct super_block *sb)
1472 1535
1473 struct inode *inode = 1536 struct inode *inode =
1474 cgroup_new_inode(S_IFDIR | S_IRUGO | S_IXUGO | S_IWUSR, sb); 1537 cgroup_new_inode(S_IFDIR | S_IRUGO | S_IXUGO | S_IWUSR, sb);
1475 struct dentry *dentry;
1476 1538
1477 if (!inode) 1539 if (!inode)
1478 return -ENOMEM; 1540 return -ENOMEM;
@@ -1481,12 +1543,9 @@ static int cgroup_get_rootdir(struct super_block *sb)
1481 inode->i_op = &cgroup_dir_inode_operations; 1543 inode->i_op = &cgroup_dir_inode_operations;
1482 /* directories start off with i_nlink == 2 (for "." entry) */ 1544 /* directories start off with i_nlink == 2 (for "." entry) */
1483 inc_nlink(inode); 1545 inc_nlink(inode);
1484 dentry = d_alloc_root(inode); 1546 sb->s_root = d_make_root(inode);
1485 if (!dentry) { 1547 if (!sb->s_root)
1486 iput(inode);
1487 return -ENOMEM; 1548 return -ENOMEM;
1488 }
1489 sb->s_root = dentry;
1490 /* for everything else we want ->d_op set */ 1549 /* for everything else we want ->d_op set */
1491 sb->s_d_op = &cgroup_dops; 1550 sb->s_d_op = &cgroup_dops;
1492 return 0; 1551 return 0;
@@ -1696,16 +1755,6 @@ static struct file_system_type cgroup_fs_type = {
1696 1755
1697static struct kobject *cgroup_kobj; 1756static struct kobject *cgroup_kobj;
1698 1757
1699static inline struct cgroup *__d_cgrp(struct dentry *dentry)
1700{
1701 return dentry->d_fsdata;
1702}
1703
1704static inline struct cftype *__d_cft(struct dentry *dentry)
1705{
1706 return dentry->d_fsdata;
1707}
1708
1709/** 1758/**
1710 * cgroup_path - generate the path of a cgroup 1759 * cgroup_path - generate the path of a cgroup
1711 * @cgrp: the cgroup in question 1760 * @cgrp: the cgroup in question
@@ -1763,6 +1812,7 @@ EXPORT_SYMBOL_GPL(cgroup_path);
1763struct task_and_cgroup { 1812struct task_and_cgroup {
1764 struct task_struct *task; 1813 struct task_struct *task;
1765 struct cgroup *cgrp; 1814 struct cgroup *cgrp;
1815 struct css_set *cg;
1766}; 1816};
1767 1817
1768struct cgroup_taskset { 1818struct cgroup_taskset {
@@ -1843,11 +1893,10 @@ EXPORT_SYMBOL_GPL(cgroup_taskset_size);
1843 * will already exist. If not set, this function might sleep, and can fail with 1893 * will already exist. If not set, this function might sleep, and can fail with
1844 * -ENOMEM. Must be called with cgroup_mutex and threadgroup locked. 1894 * -ENOMEM. Must be called with cgroup_mutex and threadgroup locked.
1845 */ 1895 */
1846static int cgroup_task_migrate(struct cgroup *cgrp, struct cgroup *oldcgrp, 1896static void cgroup_task_migrate(struct cgroup *cgrp, struct cgroup *oldcgrp,
1847 struct task_struct *tsk, bool guarantee) 1897 struct task_struct *tsk, struct css_set *newcg)
1848{ 1898{
1849 struct css_set *oldcg; 1899 struct css_set *oldcg;
1850 struct css_set *newcg;
1851 1900
1852 /* 1901 /*
1853 * We are synchronized through threadgroup_lock() against PF_EXITING 1902 * We are synchronized through threadgroup_lock() against PF_EXITING
@@ -1857,23 +1906,6 @@ static int cgroup_task_migrate(struct cgroup *cgrp, struct cgroup *oldcgrp,
1857 WARN_ON_ONCE(tsk->flags & PF_EXITING); 1906 WARN_ON_ONCE(tsk->flags & PF_EXITING);
1858 oldcg = tsk->cgroups; 1907 oldcg = tsk->cgroups;
1859 1908
1860 /* locate or allocate a new css_set for this task. */
1861 if (guarantee) {
1862 /* we know the css_set we want already exists. */
1863 struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT];
1864 read_lock(&css_set_lock);
1865 newcg = find_existing_css_set(oldcg, cgrp, template);
1866 BUG_ON(!newcg);
1867 get_css_set(newcg);
1868 read_unlock(&css_set_lock);
1869 } else {
1870 might_sleep();
1871 /* find_css_set will give us newcg already referenced. */
1872 newcg = find_css_set(oldcg, cgrp);
1873 if (!newcg)
1874 return -ENOMEM;
1875 }
1876
1877 task_lock(tsk); 1909 task_lock(tsk);
1878 rcu_assign_pointer(tsk->cgroups, newcg); 1910 rcu_assign_pointer(tsk->cgroups, newcg);
1879 task_unlock(tsk); 1911 task_unlock(tsk);
@@ -1892,7 +1924,6 @@ static int cgroup_task_migrate(struct cgroup *cgrp, struct cgroup *oldcgrp,
1892 put_css_set(oldcg); 1924 put_css_set(oldcg);
1893 1925
1894 set_bit(CGRP_RELEASABLE, &oldcgrp->flags); 1926 set_bit(CGRP_RELEASABLE, &oldcgrp->flags);
1895 return 0;
1896} 1927}
1897 1928
1898/** 1929/**
@@ -1905,11 +1936,12 @@ static int cgroup_task_migrate(struct cgroup *cgrp, struct cgroup *oldcgrp,
1905 */ 1936 */
1906int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk) 1937int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk)
1907{ 1938{
1908 int retval; 1939 int retval = 0;
1909 struct cgroup_subsys *ss, *failed_ss = NULL; 1940 struct cgroup_subsys *ss, *failed_ss = NULL;
1910 struct cgroup *oldcgrp; 1941 struct cgroup *oldcgrp;
1911 struct cgroupfs_root *root = cgrp->root; 1942 struct cgroupfs_root *root = cgrp->root;
1912 struct cgroup_taskset tset = { }; 1943 struct cgroup_taskset tset = { };
1944 struct css_set *newcg;
1913 1945
1914 /* @tsk either already exited or can't exit until the end */ 1946 /* @tsk either already exited or can't exit until the end */
1915 if (tsk->flags & PF_EXITING) 1947 if (tsk->flags & PF_EXITING)
@@ -1925,7 +1957,7 @@ int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk)
1925 1957
1926 for_each_subsys(root, ss) { 1958 for_each_subsys(root, ss) {
1927 if (ss->can_attach) { 1959 if (ss->can_attach) {
1928 retval = ss->can_attach(ss, cgrp, &tset); 1960 retval = ss->can_attach(cgrp, &tset);
1929 if (retval) { 1961 if (retval) {
1930 /* 1962 /*
1931 * Remember on which subsystem the can_attach() 1963 * Remember on which subsystem the can_attach()
@@ -1939,13 +1971,17 @@ int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk)
1939 } 1971 }
1940 } 1972 }
1941 1973
1942 retval = cgroup_task_migrate(cgrp, oldcgrp, tsk, false); 1974 newcg = find_css_set(tsk->cgroups, cgrp);
1943 if (retval) 1975 if (!newcg) {
1976 retval = -ENOMEM;
1944 goto out; 1977 goto out;
1978 }
1979
1980 cgroup_task_migrate(cgrp, oldcgrp, tsk, newcg);
1945 1981
1946 for_each_subsys(root, ss) { 1982 for_each_subsys(root, ss) {
1947 if (ss->attach) 1983 if (ss->attach)
1948 ss->attach(ss, cgrp, &tset); 1984 ss->attach(cgrp, &tset);
1949 } 1985 }
1950 1986
1951 synchronize_rcu(); 1987 synchronize_rcu();
@@ -1967,7 +2003,7 @@ out:
1967 */ 2003 */
1968 break; 2004 break;
1969 if (ss->cancel_attach) 2005 if (ss->cancel_attach)
1970 ss->cancel_attach(ss, cgrp, &tset); 2006 ss->cancel_attach(cgrp, &tset);
1971 } 2007 }
1972 } 2008 }
1973 return retval; 2009 return retval;
@@ -1997,66 +2033,6 @@ int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk)
1997} 2033}
1998EXPORT_SYMBOL_GPL(cgroup_attach_task_all); 2034EXPORT_SYMBOL_GPL(cgroup_attach_task_all);
1999 2035
2000/*
2001 * cgroup_attach_proc works in two stages, the first of which prefetches all
2002 * new css_sets needed (to make sure we have enough memory before committing
2003 * to the move) and stores them in a list of entries of the following type.
2004 * TODO: possible optimization: use css_set->rcu_head for chaining instead
2005 */
2006struct cg_list_entry {
2007 struct css_set *cg;
2008 struct list_head links;
2009};
2010
2011static bool css_set_check_fetched(struct cgroup *cgrp,
2012 struct task_struct *tsk, struct css_set *cg,
2013 struct list_head *newcg_list)
2014{
2015 struct css_set *newcg;
2016 struct cg_list_entry *cg_entry;
2017 struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT];
2018
2019 read_lock(&css_set_lock);
2020 newcg = find_existing_css_set(cg, cgrp, template);
2021 read_unlock(&css_set_lock);
2022
2023 /* doesn't exist at all? */
2024 if (!newcg)
2025 return false;
2026 /* see if it's already in the list */
2027 list_for_each_entry(cg_entry, newcg_list, links)
2028 if (cg_entry->cg == newcg)
2029 return true;
2030
2031 /* not found */
2032 return false;
2033}
2034
2035/*
2036 * Find the new css_set and store it in the list in preparation for moving the
2037 * given task to the given cgroup. Returns 0 or -ENOMEM.
2038 */
2039static int css_set_prefetch(struct cgroup *cgrp, struct css_set *cg,
2040 struct list_head *newcg_list)
2041{
2042 struct css_set *newcg;
2043 struct cg_list_entry *cg_entry;
2044
2045 /* ensure a new css_set will exist for this thread */
2046 newcg = find_css_set(cg, cgrp);
2047 if (!newcg)
2048 return -ENOMEM;
2049 /* add it to the list */
2050 cg_entry = kmalloc(sizeof(struct cg_list_entry), GFP_KERNEL);
2051 if (!cg_entry) {
2052 put_css_set(newcg);
2053 return -ENOMEM;
2054 }
2055 cg_entry->cg = newcg;
2056 list_add(&cg_entry->links, newcg_list);
2057 return 0;
2058}
2059
2060/** 2036/**
2061 * cgroup_attach_proc - attach all threads in a threadgroup to a cgroup 2037 * cgroup_attach_proc - attach all threads in a threadgroup to a cgroup
2062 * @cgrp: the cgroup to attach to 2038 * @cgrp: the cgroup to attach to
@@ -2070,20 +2046,12 @@ static int cgroup_attach_proc(struct cgroup *cgrp, struct task_struct *leader)
2070 int retval, i, group_size; 2046 int retval, i, group_size;
2071 struct cgroup_subsys *ss, *failed_ss = NULL; 2047 struct cgroup_subsys *ss, *failed_ss = NULL;
2072 /* guaranteed to be initialized later, but the compiler needs this */ 2048 /* guaranteed to be initialized later, but the compiler needs this */
2073 struct css_set *oldcg;
2074 struct cgroupfs_root *root = cgrp->root; 2049 struct cgroupfs_root *root = cgrp->root;
2075 /* threadgroup list cursor and array */ 2050 /* threadgroup list cursor and array */
2076 struct task_struct *tsk; 2051 struct task_struct *tsk;
2077 struct task_and_cgroup *tc; 2052 struct task_and_cgroup *tc;
2078 struct flex_array *group; 2053 struct flex_array *group;
2079 struct cgroup_taskset tset = { }; 2054 struct cgroup_taskset tset = { };
2080 /*
2081 * we need to make sure we have css_sets for all the tasks we're
2082 * going to move -before- we actually start moving them, so that in
2083 * case we get an ENOMEM we can bail out before making any changes.
2084 */
2085 struct list_head newcg_list;
2086 struct cg_list_entry *cg_entry, *temp_nobe;
2087 2055
2088 /* 2056 /*
2089 * step 0: in order to do expensive, possibly blocking operations for 2057 * step 0: in order to do expensive, possibly blocking operations for
@@ -2102,23 +2070,14 @@ static int cgroup_attach_proc(struct cgroup *cgrp, struct task_struct *leader)
2102 if (retval) 2070 if (retval)
2103 goto out_free_group_list; 2071 goto out_free_group_list;
2104 2072
2105 /* prevent changes to the threadgroup list while we take a snapshot. */
2106 read_lock(&tasklist_lock);
2107 if (!thread_group_leader(leader)) {
2108 /*
2109 * a race with de_thread from another thread's exec() may strip
2110 * us of our leadership, making while_each_thread unsafe to use
2111 * on this task. if this happens, there is no choice but to
2112 * throw this task away and try again (from cgroup_procs_write);
2113 * this is "double-double-toil-and-trouble-check locking".
2114 */
2115 read_unlock(&tasklist_lock);
2116 retval = -EAGAIN;
2117 goto out_free_group_list;
2118 }
2119
2120 tsk = leader; 2073 tsk = leader;
2121 i = 0; 2074 i = 0;
2075 /*
2076 * Prevent freeing of tasks while we take a snapshot. Tasks that are
2077 * already PF_EXITING could be freed from underneath us unless we
2078 * take an rcu_read_lock.
2079 */
2080 rcu_read_lock();
2122 do { 2081 do {
2123 struct task_and_cgroup ent; 2082 struct task_and_cgroup ent;
2124 2083
@@ -2128,24 +2087,24 @@ static int cgroup_attach_proc(struct cgroup *cgrp, struct task_struct *leader)
2128 2087
2129 /* as per above, nr_threads may decrease, but not increase. */ 2088 /* as per above, nr_threads may decrease, but not increase. */
2130 BUG_ON(i >= group_size); 2089 BUG_ON(i >= group_size);
2131 /*
2132 * saying GFP_ATOMIC has no effect here because we did prealloc
2133 * earlier, but it's good form to communicate our expectations.
2134 */
2135 ent.task = tsk; 2090 ent.task = tsk;
2136 ent.cgrp = task_cgroup_from_root(tsk, root); 2091 ent.cgrp = task_cgroup_from_root(tsk, root);
2137 /* nothing to do if this task is already in the cgroup */ 2092 /* nothing to do if this task is already in the cgroup */
2138 if (ent.cgrp == cgrp) 2093 if (ent.cgrp == cgrp)
2139 continue; 2094 continue;
2095 /*
2096 * saying GFP_ATOMIC has no effect here because we did prealloc
2097 * earlier, but it's good form to communicate our expectations.
2098 */
2140 retval = flex_array_put(group, i, &ent, GFP_ATOMIC); 2099 retval = flex_array_put(group, i, &ent, GFP_ATOMIC);
2141 BUG_ON(retval != 0); 2100 BUG_ON(retval != 0);
2142 i++; 2101 i++;
2143 } while_each_thread(leader, tsk); 2102 } while_each_thread(leader, tsk);
2103 rcu_read_unlock();
2144 /* remember the number of threads in the array for later. */ 2104 /* remember the number of threads in the array for later. */
2145 group_size = i; 2105 group_size = i;
2146 tset.tc_array = group; 2106 tset.tc_array = group;
2147 tset.tc_array_len = group_size; 2107 tset.tc_array_len = group_size;
2148 read_unlock(&tasklist_lock);
2149 2108
2150 /* methods shouldn't be called if no task is actually migrating */ 2109 /* methods shouldn't be called if no task is actually migrating */
2151 retval = 0; 2110 retval = 0;
@@ -2157,7 +2116,7 @@ static int cgroup_attach_proc(struct cgroup *cgrp, struct task_struct *leader)
2157 */ 2116 */
2158 for_each_subsys(root, ss) { 2117 for_each_subsys(root, ss) {
2159 if (ss->can_attach) { 2118 if (ss->can_attach) {
2160 retval = ss->can_attach(ss, cgrp, &tset); 2119 retval = ss->can_attach(cgrp, &tset);
2161 if (retval) { 2120 if (retval) {
2162 failed_ss = ss; 2121 failed_ss = ss;
2163 goto out_cancel_attach; 2122 goto out_cancel_attach;
@@ -2169,17 +2128,12 @@ static int cgroup_attach_proc(struct cgroup *cgrp, struct task_struct *leader)
2169 * step 2: make sure css_sets exist for all threads to be migrated. 2128 * step 2: make sure css_sets exist for all threads to be migrated.
2170 * we use find_css_set, which allocates a new one if necessary. 2129 * we use find_css_set, which allocates a new one if necessary.
2171 */ 2130 */
2172 INIT_LIST_HEAD(&newcg_list);
2173 for (i = 0; i < group_size; i++) { 2131 for (i = 0; i < group_size; i++) {
2174 tc = flex_array_get(group, i); 2132 tc = flex_array_get(group, i);
2175 oldcg = tc->task->cgroups; 2133 tc->cg = find_css_set(tc->task->cgroups, cgrp);
2176 2134 if (!tc->cg) {
2177 /* if we don't already have it in the list get a new one */ 2135 retval = -ENOMEM;
2178 if (!css_set_check_fetched(cgrp, tc->task, oldcg, 2136 goto out_put_css_set_refs;
2179 &newcg_list)) {
2180 retval = css_set_prefetch(cgrp, oldcg, &newcg_list);
2181 if (retval)
2182 goto out_list_teardown;
2183 } 2137 }
2184 } 2138 }
2185 2139
@@ -2190,8 +2144,7 @@ static int cgroup_attach_proc(struct cgroup *cgrp, struct task_struct *leader)
2190 */ 2144 */
2191 for (i = 0; i < group_size; i++) { 2145 for (i = 0; i < group_size; i++) {
2192 tc = flex_array_get(group, i); 2146 tc = flex_array_get(group, i);
2193 retval = cgroup_task_migrate(cgrp, tc->cgrp, tc->task, true); 2147 cgroup_task_migrate(cgrp, tc->cgrp, tc->task, tc->cg);
2194 BUG_ON(retval);
2195 } 2148 }
2196 /* nothing is sensitive to fork() after this point. */ 2149 /* nothing is sensitive to fork() after this point. */
2197 2150
@@ -2200,7 +2153,7 @@ static int cgroup_attach_proc(struct cgroup *cgrp, struct task_struct *leader)
2200 */ 2153 */
2201 for_each_subsys(root, ss) { 2154 for_each_subsys(root, ss) {
2202 if (ss->attach) 2155 if (ss->attach)
2203 ss->attach(ss, cgrp, &tset); 2156 ss->attach(cgrp, &tset);
2204 } 2157 }
2205 2158
2206 /* 2159 /*
@@ -2209,21 +2162,22 @@ static int cgroup_attach_proc(struct cgroup *cgrp, struct task_struct *leader)
2209 synchronize_rcu(); 2162 synchronize_rcu();
2210 cgroup_wakeup_rmdir_waiter(cgrp); 2163 cgroup_wakeup_rmdir_waiter(cgrp);
2211 retval = 0; 2164 retval = 0;
2212out_list_teardown: 2165out_put_css_set_refs:
2213 /* clean up the list of prefetched css_sets. */ 2166 if (retval) {
2214 list_for_each_entry_safe(cg_entry, temp_nobe, &newcg_list, links) { 2167 for (i = 0; i < group_size; i++) {
2215 list_del(&cg_entry->links); 2168 tc = flex_array_get(group, i);
2216 put_css_set(cg_entry->cg); 2169 if (!tc->cg)
2217 kfree(cg_entry); 2170 break;
2171 put_css_set(tc->cg);
2172 }
2218 } 2173 }
2219out_cancel_attach: 2174out_cancel_attach:
2220 /* same deal as in cgroup_attach_task */
2221 if (retval) { 2175 if (retval) {
2222 for_each_subsys(root, ss) { 2176 for_each_subsys(root, ss) {
2223 if (ss == failed_ss) 2177 if (ss == failed_ss)
2224 break; 2178 break;
2225 if (ss->cancel_attach) 2179 if (ss->cancel_attach)
2226 ss->cancel_attach(ss, cgrp, &tset); 2180 ss->cancel_attach(cgrp, &tset);
2227 } 2181 }
2228 } 2182 }
2229out_free_group_list: 2183out_free_group_list:
@@ -2245,22 +2199,14 @@ static int attach_task_by_pid(struct cgroup *cgrp, u64 pid, bool threadgroup)
2245 if (!cgroup_lock_live_group(cgrp)) 2199 if (!cgroup_lock_live_group(cgrp))
2246 return -ENODEV; 2200 return -ENODEV;
2247 2201
2202retry_find_task:
2203 rcu_read_lock();
2248 if (pid) { 2204 if (pid) {
2249 rcu_read_lock();
2250 tsk = find_task_by_vpid(pid); 2205 tsk = find_task_by_vpid(pid);
2251 if (!tsk) { 2206 if (!tsk) {
2252 rcu_read_unlock(); 2207 rcu_read_unlock();
2253 cgroup_unlock(); 2208 ret= -ESRCH;
2254 return -ESRCH; 2209 goto out_unlock_cgroup;
2255 }
2256 if (threadgroup) {
2257 /*
2258 * RCU protects this access, since tsk was found in the
2259 * tid map. a race with de_thread may cause group_leader
2260 * to stop being the leader, but cgroup_attach_proc will
2261 * detect it later.
2262 */
2263 tsk = tsk->group_leader;
2264 } 2210 }
2265 /* 2211 /*
2266 * even if we're attaching all tasks in the thread group, we 2212 * even if we're attaching all tasks in the thread group, we
@@ -2271,29 +2217,38 @@ static int attach_task_by_pid(struct cgroup *cgrp, u64 pid, bool threadgroup)
2271 cred->euid != tcred->uid && 2217 cred->euid != tcred->uid &&
2272 cred->euid != tcred->suid) { 2218 cred->euid != tcred->suid) {
2273 rcu_read_unlock(); 2219 rcu_read_unlock();
2274 cgroup_unlock(); 2220 ret = -EACCES;
2275 return -EACCES; 2221 goto out_unlock_cgroup;
2276 } 2222 }
2277 get_task_struct(tsk); 2223 } else
2278 rcu_read_unlock(); 2224 tsk = current;
2279 } else {
2280 if (threadgroup)
2281 tsk = current->group_leader;
2282 else
2283 tsk = current;
2284 get_task_struct(tsk);
2285 }
2286
2287 threadgroup_lock(tsk);
2288 2225
2289 if (threadgroup) 2226 if (threadgroup)
2227 tsk = tsk->group_leader;
2228 get_task_struct(tsk);
2229 rcu_read_unlock();
2230
2231 threadgroup_lock(tsk);
2232 if (threadgroup) {
2233 if (!thread_group_leader(tsk)) {
2234 /*
2235 * a race with de_thread from another thread's exec()
2236 * may strip us of our leadership, if this happens,
2237 * there is no choice but to throw this task away and
2238 * try again; this is
2239 * "double-double-toil-and-trouble-check locking".
2240 */
2241 threadgroup_unlock(tsk);
2242 put_task_struct(tsk);
2243 goto retry_find_task;
2244 }
2290 ret = cgroup_attach_proc(cgrp, tsk); 2245 ret = cgroup_attach_proc(cgrp, tsk);
2291 else 2246 } else
2292 ret = cgroup_attach_task(cgrp, tsk); 2247 ret = cgroup_attach_task(cgrp, tsk);
2293
2294 threadgroup_unlock(tsk); 2248 threadgroup_unlock(tsk);
2295 2249
2296 put_task_struct(tsk); 2250 put_task_struct(tsk);
2251out_unlock_cgroup:
2297 cgroup_unlock(); 2252 cgroup_unlock();
2298 return ret; 2253 return ret;
2299} 2254}
@@ -2305,16 +2260,7 @@ static int cgroup_tasks_write(struct cgroup *cgrp, struct cftype *cft, u64 pid)
2305 2260
2306static int cgroup_procs_write(struct cgroup *cgrp, struct cftype *cft, u64 tgid) 2261static int cgroup_procs_write(struct cgroup *cgrp, struct cftype *cft, u64 tgid)
2307{ 2262{
2308 int ret; 2263 return attach_task_by_pid(cgrp, tgid, true);
2309 do {
2310 /*
2311 * attach_proc fails with -EAGAIN if threadgroup leadership
2312 * changes in the middle of the operation, in which case we need
2313 * to find the task_struct for the new leader and start over.
2314 */
2315 ret = attach_task_by_pid(cgrp, tgid, true);
2316 } while (ret == -EAGAIN);
2317 return ret;
2318} 2264}
2319 2265
2320/** 2266/**
@@ -2710,50 +2656,191 @@ static umode_t cgroup_file_mode(const struct cftype *cft)
2710 return mode; 2656 return mode;
2711} 2657}
2712 2658
2713int cgroup_add_file(struct cgroup *cgrp, 2659static int cgroup_add_file(struct cgroup *cgrp, struct cgroup_subsys *subsys,
2714 struct cgroup_subsys *subsys, 2660 const struct cftype *cft)
2715 const struct cftype *cft)
2716{ 2661{
2717 struct dentry *dir = cgrp->dentry; 2662 struct dentry *dir = cgrp->dentry;
2663 struct cgroup *parent = __d_cgrp(dir);
2718 struct dentry *dentry; 2664 struct dentry *dentry;
2665 struct cfent *cfe;
2719 int error; 2666 int error;
2720 umode_t mode; 2667 umode_t mode;
2721
2722 char name[MAX_CGROUP_TYPE_NAMELEN + MAX_CFTYPE_NAME + 2] = { 0 }; 2668 char name[MAX_CGROUP_TYPE_NAMELEN + MAX_CFTYPE_NAME + 2] = { 0 };
2669
2670 /* does @cft->flags tell us to skip creation on @cgrp? */
2671 if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgrp->parent)
2672 return 0;
2673 if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgrp->parent)
2674 return 0;
2675
2723 if (subsys && !test_bit(ROOT_NOPREFIX, &cgrp->root->flags)) { 2676 if (subsys && !test_bit(ROOT_NOPREFIX, &cgrp->root->flags)) {
2724 strcpy(name, subsys->name); 2677 strcpy(name, subsys->name);
2725 strcat(name, "."); 2678 strcat(name, ".");
2726 } 2679 }
2727 strcat(name, cft->name); 2680 strcat(name, cft->name);
2681
2728 BUG_ON(!mutex_is_locked(&dir->d_inode->i_mutex)); 2682 BUG_ON(!mutex_is_locked(&dir->d_inode->i_mutex));
2683
2684 cfe = kzalloc(sizeof(*cfe), GFP_KERNEL);
2685 if (!cfe)
2686 return -ENOMEM;
2687
2729 dentry = lookup_one_len(name, dir, strlen(name)); 2688 dentry = lookup_one_len(name, dir, strlen(name));
2730 if (!IS_ERR(dentry)) { 2689 if (IS_ERR(dentry)) {
2731 mode = cgroup_file_mode(cft);
2732 error = cgroup_create_file(dentry, mode | S_IFREG,
2733 cgrp->root->sb);
2734 if (!error)
2735 dentry->d_fsdata = (void *)cft;
2736 dput(dentry);
2737 } else
2738 error = PTR_ERR(dentry); 2690 error = PTR_ERR(dentry);
2691 goto out;
2692 }
2693
2694 mode = cgroup_file_mode(cft);
2695 error = cgroup_create_file(dentry, mode | S_IFREG, cgrp->root->sb);
2696 if (!error) {
2697 cfe->type = (void *)cft;
2698 cfe->dentry = dentry;
2699 dentry->d_fsdata = cfe;
2700 list_add_tail(&cfe->node, &parent->files);
2701 cfe = NULL;
2702 }
2703 dput(dentry);
2704out:
2705 kfree(cfe);
2739 return error; 2706 return error;
2740} 2707}
2741EXPORT_SYMBOL_GPL(cgroup_add_file);
2742 2708
2743int cgroup_add_files(struct cgroup *cgrp, 2709static int cgroup_addrm_files(struct cgroup *cgrp, struct cgroup_subsys *subsys,
2744 struct cgroup_subsys *subsys, 2710 const struct cftype cfts[], bool is_add)
2745 const struct cftype cft[],
2746 int count)
2747{ 2711{
2748 int i, err; 2712 const struct cftype *cft;
2749 for (i = 0; i < count; i++) { 2713 int err, ret = 0;
2750 err = cgroup_add_file(cgrp, subsys, &cft[i]); 2714
2751 if (err) 2715 for (cft = cfts; cft->name[0] != '\0'; cft++) {
2752 return err; 2716 if (is_add)
2717 err = cgroup_add_file(cgrp, subsys, cft);
2718 else
2719 err = cgroup_rm_file(cgrp, cft);
2720 if (err) {
2721 pr_warning("cgroup_addrm_files: failed to %s %s, err=%d\n",
2722 is_add ? "add" : "remove", cft->name, err);
2723 ret = err;
2724 }
2725 }
2726 return ret;
2727}
2728
2729static DEFINE_MUTEX(cgroup_cft_mutex);
2730
2731static void cgroup_cfts_prepare(void)
2732 __acquires(&cgroup_cft_mutex) __acquires(&cgroup_mutex)
2733{
2734 /*
2735 * Thanks to the entanglement with vfs inode locking, we can't walk
2736 * the existing cgroups under cgroup_mutex and create files.
2737 * Instead, we increment reference on all cgroups and build list of
2738 * them using @cgrp->cft_q_node. Grab cgroup_cft_mutex to ensure
2739 * exclusive access to the field.
2740 */
2741 mutex_lock(&cgroup_cft_mutex);
2742 mutex_lock(&cgroup_mutex);
2743}
2744
2745static void cgroup_cfts_commit(struct cgroup_subsys *ss,
2746 const struct cftype *cfts, bool is_add)
2747 __releases(&cgroup_mutex) __releases(&cgroup_cft_mutex)
2748{
2749 LIST_HEAD(pending);
2750 struct cgroup *cgrp, *n;
2751
2752 /* %NULL @cfts indicates abort and don't bother if @ss isn't attached */
2753 if (cfts && ss->root != &rootnode) {
2754 list_for_each_entry(cgrp, &ss->root->allcg_list, allcg_node) {
2755 dget(cgrp->dentry);
2756 list_add_tail(&cgrp->cft_q_node, &pending);
2757 }
2758 }
2759
2760 mutex_unlock(&cgroup_mutex);
2761
2762 /*
2763 * All new cgroups will see @cfts update on @ss->cftsets. Add/rm
2764 * files for all cgroups which were created before.
2765 */
2766 list_for_each_entry_safe(cgrp, n, &pending, cft_q_node) {
2767 struct inode *inode = cgrp->dentry->d_inode;
2768
2769 mutex_lock(&inode->i_mutex);
2770 mutex_lock(&cgroup_mutex);
2771 if (!cgroup_is_removed(cgrp))
2772 cgroup_addrm_files(cgrp, ss, cfts, is_add);
2773 mutex_unlock(&cgroup_mutex);
2774 mutex_unlock(&inode->i_mutex);
2775
2776 list_del_init(&cgrp->cft_q_node);
2777 dput(cgrp->dentry);
2753 } 2778 }
2779
2780 mutex_unlock(&cgroup_cft_mutex);
2781}
2782
2783/**
2784 * cgroup_add_cftypes - add an array of cftypes to a subsystem
2785 * @ss: target cgroup subsystem
2786 * @cfts: zero-length name terminated array of cftypes
2787 *
2788 * Register @cfts to @ss. Files described by @cfts are created for all
2789 * existing cgroups to which @ss is attached and all future cgroups will
2790 * have them too. This function can be called anytime whether @ss is
2791 * attached or not.
2792 *
2793 * Returns 0 on successful registration, -errno on failure. Note that this
2794 * function currently returns 0 as long as @cfts registration is successful
2795 * even if some file creation attempts on existing cgroups fail.
2796 */
2797int cgroup_add_cftypes(struct cgroup_subsys *ss, const struct cftype *cfts)
2798{
2799 struct cftype_set *set;
2800
2801 set = kzalloc(sizeof(*set), GFP_KERNEL);
2802 if (!set)
2803 return -ENOMEM;
2804
2805 cgroup_cfts_prepare();
2806 set->cfts = cfts;
2807 list_add_tail(&set->node, &ss->cftsets);
2808 cgroup_cfts_commit(ss, cfts, true);
2809
2754 return 0; 2810 return 0;
2755} 2811}
2756EXPORT_SYMBOL_GPL(cgroup_add_files); 2812EXPORT_SYMBOL_GPL(cgroup_add_cftypes);
2813
2814/**
2815 * cgroup_rm_cftypes - remove an array of cftypes from a subsystem
2816 * @ss: target cgroup subsystem
2817 * @cfts: zero-length name terminated array of cftypes
2818 *
2819 * Unregister @cfts from @ss. Files described by @cfts are removed from
2820 * all existing cgroups to which @ss is attached and all future cgroups
2821 * won't have them either. This function can be called anytime whether @ss
2822 * is attached or not.
2823 *
2824 * Returns 0 on successful unregistration, -ENOENT if @cfts is not
2825 * registered with @ss.
2826 */
2827int cgroup_rm_cftypes(struct cgroup_subsys *ss, const struct cftype *cfts)
2828{
2829 struct cftype_set *set;
2830
2831 cgroup_cfts_prepare();
2832
2833 list_for_each_entry(set, &ss->cftsets, node) {
2834 if (set->cfts == cfts) {
2835 list_del_init(&set->node);
2836 cgroup_cfts_commit(ss, cfts, false);
2837 return 0;
2838 }
2839 }
2840
2841 cgroup_cfts_commit(ss, NULL, false);
2842 return -ENOENT;
2843}
2757 2844
2758/** 2845/**
2759 * cgroup_task_count - count the number of tasks in a cgroup. 2846 * cgroup_task_count - count the number of tasks in a cgroup.
@@ -2804,15 +2891,20 @@ static void cgroup_advance_iter(struct cgroup *cgrp,
2804 * using their cgroups capability, we don't maintain the lists running 2891 * using their cgroups capability, we don't maintain the lists running
2805 * through each css_set to its tasks until we see the list actually 2892 * through each css_set to its tasks until we see the list actually
2806 * used - in other words after the first call to cgroup_iter_start(). 2893 * used - in other words after the first call to cgroup_iter_start().
2807 *
2808 * The tasklist_lock is not held here, as do_each_thread() and
2809 * while_each_thread() are protected by RCU.
2810 */ 2894 */
2811static void cgroup_enable_task_cg_lists(void) 2895static void cgroup_enable_task_cg_lists(void)
2812{ 2896{
2813 struct task_struct *p, *g; 2897 struct task_struct *p, *g;
2814 write_lock(&css_set_lock); 2898 write_lock(&css_set_lock);
2815 use_task_css_set_links = 1; 2899 use_task_css_set_links = 1;
2900 /*
2901 * We need tasklist_lock because RCU is not safe against
2902 * while_each_thread(). Besides, a forking task that has passed
2903 * cgroup_post_fork() without seeing use_task_css_set_links = 1
2904 * is not guaranteed to have its child immediately visible in the
2905 * tasklist if we walk through it with RCU.
2906 */
2907 read_lock(&tasklist_lock);
2816 do_each_thread(g, p) { 2908 do_each_thread(g, p) {
2817 task_lock(p); 2909 task_lock(p);
2818 /* 2910 /*
@@ -2824,6 +2916,7 @@ static void cgroup_enable_task_cg_lists(void)
2824 list_add(&p->cg_list, &p->cgroups->tasks); 2916 list_add(&p->cg_list, &p->cgroups->tasks);
2825 task_unlock(p); 2917 task_unlock(p);
2826 } while_each_thread(g, p); 2918 } while_each_thread(g, p);
2919 read_unlock(&tasklist_lock);
2827 write_unlock(&css_set_lock); 2920 write_unlock(&css_set_lock);
2828} 2921}
2829 2922
@@ -3043,6 +3136,38 @@ int cgroup_scan_tasks(struct cgroup_scanner *scan)
3043 * 3136 *
3044 */ 3137 */
3045 3138
3139/* which pidlist file are we talking about? */
3140enum cgroup_filetype {
3141 CGROUP_FILE_PROCS,
3142 CGROUP_FILE_TASKS,
3143};
3144
3145/*
3146 * A pidlist is a list of pids that virtually represents the contents of one
3147 * of the cgroup files ("procs" or "tasks"). We keep a list of such pidlists,
3148 * a pair (one each for procs, tasks) for each pid namespace that's relevant
3149 * to the cgroup.
3150 */
3151struct cgroup_pidlist {
3152 /*
3153 * used to find which pidlist is wanted. doesn't change as long as
3154 * this particular list stays in the list.
3155 */
3156 struct { enum cgroup_filetype type; struct pid_namespace *ns; } key;
3157 /* array of xids */
3158 pid_t *list;
3159 /* how many elements the above list has */
3160 int length;
3161 /* how many files are using the current array */
3162 int use_count;
3163 /* each of these stored in a list by its cgroup */
3164 struct list_head links;
3165 /* pointer to the cgroup we belong to, for list removal purposes */
3166 struct cgroup *owner;
3167 /* protects the other fields */
3168 struct rw_semaphore mutex;
3169};
3170
3046/* 3171/*
3047 * The following two functions "fix" the issue where there are more pids 3172 * The following two functions "fix" the issue where there are more pids
3048 * than kmalloc will give memory for; in such cases, we use vmalloc/vfree. 3173 * than kmalloc will give memory for; in such cases, we use vmalloc/vfree.
@@ -3694,13 +3819,14 @@ static struct cftype files[] = {
3694 .read_u64 = cgroup_clone_children_read, 3819 .read_u64 = cgroup_clone_children_read,
3695 .write_u64 = cgroup_clone_children_write, 3820 .write_u64 = cgroup_clone_children_write,
3696 }, 3821 },
3697}; 3822 {
3698 3823 .name = "release_agent",
3699static struct cftype cft_release_agent = { 3824 .flags = CFTYPE_ONLY_ON_ROOT,
3700 .name = "release_agent", 3825 .read_seq_string = cgroup_release_agent_show,
3701 .read_seq_string = cgroup_release_agent_show, 3826 .write_string = cgroup_release_agent_write,
3702 .write_string = cgroup_release_agent_write, 3827 .max_write_len = PATH_MAX,
3703 .max_write_len = PATH_MAX, 3828 },
3829 { } /* terminate */
3704}; 3830};
3705 3831
3706static int cgroup_populate_dir(struct cgroup *cgrp) 3832static int cgroup_populate_dir(struct cgroup *cgrp)
@@ -3708,22 +3834,21 @@ static int cgroup_populate_dir(struct cgroup *cgrp)
3708 int err; 3834 int err;
3709 struct cgroup_subsys *ss; 3835 struct cgroup_subsys *ss;
3710 3836
3711 /* First clear out any existing files */ 3837 err = cgroup_addrm_files(cgrp, NULL, files, true);
3712 cgroup_clear_directory(cgrp->dentry);
3713
3714 err = cgroup_add_files(cgrp, NULL, files, ARRAY_SIZE(files));
3715 if (err < 0) 3838 if (err < 0)
3716 return err; 3839 return err;
3717 3840
3718 if (cgrp == cgrp->top_cgroup) { 3841 /* process cftsets of each subsystem */
3719 if ((err = cgroup_add_file(cgrp, NULL, &cft_release_agent)) < 0)
3720 return err;
3721 }
3722
3723 for_each_subsys(cgrp->root, ss) { 3842 for_each_subsys(cgrp->root, ss) {
3843 struct cftype_set *set;
3844
3724 if (ss->populate && (err = ss->populate(ss, cgrp)) < 0) 3845 if (ss->populate && (err = ss->populate(ss, cgrp)) < 0)
3725 return err; 3846 return err;
3847
3848 list_for_each_entry(set, &ss->cftsets, node)
3849 cgroup_addrm_files(cgrp, ss, set->cfts, true);
3726 } 3850 }
3851
3727 /* This cgroup is ready now */ 3852 /* This cgroup is ready now */
3728 for_each_subsys(cgrp->root, ss) { 3853 for_each_subsys(cgrp->root, ss) {
3729 struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id]; 3854 struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id];
@@ -3739,6 +3864,14 @@ static int cgroup_populate_dir(struct cgroup *cgrp)
3739 return 0; 3864 return 0;
3740} 3865}
3741 3866
3867static void css_dput_fn(struct work_struct *work)
3868{
3869 struct cgroup_subsys_state *css =
3870 container_of(work, struct cgroup_subsys_state, dput_work);
3871
3872 dput(css->cgroup->dentry);
3873}
3874
3742static void init_cgroup_css(struct cgroup_subsys_state *css, 3875static void init_cgroup_css(struct cgroup_subsys_state *css,
3743 struct cgroup_subsys *ss, 3876 struct cgroup_subsys *ss,
3744 struct cgroup *cgrp) 3877 struct cgroup *cgrp)
@@ -3751,6 +3884,16 @@ static void init_cgroup_css(struct cgroup_subsys_state *css,
3751 set_bit(CSS_ROOT, &css->flags); 3884 set_bit(CSS_ROOT, &css->flags);
3752 BUG_ON(cgrp->subsys[ss->subsys_id]); 3885 BUG_ON(cgrp->subsys[ss->subsys_id]);
3753 cgrp->subsys[ss->subsys_id] = css; 3886 cgrp->subsys[ss->subsys_id] = css;
3887
3888 /*
3889 * If !clear_css_refs, css holds an extra ref to @cgrp->dentry
3890 * which is put on the last css_put(). dput() requires process
3891 * context, which css_put() may be called without. @css->dput_work
3892 * will be used to invoke dput() asynchronously from css_put().
3893 */
3894 INIT_WORK(&css->dput_work, css_dput_fn);
3895 if (ss->__DEPRECATED_clear_css_refs)
3896 set_bit(CSS_CLEAR_CSS_REFS, &css->flags);
3754} 3897}
3755 3898
3756static void cgroup_lock_hierarchy(struct cgroupfs_root *root) 3899static void cgroup_lock_hierarchy(struct cgroupfs_root *root)
@@ -3827,7 +3970,7 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
3827 set_bit(CGRP_CLONE_CHILDREN, &cgrp->flags); 3970 set_bit(CGRP_CLONE_CHILDREN, &cgrp->flags);
3828 3971
3829 for_each_subsys(root, ss) { 3972 for_each_subsys(root, ss) {
3830 struct cgroup_subsys_state *css = ss->create(ss, cgrp); 3973 struct cgroup_subsys_state *css = ss->create(cgrp);
3831 3974
3832 if (IS_ERR(css)) { 3975 if (IS_ERR(css)) {
3833 err = PTR_ERR(css); 3976 err = PTR_ERR(css);
@@ -3841,7 +3984,7 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
3841 } 3984 }
3842 /* At error, ->destroy() callback has to free assigned ID. */ 3985 /* At error, ->destroy() callback has to free assigned ID. */
3843 if (clone_children(parent) && ss->post_clone) 3986 if (clone_children(parent) && ss->post_clone)
3844 ss->post_clone(ss, cgrp); 3987 ss->post_clone(cgrp);
3845 } 3988 }
3846 3989
3847 cgroup_lock_hierarchy(root); 3990 cgroup_lock_hierarchy(root);
@@ -3853,9 +3996,16 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
3853 if (err < 0) 3996 if (err < 0)
3854 goto err_remove; 3997 goto err_remove;
3855 3998
3999 /* If !clear_css_refs, each css holds a ref to the cgroup's dentry */
4000 for_each_subsys(root, ss)
4001 if (!ss->__DEPRECATED_clear_css_refs)
4002 dget(dentry);
4003
3856 /* The cgroup directory was pre-locked for us */ 4004 /* The cgroup directory was pre-locked for us */
3857 BUG_ON(!mutex_is_locked(&cgrp->dentry->d_inode->i_mutex)); 4005 BUG_ON(!mutex_is_locked(&cgrp->dentry->d_inode->i_mutex));
3858 4006
4007 list_add_tail(&cgrp->allcg_node, &root->allcg_list);
4008
3859 err = cgroup_populate_dir(cgrp); 4009 err = cgroup_populate_dir(cgrp);
3860 /* If err < 0, we have a half-filled directory - oh well ;) */ 4010 /* If err < 0, we have a half-filled directory - oh well ;) */
3861 4011
@@ -3875,7 +4025,7 @@ static long cgroup_create(struct cgroup *parent, struct dentry *dentry,
3875 4025
3876 for_each_subsys(root, ss) { 4026 for_each_subsys(root, ss) {
3877 if (cgrp->subsys[ss->subsys_id]) 4027 if (cgrp->subsys[ss->subsys_id])
3878 ss->destroy(ss, cgrp); 4028 ss->destroy(cgrp);
3879 } 4029 }
3880 4030
3881 mutex_unlock(&cgroup_mutex); 4031 mutex_unlock(&cgroup_mutex);
@@ -3895,18 +4045,19 @@ static int cgroup_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
3895 return cgroup_create(c_parent, dentry, mode | S_IFDIR); 4045 return cgroup_create(c_parent, dentry, mode | S_IFDIR);
3896} 4046}
3897 4047
4048/*
4049 * Check the reference count on each subsystem. Since we already
4050 * established that there are no tasks in the cgroup, if the css refcount
4051 * is also 1, then there should be no outstanding references, so the
4052 * subsystem is safe to destroy. We scan across all subsystems rather than
4053 * using the per-hierarchy linked list of mounted subsystems since we can
4054 * be called via check_for_release() with no synchronization other than
4055 * RCU, and the subsystem linked list isn't RCU-safe.
4056 */
3898static int cgroup_has_css_refs(struct cgroup *cgrp) 4057static int cgroup_has_css_refs(struct cgroup *cgrp)
3899{ 4058{
3900 /* Check the reference count on each subsystem. Since we
3901 * already established that there are no tasks in the
3902 * cgroup, if the css refcount is also 1, then there should
3903 * be no outstanding references, so the subsystem is safe to
3904 * destroy. We scan across all subsystems rather than using
3905 * the per-hierarchy linked list of mounted subsystems since
3906 * we can be called via check_for_release() with no
3907 * synchronization other than RCU, and the subsystem linked
3908 * list isn't RCU-safe */
3909 int i; 4059 int i;
4060
3910 /* 4061 /*
3911 * We won't need to lock the subsys array, because the subsystems 4062 * We won't need to lock the subsys array, because the subsystems
3912 * we're concerned about aren't going anywhere since our cgroup root 4063 * we're concerned about aren't going anywhere since our cgroup root
@@ -3915,17 +4066,21 @@ static int cgroup_has_css_refs(struct cgroup *cgrp)
3915 for (i = 0; i < CGROUP_SUBSYS_COUNT; i++) { 4066 for (i = 0; i < CGROUP_SUBSYS_COUNT; i++) {
3916 struct cgroup_subsys *ss = subsys[i]; 4067 struct cgroup_subsys *ss = subsys[i];
3917 struct cgroup_subsys_state *css; 4068 struct cgroup_subsys_state *css;
4069
3918 /* Skip subsystems not present or not in this hierarchy */ 4070 /* Skip subsystems not present or not in this hierarchy */
3919 if (ss == NULL || ss->root != cgrp->root) 4071 if (ss == NULL || ss->root != cgrp->root)
3920 continue; 4072 continue;
4073
3921 css = cgrp->subsys[ss->subsys_id]; 4074 css = cgrp->subsys[ss->subsys_id];
3922 /* When called from check_for_release() it's possible 4075 /*
4076 * When called from check_for_release() it's possible
3923 * that by this point the cgroup has been removed 4077 * that by this point the cgroup has been removed
3924 * and the css deleted. But a false-positive doesn't 4078 * and the css deleted. But a false-positive doesn't
3925 * matter, since it can only happen if the cgroup 4079 * matter, since it can only happen if the cgroup
3926 * has been deleted and hence no longer needs the 4080 * has been deleted and hence no longer needs the
3927 * release agent to be called anyway. */ 4081 * release agent to be called anyway.
3928 if (css && (atomic_read(&css->refcnt) > 1)) 4082 */
4083 if (css && css_refcnt(css) > 1)
3929 return 1; 4084 return 1;
3930 } 4085 }
3931 return 0; 4086 return 0;
@@ -3935,51 +4090,63 @@ static int cgroup_has_css_refs(struct cgroup *cgrp)
3935 * Atomically mark all (or else none) of the cgroup's CSS objects as 4090 * Atomically mark all (or else none) of the cgroup's CSS objects as
3936 * CSS_REMOVED. Return true on success, or false if the cgroup has 4091 * CSS_REMOVED. Return true on success, or false if the cgroup has
3937 * busy subsystems. Call with cgroup_mutex held 4092 * busy subsystems. Call with cgroup_mutex held
4093 *
4094 * Depending on whether a subsys has __DEPRECATED_clear_css_refs set or
4095 * not, cgroup removal behaves differently.
4096 *
4097 * If clear is set, css refcnt for the subsystem should be zero before
4098 * cgroup removal can be committed. This is implemented by
4099 * CGRP_WAIT_ON_RMDIR and retry logic around ->pre_destroy(), which may be
4100 * called multiple times until all css refcnts reach zero and is allowed to
4101 * veto removal on any invocation. This behavior is deprecated and will be
4102 * removed as soon as the existing user (memcg) is updated.
4103 *
4104 * If clear is not set, each css holds an extra reference to the cgroup's
4105 * dentry and cgroup removal proceeds regardless of css refs.
4106 * ->pre_destroy() will be called at least once and is not allowed to fail.
4107 * On the last put of each css, whenever that may be, the extra dentry ref
4108 * is put so that dentry destruction happens only after all css's are
4109 * released.
3938 */ 4110 */
3939
3940static int cgroup_clear_css_refs(struct cgroup *cgrp) 4111static int cgroup_clear_css_refs(struct cgroup *cgrp)
3941{ 4112{
3942 struct cgroup_subsys *ss; 4113 struct cgroup_subsys *ss;
3943 unsigned long flags; 4114 unsigned long flags;
3944 bool failed = false; 4115 bool failed = false;
4116
3945 local_irq_save(flags); 4117 local_irq_save(flags);
4118
4119 /*
4120 * Block new css_tryget() by deactivating refcnt. If all refcnts
4121 * for subsystems w/ clear_css_refs set were 1 at the moment of
4122 * deactivation, we succeeded.
4123 */
3946 for_each_subsys(cgrp->root, ss) { 4124 for_each_subsys(cgrp->root, ss) {
3947 struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id]; 4125 struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id];
3948 int refcnt; 4126
3949 while (1) { 4127 WARN_ON(atomic_read(&css->refcnt) < 0);
3950 /* We can only remove a CSS with a refcnt==1 */ 4128 atomic_add(CSS_DEACT_BIAS, &css->refcnt);
3951 refcnt = atomic_read(&css->refcnt); 4129
3952 if (refcnt > 1) { 4130 if (ss->__DEPRECATED_clear_css_refs)
3953 failed = true; 4131 failed |= css_refcnt(css) != 1;
3954 goto done;
3955 }
3956 BUG_ON(!refcnt);
3957 /*
3958 * Drop the refcnt to 0 while we check other
3959 * subsystems. This will cause any racing
3960 * css_tryget() to spin until we set the
3961 * CSS_REMOVED bits or abort
3962 */
3963 if (atomic_cmpxchg(&css->refcnt, refcnt, 0) == refcnt)
3964 break;
3965 cpu_relax();
3966 }
3967 } 4132 }
3968 done: 4133
4134 /*
4135 * If succeeded, set REMOVED and put all the base refs; otherwise,
4136 * restore refcnts to positive values. Either way, all in-progress
4137 * css_tryget() will be released.
4138 */
3969 for_each_subsys(cgrp->root, ss) { 4139 for_each_subsys(cgrp->root, ss) {
3970 struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id]; 4140 struct cgroup_subsys_state *css = cgrp->subsys[ss->subsys_id];
3971 if (failed) { 4141
3972 /* 4142 if (!failed) {
3973 * Restore old refcnt if we previously managed
3974 * to clear it from 1 to 0
3975 */
3976 if (!atomic_read(&css->refcnt))
3977 atomic_set(&css->refcnt, 1);
3978 } else {
3979 /* Commit the fact that the CSS is removed */
3980 set_bit(CSS_REMOVED, &css->flags); 4143 set_bit(CSS_REMOVED, &css->flags);
4144 css_put(css);
4145 } else {
4146 atomic_sub(CSS_DEACT_BIAS, &css->refcnt);
3981 } 4147 }
3982 } 4148 }
4149
3983 local_irq_restore(flags); 4150 local_irq_restore(flags);
3984 return !failed; 4151 return !failed;
3985} 4152}
@@ -4064,6 +4231,8 @@ again:
4064 list_del_init(&cgrp->sibling); 4231 list_del_init(&cgrp->sibling);
4065 cgroup_unlock_hierarchy(cgrp->root); 4232 cgroup_unlock_hierarchy(cgrp->root);
4066 4233
4234 list_del_init(&cgrp->allcg_node);
4235
4067 d = dget(cgrp->dentry); 4236 d = dget(cgrp->dentry);
4068 4237
4069 cgroup_d_remove_dir(d); 4238 cgroup_d_remove_dir(d);
@@ -4090,16 +4259,33 @@ again:
4090 return 0; 4259 return 0;
4091} 4260}
4092 4261
4262static void __init_or_module cgroup_init_cftsets(struct cgroup_subsys *ss)
4263{
4264 INIT_LIST_HEAD(&ss->cftsets);
4265
4266 /*
4267 * base_cftset is embedded in subsys itself, no need to worry about
4268 * deregistration.
4269 */
4270 if (ss->base_cftypes) {
4271 ss->base_cftset.cfts = ss->base_cftypes;
4272 list_add_tail(&ss->base_cftset.node, &ss->cftsets);
4273 }
4274}
4275
4093static void __init cgroup_init_subsys(struct cgroup_subsys *ss) 4276static void __init cgroup_init_subsys(struct cgroup_subsys *ss)
4094{ 4277{
4095 struct cgroup_subsys_state *css; 4278 struct cgroup_subsys_state *css;
4096 4279
4097 printk(KERN_INFO "Initializing cgroup subsys %s\n", ss->name); 4280 printk(KERN_INFO "Initializing cgroup subsys %s\n", ss->name);
4098 4281
4282 /* init base cftset */
4283 cgroup_init_cftsets(ss);
4284
4099 /* Create the top cgroup state for this subsystem */ 4285 /* Create the top cgroup state for this subsystem */
4100 list_add(&ss->sibling, &rootnode.subsys_list); 4286 list_add(&ss->sibling, &rootnode.subsys_list);
4101 ss->root = &rootnode; 4287 ss->root = &rootnode;
4102 css = ss->create(ss, dummytop); 4288 css = ss->create(dummytop);
4103 /* We don't handle early failures gracefully */ 4289 /* We don't handle early failures gracefully */
4104 BUG_ON(IS_ERR(css)); 4290 BUG_ON(IS_ERR(css));
4105 init_cgroup_css(css, ss, dummytop); 4291 init_cgroup_css(css, ss, dummytop);
@@ -4165,6 +4351,9 @@ int __init_or_module cgroup_load_subsys(struct cgroup_subsys *ss)
4165 return 0; 4351 return 0;
4166 } 4352 }
4167 4353
4354 /* init base cftset */
4355 cgroup_init_cftsets(ss);
4356
4168 /* 4357 /*
4169 * need to register a subsys id before anything else - for example, 4358 * need to register a subsys id before anything else - for example,
4170 * init_cgroup_css needs it. 4359 * init_cgroup_css needs it.
@@ -4188,7 +4377,7 @@ int __init_or_module cgroup_load_subsys(struct cgroup_subsys *ss)
4188 * no ss->create seems to need anything important in the ss struct, so 4377 * no ss->create seems to need anything important in the ss struct, so
4189 * this can happen first (i.e. before the rootnode attachment). 4378 * this can happen first (i.e. before the rootnode attachment).
4190 */ 4379 */
4191 css = ss->create(ss, dummytop); 4380 css = ss->create(dummytop);
4192 if (IS_ERR(css)) { 4381 if (IS_ERR(css)) {
4193 /* failure case - need to deassign the subsys[] slot. */ 4382 /* failure case - need to deassign the subsys[] slot. */
4194 subsys[i] = NULL; 4383 subsys[i] = NULL;
@@ -4206,7 +4395,7 @@ int __init_or_module cgroup_load_subsys(struct cgroup_subsys *ss)
4206 int ret = cgroup_init_idr(ss, css); 4395 int ret = cgroup_init_idr(ss, css);
4207 if (ret) { 4396 if (ret) {
4208 dummytop->subsys[ss->subsys_id] = NULL; 4397 dummytop->subsys[ss->subsys_id] = NULL;
4209 ss->destroy(ss, dummytop); 4398 ss->destroy(dummytop);
4210 subsys[i] = NULL; 4399 subsys[i] = NULL;
4211 mutex_unlock(&cgroup_mutex); 4400 mutex_unlock(&cgroup_mutex);
4212 return ret; 4401 return ret;
@@ -4304,7 +4493,7 @@ void cgroup_unload_subsys(struct cgroup_subsys *ss)
4304 * pointer to find their state. note that this also takes care of 4493 * pointer to find their state. note that this also takes care of
4305 * freeing the css_id. 4494 * freeing the css_id.
4306 */ 4495 */
4307 ss->destroy(ss, dummytop); 4496 ss->destroy(dummytop);
4308 dummytop->subsys[ss->subsys_id] = NULL; 4497 dummytop->subsys[ss->subsys_id] = NULL;
4309 4498
4310 mutex_unlock(&cgroup_mutex); 4499 mutex_unlock(&cgroup_mutex);
@@ -4580,7 +4769,7 @@ void cgroup_fork_callbacks(struct task_struct *child)
4580 for (i = 0; i < CGROUP_BUILTIN_SUBSYS_COUNT; i++) { 4769 for (i = 0; i < CGROUP_BUILTIN_SUBSYS_COUNT; i++) {
4581 struct cgroup_subsys *ss = subsys[i]; 4770 struct cgroup_subsys *ss = subsys[i];
4582 if (ss->fork) 4771 if (ss->fork)
4583 ss->fork(ss, child); 4772 ss->fork(child);
4584 } 4773 }
4585 } 4774 }
4586} 4775}
@@ -4596,6 +4785,17 @@ void cgroup_fork_callbacks(struct task_struct *child)
4596 */ 4785 */
4597void cgroup_post_fork(struct task_struct *child) 4786void cgroup_post_fork(struct task_struct *child)
4598{ 4787{
4788 /*
4789 * use_task_css_set_links is set to 1 before we walk the tasklist
4790 * under the tasklist_lock and we read it here after we added the child
4791 * to the tasklist under the tasklist_lock as well. If the child wasn't
4792 * yet in the tasklist when we walked through it from
4793 * cgroup_enable_task_cg_lists(), then use_task_css_set_links value
4794 * should be visible now due to the paired locking and barriers implied
4795 * by LOCK/UNLOCK: it is written before the tasklist_lock unlock
4796 * in cgroup_enable_task_cg_lists() and read here after the tasklist_lock
4797 * lock on fork.
4798 */
4599 if (use_task_css_set_links) { 4799 if (use_task_css_set_links) {
4600 write_lock(&css_set_lock); 4800 write_lock(&css_set_lock);
4601 if (list_empty(&child->cg_list)) { 4801 if (list_empty(&child->cg_list)) {
@@ -4682,7 +4882,7 @@ void cgroup_exit(struct task_struct *tsk, int run_callbacks)
4682 struct cgroup *old_cgrp = 4882 struct cgroup *old_cgrp =
4683 rcu_dereference_raw(cg->subsys[i])->cgroup; 4883 rcu_dereference_raw(cg->subsys[i])->cgroup;
4684 struct cgroup *cgrp = task_cgroup(tsk, i); 4884 struct cgroup *cgrp = task_cgroup(tsk, i);
4685 ss->exit(ss, cgrp, old_cgrp, tsk); 4885 ss->exit(cgrp, old_cgrp, tsk);
4686 } 4886 }
4687 } 4887 }
4688 } 4888 }
@@ -4743,21 +4943,41 @@ static void check_for_release(struct cgroup *cgrp)
4743} 4943}
4744 4944
4745/* Caller must verify that the css is not for root cgroup */ 4945/* Caller must verify that the css is not for root cgroup */
4746void __css_put(struct cgroup_subsys_state *css, int count) 4946bool __css_tryget(struct cgroup_subsys_state *css)
4947{
4948 do {
4949 int v = css_refcnt(css);
4950
4951 if (atomic_cmpxchg(&css->refcnt, v, v + 1) == v)
4952 return true;
4953 cpu_relax();
4954 } while (!test_bit(CSS_REMOVED, &css->flags));
4955
4956 return false;
4957}
4958EXPORT_SYMBOL_GPL(__css_tryget);
4959
4960/* Caller must verify that the css is not for root cgroup */
4961void __css_put(struct cgroup_subsys_state *css)
4747{ 4962{
4748 struct cgroup *cgrp = css->cgroup; 4963 struct cgroup *cgrp = css->cgroup;
4749 int val; 4964
4750 rcu_read_lock(); 4965 rcu_read_lock();
4751 val = atomic_sub_return(count, &css->refcnt); 4966 atomic_dec(&css->refcnt);
4752 if (val == 1) { 4967 switch (css_refcnt(css)) {
4968 case 1:
4753 if (notify_on_release(cgrp)) { 4969 if (notify_on_release(cgrp)) {
4754 set_bit(CGRP_RELEASABLE, &cgrp->flags); 4970 set_bit(CGRP_RELEASABLE, &cgrp->flags);
4755 check_for_release(cgrp); 4971 check_for_release(cgrp);
4756 } 4972 }
4757 cgroup_wakeup_rmdir_waiter(cgrp); 4973 cgroup_wakeup_rmdir_waiter(cgrp);
4974 break;
4975 case 0:
4976 if (!test_bit(CSS_CLEAR_CSS_REFS, &css->flags))
4977 schedule_work(&css->dput_work);
4978 break;
4758 } 4979 }
4759 rcu_read_unlock(); 4980 rcu_read_unlock();
4760 WARN_ON_ONCE(val < 1);
4761} 4981}
4762EXPORT_SYMBOL_GPL(__css_put); 4982EXPORT_SYMBOL_GPL(__css_put);
4763 4983
@@ -4876,7 +5096,7 @@ unsigned short css_id(struct cgroup_subsys_state *css)
4876 * on this or this is under rcu_read_lock(). Once css->id is allocated, 5096 * on this or this is under rcu_read_lock(). Once css->id is allocated,
4877 * it's unchanged until freed. 5097 * it's unchanged until freed.
4878 */ 5098 */
4879 cssid = rcu_dereference_check(css->id, atomic_read(&css->refcnt)); 5099 cssid = rcu_dereference_check(css->id, css_refcnt(css));
4880 5100
4881 if (cssid) 5101 if (cssid)
4882 return cssid->id; 5102 return cssid->id;
@@ -4888,7 +5108,7 @@ unsigned short css_depth(struct cgroup_subsys_state *css)
4888{ 5108{
4889 struct css_id *cssid; 5109 struct css_id *cssid;
4890 5110
4891 cssid = rcu_dereference_check(css->id, atomic_read(&css->refcnt)); 5111 cssid = rcu_dereference_check(css->id, css_refcnt(css));
4892 5112
4893 if (cssid) 5113 if (cssid)
4894 return cssid->depth; 5114 return cssid->depth;
@@ -4939,9 +5159,9 @@ void free_css_id(struct cgroup_subsys *ss, struct cgroup_subsys_state *css)
4939 5159
4940 rcu_assign_pointer(id->css, NULL); 5160 rcu_assign_pointer(id->css, NULL);
4941 rcu_assign_pointer(css->id, NULL); 5161 rcu_assign_pointer(css->id, NULL);
4942 write_lock(&ss->id_lock); 5162 spin_lock(&ss->id_lock);
4943 idr_remove(&ss->idr, id->id); 5163 idr_remove(&ss->idr, id->id);
4944 write_unlock(&ss->id_lock); 5164 spin_unlock(&ss->id_lock);
4945 kfree_rcu(id, rcu_head); 5165 kfree_rcu(id, rcu_head);
4946} 5166}
4947EXPORT_SYMBOL_GPL(free_css_id); 5167EXPORT_SYMBOL_GPL(free_css_id);
@@ -4967,10 +5187,10 @@ static struct css_id *get_new_cssid(struct cgroup_subsys *ss, int depth)
4967 error = -ENOMEM; 5187 error = -ENOMEM;
4968 goto err_out; 5188 goto err_out;
4969 } 5189 }
4970 write_lock(&ss->id_lock); 5190 spin_lock(&ss->id_lock);
4971 /* Don't use 0. allocates an ID of 1-65535 */ 5191 /* Don't use 0. allocates an ID of 1-65535 */
4972 error = idr_get_new_above(&ss->idr, newid, 1, &myid); 5192 error = idr_get_new_above(&ss->idr, newid, 1, &myid);
4973 write_unlock(&ss->id_lock); 5193 spin_unlock(&ss->id_lock);
4974 5194
4975 /* Returns error when there are no free spaces for new ID.*/ 5195 /* Returns error when there are no free spaces for new ID.*/
4976 if (error) { 5196 if (error) {
@@ -4985,9 +5205,9 @@ static struct css_id *get_new_cssid(struct cgroup_subsys *ss, int depth)
4985 return newid; 5205 return newid;
4986remove_idr: 5206remove_idr:
4987 error = -ENOSPC; 5207 error = -ENOSPC;
4988 write_lock(&ss->id_lock); 5208 spin_lock(&ss->id_lock);
4989 idr_remove(&ss->idr, myid); 5209 idr_remove(&ss->idr, myid);
4990 write_unlock(&ss->id_lock); 5210 spin_unlock(&ss->id_lock);
4991err_out: 5211err_out:
4992 kfree(newid); 5212 kfree(newid);
4993 return ERR_PTR(error); 5213 return ERR_PTR(error);
@@ -4999,7 +5219,7 @@ static int __init_or_module cgroup_init_idr(struct cgroup_subsys *ss,
4999{ 5219{
5000 struct css_id *newid; 5220 struct css_id *newid;
5001 5221
5002 rwlock_init(&ss->id_lock); 5222 spin_lock_init(&ss->id_lock);
5003 idr_init(&ss->idr); 5223 idr_init(&ss->idr);
5004 5224
5005 newid = get_new_cssid(ss, 0); 5225 newid = get_new_cssid(ss, 0);
@@ -5087,6 +5307,8 @@ css_get_next(struct cgroup_subsys *ss, int id,
5087 return NULL; 5307 return NULL;
5088 5308
5089 BUG_ON(!ss->use_id); 5309 BUG_ON(!ss->use_id);
5310 WARN_ON_ONCE(!rcu_read_lock_held());
5311
5090 /* fill start point for scan */ 5312 /* fill start point for scan */
5091 tmpid = id; 5313 tmpid = id;
5092 while (1) { 5314 while (1) {
@@ -5094,10 +5316,7 @@ css_get_next(struct cgroup_subsys *ss, int id,
5094 * scan next entry from bitmap(tree), tmpid is updated after 5316 * scan next entry from bitmap(tree), tmpid is updated after
5095 * idr_get_next(). 5317 * idr_get_next().
5096 */ 5318 */
5097 read_lock(&ss->id_lock);
5098 tmp = idr_get_next(&ss->idr, &tmpid); 5319 tmp = idr_get_next(&ss->idr, &tmpid);
5099 read_unlock(&ss->id_lock);
5100
5101 if (!tmp) 5320 if (!tmp)
5102 break; 5321 break;
5103 if (tmp->depth >= depth && tmp->stack[depth] == rootid) { 5322 if (tmp->depth >= depth && tmp->stack[depth] == rootid) {
@@ -5137,8 +5356,7 @@ struct cgroup_subsys_state *cgroup_css_from_dir(struct file *f, int id)
5137} 5356}
5138 5357
5139#ifdef CONFIG_CGROUP_DEBUG 5358#ifdef CONFIG_CGROUP_DEBUG
5140static struct cgroup_subsys_state *debug_create(struct cgroup_subsys *ss, 5359static struct cgroup_subsys_state *debug_create(struct cgroup *cont)
5141 struct cgroup *cont)
5142{ 5360{
5143 struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL); 5361 struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL);
5144 5362
@@ -5148,7 +5366,7 @@ static struct cgroup_subsys_state *debug_create(struct cgroup_subsys *ss,
5148 return css; 5366 return css;
5149} 5367}
5150 5368
5151static void debug_destroy(struct cgroup_subsys *ss, struct cgroup *cont) 5369static void debug_destroy(struct cgroup *cont)
5152{ 5370{
5153 kfree(cont->subsys[debug_subsys_id]); 5371 kfree(cont->subsys[debug_subsys_id]);
5154} 5372}
@@ -5271,19 +5489,15 @@ static struct cftype debug_files[] = {
5271 .name = "releasable", 5489 .name = "releasable",
5272 .read_u64 = releasable_read, 5490 .read_u64 = releasable_read,
5273 }, 5491 },
5274};
5275 5492
5276static int debug_populate(struct cgroup_subsys *ss, struct cgroup *cont) 5493 { } /* terminate */
5277{ 5494};
5278 return cgroup_add_files(cont, ss, debug_files,
5279 ARRAY_SIZE(debug_files));
5280}
5281 5495
5282struct cgroup_subsys debug_subsys = { 5496struct cgroup_subsys debug_subsys = {
5283 .name = "debug", 5497 .name = "debug",
5284 .create = debug_create, 5498 .create = debug_create,
5285 .destroy = debug_destroy, 5499 .destroy = debug_destroy,
5286 .populate = debug_populate,
5287 .subsys_id = debug_subsys_id, 5500 .subsys_id = debug_subsys_id,
5501 .base_cftypes = debug_files,
5288}; 5502};
5289#endif /* CONFIG_CGROUP_DEBUG */ 5503#endif /* CONFIG_CGROUP_DEBUG */
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-05 18:17:04 -0500