litmus-rt.git - The LITMUS^RT kernel.

diff options

author	Chris Mason <chris.mason@oracle.com>	2007-04-26 16:46:15 -0400
committer	David Woodhouse <dwmw2@hera.kernel.org>	2007-04-26 16:46:15 -0400
commit	9078a3e1e4e489dddc636c7bb8780349d4298743 (patch)
tree	98597a438d60fc8722c7562b79cecaa303b4721d /fs/btrfs/extent-tree.c
parent	f2458e1d8c90958ed3631654cb7fd5ab01478505 (diff)

Btrfs: start of block group code

Signed-off-by: Chris Mason <chris.mason@oracle.com>

Diffstat (limited to 'fs/btrfs/extent-tree.c')

-rw-r--r--

fs/btrfs/extent-tree.c

204

1 files changed, 188 insertions, 16 deletions


diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index e6fe3fd38819..0bb4fc83cfd6 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c
@@ -127,6 +127,105 @@ int btrfs_inc_ref(struct btrfs_trans_handle trans, struct btrfs_root root,
127	return 0;	127	return 0;
128	}	128	}
129		129
		130	static int write_one_cache_group(struct btrfs_trans_handle *trans,
		131	struct btrfs_root *root,
		132	struct btrfs_path *path,
		133	struct btrfs_block_group_cache *cache)
		134	{
		135	int ret;
		136	int pending_ret;
		137	struct btrfs_root *extent_root = root->fs_info->extent_root;
		138	struct btrfs_block_group_item *bi;
		139	struct btrfs_key ins;
		140
		141	find_free_extent(trans, extent_root, 0, 0, (u64)-1, &ins);
		142	ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
		143	BUG_ON(ret);
		144	bi = btrfs_item_ptr(btrfs_buffer_leaf(path->nodes[0]), path->slots[0],
		145	struct btrfs_block_group_item);
		146	memcpy(bi, &cache->item, sizeof(*bi));
		147	mark_buffer_dirty(path->nodes[0]);
		148	btrfs_release_path(extent_root, path);
		149
		150	finish_current_insert(trans, extent_root);
		151	pending_ret = del_pending_extents(trans, extent_root);
		152	if (ret)
		153	return ret;
		154	if (pending_ret)
		155	return pending_ret;
		156	return 0;
		157
		158	}
		159
		160	int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans,
		161	struct btrfs_root *root)
		162	{
		163	struct btrfs_block_group_cache *cache[8];
		164	int ret;
		165	int err = 0;
		166	int werr = 0;
		167	struct radix_tree_root *radix = &root->fs_info->block_group_radix;
		168	int i;
		169	struct btrfs_path *path;
		170
		171	path = btrfs_alloc_path();
		172	if (!path)
		173	return -ENOMEM;
		174
		175	while(1) {
		176	ret = radix_tree_gang_lookup_tag(radix, (void **)cache,
		177	0, ARRAY_SIZE(cache),
		178	BTRFS_BLOCK_GROUP_DIRTY);
		179	if (!ret)
		180	break;
		181	for (i = 0; i < ret; i++) {
		182	radix_tree_tag_clear(radix, cache[i]->key.objectid +
		183	cache[i]->key.offset - 1,
		184	BTRFS_BLOCK_GROUP_DIRTY);
		185	err = write_one_cache_group(trans, root,
		186	path, cache[i]);
		187	if (err)
		188	werr = err;
		189	}
		190	}
		191	btrfs_free_path(path);
		192	return werr;
		193	}
		194
		195	static int update_block_group(struct btrfs_trans_handle *trans,
		196	struct btrfs_root *root,
		197	u64 blocknr, u64 num, int alloc)
		198	{
		199	struct btrfs_block_group_cache *cache;
		200	struct btrfs_fs_info *info = root->fs_info;
		201	u64 total = num;
		202	u64 old_val;
		203	u64 block_in_group;
		204	int ret;
		205	while(total) {
		206	ret = radix_tree_gang_lookup(&info->block_group_radix,
		207	(void **)&cache, blocknr, 1);
		208	if (!ret)
		209	return -1;
		210	block_in_group = blocknr - cache->key.objectid;
		211	WARN_ON(block_in_group > cache->key.offset);
		212	radix_tree_tag_set(&info->block_group_radix,
		213	cache->key.objectid + cache->key.offset - 1,
		214	BTRFS_BLOCK_GROUP_DIRTY);
		215
		216	old_val = btrfs_block_group_used(&cache->item);
		217	num = min(total, cache->key.offset - block_in_group);
		218	total -= num;
		219	blocknr += num;
		220	if (alloc)
		221	old_val += num;
		222	else
		223	old_val -= num;
		224	btrfs_set_block_group_used(&cache->item, old_val);
		225	}
		226	return 0;
		227	}
		228
130	int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans, struct	229	int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans, struct
131	btrfs_root *root)	230	btrfs_root *root)
132	{	231	{
@@ -264,6 +363,8 @@ static int __free_extent(struct btrfs_trans_handle *trans, struct btrfs_root
264	ret = btrfs_del_item(trans, extent_root, path);	363	ret = btrfs_del_item(trans, extent_root, path);
265	if (ret)	364	if (ret)
266	BUG();	365	BUG();
		366	ret = update_block_group(trans, root, blocknr, num_blocks, 0);
		367	BUG_ON(ret);
267	}	368	}
268	btrfs_release_path(extent_root, path);	369	btrfs_release_path(extent_root, path);
269	btrfs_free_path(path);	370	btrfs_free_path(path);
@@ -365,21 +466,6 @@ static int find_free_extent(struct btrfs_trans_handle *trans, struct btrfs_root
365	num_blocks = 1;	466	num_blocks = 1;
366	total_needed = min(level + 2, BTRFS_MAX_LEVEL) * 3;	467	total_needed = min(level + 2, BTRFS_MAX_LEVEL) * 3;
367	}	468	}
368	if (info->last_insert.objectid == 0 && search_end == (u64)-1) {
369	struct btrfs_disk_key *last_key;
370	btrfs_init_path(path);
371	ins->objectid = (u64)-1;
372	ins->offset = (u64)-1;
373	ret = btrfs_search_slot(trans, root, ins, path, 0, 0);
374	if (ret < 0)
375	goto error;
376	BUG_ON(ret == 0);
377	if (path->slots[0] > 0)
378	path->slots[0]--;
379	l = btrfs_buffer_leaf(path->nodes[0]);
380	last_key = &l->items[path->slots[0]].key;
381	search_start = btrfs_disk_key_objectid(last_key);
382	}
383	if (info->last_insert.objectid > search_start)	469	if (info->last_insert.objectid > search_start)
384	search_start = info->last_insert.objectid;	470	search_start = info->last_insert.objectid;
385		471
@@ -420,6 +506,8 @@ check_failed:
420	goto check_pending;	506	goto check_pending;
421	}	507	}
422	btrfs_disk_key_to_cpu(&key, &l->items[slot].key);	508	btrfs_disk_key_to_cpu(&key, &l->items[slot].key);
		509	if (btrfs_key_type(&key) != BTRFS_EXTENT_ITEM_KEY)
		510	goto next;
423	if (key.objectid >= search_start) {	511	if (key.objectid >= search_start) {
424	if (start_found) {	512	if (start_found) {
425	if (last_block < search_start)	513	if (last_block < search_start)
@@ -434,6 +522,7 @@ check_failed:
434	}	522	}
435	start_found = 1;	523	start_found = 1;
436	last_block = key.objectid + key.offset;	524	last_block = key.objectid + key.offset;
		525	next:
437	path->slots[0]++;	526	path->slots[0]++;
438	}	527	}
439	// FIXME -ENOSPC	528	// FIXME -ENOSPC
@@ -498,7 +587,6 @@ error:
498	btrfs_free_path(path);	587	btrfs_free_path(path);
499	return ret;	588	return ret;
500	}	589	}
501
502	/*	590	/*
503	* finds a free extent and does all the dirty work required for allocation	591	* finds a free extent and does all the dirty work required for allocation
504	* returns the key for the extent through ins, and a tree buffer for	592	* returns the key for the extent through ins, and a tree buffer for
@@ -532,6 +620,9 @@ int btrfs_alloc_extent(struct btrfs_trans_handle *trans,
532	ins->objectid = info->extent_tree_prealloc[nr];	620	ins->objectid = info->extent_tree_prealloc[nr];
533	info->extent_tree_insert[info->extent_tree_insert_nr++] =	621	info->extent_tree_insert[info->extent_tree_insert_nr++] =
534	ins->objectid;	622	ins->objectid;
		623	ret = update_block_group(trans, root,
		624	ins->objectid, ins->offset, 1);
		625	BUG_ON(ret);
535	return 0;	626	return 0;
536	}	627	}
537	/* do the real allocation */	628	/* do the real allocation */
@@ -558,6 +649,7 @@ int btrfs_alloc_extent(struct btrfs_trans_handle *trans,
558	return ret;	649	return ret;
559	if (pending_ret)	650	if (pending_ret)
560	return pending_ret;	651	return pending_ret;
		652	ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
561	return 0;	653	return 0;
562	}	654	}
563		655
@@ -578,6 +670,7 @@ struct buffer_head btrfs_alloc_free_block(struct btrfs_trans_handle trans,
578	BUG();	670	BUG();
579	return NULL;	671	return NULL;
580	}	672	}
		673	BUG_ON(ret);
581	buf = btrfs_find_create_tree_block(root, ins.objectid);	674	buf = btrfs_find_create_tree_block(root, ins.objectid);
582	set_buffer_uptodate(buf);	675	set_buffer_uptodate(buf);
583	return buf;	676	return buf;
@@ -758,3 +851,82 @@ int btrfs_drop_snapshot(struct btrfs_trans_handle *trans, struct btrfs_root
758	btrfs_free_path(path);	851	btrfs_free_path(path);
759	return ret;	852	return ret;
760	}	853	}
		854
		855	int btrfs_free_block_groups(struct btrfs_fs_info *info)
		856	{
		857	int ret;
		858	struct btrfs_block_group_cache *cache[8];
		859	int i;
		860
		861	while(1) {
		862	ret = radix_tree_gang_lookup(&info->block_group_radix,
		863	(void **)cache, 0,
		864	ARRAY_SIZE(cache));
		865	if (!ret)
		866	break;
		867	for (i = 0; i < ret; i++) {
		868	radix_tree_delete(&info->block_group_radix,
		869	cache[i]->key.objectid +
		870	cache[i]->key.offset - 1);
		871	kfree(cache[i]);
		872	}
		873	}
		874	return 0;
		875	}
		876
		877	int btrfs_read_block_groups(struct btrfs_root *root)
		878	{
		879	struct btrfs_path *path;
		880	int ret;
		881	int err = 0;
		882	struct btrfs_block_group_item *bi;
		883	struct btrfs_block_group_cache *cache;
		884	struct btrfs_key key;
		885	struct btrfs_key found_key;
		886	struct btrfs_leaf *leaf;
		887	u64 group_size_blocks = BTRFS_BLOCK_GROUP_SIZE / root->blocksize;
		888
		889	root = root->fs_info->extent_root;
		890	key.objectid = 0;
		891	key.offset = group_size_blocks;
		892	key.flags = 0;
		893	btrfs_set_key_type(&key, BTRFS_BLOCK_GROUP_ITEM_KEY);
		894
		895	path = btrfs_alloc_path();
		896	if (!path)
		897	return -ENOMEM;
		898
		899	while(1) {
		900	ret = btrfs_search_slot(NULL, root->fs_info->extent_root,
		901	&key, path, 0, 0);
		902	if (ret != 0) {
		903	err = ret;
		904	break;
		905	}
		906	leaf = btrfs_buffer_leaf(path->nodes[0]);
		907	btrfs_disk_key_to_cpu(&found_key,
		908	&leaf->items[path->slots[0]].key);
		909	cache = kmalloc(sizeof(*cache), GFP_NOFS);
		910	if (!cache) {
		911	err = -1;
		912	break;
		913	}
		914	bi = btrfs_item_ptr(leaf, path->slots[0],
		915	struct btrfs_block_group_item);
		916	memcpy(&cache->item, bi, sizeof(*bi));
		917	memcpy(&cache->key, &found_key, sizeof(found_key));
		918	key.objectid = found_key.objectid + found_key.offset;
		919	btrfs_release_path(root, path);
		920	ret = radix_tree_insert(&root->fs_info->block_group_radix,
		921	found_key.objectid +
		922	found_key.offset - 1,
		923	(void *)cache);
		924	BUG_ON(ret);
		925	if (key.objectid >=
		926	btrfs_super_total_blocks(root->fs_info->disk_super))
		927	break;
		928	}
		929
		930	btrfs_free_path(path);
		931	return 0;
		932	}

902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044

/* * padata.c - generic interface to process data streams in parallel * * Copyright (C) 2008, 2009 secunet Security Networks AG * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com> * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. * * This program is distributed in the hope it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License along with * this program; if not, write to the Free Software Foundation, Inc., * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. */ #include <linux/module.h> #include <linux/cpumask.h> #include <linux/err.h> #include <linux/cpu.h> #include <linux/padata.h> #include <linux/mutex.h> #include <linux/sched.h> #include <linux/slab.h> #include <linux/sysfs.h> #include <linux/rcupdate.h> #define MAX_SEQ_NR (INT_MAX - NR_CPUS) #define MAX_OBJ_NUM 1000 static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index) { int cpu, target_cpu; target_cpu = cpumask_first(pd->cpumask.pcpu); for (cpu = 0; cpu < cpu_index; cpu++) target_cpu = cpumask_next(target_cpu, pd->cpumask.pcpu); return target_cpu; } static int padata_cpu_hash(struct padata_priv *padata) { int cpu_index; struct parallel_data *pd; pd = padata->pd; /* * Hash the sequence numbers to the cpus by taking * seq_nr mod. number of cpus in use. */ cpu_index = padata->seq_nr % cpumask_weight(pd->cpumask.pcpu); return padata_index_to_cpu(pd, cpu_index); } static void padata_parallel_worker(struct work_struct *parallel_work) { struct padata_parallel_queue *pqueue; struct parallel_data *pd; struct padata_instance *pinst; LIST_HEAD(local_list); local_bh_disable(); pqueue = container_of(parallel_work, struct padata_parallel_queue, work); pd = pqueue->pd; pinst = pd->pinst; spin_lock(&pqueue->parallel.lock); list_replace_init(&pqueue->parallel.list, &local_list); spin_unlock(&pqueue->parallel.lock); while (!list_empty(&local_list)) { struct padata_priv *padata; padata = list_entry(local_list.next, struct padata_priv, list); list_del_init(&padata->list); padata->parallel(padata); } local_bh_enable(); } /** * padata_do_parallel - padata parallelization function * * @pinst: padata instance * @padata: object to be parallelized * @cb_cpu: cpu the serialization callback function will run on, * must be in the serial cpumask of padata(i.e. cpumask.cbcpu). * * The parallelization callback function will run with BHs off. * Note: Every object which is parallelized by padata_do_parallel * must be seen by padata_do_serial. */ int padata_do_parallel(struct padata_instance *pinst, struct padata_priv *padata, int cb_cpu) { int target_cpu, err; struct padata_parallel_queue *queue; struct parallel_data *pd; rcu_read_lock_bh(); pd = rcu_dereference(pinst->pd); err = -EINVAL; if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID) goto out; if (!cpumask_test_cpu(cb_cpu, pd->cpumask.cbcpu)) goto out; err = -EBUSY; if ((pinst->flags & PADATA_RESET)) goto out; if (atomic_read(&pd->refcnt) >= MAX_OBJ_NUM) goto out; err = 0; atomic_inc(&pd->refcnt); padata->pd = pd; padata->cb_cpu = cb_cpu; if (unlikely(atomic_read(&pd->seq_nr) == pd->max_seq_nr)) atomic_set(&pd->seq_nr, -1); padata->seq_nr = atomic_inc_return(&pd->seq_nr); target_cpu = padata_cpu_hash(padata); queue = per_cpu_ptr(pd->pqueue, target_cpu); spin_lock(&queue->parallel.lock); list_add_tail(&padata->list, &queue->parallel.list); spin_unlock(&queue->parallel.lock); queue_work_on(target_cpu, pinst->wq, &queue->work); out: rcu_read_unlock_bh(); return err; } EXPORT_SYMBOL(padata_do_parallel); /* * padata_get_next - Get the next object that needs serialization. * * Return values are: * * A pointer to the control struct of the next object that needs * serialization, if present in one of the percpu reorder queues. * * NULL, if all percpu reorder queues are empty. * * -EINPROGRESS, if the next object that needs serialization will * be parallel processed by another cpu and is not yet present in * the cpu's reorder queue. * * -ENODATA, if this cpu has to do the parallel processing for * the next object. */ static struct padata_priv *padata_get_next(struct parallel_data *pd) { int cpu, num_cpus; int next_nr, next_index; struct padata_parallel_queue *queue, *next_queue; struct padata_priv *padata; struct padata_list *reorder; num_cpus = cpumask_weight(pd->cpumask.pcpu); /* * Calculate the percpu reorder queue and the sequence * number of the next object. */ next_nr = pd->processed; next_index = next_nr % num_cpus; cpu = padata_index_to_cpu(pd, next_index); next_queue = per_cpu_ptr(pd->pqueue, cpu); if (unlikely(next_nr > pd->max_seq_nr)) { next_nr = next_nr - pd->max_seq_nr - 1; next_index = next_nr % num_cpus; cpu = padata_index_to_cpu(pd, next_index); next_queue = per_cpu_ptr(pd->pqueue, cpu); pd->processed = 0; } padata = NULL; reorder = &next_queue->reorder; if (!list_empty(&reorder->list)) { padata = list_entry(reorder->list.next, struct padata_priv, list); BUG_ON(next_nr != padata->seq_nr); spin_lock(&reorder->lock); list_del_init(&padata->list); atomic_dec(&pd->reorder_objects); spin_unlock(&reorder->lock); pd->processed++; goto out; } queue = per_cpu_ptr(pd->pqueue, smp_processor_id()); if (queue->cpu_index == next_queue->cpu_index) { padata = ERR_PTR(-ENODATA); goto out; } padata = ERR_PTR(-EINPROGRESS); out: return padata; } static void padata_reorder(struct parallel_data *pd) { struct padata_priv *padata; struct padata_serial_queue *squeue; struct padata_instance *pinst = pd->pinst; /* * We need to ensure that only one cpu can work on dequeueing of * the reorder queue the time. Calculating in which percpu reorder * queue the next object will arrive takes some time. A spinlock * would be highly contended. Also it is not clear in which order * the objects arrive to the reorder queues. So a cpu could wait to * get the lock just to notice that there is nothing to do at the * moment. Therefore we use a trylock and let the holder of the lock * care for all the objects enqueued during the holdtime of the lock. */ if (!spin_trylock_bh(&pd->lock)) return; while (1) { padata = padata_get_next(pd); /* * All reorder queues are empty, or the next object that needs * serialization is parallel processed by another cpu and is * still on it's way to the cpu's reorder queue, nothing to * do for now. */ if (!padata || PTR_ERR(padata) == -EINPROGRESS) break; /* * This cpu has to do the parallel processing of the next * object. It's waiting in the cpu's parallelization queue, * so exit imediately. */ if (PTR_ERR(padata) == -ENODATA) { del_timer(&pd->timer); spin_unlock_bh(&pd->lock); return; } squeue = per_cpu_ptr(pd->squeue, padata->cb_cpu); spin_lock(&squeue->serial.lock); list_add_tail(&padata->list, &squeue->serial.list); spin_unlock(&squeue->serial.lock); queue_work_on(padata->cb_cpu, pinst->wq, &squeue->work); } spin_unlock_bh(&pd->lock); /* * The next object that needs serialization might have arrived to * the reorder queues in the meantime, we will be called again * from the timer function if noone else cares for it. */ if (atomic_read(&pd->reorder_objects) && !(pinst->flags & PADATA_RESET)) mod_timer(&pd->timer, jiffies + HZ); else del_timer(&pd->timer); return; } static void padata_reorder_timer(unsigned long arg) { struct parallel_data *pd = (struct parallel_data *)arg; padata_reorder(pd); } static void padata_serial_worker(struct work_struct *serial_work) { struct padata_serial_queue *squeue; struct parallel_data *pd; LIST_HEAD(local_list); local_bh_disable(); squeue = container_of(serial_work, struct padata_serial_queue, work); pd = squeue->pd; spin_lock(&squeue->serial.lock); list_replace_init(&squeue->serial.list, &local_list); spin_unlock(&squeue->serial.lock); while (!list_empty(&local_list)) { struct padata_priv *padata; padata = list_entry(local_list.next, struct padata_priv, list); list_del_init(&padata->list); padata->serial(padata); atomic_dec(&pd->refcnt); } local_bh_enable(); } /** * padata_do_serial - padata serialization function * * @padata: object to be serialized. * * padata_do_serial must be called for every parallelized object. * The serialization callback function will run with BHs off. */ void padata_do_serial(struct padata_priv *padata) { int cpu; struct padata_parallel_queue *pqueue; struct parallel_data *pd; pd = padata->pd; cpu = get_cpu(); pqueue = per_cpu_ptr(pd->pqueue, cpu); spin_lock(&pqueue->reorder.lock); atomic_inc(&pd->reorder_objects); list_add_tail(&padata->list, &pqueue->reorder.list); spin_unlock(&pqueue->reorder.lock); put_cpu(); padata_reorder(pd); } EXPORT_SYMBOL(padata_do_serial); static int padata_setup_cpumasks(struct parallel_data *pd, const struct cpumask *pcpumask, const struct cpumask *cbcpumask) { if (!alloc_cpumask_var(&pd->cpumask.pcpu, GFP_KERNEL)) return -ENOMEM; cpumask_and(pd->cpumask.pcpu, pcpumask, cpu_active_mask); if (!alloc_cpumask_var(&pd->cpumask.cbcpu, GFP_KERNEL)) { free_cpumask_var(pd->cpumask.cbcpu); return -ENOMEM; } cpumask_and(pd->cpumask.cbcpu, cbcpumask, cpu_active_mask); return 0; } static void __padata_list_init(struct padata_list *pd_list) { INIT_LIST_HEAD(&pd_list->list); spin_lock_init(&pd_list->lock); } /* Initialize all percpu queues used by serial workers */ static void padata_init_squeues(struct parallel_data *pd) { int cpu; struct padata_serial_queue *squeue; for_each_cpu(cpu, pd->cpumask.cbcpu) { squeue = per_cpu_ptr(pd->squeue, cpu); squeue->pd = pd; __padata_list_init(&squeue->serial); INIT_WORK(&squeue->work, padata_serial_worker); } } /* Initialize all percpu queues used by parallel workers */ static void padata_init_pqueues(struct parallel_data *pd) { int cpu_index, num_cpus, cpu; struct padata_parallel_queue *pqueue; cpu_index = 0; for_each_cpu(cpu, pd->cpumask.pcpu) { pqueue = per_cpu_ptr(pd->pqueue, cpu); pqueue->pd = pd; pqueue->cpu_index = cpu_index; cpu_index++; __padata_list_init(&pqueue->reorder); __padata_list_init(&pqueue->parallel); INIT_WORK(&pqueue->work, padata_parallel_worker); atomic_set(&pqueue->num_obj, 0); } num_cpus = cpumask_weight(pd->cpumask.pcpu); pd->max_seq_nr = num_cpus ? (MAX_SEQ_NR / num_cpus) * num_cpus - 1 : 0; } /* Allocate and initialize the internal cpumask dependend resources. */ static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst, const struct cpumask *pcpumask, const struct cpumask *cbcpumask) { struct parallel_data *pd; pd = kzalloc(sizeof(struct parallel_data), GFP_KERNEL); if (!pd) goto err; pd->pqueue = alloc_percpu(struct padata_parallel_queue); if (!pd->pqueue) goto err_free_pd; pd->squeue = alloc_percpu(struct padata_serial_queue); if (!pd->squeue) goto err_free_pqueue; if (padata_setup_cpumasks(pd, pcpumask, cbcpumask) < 0) goto err_free_squeue; padata_init_pqueues(pd); padata_init_squeues(pd); setup_timer(&pd->timer, padata_reorder_timer, (unsigned long)pd); atomic_set(&pd->seq_nr, -1); atomic_set(&pd->reorder_objects, 0); atomic_set(&pd->refcnt, 0); pd->pinst = pinst; spin_lock_init(&pd->lock); return pd; err_free_squeue: free_percpu(pd->squeue); err_free_pqueue: free_percpu(pd->pqueue); err_free_pd: kfree(pd); err: return NULL; } static void padata_free_pd(struct parallel_data *pd) { free_cpumask_var(pd->cpumask.pcpu); free_cpumask_var(pd->cpumask.cbcpu); free_percpu(pd->pqueue); free_percpu(pd->squeue); kfree(pd); } /* Flush all objects out of the padata queues. */ static void padata_flush_queues(struct parallel_data *pd) { int cpu; struct padata_parallel_queue *pqueue; struct padata_serial_queue *squeue; for_each_cpu(cpu, pd->cpumask.pcpu) { pqueue = per_cpu_ptr(pd->pqueue, cpu); flush_work(&pqueue->work); } del_timer_sync(&pd->timer); if (atomic_read(&pd->reorder_objects)) padata_reorder(pd); for_each_cpu(cpu, pd->cpumask.cbcpu) { squeue = per_cpu_ptr(pd->squeue, cpu); flush_work(&squeue->work); } BUG_ON(atomic_read(&pd->refcnt) != 0); } static void __padata_start(struct padata_instance *pinst) { pinst->flags |= PADATA_INIT; } static void __padata_stop(struct padata_instance *pinst) { if (!(pinst->flags & PADATA_INIT)) return; pinst->flags &= ~PADATA_INIT; synchronize_rcu(); get_online_cpus(); padata_flush_queues(pinst->pd); put_online_cpus(); } /* Replace the internal control stucture with a new one. */ static void padata_replace(struct padata_instance *pinst, struct parallel_data *pd_new) { struct parallel_data *pd_old = pinst->pd; int notification_mask = 0; pinst->flags |= PADATA_RESET; rcu_assign_pointer(pinst->pd, pd_new); synchronize_rcu(); if (!cpumask_equal(pd_old->cpumask.pcpu, pd_new->cpumask.pcpu)) notification_mask |= PADATA_CPU_PARALLEL; if (!cpumask_equal(pd_old->cpumask.cbcpu, pd_new->cpumask.cbcpu)) notification_mask |= PADATA_CPU_SERIAL; padata_flush_queues(pd_old); padata_free_pd(pd_old); if (notification_mask) blocking_notifier_call_chain(&pinst->cpumask_change_notifier, notification_mask, &pd_new->cpumask); pinst->flags &= ~PADATA_RESET; } /** * padata_register_cpumask_notifier - Registers a notifier that will be called * if either pcpu or cbcpu or both cpumasks change. * * @pinst: A poineter to padata instance * @nblock: A pointer to notifier block. */ int padata_register_cpumask_notifier(struct padata_instance *pinst, struct notifier_block *nblock) { return blocking_notifier_chain_register(&pinst->cpumask_change_notifier, nblock); } EXPORT_SYMBOL(padata_register_cpumask_notifier); /** * padata_unregister_cpumask_notifier - Unregisters cpumask notifier * registered earlier using padata_register_cpumask_notifier * * @pinst: A pointer to data instance. * @nlock: A pointer to notifier block. */ int padata_unregister_cpumask_notifier(struct padata_instance *pinst, struct notifier_block *nblock) { return blocking_notifier_chain_unregister( &pinst->cpumask_change_notifier, nblock); } EXPORT_SYMBOL(padata_unregister_cpumask_notifier); /* If cpumask contains no active cpu, we mark the instance as invalid. */ static bool padata_validate_cpumask(struct padata_instance *pinst, const struct cpumask *cpumask) { if (!cpumask_intersects(cpumask, cpu_active_mask)) { pinst->flags |= PADATA_INVALID; return false; } pinst->flags &= ~PADATA_INVALID; return true; } static int __padata_set_cpumasks(struct padata_instance *pinst, cpumask_var_t pcpumask, cpumask_var_t cbcpumask) { int valid; struct parallel_data *pd; valid = padata_validate_cpumask(pinst, pcpumask); if (!valid) { __padata_stop(pinst); goto out_replace; } valid = padata_validate_cpumask(pinst, cbcpumask); if (!valid) __padata_stop(pinst); out_replace: pd = padata_alloc_pd(pinst, pcpumask, cbcpumask); if (!pd) return -ENOMEM; cpumask_copy(pinst->cpumask.pcpu, pcpumask); cpumask_copy(pinst->cpumask.cbcpu, cbcpumask); padata_replace(pinst, pd); if (valid) __padata_start(pinst); return 0; } /** * padata_set_cpumasks - Set both parallel and serial cpumasks. The first * one is used by parallel workers and the second one * by the wokers doing serialization. * * @pinst: padata instance * @pcpumask: the cpumask to use for parallel workers * @cbcpumask: the cpumsak to use for serial workers */ int padata_set_cpumasks(struct padata_instance *pinst, cpumask_var_t pcpumask, cpumask_var_t cbcpumask) { int err; mutex_lock(&pinst->lock); get_online_cpus(); err = __padata_set_cpumasks(pinst, pcpumask, cbcpumask); put_online_cpus(); mutex_unlock(&pinst->lock); return err; } EXPORT_SYMBOL(padata_set_cpumasks); /** * padata_set_cpumask: Sets specified by @cpumask_type cpumask to the value * equivalent to @cpumask. * * @pinst: padata instance * @cpumask_type: PADATA_CPU_SERIAL or PADATA_CPU_PARALLEL corresponding * to parallel and serial cpumasks respectively. * @cpumask: the cpumask to use */ int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, cpumask_var_t cpumask) { struct cpumask *serial_mask, *parallel_mask; int err = -EINVAL; mutex_lock(&pinst->lock); get_online_cpus(); switch (cpumask_type) { case PADATA_CPU_PARALLEL: serial_mask = pinst->cpumask.cbcpu; parallel_mask = cpumask; break; case PADATA_CPU_SERIAL: parallel_mask = pinst->cpumask.pcpu; serial_mask = cpumask; break; default: goto out; } err = __padata_set_cpumasks(pinst, parallel_mask, serial_mask); out: put_online_cpus(); mutex_unlock(&pinst->lock); return err; } EXPORT_SYMBOL(padata_set_cpumask); static int __padata_add_cpu(struct padata_instance *pinst, int cpu) { struct parallel_data *pd; if (cpumask_test_cpu(cpu, cpu_active_mask)) { pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu, pinst->cpumask.cbcpu); if (!pd) return -ENOMEM; padata_replace(pinst, pd); if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) && padata_validate_cpumask(pinst, pinst->cpumask.cbcpu)) __padata_start(pinst); } return 0; } /** * padata_add_cpu - add a cpu to one or both(parallel and serial) * padata cpumasks. * * @pinst: padata instance * @cpu: cpu to add * @mask: bitmask of flags specifying to which cpumask @cpu shuld be added. * The @mask may be any combination of the following flags: * PADATA_CPU_SERIAL - serial cpumask * PADATA_CPU_PARALLEL - parallel cpumask */ int padata_add_cpu(struct padata_instance *pinst, int cpu, int mask) { int err; if (!(mask & (PADATA_CPU_SERIAL | PADATA_CPU_PARALLEL))) return -EINVAL; mutex_lock(&pinst->lock); get_online_cpus(); if (mask & PADATA_CPU_SERIAL) cpumask_set_cpu(cpu, pinst->cpumask.cbcpu); if (mask & PADATA_CPU_PARALLEL) cpumask_set_cpu(cpu, pinst->cpumask.pcpu); err = __padata_add_cpu(pinst, cpu); put_online_cpus(); mutex_unlock(&pinst->lock); return err; } EXPORT_SYMBOL(padata_add_cpu); static int __padata_remove_cpu(struct padata_instance *pinst, int cpu) { struct parallel_data *pd = NULL; if (cpumask_test_cpu(cpu, cpu_online_mask)) { if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) || !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu)) __padata_stop(pinst); pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu, pinst->cpumask.cbcpu); if (!pd) return -ENOMEM; padata_replace(pinst, pd); } return 0; } /** * padata_remove_cpu - remove a cpu from the one or both(serial and paralell) * padata cpumasks. * * @pinst: padata instance * @cpu: cpu to remove * @mask: bitmask specifying from which cpumask @cpu should be removed * The @mask may be any combination of the following flags: * PADATA_CPU_SERIAL - serial cpumask * PADATA_CPU_PARALLEL - parallel cpumask */ int padata_remove_cpu(struct padata_instance *pinst, int cpu, int mask) { int err; if (!(mask & (PADATA_CPU_SERIAL | PADATA_CPU_PARALLEL))) return -EINVAL; mutex_lock(&pinst->lock); get_online_cpus(); if (mask & PADATA_CPU_SERIAL) cpumask_clear_cpu(cpu, pinst->cpumask.cbcpu); if (mask & PADATA_CPU_PARALLEL) cpumask_clear_cpu(cpu, pinst->cpumask.pcpu); err = __padata_remove_cpu(pinst, cpu); put_online_cpus(); mutex_unlock(&pinst->lock); return err; } EXPORT_SYMBOL(padata_remove_cpu); /** * padata_start - start the parallel processing * * @pinst: padata instance to start */ int padata_start(struct padata_instance *pinst) { int err = 0; mutex_lock(&pinst->lock); if (pinst->flags & PADATA_INVALID) err =-EINVAL; __padata_start(pinst); mutex_unlock(&pinst->lock); return err; } EXPORT_SYMBOL(padata_start); /** * padata_stop - stop the parallel processing * * @pinst: padata instance to stop */ void padata_stop(struct padata_instance *pinst) { mutex_lock(&pinst->lock); __padata_stop(pinst); mutex_unlock(&pinst->lock); } EXPORT_SYMBOL(padata_stop); #ifdef CONFIG_HOTPLUG_CPU static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu) { return cpumask_test_cpu(cpu, pinst->cpumask.pcpu) || cpumask_test_cpu(cpu, pinst->cpumask.cbcpu); } static int padata_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { int err; struct padata_instance *pinst; int cpu = (unsigned long)hcpu; pinst = container_of(nfb, struct padata_instance, cpu_notifier); switch (action) { case CPU_ONLINE: case CPU_ONLINE_FROZEN: if (!pinst_has_cpu(pinst, cpu)) break; mutex_lock(&pinst->lock); err = __padata_add_cpu(pinst, cpu); mutex_unlock(&pinst->lock); if (err) return notifier_from_errno(err); break; case CPU_DOWN_PREPARE: case CPU_DOWN_PREPARE_FROZEN: if (!pinst_has_cpu(pinst, cpu)) break; mutex_lock(&pinst->lock); err = __padata_remove_cpu(pinst, cpu); mutex_unlock(&pinst->lock); if (err) return notifier_from_errno(err); break; case CPU_UP_CANCELED: case CPU_UP_CANCELED_FROZEN: if (!pinst_has_cpu(pinst, cpu)) break; mutex_lock(&pinst->lock); __padata_remove_cpu(pinst, cpu); mutex_unlock(&pinst->lock); case CPU_DOWN_FAILED: case CPU_DOWN_FAILED_FROZEN: if (!pinst_has_cpu(pinst, cpu)) break; mutex_lock(&pinst->lock); __padata_add_cpu(pinst, cpu); mutex_unlock(&pinst->lock); } return NOTIFY_OK; } #endif static void __padata_free(struct padata_instance *pinst) { #ifdef CONFIG_HOTPLUG_CPU unregister_hotcpu_notifier(&pinst->cpu_notifier); #endif padata_stop(pinst); padata_free_pd(pinst->pd); free_cpumask_var(pinst->cpumask.pcpu); free_cpumask_var(pinst->cpumask.cbcpu); kfree(pinst); } #define kobj2pinst(_kobj) \ container_of(_kobj, struct padata_instance, kobj) #define attr2pentry(_attr) \ container_of(_attr, struct padata_sysfs_entry, attr) static void padata_sysfs_release(struct kobject *kobj) { struct padata_instance *pinst = kobj2pinst(kobj); __padata_free(pinst); } struct padata_sysfs_entry { struct attribute attr; ssize_t (*show)(struct padata_instance *, struct attribute *, char *); ssize_t (*store)(struct padata_instance *, struct attribute *, const char *, size_t); }; static ssize_t show_cpumask(struct padata_instance *pinst, struct attribute *attr, char *buf) { struct cpumask *cpumask; ssize_t len; mutex_lock(&pinst->lock); if (!strcmp(attr->name, "serial_cpumask")) cpumask = pinst->cpumask.cbcpu; else cpumask = pinst->cpumask.pcpu; len = bitmap_scnprintf(buf, PAGE_SIZE, cpumask_bits(cpumask), nr_cpu_ids); if (PAGE_SIZE - len < 2) len = -EINVAL; else len += sprintf(buf + len, "\n"); mutex_unlock(&pinst->lock); return len; } static ssize_t store_cpumask(struct padata_instance *pinst, struct attribute *attr, const char *buf, size_t count) { cpumask_var_t new_cpumask; ssize_t ret; int mask_type; if (!alloc_cpumask_var(&new_cpumask, GFP_KERNEL)) return -ENOMEM; ret = bitmap_parse(buf, count, cpumask_bits(new_cpumask), nr_cpumask_bits); if (ret < 0) goto out; mask_type = !strcmp(attr->name, "serial_cpumask") ? PADATA_CPU_SERIAL : PADATA_CPU_PARALLEL; ret = padata_set_cpumask(pinst, mask_type, new_cpumask); if (!ret) ret = count; out: free_cpumask_var(new_cpumask); return ret; } #define PADATA_ATTR_RW(_name, _show_name, _store_name) \ static struct padata_sysfs_entry _name##_attr = \ __ATTR(_name, 0644, _show_name, _store_name) #define PADATA_ATTR_RO(_name, _show_name) \ static struct padata_sysfs_entry _name##_attr = \ __ATTR(_name, 0400, _show_name, NULL) PADATA_ATTR_RW(serial_cpumask, show_cpumask, store_cpumask); PADATA_ATTR_RW(parallel_cpumask, show_cpumask, store_cpumask); /* * Padata sysfs provides the following objects: * serial_cpumask [RW] - cpumask for serial workers * parallel_cpumask [RW] - cpumask for parallel workers */ static struct attribute *padata_default_attrs[] = { &serial_cpumask_attr.attr, &parallel_cpumask_attr.attr, NULL, }; static ssize_t padata_sysfs_show(struct kobject *kobj, struct attribute *attr, char *buf) { struct padata_instance *pinst; struct padata_sysfs_entry *pentry; ssize_t ret = -EIO; pinst = kobj2pinst(kobj); pentry = attr2pentry(attr); if (pentry->show) ret = pentry->show(pinst, attr, buf); return ret; } static ssize_t padata_sysfs_store(struct kobject *kobj, struct attribute *attr, const char *buf, size_t count) { struct padata_instance *pinst; struct padata_sysfs_entry *pentry; ssize_t ret = -EIO; pinst = kobj2pinst(kobj); pentry = attr2pentry(attr); if (pentry->show) ret = pentry->store(pinst, attr, buf, count); return ret; } static const struct sysfs_ops padata_sysfs_ops = { .show = padata_sysfs_show, .store = padata_sysfs_store, }; static struct kobj_type padata_attr_type = { .sysfs_ops = &padata_sysfs_ops, .default_attrs = padata_default_attrs, .release = padata_sysfs_release, }; /** * padata_alloc_possible - Allocate and initialize padata instance. * Use the cpu_possible_mask for serial and * parallel workers. * * @wq: workqueue to use for the allocated padata instance */ struct padata_instance *padata_alloc_possible(struct workqueue_struct *wq) { return padata_alloc(wq, cpu_possible_mask, cpu_possible_mask); } EXPORT_SYMBOL(padata_alloc_possible); /** * padata_alloc - allocate and initialize a padata instance and specify * cpumasks for serial and parallel workers. * * @wq: workqueue to use for the allocated padata instance * @pcpumask: cpumask that will be used for padata parallelization * @cbcpumask: cpumask that will be used for padata serialization */ struct padata_instance *padata_alloc(struct workqueue_struct *wq, const struct cpumask *pcpumask, const struct cpumask *cbcpumask) { struct padata_instance *pinst; struct parallel_data *pd = NULL; pinst = kzalloc(sizeof(struct padata_instance), GFP_KERNEL); if (!pinst) goto err; get_online_cpus(); if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL)) goto err_free_inst; if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) { free_cpumask_var(pinst->cpumask.pcpu); goto err_free_inst; } if (!padata_validate_cpumask(pinst, pcpumask) || !padata_validate_cpumask(pinst, cbcpumask)) goto err_free_masks; pd = padata_alloc_pd(pinst, pcpumask, cbcpumask); if (!pd) goto err_free_masks; rcu_assign_pointer(pinst->pd, pd); pinst->wq = wq; cpumask_copy(pinst->cpumask.pcpu, pcpumask); cpumask_copy(pinst->cpumask.cbcpu, cbcpumask); pinst->flags = 0; #ifdef CONFIG_HOTPLUG_CPU pinst->cpu_notifier.notifier_call = padata_cpu_callback; pinst->cpu_notifier.priority = 0; register_hotcpu_notifier(&pinst->cpu_notifier); #endif put_online_cpus(); BLOCKING_INIT_NOTIFIER_HEAD(&pinst->cpumask_change_notifier); kobject_init(&pinst->kobj, &padata_attr_type); mutex_init(&pinst->lock); return pinst; err_free_masks: free_cpumask_var(pinst->cpumask.pcpu); free_cpumask_var(pinst->cpumask.cbcpu); err_free_inst: kfree(pinst); put_online_cpus(); err: return NULL; } EXPORT_SYMBOL(padata_alloc); /** * padata_free - free a padata instance * * @padata_inst: padata instance to free */ void padata_free(struct padata_instance *pinst) { kobject_put(&pinst->kobj); } EXPORT_SYMBOL(padata_free);


context:
space:
mode: