/* * Generic helpers for smp ipi calls * * (C) Jens Axboe <jens.axboe@oracle.com> 2008 * */ #include <linux/init.h> #include <linux/module.h> #include <linux/percpu.h> #include <linux/rcupdate.h> #include <linux/rculist.h> #include <linux/smp.h> static DEFINE_PER_CPU(struct call_single_queue, call_single_queue); static LIST_HEAD(call_function_queue); __cacheline_aligned_in_smp DEFINE_SPINLOCK(call_function_lock); enum { CSD_FLAG_WAIT = 0x01, CSD_FLAG_ALLOC = 0x02, }; struct call_function_data { struct call_single_data csd; spinlock_t lock; unsigned int refs; cpumask_t cpumask; struct rcu_head rcu_head; }; struct call_single_queue { struct list_head list; spinlock_t lock; }; static int __cpuinit init_call_single_data(void) { int i; for_each_possible_cpu(i) { struct call_single_queue *q = &per_cpu(call_single_queue, i); spin_lock_init(&q->lock); INIT_LIST_HEAD(&q->list); } return 0; } early_initcall(init_call_single_data); static void csd_flag_wait(struct call_single_data *data) { /* Wait for response */ do { if (!(data->flags & CSD_FLAG_WAIT)) break; cpu_relax(); } while (1); } /* * Insert a previously allocated call_single_data element for execution * on the given CPU. data must already have ->func, ->info, and ->flags set. */ static void generic_exec_single(int cpu, struct call_single_data *data) { struct call_single_queue *dst = &per_cpu(call_single_queue, cpu); int wait = data->flags & CSD_FLAG_WAIT, ipi; unsigned long flags; spin_lock_irqsave(&dst->lock, flags); ipi = list_empty(&dst->list); list_add_tail(&data->list, &dst->list); spin_unlock_irqrestore(&dst->lock, flags); /* * Make the list addition visible before sending the ipi. */ smp_mb(); if (ipi) arch_send_call_function_single_ipi(cpu); if (wait) csd_flag_wait(data); } static void rcu_free_call_data(struct rcu_head *head) { struct call_function_data *data; data = container_of(head, struct call_function_data, rcu_head); kfree(data); } /* * Invoked by arch to handle an IPI for call function. Must be called with * interrupts disabled. */ void generic_smp_call_function_interrupt(void) { struct call_function_data *data; int cpu = get_cpu(); /* * It's ok to use list_for_each_rcu() here even though we may delete * 'pos', since list_del_rcu() doesn't clear ->next */ rcu_read_lock(); list_for_each_entry_rcu(data, &call_function_queue, csd.list) { int refs; if (!cpu_isset(cpu, data->cpumask)) continue; data->csd.func(data->csd.info); spin_lock(&data->lock); cpu_clear(cpu, data->cpumask); WARN_ON(data->refs == 0); data->refs--; refs = data->refs; spin_unlock(&data->lock); if (refs) continue; spin_lock(&call_function_lock); list_del_rcu(&data->csd.list); spin_unlock(&call_function_lock); if (data->csd.flags & CSD_FLAG_WAIT) { /* * serialize stores to data with the flag clear * and wakeup */ smp_wmb(); data->csd.flags &= ~CSD_FLAG_WAIT; } if (data->csd.flags & CSD_FLAG_ALLOC) call_rcu(&data->rcu_head, rcu_free_call_data); } rcu_read_unlock(); put_cpu(); } /* * Invoked by arch to handle an IPI for call function single. Must be called * from the arch with interrupts disabled. */ void generic_smp_call_function_single_interrupt(void) { struct call_single_queue *q = &__get_cpu_var(call_single_queue); LIST_HEAD(list); /* * Need to see other stores to list head for checking whether * list is empty without holding q->lock */ smp_read_barrier_depends(); while (!list_empty(&q->list)) { unsigned int data_flags; spin_lock(&q->lock); list_replace_init(&q->list, &list); spin_unlock(&q->lock); while (!list_empty(&list)) { struct call_single_data *data; data = list_entry(list.next, struct call_single_data, list); list_del(&data->list); /* * 'data' can be invalid after this call if * flags == 0 (when called through * generic_exec_single(), so save them away before * making the call. */ data_flags = data->flags; data->func(data->info); if (data_flags & CSD_FLAG_WAIT) { smp_wmb(); data->flags &= ~CSD_FLAG_WAIT; } else if (data_flags & CSD_FLAG_ALLOC) kfree(data); } /* * See comment on outer loop */ smp_read_barrier_depends(); } } /* * smp_call_function_single - Run a function on a specific CPU * @func: The function to run. This must be fast and non-blocking. * @info: An arbitrary pointer to pass to the function. * @wait: If true, wait until function has completed on other CPUs. * * Returns 0 on success, else a negative status code. Note that @wait * will be implicitly turned on in case of allocation failures, since * we fall back to on-stack allocation. */ int smp_call_function_single(int cpu, void (*func) (void *info), void *info, int wait) { struct call_single_data d; unsigned long flags; /* prevent preemption and reschedule on another processor, as well as CPU removal */ int me = get_cpu(); int err = 0; /* Can deadlock when called with interrupts disabled */ WARN_ON(irqs_disabled()); if (cpu == me) { local_irq_save(flags); func(info); local_irq_restore(flags); } else if ((unsigned)cpu < NR_CPUS && cpu_online(cpu)) { struct call_single_data *data = NULL; if (!wait) { data = kmalloc(sizeof(*data), GFP_ATOMIC); if (data) data->flags = CSD_FLAG_ALLOC; } if (!data) { data = &d; data->flags = CSD_FLAG_WAIT; } data->func = func; data->info = info; generic_exec_single(cpu, data); } else { err = -ENXIO; /* CPU not online */ } put_cpu(); return err; } EXPORT_SYMBOL(smp_call_function_single); /** * __smp_call_function_single(): Run a function on another CPU * @cpu: The CPU to run on. * @data: Pre-allocated and setup data structure * * Like smp_call_function_single(), but allow caller to pass in a pre-allocated * data structure. Useful for embedding @data inside other structures, for * instance. * */ void __smp_call_function_single(int cpu, struct call_single_data *data) { /* Can deadlock when called with interrupts disabled */ WARN_ON((data->flags & CSD_FLAG_WAIT) && irqs_disabled()); generic_exec_single(cpu, data); } /* Dummy function */ static void quiesce_dummy(void *unused) { } /* * Ensure stack based data used in call function mask is safe to free. * * This is needed by smp_call_function_mask when using on-stack data, because * a single call function queue is shared by all CPUs, and any CPU may pick up * the data item on the queue at any time before it is deleted. So we need to * ensure that all CPUs have transitioned through a quiescent state after * this call. * * This is a very slow function, implemented by sending synchronous IPIs to * all possible CPUs. For this reason, we have to alloc data rather than use * stack based data even in the case of synchronous calls. The stack based * data is then just used for deadlock/oom fallback which will be very rare. * * If a faster scheme can be made, we could go back to preferring stack based * data -- the data allocation/free is non-zero cost. */ static void smp_call_function_mask_quiesce_stack(cpumask_t mask) { struct call_single_data data; int cpu; data.func = quiesce_dummy; data.info = NULL; for_each_cpu_mask(cpu, mask) { data.flags = CSD_FLAG_WAIT; generic_exec_single(cpu, &data); } } /** * smp_call_function_mask(): Run a function on a set of other CPUs. * @mask: The set of cpus to run on. * @func: The function to run. This must be fast and non-blocking. * @info: An arbitrary pointer to pass to the function. * @wait: If true, wait (atomically) until function has completed on other CPUs. * * Returns 0 on success, else a negative status code. * * If @wait is true, then returns once @func has returned. Note that @wait * will be implicitly turned on in case of allocation failures, since * we fall back to on-stack allocation. * * You must not call this function with disabled interrupts or from a * hardware interrupt handler or from a bottom half handler. Preemption * must be disabled when calling this function. */ int smp_call_function_mask(cpumask_t mask, void (*func)(void *), void *info, int wait) { struct call_function_data d; struct call_function_data *data = NULL; cpumask_t allbutself; unsigned long flags; int cpu, num_cpus; int slowpath = 0; /* Can deadlock when called with interrupts disabled */ WARN_ON(irqs_disabled()); cpu = smp_processor_id(); allbutself = cpu_online_map; cpu_clear(cpu, allbutself); cpus_and(mask, mask, allbutself); num_cpus = cpus_weight(mask); /* * If zero CPUs, return. If just a single CPU, turn this request * into a targetted single call instead since it's faster. */ if (!num_cpus) return 0; else if (num_cpus == 1) { cpu = first_cpu(mask); return smp_call_function_single(cpu, func, info, wait); } data = kmalloc(sizeof(*data), GFP_ATOMIC); if (data) { data->csd.flags = CSD_FLAG_ALLOC; if (wait) data->csd.flags |= CSD_FLAG_WAIT; } else { data = &d; data->csd.flags = CSD_FLAG_WAIT; wait = 1; slowpath = 1; } spin_lock_init(&data->lock); data->csd.func = func; data->csd.info = info; data->refs = num_cpus; data->cpumask = mask; spin_lock_irqsave(&call_function_lock, flags); list_add_tail_rcu(&data->csd.list, &call_function_queue); spin_unlock_irqrestore(&call_function_lock, flags); /* * Make the list addition visible before sending the ipi. */ smp_mb(); /* Send a message to all CPUs in the map */ arch_send_call_function_ipi(mask); /* optionally wait for the CPUs to complete */ if (wait) { csd_flag_wait(&data->csd); if (unlikely(slowpath)) smp_call_function_mask_quiesce_stack(mask); } return 0; } EXPORT_SYMBOL(smp_call_function_mask); /** * smp_call_function(): Run a function on all other CPUs. * @func: The function to run. This must be fast and non-blocking. * @info: An arbitrary pointer to pass to the function. * @wait: If true, wait (atomically) until function has completed on other CPUs. * * Returns 0 on success, else a negative status code. * * If @wait is true, then returns once @func has returned; otherwise * it returns just before the target cpu calls @func. In case of allocation * failure, @wait will be implicitly turned on. * * You must not call this function with disabled interrupts or from a * hardware interrupt handler or from a bottom half handler. */ int smp_call_function(void (*func)(void *), void *info, int wait) { int ret; preempt_disable(); ret = smp_call_function_mask(cpu_online_map, func, info, wait); preempt_enable(); return ret; } EXPORT_SYMBOL(smp_call_function); void ipi_call_lock(void) { spin_lock(&call_function_lock); } void ipi_call_unlock(void) { spin_unlock(&call_function_lock); } void ipi_call_lock_irq(void) { spin_lock_irq(&call_function_lock); } void ipi_call_unlock_irq(void) { spin_unlock_irq(&call_function_lock); }