8 files changed, 132 insertions, 111 deletions

diff --git a/crypto/async_tx/async_tx.c b/crypto/async_tx/async_tx.c
index 8cfac182165d..43fe4cbe71e6 100644
--- a/crypto/async_tx/async_tx.c
+++ b/crypto/async_tx/async_tx.c
@@ -198,8 +198,6 @@ dma_channel_add_remove(struct dma_client *client,
                 /* add the channel to the generic management list */
                 master_ref = kmalloc(sizeof(*master_ref), GFP_KERNEL);
                 if (master_ref) {
-                        /* keep a reference until async_tx is unloaded */
-                        dma_chan_get(chan);
                         init_dma_chan_ref(master_ref, chan);
                         spin_lock_irqsave(&async_tx_lock, flags);
                         list_add_tail_rcu(&master_ref->node,
@@ -221,8 +219,6 @@ dma_channel_add_remove(struct dma_client *client,
                 spin_lock_irqsave(&async_tx_lock, flags);
                 list_for_each_entry(ref, &async_tx_master_list, node)
                         if (ref->chan == chan) {
-                                /* permit backing devices to go away */
-                                dma_chan_put(ref->chan);
                                 list_del_rcu(&ref->node);
                                 call_rcu(&ref->rcu, free_dma_chan_ref);
                                 found = 1;
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index b9008932a8f3..d4d925912c47 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -74,6 +74,7 @@
 static DEFINE_MUTEX(dma_list_mutex);
 static LIST_HEAD(dma_device_list);
 static LIST_HEAD(dma_client_list);
+static long dmaengine_ref_count;
 
 /* --- sysfs implementation --- */
 
@@ -105,19 +106,8 @@ static ssize_t show_bytes_transferred(struct device *dev, struct device_attribut
 static ssize_t show_in_use(struct device *dev, struct device_attribute *attr, char *buf)
 {
         struct dma_chan *chan = to_dma_chan(dev);
-        int in_use = 0;
-
-        if (unlikely(chan->slow_ref) &&
-                atomic_read(&chan->refcount.refcount) > 1)
-                in_use = 1;
-        else {
-                if (local_read(&(per_cpu_ptr(chan->local,
-                        get_cpu())->refcount)) > 0)
-                        in_use = 1;
-                put_cpu();
-        }
 
-        return sprintf(buf, "%d\n", in_use);
+        return sprintf(buf, "%d\n", chan->client_count);
 }
 
 static struct device_attribute dma_attrs[] = {
@@ -155,6 +145,78 @@ __dma_chan_satisfies_mask(struct dma_chan *chan, dma_cap_mask_t *want)
         return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END);
 }
 
+static struct module *dma_chan_to_owner(struct dma_chan *chan)
+{
+        return chan->device->dev->driver->owner;
+}
+
+/**
+ * balance_ref_count - catch up the channel reference count
+ * @chan - channel to balance ->client_count versus dmaengine_ref_count
+ *
+ * balance_ref_count must be called under dma_list_mutex
+ */
+static void balance_ref_count(struct dma_chan *chan)
+{
+        struct module *owner = dma_chan_to_owner(chan);
+
+        while (chan->client_count < dmaengine_ref_count) {
+                __module_get(owner);
+                chan->client_count++;
+        }
+}
+
+/**
+ * dma_chan_get - try to grab a dma channel's parent driver module
+ * @chan - channel to grab
+ *
+ * Must be called under dma_list_mutex
+ */
+static int dma_chan_get(struct dma_chan *chan)
+{
+        int err = -ENODEV;
+        struct module *owner = dma_chan_to_owner(chan);
+
+        if (chan->client_count) {
+                __module_get(owner);
+                err = 0;
+        } else if (try_module_get(owner))
+                err = 0;
+
+        if (err == 0)
+                chan->client_count++;
+
+        /* allocate upon first client reference */
+        if (chan->client_count == 1 && err == 0) {
+                int desc_cnt = chan->device->device_alloc_chan_resources(chan, NULL);
+
+                if (desc_cnt < 0) {
+                        err = desc_cnt;
+                        chan->client_count = 0;
+                        module_put(owner);
+                } else
+                        balance_ref_count(chan);
+        }
+
+        return err;
+}
+
+/**
+ * dma_chan_put - drop a reference to a dma channel's parent driver module
+ * @chan - channel to release
+ *
+ * Must be called under dma_list_mutex
+ */
+static void dma_chan_put(struct dma_chan *chan)
+{
+        if (!chan->client_count)
+                return; /* this channel failed alloc_chan_resources */
+        chan->client_count--;
+        module_put(dma_chan_to_owner(chan));
+        if (chan->client_count == 0)
+                chan->device->device_free_chan_resources(chan);
+}
+
 /**
  * dma_client_chan_alloc - try to allocate channels to a client
  * @client: &dma_client
@@ -165,7 +227,6 @@ static void dma_client_chan_alloc(struct dma_client *client)
 {
         struct dma_device *device;
         struct dma_chan *chan;
-        int desc;       /* allocated descriptor count */
         enum dma_state_client ack;
 
         /* Find a channel */
@@ -178,23 +239,16 @@ static void dma_client_chan_alloc(struct dma_client *client)
                 list_for_each_entry(chan, &device->channels, device_node) {
                         if (!dma_chan_satisfies_mask(chan, client->cap_mask))
                                 continue;
+                        if (!chan->client_count)
+                                continue;
+                        ack = client->event_callback(client, chan,
+                                                     DMA_RESOURCE_AVAILABLE);
 
-                        desc = chan->device->device_alloc_chan_resources(
-                                        chan, client);
-                        if (desc >= 0) {
-                                ack = client->event_callback(client,
-                                                chan,
-                                                DMA_RESOURCE_AVAILABLE);
-
-                                /* we are done once this client rejects
-                                 * an available resource
-                                 */
-                                if (ack == DMA_ACK) {
-                                        dma_chan_get(chan);
-                                        chan->client_count++;
-                                } else if (ack == DMA_NAK)
-                                        return;
-                        }
+                        /* we are done once this client rejects
+                         * an available resource
+                         */
+                        if (ack == DMA_NAK)
+                                return;
                 }
         }
 }
@@ -224,7 +278,6 @@ EXPORT_SYMBOL(dma_sync_wait);
 void dma_chan_cleanup(struct kref *kref)
 {
         struct dma_chan *chan = container_of(kref, struct dma_chan, refcount);
-        chan->device->device_free_chan_resources(chan);
         kref_put(&chan->device->refcount, dma_async_device_cleanup);
 }
 EXPORT_SYMBOL(dma_chan_cleanup);
@@ -232,18 +285,12 @@ EXPORT_SYMBOL(dma_chan_cleanup);
 static void dma_chan_free_rcu(struct rcu_head *rcu)
 {
         struct dma_chan *chan = container_of(rcu, struct dma_chan, rcu);
-        int bias = 0x7FFFFFFF;
-        int i;
-        for_each_possible_cpu(i)
-                bias -= local_read(&per_cpu_ptr(chan->local, i)->refcount);
-        atomic_sub(bias, &chan->refcount.refcount);
+
         kref_put(&chan->refcount, dma_chan_cleanup);
 }
 
 static void dma_chan_release(struct dma_chan *chan)
 {
-        atomic_add(0x7FFFFFFF, &chan->refcount.refcount);
-        chan->slow_ref = 1;
         call_rcu(&chan->rcu, dma_chan_free_rcu);
 }
 
@@ -263,43 +310,36 @@ static void dma_clients_notify_available(void)
 }
 
 /**
- * dma_chans_notify_available - tell the clients that a channel is going away
- * @chan: channel on its way out
- */
-static void dma_clients_notify_removed(struct dma_chan *chan)
-{
-        struct dma_client *client;
-        enum dma_state_client ack;
-
-        mutex_lock(&dma_list_mutex);
-
-        list_for_each_entry(client, &dma_client_list, global_node) {
-                ack = client->event_callback(client, chan,
-                                DMA_RESOURCE_REMOVED);
-
-                /* client was holding resources for this channel so
-                 * free it
-                 */
-                if (ack == DMA_ACK) {
-                        dma_chan_put(chan);
-                        chan->client_count--;
-                }
-        }
-
-        mutex_unlock(&dma_list_mutex);
-}
-
-/**
  * dma_async_client_register - register a &dma_client
  * @client: ptr to a client structure with valid 'event_callback' and 'cap_mask'
  */
 void dma_async_client_register(struct dma_client *client)
 {
+        struct dma_device *device, *_d;
+        struct dma_chan *chan;
+        int err;
+
         /* validate client data */
         BUG_ON(dma_has_cap(DMA_SLAVE, client->cap_mask) &&
                 !client->slave);
 
         mutex_lock(&dma_list_mutex);
+        dmaengine_ref_count++;
+
+        /* try to grab channels */
+        list_for_each_entry_safe(device, _d, &dma_device_list, global_node)
+                list_for_each_entry(chan, &device->channels, device_node) {
+                        err = dma_chan_get(chan);
+                        if (err == -ENODEV) {
+                                /* module removed before we could use it */
+                                list_del_init(&device->global_node);
+                                break;
+                        } else if (err)
+                                pr_err("dmaengine: failed to get %s: (%d)\n",
+                                       dev_name(&chan->dev), err);
+                }
+
+
         list_add_tail(&client->global_node, &dma_client_list);
         mutex_unlock(&dma_list_mutex);
 }
@@ -315,23 +355,17 @@ void dma_async_client_unregister(struct dma_client *client)
 {
         struct dma_device *device;
         struct dma_chan *chan;
-        enum dma_state_client ack;
 
         if (!client)
                 return;
 
         mutex_lock(&dma_list_mutex);
-        /* free all channels the client is holding */
+        dmaengine_ref_count--;
+        BUG_ON(dmaengine_ref_count < 0);
+        /* drop channel references */
         list_for_each_entry(device, &dma_device_list, global_node)
-                list_for_each_entry(chan, &device->channels, device_node) {
-                        ack = client->event_callback(client, chan,
-                                DMA_RESOURCE_REMOVED);
-
-                        if (ack == DMA_ACK) {
-                                dma_chan_put(chan);
-                                chan->client_count--;
-                        }
-                }
+                list_for_each_entry(chan, &device->channels, device_node)
+                        dma_chan_put(chan);
 
         list_del(&client->global_node);
         mutex_unlock(&dma_list_mutex);
@@ -423,6 +457,21 @@ int dma_async_device_register(struct dma_device *device)
         }
 
         mutex_lock(&dma_list_mutex);
+        if (dmaengine_ref_count)
+                list_for_each_entry(chan, &device->channels, device_node) {
+                        /* if clients are already waiting for channels we need
+                         * to take references on their behalf
+                         */
+                        if (dma_chan_get(chan) == -ENODEV) {
+                                /* note we can only get here for the first
+                                 * channel as the remaining channels are
+                                 * guaranteed to get a reference
+                                 */
+                                rc = -ENODEV;
+                                mutex_unlock(&dma_list_mutex);
+                                goto err_out;
+                        }
+                }
         list_add_tail(&device->global_node, &dma_device_list);
         mutex_unlock(&dma_list_mutex);
 
@@ -456,7 +505,7 @@ static void dma_async_device_cleanup(struct kref *kref)
 }
 
 /**
- * dma_async_device_unregister - unregisters DMA devices
+ * dma_async_device_unregister - unregister a DMA device
  * @device: &dma_device
  */
 void dma_async_device_unregister(struct dma_device *device)
@@ -468,7 +517,9 @@ void dma_async_device_unregister(struct dma_device *device)
         mutex_unlock(&dma_list_mutex);
 
         list_for_each_entry(chan, &device->channels, device_node) {
-                dma_clients_notify_removed(chan);
+                WARN_ONCE(chan->client_count,
+                          "%s called while %d clients hold a reference\n",
+                          __func__, chan->client_count);
                 device_unregister(&chan->dev);
                 dma_chan_release(chan);
         }
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index ed9636bfb54a..db4050884713 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -215,7 +215,6 @@ static int dmatest_func(void *data)
 
         smp_rmb();
         chan = thread->chan;
-        dma_chan_get(chan);
 
         while (!kthread_should_stop()) {
                 total_tests++;
@@ -293,7 +292,6 @@ static int dmatest_func(void *data)
         }
 
         ret = 0;
-        dma_chan_put(chan);
         kfree(thread->dstbuf);
 err_dstbuf:
         kfree(thread->srcbuf);
diff --git a/drivers/dma/dw_dmac.c b/drivers/dma/dw_dmac.c
index 0778d99aea7c..377dafa37a20 100644
--- a/drivers/dma/dw_dmac.c
+++ b/drivers/dma/dw_dmac.c
@@ -773,7 +773,7 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan,
         dev_vdbg(&chan->dev, "alloc_chan_resources\n");
 
         /* Channels doing slave DMA can only handle one client. */
-        if (dwc->dws || client->slave) {
+        if (dwc->dws || (client && client->slave)) {
                 if (chan->client_count)
                         return -EBUSY;
         }
diff --git a/drivers/mmc/host/atmel-mci.c b/drivers/mmc/host/atmel-mci.c
index 7a3f2436b011..6c11f4d4c4e9 100644
--- a/drivers/mmc/host/atmel-mci.c
+++ b/drivers/mmc/host/atmel-mci.c
@@ -593,10 +593,8 @@ atmci_submit_data_dma(struct atmel_mci *host, struct mmc_data *data)
 
         /* If we don't have a channel, we can't do DMA */
         chan = host->dma.chan;
-        if (chan) {
-                dma_chan_get(chan);
+        if (chan)
                 host->data_chan = chan;
-        }
 
         if (!chan)
                 return -ENODEV;
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
index e4ec7e7b8056..d18d37d1015d 100644
--- a/include/linux/dmaengine.h
+++ b/include/linux/dmaengine.h
@@ -165,7 +165,6 @@ struct dma_slave {
  */
 
 struct dma_chan_percpu {
-        local_t refcount;
         /* stats */
         unsigned long memcpy_count;
         unsigned long bytes_transferred;
@@ -205,26 +204,6 @@ struct dma_chan {
 
 void dma_chan_cleanup(struct kref *kref);
 
-static inline void dma_chan_get(struct dma_chan *chan)
-{
-        if (unlikely(chan->slow_ref))
-                kref_get(&chan->refcount);
-        else {
-                local_inc(&(per_cpu_ptr(chan->local, get_cpu())->refcount));
-                put_cpu();
-        }
-}
-
-static inline void dma_chan_put(struct dma_chan *chan)
-{
-        if (unlikely(chan->slow_ref))
-                kref_put(&chan->refcount, dma_chan_cleanup);
-        else {
-                local_dec(&(per_cpu_ptr(chan->local, get_cpu())->refcount));
-                put_cpu();
-        }
-}
-
 /*
  * typedef dma_event_callback - function pointer to a DMA event callback
  * For each channel added to the system this routine is called for each client.
diff --git a/include/net/netdma.h b/include/net/netdma.h
index f28c6e064e8f..cbe2737f4a61 100644
--- a/include/net/netdma.h
+++ b/include/net/netdma.h
@@ -27,11 +27,11 @@
 static inline struct dma_chan *get_softnet_dma(void)
 {
         struct dma_chan *chan;
+
         rcu_read_lock();
         chan = rcu_dereference(__get_cpu_var(softnet_data).net_dma);
-        if (chan)
-                dma_chan_get(chan);
         rcu_read_unlock();
+
         return chan;
 }
 
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index f28acf11fc67..75e0e0a2d8db 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -1632,7 +1632,6 @@ skip_copy:
 
                 /* Safe to free early-copied skbs now */
                 __skb_queue_purge(&sk->sk_async_wait_queue);
-                dma_chan_put(tp->ucopy.dma_chan);
                 tp->ucopy.dma_chan = NULL;
         }
         if (tp->ucopy.pinned_list) {