1 files changed, 223 insertions, 13 deletions

diff --git a/lib/dma-debug.c b/lib/dma-debug.c
index d87a17a819d0..98f2d7e91a91 100644
--- a/lib/dma-debug.c
+++ b/lib/dma-debug.c
@@ -53,11 +53,26 @@ enum map_err_types {
 
 #define DMA_DEBUG_STACKTRACE_ENTRIES 5
 
+/**
+ * struct dma_debug_entry - track a dma_map* or dma_alloc_coherent mapping
+ * @list: node on pre-allocated free_entries list
+ * @dev: 'dev' argument to dma_map_{page|single|sg} or dma_alloc_coherent
+ * @type: single, page, sg, coherent
+ * @pfn: page frame of the start address
+ * @offset: offset of mapping relative to pfn
+ * @size: length of the mapping
+ * @direction: enum dma_data_direction
+ * @sg_call_ents: 'nents' from dma_map_sg
+ * @sg_mapped_ents: 'mapped_ents' from dma_map_sg
+ * @map_err_type: track whether dma_mapping_error() was checked
+ * @stacktrace: support backtraces when a violation is detected
+ */
 struct dma_debug_entry {
         struct list_head list;
         struct device    *dev;
         int              type;
-        phys_addr_t      paddr;
+        unsigned long    pfn;
+        size_t           offset;
         u64              dev_addr;
         u64              size;
         int              direction;
@@ -372,6 +387,11 @@ static void hash_bucket_del(struct dma_debug_entry *entry)
         list_del(&entry->list);
 }
 
+static unsigned long long phys_addr(struct dma_debug_entry *entry)
+{
+        return page_to_phys(pfn_to_page(entry->pfn)) + entry->offset;
+}
+
 /*
  * Dump mapping entries for debugging purposes
  */
@@ -389,9 +409,9 @@ void debug_dma_dump_mappings(struct device *dev)
                 list_for_each_entry(entry, &bucket->list, list) {
                         if (!dev || dev == entry->dev) {
                                 dev_info(entry->dev,
-                                         "%s idx %d P=%Lx D=%Lx L=%Lx %s %s\n",
+                                         "%s idx %d P=%Lx N=%lx D=%Lx L=%Lx %s %s\n",
                                          type2name[entry->type], idx,
-                                         (unsigned long long)entry->paddr,
+                                         phys_addr(entry), entry->pfn,
                                          entry->dev_addr, entry->size,
                                          dir2name[entry->direction],
                                          maperr2str[entry->map_err_type]);
@@ -404,6 +424,176 @@ void debug_dma_dump_mappings(struct device *dev)
 EXPORT_SYMBOL(debug_dma_dump_mappings);
 
 /*
+ * For each mapping (initial cacheline in the case of
+ * dma_alloc_coherent/dma_map_page, initial cacheline in each page of a
+ * scatterlist, or the cacheline specified in dma_map_single) insert
+ * into this tree using the cacheline as the key. At
+ * dma_unmap_{single|sg|page} or dma_free_coherent delete the entry.  If
+ * the entry already exists at insertion time add a tag as a reference
+ * count for the overlapping mappings.  For now, the overlap tracking
+ * just ensures that 'unmaps' balance 'maps' before marking the
+ * cacheline idle, but we should also be flagging overlaps as an API
+ * violation.
+ *
+ * Memory usage is mostly constrained by the maximum number of available
+ * dma-debug entries in that we need a free dma_debug_entry before
+ * inserting into the tree.  In the case of dma_map_page and
+ * dma_alloc_coherent there is only one dma_debug_entry and one
+ * dma_active_cacheline entry to track per event.  dma_map_sg(), on the
+ * other hand, consumes a single dma_debug_entry, but inserts 'nents'
+ * entries into the tree.
+ *
+ * At any time debug_dma_assert_idle() can be called to trigger a
+ * warning if any cachelines in the given page are in the active set.
+ */
+static RADIX_TREE(dma_active_cacheline, GFP_NOWAIT);
+static DEFINE_SPINLOCK(radix_lock);
+#define ACTIVE_CACHELINE_MAX_OVERLAP ((1 << RADIX_TREE_MAX_TAGS) - 1)
+#define CACHELINE_PER_PAGE_SHIFT (PAGE_SHIFT - L1_CACHE_SHIFT)
+#define CACHELINES_PER_PAGE (1 << CACHELINE_PER_PAGE_SHIFT)
+
+static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry)
+{
+        return (entry->pfn << CACHELINE_PER_PAGE_SHIFT) +
+                (entry->offset >> L1_CACHE_SHIFT);
+}
+
+static int active_cacheline_read_overlap(phys_addr_t cln)
+{
+        int overlap = 0, i;
+
+        for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
+                if (radix_tree_tag_get(&dma_active_cacheline, cln, i))
+                        overlap |= 1 << i;
+        return overlap;
+}
+
+static int active_cacheline_set_overlap(phys_addr_t cln, int overlap)
+{
+        int i;
+
+        if (overlap > ACTIVE_CACHELINE_MAX_OVERLAP || overlap < 0)
+                return overlap;
+
+        for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
+                if (overlap & 1 << i)
+                        radix_tree_tag_set(&dma_active_cacheline, cln, i);
+                else
+                        radix_tree_tag_clear(&dma_active_cacheline, cln, i);
+
+        return overlap;
+}
+
+static void active_cacheline_inc_overlap(phys_addr_t cln)
+{
+        int overlap = active_cacheline_read_overlap(cln);
+
+        overlap = active_cacheline_set_overlap(cln, ++overlap);
+
+        /* If we overflowed the overlap counter then we're potentially
+         * leaking dma-mappings.  Otherwise, if maps and unmaps are
+         * balanced then this overflow may cause false negatives in
+         * debug_dma_assert_idle() as the cacheline may be marked idle
+         * prematurely.
+         */
+        WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP,
+                  "DMA-API: exceeded %d overlapping mappings of cacheline %pa\n",
+                  ACTIVE_CACHELINE_MAX_OVERLAP, &cln);
+}
+
+static int active_cacheline_dec_overlap(phys_addr_t cln)
+{
+        int overlap = active_cacheline_read_overlap(cln);
+
+        return active_cacheline_set_overlap(cln, --overlap);
+}
+
+static int active_cacheline_insert(struct dma_debug_entry *entry)
+{
+        phys_addr_t cln = to_cacheline_number(entry);
+        unsigned long flags;
+        int rc;
+
+        /* If the device is not writing memory then we don't have any
+         * concerns about the cpu consuming stale data.  This mitigates
+         * legitimate usages of overlapping mappings.
+         */
+        if (entry->direction == DMA_TO_DEVICE)
+                return 0;
+
+        spin_lock_irqsave(&radix_lock, flags);
+        rc = radix_tree_insert(&dma_active_cacheline, cln, entry);
+        if (rc == -EEXIST)
+                active_cacheline_inc_overlap(cln);
+        spin_unlock_irqrestore(&radix_lock, flags);
+
+        return rc;
+}
+
+static void active_cacheline_remove(struct dma_debug_entry *entry)
+{
+        phys_addr_t cln = to_cacheline_number(entry);
+        unsigned long flags;
+
+        /* ...mirror the insert case */
+        if (entry->direction == DMA_TO_DEVICE)
+                return;
+
+        spin_lock_irqsave(&radix_lock, flags);
+        /* since we are counting overlaps the final put of the
+         * cacheline will occur when the overlap count is 0.
+         * active_cacheline_dec_overlap() returns -1 in that case
+         */
+        if (active_cacheline_dec_overlap(cln) < 0)
+                radix_tree_delete(&dma_active_cacheline, cln);
+        spin_unlock_irqrestore(&radix_lock, flags);
+}
+
+/**
+ * debug_dma_assert_idle() - assert that a page is not undergoing dma
+ * @page: page to lookup in the dma_active_cacheline tree
+ *
+ * Place a call to this routine in cases where the cpu touching the page
+ * before the dma completes (page is dma_unmapped) will lead to data
+ * corruption.
+ */
+void debug_dma_assert_idle(struct page *page)
+{
+        static struct dma_debug_entry *ents[CACHELINES_PER_PAGE];
+        struct dma_debug_entry *entry = NULL;
+        void **results = (void **) &ents;
+        unsigned int nents, i;
+        unsigned long flags;
+        phys_addr_t cln;
+
+        if (!page)
+                return;
+
+        cln = (phys_addr_t) page_to_pfn(page) << CACHELINE_PER_PAGE_SHIFT;
+        spin_lock_irqsave(&radix_lock, flags);
+        nents = radix_tree_gang_lookup(&dma_active_cacheline, results, cln,
+                                       CACHELINES_PER_PAGE);
+        for (i = 0; i < nents; i++) {
+                phys_addr_t ent_cln = to_cacheline_number(ents[i]);
+
+                if (ent_cln == cln) {
+                        entry = ents[i];
+                        break;
+                } else if (ent_cln >= cln + CACHELINES_PER_PAGE)
+                        break;
+        }
+        spin_unlock_irqrestore(&radix_lock, flags);
+
+        if (!entry)
+                return;
+
+        cln = to_cacheline_number(entry);
+        err_printk(entry->dev, entry,
+                   "DMA-API: cpu touching an active dma mapped cacheline [cln=%pa]\n",
+                   &cln);
+}
+
+/*
  * Wrapper function for adding an entry to the hash.
  * This function takes care of locking itself.
  */
@@ -411,10 +601,21 @@ static void add_dma_entry(struct dma_debug_entry *entry)
 {
         struct hash_bucket *bucket;
         unsigned long flags;
+        int rc;
 
         bucket = get_hash_bucket(entry, &flags);
         hash_bucket_add(bucket, entry);
         put_hash_bucket(bucket, &flags);
+
+        rc = active_cacheline_insert(entry);
+        if (rc == -ENOMEM) {
+                pr_err("DMA-API: cacheline tracking ENOMEM, dma-debug disabled\n");
+                global_disable = true;
+        }
+
+        /* TODO: report -EEXIST errors here as overlapping mappings are
+         * not supported by the DMA API
+         */
 }
 
 static struct dma_debug_entry *__dma_entry_alloc(void)
@@ -469,6 +670,8 @@ static void dma_entry_free(struct dma_debug_entry *entry)
 {
         unsigned long flags;
 
+        active_cacheline_remove(entry);
+
         /*
          * add to beginning of the list - this way the entries are
          * more likely cache hot when they are reallocated.
@@ -895,15 +1098,15 @@ static void check_unmap(struct dma_debug_entry *ref)
                            ref->dev_addr, ref->size,
                            type2name[entry->type], type2name[ref->type]);
         } else if ((entry->type == dma_debug_coherent) &&
-                   (ref->paddr != entry->paddr)) {
+                   (phys_addr(ref) != phys_addr(entry))) {
                 err_printk(ref->dev, entry, "DMA-API: device driver frees "
                            "DMA memory with different CPU address "
                            "[device address=0x%016llx] [size=%llu bytes] "
                            "[cpu alloc address=0x%016llx] "
                            "[cpu free address=0x%016llx]",
                            ref->dev_addr, ref->size,
-                           (unsigned long long)entry->paddr,
-                           (unsigned long long)ref->paddr);
+                           phys_addr(entry),
+                           phys_addr(ref));
         }
 
         if (ref->sg_call_ents && ref->type == dma_debug_sg &&
@@ -1052,7 +1255,8 @@ void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
 
         entry->dev       = dev;
         entry->type      = dma_debug_page;
-        entry->paddr     = page_to_phys(page) + offset;
+        entry->pfn       = page_to_pfn(page);
+        entry->offset    = offset,
         entry->dev_addr  = dma_addr;
         entry->size      = size;
         entry->direction = direction;
@@ -1148,7 +1352,8 @@ void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
 
                 entry->type           = dma_debug_sg;
                 entry->dev            = dev;
-                entry->paddr          = sg_phys(s);
+                entry->pfn            = page_to_pfn(sg_page(s));
+                entry->offset         = s->offset,
                 entry->size           = sg_dma_len(s);
                 entry->dev_addr       = sg_dma_address(s);
                 entry->direction      = direction;
@@ -1198,7 +1403,8 @@ void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
                 struct dma_debug_entry ref = {
                         .type           = dma_debug_sg,
                         .dev            = dev,
-                        .paddr          = sg_phys(s),
+                        .pfn            = page_to_pfn(sg_page(s)),
+                        .offset         = s->offset,
                         .dev_addr       = sg_dma_address(s),
                         .size           = sg_dma_len(s),
                         .direction      = dir,
@@ -1233,7 +1439,8 @@ void debug_dma_alloc_coherent(struct device *dev, size_t size,
 
         entry->type      = dma_debug_coherent;
         entry->dev       = dev;
-        entry->paddr     = virt_to_phys(virt);
+        entry->pfn       = page_to_pfn(virt_to_page(virt));
+        entry->offset    = (size_t) virt & PAGE_MASK;
         entry->size      = size;
         entry->dev_addr  = dma_addr;
         entry->direction = DMA_BIDIRECTIONAL;
@@ -1248,7 +1455,8 @@ void debug_dma_free_coherent(struct device *dev, size_t size,
         struct dma_debug_entry ref = {
                 .type           = dma_debug_coherent,
                 .dev            = dev,
-                .paddr          = virt_to_phys(virt),
+                .pfn            = page_to_pfn(virt_to_page(virt)),
+                .offset         = (size_t) virt & PAGE_MASK,
                 .dev_addr       = addr,
                 .size           = size,
                 .direction      = DMA_BIDIRECTIONAL,
@@ -1356,7 +1564,8 @@ void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
                 struct dma_debug_entry ref = {
                         .type           = dma_debug_sg,
                         .dev            = dev,
-                        .paddr          = sg_phys(s),
+                        .pfn            = page_to_pfn(sg_page(s)),
+                        .offset         = s->offset,
                         .dev_addr       = sg_dma_address(s),
                         .size           = sg_dma_len(s),
                         .direction      = direction,
@@ -1388,7 +1597,8 @@ void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
                 struct dma_debug_entry ref = {
                         .type           = dma_debug_sg,
                         .dev            = dev,
-                        .paddr          = sg_phys(s),
+                        .pfn            = page_to_pfn(sg_page(s)),
+                        .offset         = s->offset,
                         .dev_addr       = sg_dma_address(s),
                         .size           = sg_dma_len(s),
                         .direction      = direction,