path: root/lib/dma-debug.c

/*
 * Copyright (C) 2008 Advanced Micro Devices, Inc.
 *
 * Author: Joerg Roedel <joerg.roedel@amd.com>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 */

#include <linux/scatterlist.h>
#include <linux/dma-mapping.h>
#include <linux/stacktrace.h>
#include <linux/dma-debug.h>
#include <linux/spinlock.h>
#include <linux/debugfs.h>
#include <linux/device.h>
#include <linux/types.h>
#include <linux/sched.h>
#include <linux/list.h>
#include <linux/slab.h>

#include <asm/sections.h>

#define HASH_SIZE       1024ULL
#define HASH_FN_SHIFT   13
#define HASH_FN_MASK    (HASH_SIZE - 1)

enum {
	dma_debug_single,
	dma_debug_page,
	dma_debug_sg,
	dma_debug_coherent,
};

#define DMA_DEBUG_STACKTRACE_ENTRIES 5

struct dma_debug_entry {
	struct list_head list;
	struct device    *dev;
	int              type;
	phys_addr_t      paddr;
	u64              dev_addr;
	u64              size;
	int              direction;
	int		 sg_call_ents;
	int		 sg_mapped_ents;
#ifdef CONFIG_STACKTRACE
	struct		 stack_trace stacktrace;
	unsigned long	 st_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
#endif
};

struct hash_bucket {
	struct list_head list;
	spinlock_t lock;
} ____cacheline_aligned_in_smp;

/* Hash list to save the allocated dma addresses */
static struct hash_bucket dma_entry_hash[HASH_SIZE];
/* List of pre-allocated dma_debug_entry's */
static LIST_HEAD(free_entries);
/* Lock for the list above */
static DEFINE_SPINLOCK(free_entries_lock);

/* Global disable flag - will be set in case of an error */
static bool global_disable __read_mostly;

/* Global error count */
static u32 error_count;

/* Global error show enable*/
static u32 show_all_errors __read_mostly;
/* Number of errors to show */
static u32 show_num_errors = 1;

static u32 num_free_entries;
static u32 min_free_entries;

/* number of preallocated entries requested by kernel cmdline */
static u32 req_entries;

/* debugfs dentry's for the stuff above */
static struct dentry *dma_debug_dent        __read_mostly;
static struct dentry *global_disable_dent   __read_mostly;
static struct dentry *error_count_dent      __read_mostly;
static struct dentry *show_all_errors_dent  __read_mostly;
static struct dentry *show_num_errors_dent  __read_mostly;
static struct dentry *num_free_entries_dent __read_mostly;
static struct dentry *min_free_entries_dent __read_mostly;

static const char *type2name[4] = { "single", "page",
				    "scather-gather", "coherent" };

static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
				   "DMA_FROM_DEVICE", "DMA_NONE" };

/*
 * The access to some variables in this macro is racy. We can't use atomic_t
 * here because all these variables are exported to debugfs. Some of them even
 * writeable. This is also the reason why a lock won't help much. But anyway,
 * the races are no big deal. Here is why:
 *
 *   error_count: the addition is racy, but the worst thing that can happen is
 *                that we don't count some errors
 *   show_num_errors: the subtraction is racy. Also no big deal because in
 *                    worst case this will result in one warning more in the
 *                    system log than the user configured. This variable is
 *                    writeable via debugfs.
 */
static inline void dump_entry_trace(struct dma_debug_entry *entry)
{
#ifdef CONFIG_STACKTRACE
	if (entry) {
		printk(KERN_WARNING "Mapped at:\n");
		print_stack_trace(&entry->stacktrace, 0);
	}
#endif
}

#define err_printk(dev, entry, format, arg...) do {		\
		error_count += 1;				\
		if (show_all_errors || show_num_errors > 0) {	\
			WARN(1, "%s %s: " format,		\
			     dev_driver_string(dev),		\
			     dev_name(dev) , ## arg);		\
			dump_entry_trace(entry);		\
		}						\
		if (!show_all_errors && show_num_errors > 0)	\
			show_num_errors -= 1;			\
	} while (0);

/*
 * Hash related functions
 *
 * Every DMA-API request is saved into a struct dma_debug_entry. To
 * have quick access to these structs they are stored into a hash.
 */
static int hash_fn(struct dma_debug_entry *entry)
{
	/*
	 * Hash function is based on the dma address.
	 * We use bits 20-27 here as the index into the hash
	 */
	return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;
}

/*
 * Request exclusive access to a hash bucket for a given dma_debug_entry.
 */
static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,
					   unsigned long *flags)
{
	int idx = hash_fn(entry);
	unsigned long __flags;

	spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);
	*flags = __flags;
	return &dma_entry_hash[idx];
}

/*
 * Give up exclusive access to the hash bucket
 */
static void put_hash_bucket(struct hash_bucket *bucket,
			    unsigned long *flags)
{
	unsigned long __flags = *flags;

	spin_unlock_irqrestore(&bucket->lock, __flags);
}

/*
 * Search a given entry in the hash bucket list
 */
static struct dma_debug_entry *hash_bucket_find(struct hash_bucket *bucket,
						struct dma_debug_entry *ref)
{
	struct dma_debug_entry *entry;

	list_for_each_entry(entry, &bucket->list, list) {
		if ((entry->dev_addr == ref->dev_addr) &&
		    (entry->dev == ref->dev))
			return entry;
	}

	return NULL;
}

/*
 * Add an entry to a hash bucket
 */
static void hash_bucket_add(struct hash_bucket *bucket,
			    struct dma_debug_entry *entry)
{
	list_add_tail(&entry->list, &bucket->list);
}

/*
 * Remove entry from a hash bucket list
 */
static void hash_bucket_del(struct dma_debug_entry *entry)
{
	list_del(&entry->list);
}

/*
 * Dump mapping entries for debugging purposes
 */
void debug_dma_dump_mappings(struct device *dev)
{
	int idx;

	for (idx = 0; idx < HASH_SIZE; idx++) {
		struct hash_bucket *bucket = &dma_entry_hash[idx];
		struct dma_debug_entry *entry;
		unsigned long flags;

		spin_lock_irqsave(&bucket->lock, flags);

		list_for_each_entry(entry, &bucket->list, list) {
			if (!dev || dev == entry->dev) {
				dev_info(entry->dev,
					 "%s idx %d P=%Lx D=%Lx L=%Lx %s\n",
					 type2name[entry->type], idx,
					 (unsigned long long)entry->paddr,
					 entry->dev_addr, entry->size,
					 dir2name[entry->direction]);
			}
		}

		spin_unlock_irqrestore(&bucket->lock, flags);
	}
}
EXPORT_SYMBOL(debug_dma_dump_mappings);

/*
 * Wrapper function for adding an entry to the hash.
 * This function takes care of locking itself.
 */
static void add_dma_entry(struct dma_debug_entry *entry)
{
	struct hash_bucket *bucket;
	unsigned long flags;

	bucket = get_hash_bucket(entry, &flags);
	hash_bucket_add(bucket, entry);
	put_hash_bucket(bucket, &flags);
}

/* struct dma_entry allocator
 *
 * The next two functions implement the allocator for
 * struct dma_debug_entries.
 */
static struct dma_debug_entry *dma_entry_alloc(void)
{
	struct dma_debug_entry *entry = NULL;
	unsigned long flags;

	spin_lock_irqsave(&free_entries_lock, flags);

	if (list_empty(&free_entries)) {
		printk(KERN_ERR "DMA-API: debugging out of memory "
				"- disabling\n");
		global_disable = true;
		goto out;
	}

	entry = list_entry(free_entries.next, struct dma_debug_entry, list);
	list_del(&entry->list);
	memset(entry, 0, sizeof(*entry));

#ifdef CONFIG_STACKTRACE
	entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES;
	entry->stacktrace.entries = entry->st_entries;
	entry->stacktrace.skip = 2;
	save_stack_trace(&entry->stacktrace);
#endif
	num_free_entries -= 1;
	if (num_free_entries < min_free_entries)
		min_free_entries = num_free_entries;

out:
	spin_unlock_irqrestore(&free_entries_lock, flags);

	return entry;
}

static void dma_entry_free(struct dma_debug_entry *entry)
{
	unsigned long fl
	spin_lock_irqsave(&bc->lock, flags);
	if (CIRC_CNT(bc->head, bc->tail, BUF_SIZE))
		/*
		 * If we haven't emptied the buffer, then enable the TX IRQ.
		 * We'll get an interrupt when there's more room in the
		 * hypervisor's output buffer.
		 */
		enable_tx_interrupt(bc);
	else
		disable_tx_interrupt(bc);
	spin_unlock_irqrestore(&bc->lock, flags);
}

/*
 * byte channel transmit interupt handler
 *
 * This ISR is called whenever space becomes available for transmitting
 * characters on a byte channel.
 */
static irqreturn_t ehv_bc_tty_tx_isr(int irq, void *data)
{
	struct ehv_bc_data *bc = data;

	ehv_bc_tx_dequeue(bc);
	tty_port_tty_wakeup(&bc->port);

	return IRQ_HANDLED;
}

/*
 * This function is called when the tty layer has data for us send.  We store
 * the data first in a circular buffer, and then dequeue as much of that data
 * as possible.
 *
 * We don't need to worry about whether there is enough room in the buffer for
 * all the data.  The purpose of ehv_bc_tty_write_room() is to tell the tty
 * layer how much data it can safely send to us.  We guarantee that
 * ehv_bc_tty_write_room() will never lie, so the tty layer will never send us
 * too much data.
 */
static int ehv_bc_tty_write(struct tty_struct *ttys, const unsigned char *s,
			    int count)
{
	
	}

	return -ENOMEM;
}

static int dma_debug_fs_init(void)
{
	dma_debug_dent = debugfs_create_dir("dma-api", NULL);
	if (!dma_debug_dent) {
		printk(KERN_ERR "DMA-API: can not create debugfs directory\n");
		return -ENOMEM;
	}

	global_disable_dent = debugfs_create_bool("disabled", 0444,
			dma_debug_dent,
			(u32 *)&global_disable);
	if (!global_disable_dent)
		
			bc->head = (bc->head + len) & (BUF_SIZE - 1);
		}
		spin_unlock_irqrestore(&bc->lock, flags);
		if (!len)
			break;

		s += len;
		count -= len;
		written += len;
	}

	ehv_bc_tx_dequeue(bc);

	return written;
}

/*
 * This function can be called multiple times for a given tty_struct, which is
 * why we initialize bc->ttys in ehv_bc_tty_port_activate() instead.
 *
 * The tty layer will still call this function even if the device was not
 * registered (i.e. tty_register_device() was not called).  This happens
 * because tty_register_device() is optional and some legacy drivers don't
 * use it.  So we need to check for that.
 */
static int ehv_bc_tty_open(struct tty_struct *ttys, struct file *filp)
{
	struct ehv_bc_data *bc = &bcs[ttys->index];

	if (!bc->dev)
		return -ENODEV;

	return tty_port_open(&bc->port, ttys, filp);
}

/*
 * Amazingly, if ehv_bc_tty_open() returns an error code, the tty layer will
 * still call this function to close the tty device.  So we can't assume that
 * the tty port has been initialized.
 */
static void ehv_bc_tty_close(struct tty_struct *ttys, struct file *filp)
{
	struct ehv_bc_data *bc = &bcs[ttys->index];

	if (bc->dev)
		tty_port_close(&bc->port, ttys, filp);
}

/*
 * Return the amount of space in the output buffer
 *
 * This is actually a contract between the driver and the tty layer outlining
 * how much write room the driver can guarantee will be sent OR BUFFERED.  This
 * driver MUST honor the return value.
 */
static int ehv_bc_tty_write_room(struct tty_struct *ttys)
{
	struct ehv_bc_data *bc = ttys->driver_data;
	unsigned long flags;
	int count;

	spin_lock_irqsave(&bc->lock, flags);
	count = CIRC_SPACE(bc->head, bc->tail, BUF_SIZE);
	spin_unlock_irqrestore(&bc->lock, flags);

	return count;
}

/*
 * Stop sending data to the tty layer
 *
 * This function is called when the tty layer's input buffers are getting full,
 * so the driver should stop sending it data.  The easiest way to do this is to
 * disable the RX IRQ, which will prevent ehv_bc_tty_rx_isr() from being
 * called.
 *
 * The hypervisor will continue to queue up any incoming data.  If there is any
 * data in the queue when the RX interrupt is enabled, we'll immediately get an
 * RX interrupt.
 */
static void ehv_bc_tty_throttle(struct tty_struct *ttys)
{
	struct ehv_bc_data *bc = ttys->driver_data;

	disable_irq(bc->rx_irq);
}

/*
 * Resume sending data to the tty layer
 *
 * This function is called after previously calling ehv_bc_tty_throttle().  The
 * tty layer's input buffers now have more room, so the driver can resume
 * sending it data.
 */
static void ehv_bc_tty_unthrottle(struct tty_struct *ttys)
{
	struct ehv_bc_data *bc = ttys->driver_data;

	/* If there is any data in the queue when the RX interrupt is enabled,
	 * we'll immediately get an RX interrupt.
	 */
	enable_irq(bc->rx_irq);
}

static void ehv_bc_tty_hangup(struct tty_struct *ttys)
{
	struct ehv_bc_data *bc = ttys->driver_data;

	ehv_bc_tx_dequeue(bc);
	tty_port_hangup(&bc->port);
}

/*
 * TTY driver operations
 *
 * If we could ask the hypervisor how much data is still in the TX buffer, or
 * at least how big the TX buffers are, then we could implement the
 * .wait_until_sent and .chars_in_buffer functions.
 */
static const struct tty_operations ehv_bc_ops = {
	.open		= ehv_bc_tty_open,
	.close		= ehv_bc_tty_close,
	.write		= ehv_bc_tty_write,
	.write_room	= ehv_bc_tty_write_room,
	.throttle	= ehv_bc_tty_throttle,
	.unthrottle	= ehv_bc_tty_unthrottle,
	.hangup		= ehv_bc_tty_hangup,
};

/*
 * initialize the TTY port
 *
 * This function will only be called once, no matter how many times
 * ehv_bc_tty_open() is called.  That's why we register the ISR here, and also
 * why we initialize tty_struct-related variables here.
 */
static int ehv_bc_tty_port_activate(struct tty_port *port,
				    struct tty_struct *ttys)
{
	struct ehv_bc_data *bc = container_of(port, struct ehv_bc_data, port);
	int ret;

	ttys->driver_data = bc;

	ret = request_irq(bc->rx_irq, ehv_bc_tty_rx_isr, 0, "ehv-bc", bc);
	if (ret < 0) {
		dev_err(bc->dev, "could not request rx irq %u (ret=%i)\n",
		       bc->rx_irq, ret);
		return ret;
	}

	/* request_irq also enables the IRQ */
	bc->tx_irq_enabled = 1;

	ret = request_irq(bc->tx_irq, ehv_bc_tty_tx_isr, 0, "ehv-bc", bc);
	if (ret < 0) {
		dev_err(bc->dev, "could not request tx irq %u (ret=%i)\n",
		       bc->tx_irq, ret);
		free_irq(bc->rx_irq, bc);
		return ret;
	}

	/* The TX IRQ is enabled only when we can't write all the data to the
	 * byte channel at once, so by default it's disabled.
	 */
	disable_tx_interrupt(bc);

	return 0;
}

static void ehv_bc_tty_port_shutdown(struct tty_port *port)
{
	struct ehv_bc_data *bc = container_of(port, struct ehv_bc_data, port);

	free_irq(bc->tx_irq, bc);
	free_irq(bc->rx_irq, bc);
}

static const struct tty_port_operations ehv_bc_tty_port_ops = {
	.activate = ehv_bc_tty_port_activate,
	.shutdown = ehv_bc_tty_port_shutdown,
};

static int ehv_bc_tty_probe(struct platform_device *pdev)
{
	struct device_node *np = pdev->dev.of_node;
	struct ehv_bc_data *bc;
	const uint32_t *iprop;
	unsigned int handle;
	int ret;
	static unsigned int index = 1;
	unsigned int i;

	iprop = of_get_property(np, "hv-handle", NULL);
	if (!iprop) {
		dev_err(&pdev->dev, "no 'hv-handle' property in %s node\n",
			np->name);
		return -ENODEV;
	}

	/* We already told the console layer that the index for the console
	 * device is zero, so we need to make sure that we use that index when
	 * we probe the console byte channel node.
	 */
	handle = be32_to_cpu(*iprop);
	i = (handle == stdout_bc) ? 0 : index++;
	bc = &bcs[i];

	bc->handle = handle;
	bc->head = 0;
	bc->tail = 0;
	spin_lock_init(&bc->lock);

	bc->rx_irq = irq_of_parse_and_map(np, 0);
	bc->tx_irq = irq_of_parse_and_map(np, 1);
	if ((bc->rx_irq == NO_IRQ) || (bc->tx_irq == NO_IRQ)) {
		dev_err(&pdev->dev, "no 'interrupts' property in %s node\n",
			np->name);
		ret = -ENODEV;
		goto error;
	}

	tty_port_init(&bc->port);
	bc->port.ops = &ehv_bc_tty_port_ops;

	bc->dev = tty_port_register_device(&bc->port, ehv_bc_driver, i,
			&pdev->dev);
	if (IS_ERR(bc->dev)) {
		ret = PTR_ERR(bc->dev);
		dev_err(&pdev->dev, "could not register tty (ret=%i)\n", ret);
		goto error;
	}

	dev_set_drvdata(&pdev->dev, bc);

	dev_info(&pdev->dev, "registered /dev/%s%u for byte channel %u\n",
		ehv_bc_driver->name, i, bc->handle);

	return 0;

error:
	tty_port_destroy(&bc->port);
	irq_dispose_mapping(bc->tx_irq);
	irq_dispose_mapping(bc->rx_irq);

	memset(bc, 0, sizeof(struct ehv_bc_data));
	return ret;
}

static int ehv_bc_tty_remove(struct platform_device *pdev)
{
	struct ehv_bc_data *bc = dev_get_drvdata(&pdev->dev);

	tty_unregister_device(ehv_bc_driver, bc - bcs);

	tty_port_destroy(&bc->port);
	irq_dispose_mapping(bc->tx_irq);
	irq_dispose_mapping(bc->rx_irq);

	return 0;
}

static const struct of_device_id ehv_bc_tty_of_ids[] = {
	{ .compatible = "epapr,hv-byte-channel" },
	{}
};

static struct platform_driver ehv_bc_tty_driver = {
	.driver = {
		.name = "ehv-bc",
		.of_match_table = ehv_bc_tty_of_ids,
	},
	.probe		= ehv_bc_tty_probe,
	.remove		= ehv_bc_tty_remove,
};

/**
 * ehv_bc_init - ePAPR hypervisor byte channel driver initialization
 *
 * This function is called when this module is loaded.
 */
static int __init ehv_bc_init(void)
{
	struct device_node *np;
	unsigned int count = 0; /* Number of elements in bcs[] */
	int ret;

	pr_info("ePAPR hypervisor byte channel driver\n");

	/* Count the number of byte channels */
	for_each_compatible_node(np, NULL, "epapr,hv-byte-channel")
		count++;

	if (!count)
		return -ENODEV;

	/* The array index of an element in bcs[] is the same as the tty index
	 * for that element.  If you know the address of an element in the
	 * array, then you can use pointer math (e.g. "bc - bcs") to get its
	 * tty index.
	 */
	bcs = kzalloc(count * sizeof(struct ehv_bc_data), GFP_KERNEL);
	if (!bcs)
		return -ENOMEM;

	ehv_bc_driver = alloc_tty_driver(count);
	if (!ehv_bc_driver) {
		ret = -ENOMEM;
		goto error;
	}

	ehv_bc_driver->driver_name = "ehv-bc";
	ehv_bc_driver->name = ehv_bc_console.name;
	ehv_bc_driver->type = TTY_DRIVER_TYPE_CONSOLE;
	ehv_bc_driver->subtype = SYSTEM_TYPE_CONSOLE;
	ehv_bc_driver->init_termios = tty_std_termios;
	ehv_bc_driver->flags = TTY_DRIVER_REAL_RAW | TTY_DRIVER_DYNAMIC_DEV;
	tty_set_operations(ehv_bc_driver, &ehv_bc_ops);

	ret = tty_register_driver(ehv_bc_driver);
	if (ret) {
		pr_err("ehv-bc: could not register tty driver (ret=%i)\n", ret);
		goto error;
	}

	ret = platform_driver_register(&ehv_bc_tty_driver);
	if (ret) {
		pr_err("ehv-bc: could not register platform driver (ret=%i)\n",
		       ret);
		goto error;
	}

	return 0;

error:
	if (ehv_bc_driver) {
		tty_unregister_driver(ehv_bc_driver);
		put_tty_driver(ehv_bc_driver);
	}

	kfree(bcs);

	return ret;
}


/**
 * ehv_bc_exit - ePAPR hypervisor byte channel driver termination
 *
 * This function is called when this driver is unloaded.
 */
static void __exit ehv_bc_exit(void)
{
	platform_driver_unregister(&ehv_bc_tty_driver);
	tty_unregister_driver(ehv_bc_driver);
	put_tty_driver(ehv_bc_driver);
	kfree(bcs);
}

module_init(ehv_bc_init);
module_exit(ehv_bc_exit);

MODULE_AUTHOR("Timur Tabi <timur@freescale.com>");
MODULE_DESCRIPTION("ePAPR hypervisor byte channel driver");
MODULE_LICENSE("GPL v2");
pt">]);

	if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
		       !(direction == DMA_FROM_DEVICE))
		err_printk(dev, entry, "DMA-API: device driver syncs "
				"device write-only DMA memory to device "
				"[device address=0x%016llx] [size=%llu bytes] "
				"[mapped with %s] [synced with %s]\n",
				(unsigned long long)addr, entry->size,
				dir2name[entry->direction],
				dir2name[direction]);

out:
	put_hash_bucket(bucket, &flags);

}

void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
			size_t size, int direction, dma_addr_t dma_addr,
			bool map_single)
{
	struct dma_debug_entry *entry;

	if (unlikely(global_disable))
		return;

	if (unlikely(dma_mapping_error(dev, dma_addr)))
		return;

	entry = dma_entry_alloc();
	if (!entry)
		return;

	entry->dev       = dev;
	entry->type      = dma_debug_page;
	entry->paddr     = page_to_phys(page) + offset;
	entry->dev_addr  = dma_addr;
	entry->size      = size;
	entry->direction = direction;

	if (map_single)
		entry->type = dma_debug_single;

	if (!PageHighMem(page)) {
		void *addr = ((char *)page_address(page)) + offset;
		check_for_stack(dev, addr);
		check_for_illegal_area(dev, addr, size);
	}

	add_dma_entry(entry);
}
EXPORT_SYMBOL(debug_dma_map_page);

void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
			  size_t size, int direction, bool map_single)
{
	struct dma_debug_entry ref = {
		.type           = dma_debug_page,
		.dev            = dev,
		.dev_addr       = addr,
		.size           = size,
		.direction      = direction,
	};

	if (unlikely(global_disable))
		return;

	if (map_single)
		ref.type = dma_debug_single;

	check_unmap(&ref);
}
EXPORT_SYMBOL(debug_dma_unmap_page);

void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
		      int nents, int mapped_ents, int direction)
{
	struct dma_debug_entry *entry;
	struct scatterlist *s;
	int i;

	if (unlikely(global_disable))
		return;

	for_each_sg(sg, s, mapped_ents, i) {
		entry = dma_entry_alloc();
		if (!entry)
			return;

		entry->type           = dma_debug_sg;
		entry->dev            = dev;
		entry->paddr          = sg_phys(s);
		entry->size           = s->length;
		entry->dev_addr       = s->dma_address;
		entry->direction      = direction;
		entry->sg_call_ents   = nents;
		entry->sg_mapped_ents = mapped_ents;

		if (!PageHighMem(sg_page(s))) {
			check_for_stack(dev, sg_virt(s));
			check_for_illegal_area(dev, sg_virt(s), s->length);
		}

		add_dma_entry(entry);
	}
}
EXPORT_SYMBOL(debug_dma_map_sg);

void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
			int nelems, int dir)
{
	struct dma_debug_entry *entry;
	struct scatterlist *s;
	int mapped_ents = 0, i;
	unsigned long flags;

	if (unlikely(global_disable))
		return;

	for_each_sg(sglist, s, nelems, i) {

		struct dma_debug_entry ref = {
			.type           = dma_debug_sg,
			.dev            = dev,
			.paddr          = sg_phys(s),
			.dev_addr       = s->dma_address,
			.size           = s->length,
			.direction      = dir,
			.sg_call_ents   = 0,
		};

		if (mapped_ents && i >= mapped_ents)
			break;

		if (mapped_ents == 0) {
			struct hash_bucket *bucket;
			ref.sg_call_ents = nelems;
			bucket = get_hash_bucket(&ref, &flags);
			entry = hash_bucket_find(bucket, &ref);
			if (entry)
				mapped_ents = entry->sg_mapped_ents;
			put_hash_bucket(bucket, &flags);
		}

		check_unmap(&ref);
	}
}
EXPORT_SYMBOL(debug_dma_unmap_sg);

void debug_dma_alloc_coherent(struct device *dev, size_t size,
			      dma_addr_t dma_addr, void *virt)
{
	struct dma_debug_entry *entry;

	if (unlikely(global_disable))
		return;

	if (unlikely(virt == NULL))
		return;

	entry = dma_entry_alloc();
	if (!entry)
		return;

	entry->type      = dma_debug_coherent;
	entry->dev       = dev;
	entry->paddr     = virt_to_phys(virt);
	entry->size      = size;
	entry->dev_addr  = dma_addr;
	entry->direction = DMA_BIDIRECTIONAL;

	add_dma_entry(entry);
}
EXPORT_SYMBOL(debug_dma_alloc_coherent);

void debug_dma_free_coherent(struct device *dev, size_t size,
			 void *virt, dma_addr_t addr)
{
	struct dma_debug_entry ref = {
		.type           = dma_debug_coherent,
		.dev            = dev,
		.paddr          = virt_to_phys(virt),
		.dev_addr       = addr,
		.size           = size,
		.direction      = DMA_BIDIRECTIONAL,
	};

	if (unlikely(global_disable))
		return;

	check_unmap(&ref);
}
EXPORT_SYMBOL(debug_dma_free_coherent);

void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
				   size_t size, int direction)
{
	if (unlikely(global_disable))
		return;

	check_sync(dev, dma_handle, size, 0, direction, true);
}
EXPORT_SYMBOL(debug_dma_sync_single_for_cpu);

void debug_dma_sync_single_for_device(struct device *dev,
				      dma_addr_t dma_handle, size_t size,
				      int direction)
{
	if (unlikely(global_disable))
		return;

	check_sync(dev, dma_handle, size, 0, direction, false);
}
EXPORT_SYMBOL(debug_dma_sync_single_for_device);

void debug_dma_sync_single_range_for_cpu(struct device *dev,
					 dma_addr_t dma_handle,
					 unsigned long offset, size_t size,
					 int direction)
{
	if (unlikely(global_disable))
		return;

	check_sync(dev, dma_handle, size, offset, direction, true);
}
EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu);

void debug_dma_sync_single_range_for_device(struct device *dev,
					    dma_addr_t dma_handle,
					    unsigned long offset,
					    size_t size, int direction)
{
	if (unlikely(global_disable))
		return;

	check_sync(dev, dma_handle, size, offset, direction, false);
}
EXPORT_SYMBOL(debug_dma_sync_single_range_for_device);

void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
			       int nelems, int direction)
{
	struct scatterlist *s;
	int i;

	if (unlikely(global_disable))
		return;

	for_each_sg(sg, s, nelems, i) {
		check_sync(dev, s->dma_address, s->dma_length, 0,
				direction, true);
	}
}
EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu);

void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
				  int nelems, int direction)
{
	struct scatterlist *s;
	int i;

	if (unlikely(global_disable))
		return;

	for_each_sg(sg, s, nelems, i) {
		check_sync(dev, s->dma_address, s->dma_length, 0,
				direction, false);
	}
}
EXPORT_SYMBOL(debug_dma_sync_sg_for_device);