litmus-rt.git - The LITMUS^RT kernel.

diff options

Diffstat (limited to 'include/linux/hrtimer.h')

0 files changed, 0 insertions, 0 deletions

/* * DMA driver for Nvidia's Tegra186 GPC DMA controller. * * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. * * This program is distributed in the hope it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ #include <linux/bitops.h> #include <linux/delay.h> #include <linux/dmaengine.h> #include <linux/dma-mapping.h> #include <linux/err.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/mm.h> #include <linux/module.h> #include <linux/of.h> #include <linux/of_device.h> #include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm.h> #include <linux/slab.h> #include <linux/tegra_pm_domains.h> #include <linux/version.h> #include <linux/reset.h> #include <dt-bindings/memory/tegra186-swgroup.h> #include "dmaengine.h" /* CSR register */ #define TEGRA_GPCDMA_CHAN_CSR 0x00 #define TEGRA_GPCDMA_CSR_ENB BIT(31) #define TEGRA_GPCDMA_CSR_IE_EOC BIT(30) #define TEGRA_GPCDMA_CSR_ONCE BIT(27) #define TEGRA_GPCDMA_CSR_FC_MODE_NO_MMIO (0 << 24) #define TEGRA_GPCDMA_CSR_FC_MODE_ONE_MMIO (1 << 24) #define TEGRA_GPCDMA_CSR_FC_MODE_TWO_MMIO (2 << 24) #define TEGRA_GPCDMA_CSR_FC_MODE_FOUR_MMIO (3 << 24) #define TEGRA_GPCDMA_CSR_DMA_IO2MEM_NO_FC (0 << 21) #define TEGRA_GPCDMA_CSR_DMA_IO2MEM_FC (1 << 21) #define TEGRA_GPCDMA_CSR_DMA_MEM2IO_NO_FC (2 << 21) #define TEGRA_GPCDMA_CSR_DMA_MEM2IO_FC (3 << 21) #define TEGRA_GPCDMA_CSR_DMA_MEM2MEM (4 << 21) #define TEGRA_GPCDMA_CSR_DMA_FIXED_PAT (6 << 21) #define TEGRA_GPCDMA_CSR_REQ_SEL_SHIFT 16 #define TEGRA_GPCDMA_CSR_IRQ_MASK BIT(15) #define TEGRA_GPCDMA_CSR_WEIGHT_SHIFT 10 /* STATUS register */ #define TEGRA_GPCDMA_CHAN_STATUS 0x004 #define TEGRA_GPCDMA_STATUS_BUSY BIT(31) #define TEGRA_GPCDMA_STATUS_ISE_EOC BIT(30) #define TEGRA_GPCDMA_STATUS_PING_PONG BIT(28) #define TEGRA_GPCDMA_STATUS_DMA_ACTIVITY BIT(27) #define TEGRA_GPCDMA_STATUS_CHANNEL_PAUSE BIT(26) #define TEGRA_GPCDMA_STATUS_CHANNEL_RX BIT(25) #define TEGRA_GPCDMA_STATUS_CHANNEL_TX BIT(24) #define TEGRA_GPCDMA_STATUS_IRQ_INTR_STA BIT(23) #define TEGRA_GPCDMA_STATUS_IRQ_STA BIT(21) #define TEGRA_GPCDMA_STATUS_IRQ_TRIG_STA BIT(20) #define TEGRA_GPCDMA_CHAN_CSRE 0x008 #define TEGRA_GPCDMA_CHAN_CSRE_PAUSE BIT(31) /* Source address */ #define TEGRA_GPCDMA_CHAN_SRC_PTR 0x00C /* Destination address */ #define TEGRA_GPCDMA_CHAN_DST_PTR 0x010 /* High address pointer */ #define TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR 0x014 #define TEGRA_GPCDMA_HIGH_ADDR_SCR_PTR_SHIFT 0 #define TEGRA_GPCDMA_HIGH_ADDR_SCR_PTR_MASK 0xFF #define TEGRA_GPCDMA_HIGH_ADDR_DST_PTR_SHIFT 16 #define TEGRA_GPCDMA_HIGH_ADDR_DST_PTR_MASK 0xFF /* MC sequence register */ #define TEGRA_GPCDMA_CHAN_MCSEQ 0x18 #define TEGRA_GPCDMA_MCSEQ_DATA_SWAP BIT(31) #define TEGRA_GPCDMA_MCSEQ_REQ_COUNT_SHIFT 25 #define TEGRA_GPCDMA_MCSEQ_BURST_2 (0 << 23) #define TEGRA_GPCDMA_MCSEQ_BURST_16 (3 << 23) #define TEGRA_GPCDMA_MCSEQ_WRAP1_SHIFT 20 #define TEGRA_GPCDMA_MCSEQ_WRAP0_SHIFT 17 #define TEGRA_GPCDMA_MCSEQ_WRAP_NONE 0 #define TEGRA_GPCDMA_MCSEQ_MC_PROT_SHIFT 14 #define TEGRA_GPCDMA_MCSEQ_STREAM_ID1_SHIFT 7 #define TEGRA_GPCDMA_MCSEQ_STREAM_ID0_SHIFT 0 #define TEGRA_GPCDMA_MCSEQ_STREAM_ID_MASK 0x7F /* MMIO sequence register */ #define TEGRA_GPCDMA_CHAN_MMIOSEQ 0x01c #define TEGRA_GPCDMA_MMIOSEQ_DBL_BUF BIT(31) #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8 (0 << 28) #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_16 (1 << 28) #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32 (2 << 28) #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_64 (3 << 28) #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_128 (4 << 28) #define TEGRA_GPCDMA_MMIOSEQ_DATA_SWAP BIT(27) #define TEGRA_GPCDMA_MMIOSEQ_BURST_1 (0 << 23) #define TEGRA_GPCDMA_MMIOSEQ_BURST_2 (1 << 23) #define TEGRA_GPCDMA_MMIOSEQ_BURST_4 (3 << 23) #define TEGRA_GPCDMA_MMIOSEQ_BURST_8 (7 << 23) #define TEGRA_GPCDMA_MMIOSEQ_BURST_16 (15 << 23) #define TEGRA_GPCDMA_MMIOSEQ_MASTER_ID_SHIFT 19 #define TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD_SHIFT 16 #define TEGRA_GPCDMA_MMIOSEQ_MMIO_PROT_SHIFT 7 /* Channel WCOUNT */ #define TEGRA_GPCDMA_CHAN_WCOUNT 0x20 /* Transfer count */ #define TEGRA_GPCDMA_CHAN_XFER_COUNT 0x24 /* DMA byte count status */ #define TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS 0x28 /* Error Status Register */ #define TEGRA_GPCDMA_CHAN_ERR_STATUS 0x30 #define TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT (8) #define TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK (0xF) #define TEGRA_GPCDMA_CHAN_ERR_TYPE(err) ((err >> TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT) & TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK) #define TEGRA_DMA_BM_FIFO_FULL_ERR (0xF) #define TEGRA_DMA_PERIPH_FIFO_FULL_ERR (0xE) #define TEGRA_DMA_PERIPH_ID_ERR (0xD) #define TEGRA_DMA_STREAM_ID_ERR (0xC) #define TEGRA_DMA_MC_SLAVE_ERR (0xB) #define TEGRA_DMA_MMIO_SLAVE_ERR (0xA) /* Fixed Pattern */ #define TEGRA_GPCDMA_CHAN_FIXED_PATTERN 0x34 #define TEGRA_GPCDMA_CHAN_TZ 0x38 #define TEGRA_GPCDMA_CHAN_TZ_MMIO_PROT_1 BIT(0) #define TEGRA_GPCDMA_CHAN_TZ_MC_PROT_1 BIT(1) #define TEGRA_GPCDMA_CHAN_SPARE 0x3c #define TEGRA_GPCDMA_CHAN_SPARE_EN_LEGACY_FC BIT(16) /* * If any burst is in flight and DMA paused then this is the time to complete * on-flight burst and update DMA status register. */ #define TEGRA_GPCDMA_BURST_COMPLETE_TIME 20 /* Channel base address offset from GPCDMA base address */ #define TEGRA_GPCDMA_CHANNEL_BASE_ADD_OFFSET 0x10000 struct tegra_dma; /* * tegra_dma_chip_data Tegra chip specific DMA data * @nr_channels: Number of channels available in the controller. * @channel_reg_size: Channel register size. * @max_dma_count: Maximum DMA transfer count supported by DMA controller. */ struct tegra_dma_chip_data { int nr_channels; int channel_reg_size; int max_dma_count; }; /* DMA channel registers */ struct tegra_dma_channel_regs { unsigned long csr; unsigned long src_ptr; unsigned long dst_ptr; unsigned long high_addr_ptr; unsigned long mc_seq; unsigned long mmio_seq; unsigned long wcount; unsigned long fixed_pattern; }; /* * tegra_dma_sg_req: Dma request details to configure hardware. This * contains the details for one transfer to configure DMA hw. * The client's request for data transfer can be broken into multiple * sub-transfer as per requester details and hw support. * This sub transfer get added in the list of transfer and point to Tegra * DMA descriptor which manages the transfer details. */ struct tegra_dma_sg_req { struct tegra_dma_channel_regs ch_regs; int req_len; bool configured; bool skipped; bool last_sg; bool half_done; struct list_head node; struct tegra_dma_desc *dma_desc; }; /* * tegra_dma_desc: Tegra DMA descriptors which manages the client requests. * This descriptor keep track of transfer status, callbacks and request * counts etc. */ struct tegra_dma_desc { struct dma_async_tx_descriptor txd; int bytes_requested; int bytes_transferred; enum dma_status dma_status; struct list_head node; struct list_head tx_list; struct list_head cb_node; int cb_count; }; struct tegra_dma_channel; typedef void (*dma_isr_handler)(struct tegra_dma_channel *tdc, bool to_terminate); /* tegra_dma_channel: Channel specific information */ struct tegra_dma_channel { struct dma_chan dma_chan; char name[30]; bool config_init; int id; int irq; unsigned long chan_base_offset; spinlock_t lock; bool busy; bool cyclic; struct tegra_dma *tdma; /* Different lists for managing the requests */ struct list_head free_sg_req; struct list_head pending_sg_req; struct list_head free_dma_desc; struct list_head cb_desc; /* ISR handler and tasklet for bottom half of isr handling */ dma_isr_handler isr_handler; struct tasklet_struct tasklet; dma_async_tx_callback callback; void *callback_param; /* Channel-slave specific configuration */ unsigned int slave_id; struct dma_slave_config dma_sconfig; struct tegra_dma_channel_regs channel_reg; }; /* tegra_dma: Tegra DMA specific information */ struct tegra_dma { struct dma_device dma_dev; struct device *dev; spinlock_t global_lock; void __iomem *base_addr; const struct tegra_dma_chip_data *chip_data; struct reset_control *rst; /* Last member of the structure */ struct tegra_dma_channel channels[0]; }; static inline void tdma_write(struct tegra_dma *tdma, u32 reg, u32 val) { writel(val, tdma->base_addr + reg); } static inline u32 tdma_read(struct tegra_dma *tdma, u32 reg) { return readl(tdma->base_addr + reg); } static inline void tdc_write(struct tegra_dma_channel *tdc, u32 reg, u32 val) { writel(val, tdc->tdma->base_addr + tdc->chan_base_offset + reg); } static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg) { return readl(tdc->tdma->base_addr + tdc->chan_base_offset + reg); } static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc) { return container_of(dc, struct tegra_dma_channel, dma_chan); } static inline struct tegra_dma_desc *txd_to_tegra_dma_desc( struct dma_async_tx_descriptor *td) { return container_of(td, struct tegra_dma_desc, txd); } static inline struct device *tdc2dev(struct tegra_dma_channel *tdc) { return &tdc->dma_chan.dev->device; } static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *tx); /* Get DMA desc from free list, if not there then allocate it. */ static struct tegra_dma_desc *tegra_dma_desc_get( struct tegra_dma_channel *tdc) { struct tegra_dma_desc *dma_desc; unsigned long flags; spin_lock_irqsave(&tdc->lock, flags); /* Do not allocate if desc are waiting for ack */ list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) { if (async_tx_test_ack(&dma_desc->txd)) { list_del(&dma_desc->node); spin_unlock_irqrestore(&tdc->lock, flags); dma_desc->txd.flags = 0; return dma_desc; } } spin_unlock_irqrestore(&tdc->lock, flags); /* Allocate DMA desc */ dma_desc = kzalloc(sizeof(*dma_desc), GFP_ATOMIC); if (!dma_desc) { dev_err(tdc2dev(tdc), "dma_desc alloc failed\n"); return NULL; } dma_async_tx_descriptor_init(&dma_desc->txd, &tdc->dma_chan); dma_desc->txd.tx_submit = tegra_dma_tx_submit; dma_desc->txd.flags = 0; return dma_desc; } static void tegra_dma_desc_put(struct tegra_dma_channel *tdc, struct tegra_dma_desc *dma_desc) { unsigned long flags; spin_lock_irqsave(&tdc->lock, flags); if (!list_empty(&dma_desc->tx_list)) list_splice_init(&dma_desc->tx_list, &tdc->free_sg_req); list_add_tail(&dma_desc->node, &tdc->free_dma_desc); spin_unlock_irqrestore(&tdc->lock, flags); } static struct tegra_dma_sg_req *tegra_dma_sg_req_get( struct tegra_dma_channel *tdc) { struct tegra_dma_sg_req *sg_req = NULL; unsigned long flags; spin_lock_irqsave(&tdc->lock, flags); if (!list_empty(&tdc->free_sg_req)) { sg_req = list_first_entry(&tdc->free_sg_req, typeof(*sg_req), node); list_del(&sg_req->node); spin_unlock_irqrestore(&tdc->lock, flags); return sg_req; } spin_unlock_irqrestore(&tdc->lock, flags); sg_req = kzalloc(sizeof(struct tegra_dma_sg_req), GFP_ATOMIC); if (!sg_req) dev_err(tdc2dev(tdc), "sg_req alloc failed\n"); return sg_req; } static int tegra_dma_slave_config(struct dma_chan *dc, struct dma_slave_config *sconfig) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); if (!list_empty(&tdc->pending_sg_req)) { dev_err(tdc2dev(tdc), "Configuration not allowed\n"); return -EBUSY; } memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig)); if (!tdc->slave_id) tdc->slave_id = sconfig->slave_id; tdc->config_init = true; return 0; } static void tegra_dma_pause(struct tegra_dma_channel *tdc, bool wait_for_burst_complete) { tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, TEGRA_GPCDMA_CHAN_CSRE_PAUSE); if (wait_for_burst_complete) udelay(TEGRA_GPCDMA_BURST_COMPLETE_TIME); } static void tegra_dma_resume(struct tegra_dma_channel *tdc) { tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, 0); } static void tegra_dma_stop(struct tegra_dma_channel *tdc) { u32 csr; u32 status; /* Disable interrupts */ csr = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR); csr &= ~TEGRA_GPCDMA_CSR_IE_EOC; tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr); /* Disable DMA */ csr &= ~TEGRA_GPCDMA_CSR_ENB; tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr); /* Clear interrupt status if it is there */ status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS); if (status & TEGRA_GPCDMA_STATUS_ISE_EOC) { dev_dbg(tdc2dev(tdc), "%s():clearing interrupt\n", __func__); tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS, status); } tdc->busy = false; } static void tegra_dma_start(struct tegra_dma_channel *tdc, struct tegra_dma_sg_req *sg_req) { struct tegra_dma_channel_regs *ch_regs = &sg_req->ch_regs; tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, ch_regs->wcount); tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, 0); tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, ch_regs->src_ptr); tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, ch_regs->dst_ptr); tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, ch_regs->high_addr_ptr); tdc_write(tdc, TEGRA_GPCDMA_CHAN_FIXED_PATTERN, ch_regs->fixed_pattern); tdc_write(tdc, TEGRA_GPCDMA_CHAN_MMIOSEQ, ch_regs->mmio_seq); tdc_write(tdc, TEGRA_GPCDMA_CHAN_MCSEQ, ch_regs->mc_seq); tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, ch_regs->csr); /* Start DMA */ tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, ch_regs->csr | TEGRA_GPCDMA_CSR_ENB); } static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc, struct tegra_dma_sg_req *nsg_req) { unsigned long status; status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS); /* * If interrupt is pending then do nothing as the ISR will handle * the programing for new request. */ if (status & TEGRA_GPCDMA_STATUS_ISE_EOC) { dev_err(tdc2dev(tdc), "Skipping new configuration as interrupt is pending\n"); nsg_req->skipped = true; tegra_dma_resume(tdc); return; } /* Safe to program new configuration */ tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, nsg_req->ch_regs.src_ptr); tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, nsg_req->ch_regs.dst_ptr); tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, nsg_req->ch_regs.high_addr_ptr); tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, nsg_req->ch_regs.wcount); tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, nsg_req->ch_regs.csr | TEGRA_GPCDMA_CSR_ENB); nsg_req->configured = true; nsg_req->skipped = false; } static void tdc_start_head_req(struct tegra_dma_channel *tdc) { struct tegra_dma_sg_req *sg_req; if (list_empty(&tdc->pending_sg_req)) return; sg_req = list_first_entry(&tdc->pending_sg_req, typeof(*sg_req), node); tegra_dma_start(tdc, sg_req); sg_req->configured = true; sg_req->skipped = false; tdc->busy = true; } static void tdc_configure_next_head_desc(struct tegra_dma_channel *tdc) { struct tegra_dma_sg_req *hsgreq; struct tegra_dma_sg_req *hnsgreq; if (list_empty(&tdc->pending_sg_req)) return; hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node); if (!list_is_last(&hsgreq->node, &tdc->pending_sg_req)) { hnsgreq = list_first_entry(&hsgreq->node, typeof(*hnsgreq), node); tegra_dma_configure_for_next(tdc, hnsgreq); } } static void tegra_dma_abort_all(struct tegra_dma_channel *tdc) { struct tegra_dma_sg_req *sgreq; struct tegra_dma_desc *dma_desc; while (!list_empty(&tdc->pending_sg_req)) { sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); list_move_tail(&sgreq->node, &tdc->free_sg_req); if (sgreq->last_sg) { dma_desc = sgreq->dma_desc; dma_desc->dma_status = DMA_ERROR; list_add_tail(&dma_desc->node, &tdc->free_dma_desc); /* Add in cb list if it is not there. */ if (!dma_desc->cb_count) list_add_tail(&dma_desc->cb_node, &tdc->cb_desc); dma_desc->cb_count++; } } tdc->isr_handler = NULL; } static bool handle_continuous_head_request(struct tegra_dma_channel *tdc, struct tegra_dma_sg_req *last_sg_req, bool to_terminate) { struct tegra_dma_sg_req *hsgreq = NULL; if (list_empty(&tdc->pending_sg_req)) { dev_err(tdc2dev(tdc), "Dma is running without req\n"); tegra_dma_stop(tdc); return false; } /* * Check that head req on list should be in flight. * If it is not in flight then abort transfer as * looping of transfer can not continue. */ hsgreq = list_first_entry(&tdc->pending_sg_req, typeof(*hsgreq), node); if (!hsgreq->configured && !hsgreq->skipped) { tegra_dma_stop(tdc); dev_err(tdc2dev(tdc), "Error in dma transfer, aborting dma\n"); tegra_dma_abort_all(tdc); return false; } /* Configure next request */ if (!to_terminate) tdc_configure_next_head_desc(tdc); return true; } static void handle_once_dma_done(struct tegra_dma_channel *tdc, bool to_terminate) { struct tegra_dma_sg_req *sgreq; struct tegra_dma_desc *dma_desc; tdc->busy = false; sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); dma_desc = sgreq->dma_desc; dma_desc->bytes_transferred += sgreq->req_len; list_del(&sgreq->node); if (sgreq->last_sg) { #if LINUX_VERSION_CODE < KERNEL_VERSION(3,18,0) dma_desc->dma_status = DMA_SUCCESS; #else dma_desc->dma_status = DMA_COMPLETE; #endif dma_cookie_complete(&dma_desc->txd); if (!dma_desc->cb_count) list_add_tail(&dma_desc->cb_node, &tdc->cb_desc); dma_desc->cb_count++; list_add_tail(&dma_desc->node, &tdc->free_dma_desc); } list_add_tail(&sgreq->node, &tdc->free_sg_req); if (to_terminate || list_empty(&tdc->pending_sg_req)) return; tdc_start_head_req(tdc); return; } static void handle_cont_sngl_cycle_dma_done(struct tegra_dma_channel *tdc, bool to_terminate) { struct tegra_dma_sg_req *sgreq; struct tegra_dma_desc *dma_desc; bool st; sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); dma_desc = sgreq->dma_desc; dma_desc->bytes_transferred += sgreq->req_len; /* Callback need to be call */ if (!dma_desc->cb_count) list_add_tail(&dma_desc->cb_node, &tdc->cb_desc); dma_desc->cb_count++; /* If not last req then put at end of pending list */ if (!list_is_last(&sgreq->node, &tdc->pending_sg_req)) { list_move_tail(&sgreq->node, &tdc->pending_sg_req); sgreq->configured = false; sgreq->skipped = false; st = handle_continuous_head_request(tdc, sgreq, to_terminate); if (!st) dma_desc->dma_status = DMA_ERROR; } return; } static void tegra_dma_tasklet(unsigned long data) { struct tegra_dma_channel *tdc = (struct tegra_dma_channel *)data; dma_async_tx_callback callback = NULL; void *callback_param = NULL; struct tegra_dma_desc *dma_desc; unsigned long flags; int cb_count; spin_lock_irqsave(&tdc->lock, flags); while (!list_empty(&tdc->cb_desc)) { dma_desc = list_first_entry(&tdc->cb_desc, typeof(*dma_desc), cb_node); list_del(&dma_desc->cb_node); callback = dma_desc->txd.callback; callback_param = dma_desc->txd.callback_param; cb_count = dma_desc->cb_count; dma_desc->cb_count = 0; spin_unlock_irqrestore(&tdc->lock, flags); while (cb_count-- && callback) callback(callback_param); spin_lock_irqsave(&tdc->lock, flags); } spin_unlock_irqrestore(&tdc->lock, flags); } static void tegra_dma_chan_decode_error(struct tegra_dma_channel *tdc, unsigned int err_status) { switch(TEGRA_GPCDMA_CHAN_ERR_TYPE(err_status)) { case TEGRA_DMA_BM_FIFO_FULL_ERR: dev_info(tdc2dev(tdc), "bm fifo full\n"); break; case TEGRA_DMA_PERIPH_FIFO_FULL_ERR: dev_info(tdc2dev(tdc), "peripheral fifo full\n"); break; case TEGRA_DMA_PERIPH_ID_ERR: dev_info(tdc2dev(tdc), "illegal peripheral id\n"); break; case TEGRA_DMA_STREAM_ID_ERR: dev_info(tdc2dev(tdc), "illegal stream id\n"); break; case TEGRA_DMA_MC_SLAVE_ERR: dev_info(tdc2dev(tdc), "mc slave error\n"); break; case TEGRA_DMA_MMIO_SLAVE_ERR: dev_info(tdc2dev(tdc), "mmio slave error\n"); break; default: dev_info(tdc2dev(tdc), "security violation %x\n", err_status); } } static irqreturn_t tegra_dma_isr(int irq, void *dev_id) { struct tegra_dma_channel *tdc = dev_id; unsigned long status; unsigned long flags; unsigned int err_status; spin_lock_irqsave(&tdc->lock, flags); status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS); err_status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS); if (err_status) { tegra_dma_chan_decode_error(tdc, err_status); tdc_write(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS, 0xFFFFFFFF); } if (status & TEGRA_GPCDMA_STATUS_ISE_EOC) { tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS, TEGRA_GPCDMA_STATUS_ISE_EOC); tdc->isr_handler(tdc, false); tasklet_schedule(&tdc->tasklet); spin_unlock_irqrestore(&tdc->lock, flags); return IRQ_HANDLED; } spin_unlock_irqrestore(&tdc->lock, flags); dev_info(tdc2dev(tdc), "Interrupt already served status 0x%08lx\n", status); return IRQ_NONE; } static dma_cookie_t tegra_dma_tx_submit(struct dma_async_tx_descriptor *txd) { struct tegra_dma_desc *dma_desc = txd_to_tegra_dma_desc(txd); struct tegra_dma_channel *tdc = to_tegra_dma_chan(txd->chan); unsigned long flags; dma_cookie_t cookie; spin_lock_irqsave(&tdc->lock, flags); dma_desc->dma_status = DMA_IN_PROGRESS; cookie = dma_cookie_assign(&dma_desc->txd); list_splice_tail_init(&dma_desc->tx_list, &tdc->pending_sg_req); spin_unlock_irqrestore(&tdc->lock, flags); return cookie; } static void tegra_dma_issue_pending(struct dma_chan *dc) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); unsigned long flags; unsigned long status; int count; spin_lock_irqsave(&tdc->lock, flags); if (list_empty(&tdc->pending_sg_req)) { dev_err(tdc2dev(tdc), "No DMA request\n"); goto end; } if (!tdc->busy) { tdc_start_head_req(tdc); /* Continuous mode: Configure next req */ if (tdc->cyclic) { /* * For cyclic dma transfers, program the second transfer * parameters as soon as the first dma transfer is * started inorder for the dma controller to trigger the * second transfer with the correct parameters. Poll * for the channel busy bit and start the transfer. */ count = 20; do { status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS); if (status & TEGRA_GPCDMA_STATUS_BUSY) break; udelay(1); count--; } while(count); tdc_configure_next_head_desc(tdc); } } end: spin_unlock_irqrestore(&tdc->lock, flags); return; } static void tegra_dma_terminate_all(struct dma_chan *dc) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); struct tegra_dma_sg_req *sgreq; struct tegra_dma_desc *dma_desc; unsigned long flags; unsigned long status; unsigned long wcount = 0; bool was_busy; spin_lock_irqsave(&tdc->lock, flags); if (list_empty(&tdc->pending_sg_req)) { spin_unlock_irqrestore(&tdc->lock, flags); return; } if (!tdc->busy) goto skip_dma_stop; /* Pause DMA before checking the queue status */ tegra_dma_pause(tdc, true); status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS); wcount = tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT); if (status & TEGRA_GPCDMA_STATUS_ISE_EOC) { dev_dbg(tdc2dev(tdc), "%s():handling isr\n", __func__); tdc->isr_handler(tdc, true); status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS); wcount = tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT); } was_busy = tdc->busy; tegra_dma_stop(tdc); if (!list_empty(&tdc->pending_sg_req) && was_busy) { sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); sgreq->dma_desc->bytes_transferred += sgreq->req_len - (wcount * 4); } tegra_dma_resume(tdc); skip_dma_stop: tegra_dma_abort_all(tdc); while (!list_empty(&tdc->cb_desc)) { dma_desc = list_first_entry(&tdc->cb_desc, typeof(*dma_desc), cb_node); list_del(&dma_desc->cb_node); dma_desc->cb_count = 0; } spin_unlock_irqrestore(&tdc->lock, flags); } static enum dma_status tegra_dma_tx_status(struct dma_chan *dc, dma_cookie_t cookie, struct dma_tx_state *txstate) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); struct tegra_dma_desc *dma_desc; struct tegra_dma_sg_req *sg_req; enum dma_status ret; unsigned long flags; unsigned int residual; spin_lock_irqsave(&tdc->lock, flags); ret = dma_cookie_status(dc, cookie, txstate); #if LINUX_VERSION_CODE < KERNEL_VERSION(3,18,0) if (ret == DMA_SUCCESS) { #else if (ret == DMA_COMPLETE) { #endif spin_unlock_irqrestore(&tdc->lock, flags); return ret; } /* Check on wait_ack desc status */ list_for_each_entry(dma_desc, &tdc->free_dma_desc, node) { if (dma_desc->txd.cookie == cookie) { residual = dma_desc->bytes_requested - (dma_desc->bytes_transferred % dma_desc->bytes_requested); dma_set_residue(txstate, residual); ret = dma_desc->dma_status; spin_unlock_irqrestore(&tdc->lock, flags); return ret; } } /* Check in pending list */ list_for_each_entry(sg_req, &tdc->pending_sg_req, node) { dma_desc = sg_req->dma_desc; if (dma_desc->txd.cookie == cookie) { residual = dma_desc->bytes_requested - (dma_desc->bytes_transferred % dma_desc->bytes_requested); dma_set_residue(txstate, residual); ret = dma_desc->dma_status; spin_unlock_irqrestore(&tdc->lock, flags); return ret; } } dev_dbg(tdc2dev(tdc), "cookie %d does not found\n", cookie); spin_unlock_irqrestore(&tdc->lock, flags); return ret; } static int tegra_dma_device_control(struct dma_chan *dc, enum dma_ctrl_cmd cmd, unsigned long arg) { switch (cmd) { case DMA_SLAVE_CONFIG: return tegra_dma_slave_config(dc, (struct dma_slave_config *)arg); case DMA_TERMINATE_ALL: tegra_dma_terminate_all(dc); return 0; default: break; } return -ENXIO; } static inline int get_bus_width(struct tegra_dma_channel *tdc, enum dma_slave_buswidth slave_bw) { switch (slave_bw) { case DMA_SLAVE_BUSWIDTH_1_BYTE: return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8; case DMA_SLAVE_BUSWIDTH_2_BYTES: return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_16; case DMA_SLAVE_BUSWIDTH_4_BYTES: return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32; case DMA_SLAVE_BUSWIDTH_8_BYTES: return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_64; default: dev_warn(tdc2dev(tdc), "slave bw is not supported, using 32bits\n"); return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32; } } static inline int get_burst_size(struct tegra_dma_channel *tdc, u32 burst_size, enum dma_slave_buswidth slave_bw, int len) { int burst_byte; int burst_mmio_width; /* * burst_size from client is in terms of the bus_width. * convert that into words. */ burst_byte = burst_size * slave_bw; burst_mmio_width = burst_byte / 4; /* If burst size is 0 then calculate the burst size based on length */ if (!burst_mmio_width) { if (len & 0xF) return TEGRA_GPCDMA_MMIOSEQ_BURST_1; else if ((len >> 3) & 0x1) return TEGRA_GPCDMA_MMIOSEQ_BURST_2; else if ((len >> 4) & 0x1) return TEGRA_GPCDMA_MMIOSEQ_BURST_4; else if ((len >> 5) & 0x1) return TEGRA_GPCDMA_MMIOSEQ_BURST_8; else return TEGRA_GPCDMA_MMIOSEQ_BURST_16; } if (burst_mmio_width < 2) return TEGRA_GPCDMA_MMIOSEQ_BURST_1; else if (burst_mmio_width < 4) return TEGRA_GPCDMA_MMIOSEQ_BURST_2; else if (burst_mmio_width < 8) return TEGRA_GPCDMA_MMIOSEQ_BURST_4; else if (burst_mmio_width < 16) return TEGRA_GPCDMA_MMIOSEQ_BURST_8; else return TEGRA_GPCDMA_MMIOSEQ_BURST_16; } static int get_transfer_param(struct tegra_dma_channel *tdc, enum dma_transfer_direction direction, unsigned long *apb_addr, unsigned long *mmio_seq, unsigned long *csr, unsigned int *burst_size, enum dma_slave_buswidth *slave_bw) { switch (direction) { case DMA_MEM_TO_DEV: *apb_addr = tdc->dma_sconfig.dst_addr; *mmio_seq = get_bus_width(tdc, tdc->dma_sconfig.dst_addr_width); *burst_size = tdc->dma_sconfig.dst_maxburst; *slave_bw = tdc->dma_sconfig.dst_addr_width; *csr = TEGRA_GPCDMA_CSR_DMA_MEM2IO_FC; return 0; case DMA_DEV_TO_MEM: *apb_addr = tdc->dma_sconfig.src_addr; *mmio_seq = get_bus_width(tdc, tdc->dma_sconfig.src_addr_width); *burst_size = tdc->dma_sconfig.src_maxburst; *slave_bw = tdc->dma_sconfig.src_addr_width; *csr = TEGRA_GPCDMA_CSR_DMA_IO2MEM_FC; return 0; case DMA_MEM_TO_MEM: *burst_size = tdc->dma_sconfig.src_addr_width; *csr = TEGRA_GPCDMA_CSR_DMA_MEM2MEM; return 0; default: dev_err(tdc2dev(tdc), "Dma direction is not supported\n"); return -EINVAL; } return -EINVAL; } static struct dma_async_tx_descriptor *tegra_dma_prep_dma_memset( struct dma_chan *dc, dma_addr_t dest, int value, size_t len, unsigned long flags) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); struct tegra_dma_desc *dma_desc; struct list_head req_list; struct tegra_dma_sg_req *sg_req = NULL; unsigned long csr, mc_seq; INIT_LIST_HEAD(&req_list); /* Set dma mode to fixed pattern */ csr = TEGRA_GPCDMA_CSR_DMA_FIXED_PAT; /* Enable once or continuous mode */ csr |= TEGRA_GPCDMA_CSR_ONCE; /* Enable IRQ mask */ csr |= TEGRA_GPCDMA_CSR_IRQ_MASK; /* Enable the dma interrupt */ if (flags & DMA_PREP_INTERRUPT) csr |= TEGRA_GPCDMA_CSR_IE_EOC; /* Configure default priority weight for the channel */ csr |= (1 << TEGRA_GPCDMA_CSR_WEIGHT_SHIFT); mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ); /* retain stream-id and clean rest */ mc_seq &= (TEGRA_GPCDMA_MCSEQ_STREAM_ID_MASK << TEGRA_GPCDMA_MCSEQ_STREAM_ID0_SHIFT); /* Set the address wrapping */ mc_seq |= TEGRA_GPCDMA_MCSEQ_WRAP_NONE << TEGRA_GPCDMA_MCSEQ_WRAP0_SHIFT; mc_seq |= TEGRA_GPCDMA_MCSEQ_WRAP_NONE << TEGRA_GPCDMA_MCSEQ_WRAP1_SHIFT; /* Program outstanding MC requests */ mc_seq |= (1 << TEGRA_GPCDMA_MCSEQ_REQ_COUNT_SHIFT); /* Set burst size */ mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16; dma_desc = tegra_dma_desc_get(tdc); if (!dma_desc) { dev_err(tdc2dev(tdc), "Dma descriptors not available\n"); return NULL; } INIT_LIST_HEAD(&dma_desc->tx_list); INIT_LIST_HEAD(&dma_desc->cb_node); dma_desc->cb_count = 0; dma_desc->bytes_requested = 0; dma_desc->bytes_transferred = 0; dma_desc->dma_status = DMA_IN_PROGRESS; if ((len & 3) || (dest & 3) || (len > tdc->tdma->chip_data->max_dma_count)) { dev_err(tdc2dev(tdc), "Dma length/memory address is not supported\n"); tegra_dma_desc_put(tdc, dma_desc); return NULL; } sg_req = tegra_dma_sg_req_get(tdc); if (!sg_req) { dev_err(tdc2dev(tdc), "Dma sg-req not available\n"); tegra_dma_desc_put(tdc, dma_desc); return NULL; } dma_desc->bytes_requested += len; sg_req->ch_regs.src_ptr = 0; sg_req->ch_regs.dst_ptr = dest; sg_req->ch_regs.high_addr_ptr = ((dest >> 32) & TEGRA_GPCDMA_HIGH_ADDR_DST_PTR_MASK) << TEGRA_GPCDMA_HIGH_ADDR_DST_PTR_SHIFT; sg_req->ch_regs.fixed_pattern = value; /* Word count reg takes value as (N +1) words */ sg_req->ch_regs.wcount = ((len - 4) >> 2); sg_req->ch_regs.csr = csr; sg_req->ch_regs.mmio_seq = 0; sg_req->ch_regs.mc_seq = mc_seq; sg_req->configured = false; sg_req->skipped = false; sg_req->last_sg = false; sg_req->dma_desc = dma_desc; sg_req->req_len = len; sg_req->last_sg = true; list_add_tail(&sg_req->node, &dma_desc->tx_list); if (flags & DMA_CTRL_ACK) dma_desc->txd.flags = DMA_CTRL_ACK; if (!tdc->isr_handler) tdc->isr_handler = handle_once_dma_done; return &dma_desc->txd; } static struct dma_async_tx_descriptor *tegra_dma_prep_dma_memcpy( struct dma_chan *dc, dma_addr_t dest, dma_addr_t src, size_t len, unsigned long flags) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); struct tegra_dma_desc *dma_desc; struct list_head req_list; struct tegra_dma_sg_req *sg_req = NULL; unsigned long csr, mc_seq; INIT_LIST_HEAD(&req_list); /* Set dma mode to memory to memory transfer */ csr = TEGRA_GPCDMA_CSR_DMA_MEM2MEM; /* Enable once or continuous mode */ csr |= TEGRA_GPCDMA_CSR_ONCE; /* Enable IRQ mask */ csr |= TEGRA_GPCDMA_CSR_IRQ_MASK; /* Enable the dma interrupt */ if (flags & DMA_PREP_INTERRUPT) csr |= TEGRA_GPCDMA_CSR_IE_EOC; /* Configure default priority weight for the channel */ csr |= (1 << TEGRA_GPCDMA_CSR_WEIGHT_SHIFT); mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ); /* retain stream-id and clean rest */ mc_seq &= (TEGRA_GPCDMA_MCSEQ_STREAM_ID_MASK << TEGRA_GPCDMA_MCSEQ_STREAM_ID0_SHIFT); /* Set the address wrapping */ mc_seq |= TEGRA_GPCDMA_MCSEQ_WRAP_NONE << TEGRA_GPCDMA_MCSEQ_WRAP0_SHIFT; mc_seq |= TEGRA_GPCDMA_MCSEQ_WRAP_NONE << TEGRA_GPCDMA_MCSEQ_WRAP1_SHIFT; /* Program outstanding MC requests */ mc_seq |= (1 << TEGRA_GPCDMA_MCSEQ_REQ_COUNT_SHIFT); /* Set burst size */ mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16; dma_desc = tegra_dma_desc_get(tdc); if (!dma_desc) { dev_err(tdc2dev(tdc), "Dma descriptors not available\n"); return NULL; } INIT_LIST_HEAD(&dma_desc->tx_list); INIT_LIST_HEAD(&dma_desc->cb_node); dma_desc->cb_count = 0; dma_desc->bytes_requested = 0; dma_desc->bytes_transferred = 0; dma_desc->dma_status = DMA_IN_PROGRESS; if ((len & 3) || (src & 3) || (dest & 3) || (len > tdc->tdma->chip_data->max_dma_count)) { dev_err(tdc2dev(tdc), "Dma length/memory address is not supported\n"); tegra_dma_desc_put(tdc, dma_desc); return NULL; } sg_req = tegra_dma_sg_req_get(tdc); if (!sg_req) { dev_err(tdc2dev(tdc), "Dma sg-req not available\n"); tegra_dma_desc_put(tdc, dma_desc); return NULL; } dma_desc->bytes_requested += len; sg_req->ch_regs.src_ptr = src; sg_req->ch_regs.dst_ptr = dest; sg_req->ch_regs.high_addr_ptr = (src >> 32) & TEGRA_GPCDMA_HIGH_ADDR_SCR_PTR_MASK; sg_req->ch_regs.high_addr_ptr |= ((dest >> 32) & TEGRA_GPCDMA_HIGH_ADDR_DST_PTR_MASK) << TEGRA_GPCDMA_HIGH_ADDR_DST_PTR_SHIFT; /* Word count reg takes value as (N +1) words */ sg_req->ch_regs.wcount = ((len - 4) >> 2); sg_req->ch_regs.csr = csr; sg_req->ch_regs.mmio_seq = 0; sg_req->ch_regs.mc_seq = mc_seq; sg_req->configured = false; sg_req->skipped = false; sg_req->last_sg = false; sg_req->dma_desc = dma_desc; sg_req->req_len = len; sg_req->last_sg = true; list_add_tail(&sg_req->node, &dma_desc->tx_list); if (flags & DMA_CTRL_ACK) dma_desc->txd.flags = DMA_CTRL_ACK; if (!tdc->isr_handler) tdc->isr_handler = handle_once_dma_done; return &dma_desc->txd; } static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( struct dma_chan *dc, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction direction, unsigned long flags, void *context) { struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); struct tegra_dma_desc *dma_desc; unsigned int i; struct scatterlist *sg; unsigned long csr, mc_seq, apb_ptr = 0, mmio_seq = 0; struct list_head req_list; struct tegra_dma_sg_req *sg_req = NULL; u32 burst_size; enum dma_slave_buswidth slave_bw = 0; int ret; if (!tdc->config_init) { dev_err(tdc2dev(tdc), "dma channel is not configured\n"); return NULL; } if (sg_len < 1) { dev_err(tdc2dev(tdc), "Invalid segment length %d\n", sg_len); return NULL; } ret = get_transfer_param(tdc, direction, &apb_ptr, &mmio_seq, &csr, &burst_size, &slave_bw); if (ret < 0) return NULL; INIT_LIST_HEAD(&req_list); /* Enable once or continuous mode */ csr |= TEGRA_GPCDMA_CSR_ONCE; /* Program the slave id in requestor select */ csr |= tdc->slave_id << TEGRA_GPCDMA_CSR_REQ_SEL_SHIFT; /* Enable IRQ mask */ csr |= TEGRA_GPCDMA_CSR_IRQ_MASK; /* Configure default priority weight for the channel*/ csr |= (1 << TEGRA_GPCDMA_CSR_WEIGHT_SHIFT); /* Enable the dma interrupt */ if (flags & DMA_PREP_INTERRUPT) csr |= TEGRA_GPCDMA_CSR_IE_EOC; mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ); /* retain stream-id and clean rest */ mc_seq &= (TEGRA_GPCDMA_MCSEQ_STREAM_ID_MASK << TEGRA_GPCDMA_MCSEQ_STREAM_ID0_SHIFT); /* Set the address wrapping on both MC and MMIO side */ mc_seq |= TEGRA_GPCDMA_MCSEQ_WRAP_NONE << TEGRA_GPCDMA_MCSEQ_WRAP0_SHIFT; mc_seq |= TEGRA_GPCDMA_MCSEQ_WRAP_NONE << TEGRA_GPCDMA_MCSEQ_WRAP1_SHIFT; mmio_seq |= (1 << TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD_SHIFT); /* Program 2 MC outstanding requests by default. */ mc_seq |= (1 << TEGRA_GPCDMA_MCSEQ_REQ_COUNT_SHIFT); /* Setting MC burst size depending on MMIO burst size */ if (burst_size == 64) mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16; else


context:
space:
mode: