From e22dde9904c2d26a522f1a2b89854a8238bf0933 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Tue, 17 Nov 2009 11:34:31 -0700 Subject: ioat: silence "dca disabled" messages Turning off dca is not an "error", and the dca-enabled state can be viewed from sysfs. Signed-off-by: Dan Williams --- drivers/dma/ioat/dca.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/ioat/dca.c b/drivers/dma/ioat/dca.c index 69d02615c4d6..14f0a782df44 100644 --- a/drivers/dma/ioat/dca.c +++ b/drivers/dma/ioat/dca.c @@ -98,7 +98,7 @@ static int dca_enabled_in_bios(struct pci_dev *pdev) cpuid_level_9 = cpuid_eax(9); res = test_bit(0, &cpuid_level_9); if (!res) - dev_err(&pdev->dev, "DCA is disabled in BIOS\n"); + dev_dbg(&pdev->dev, "DCA is disabled in BIOS\n"); return res; } @@ -108,7 +108,7 @@ static int system_has_dca_enabled(struct pci_dev *pdev) if (boot_cpu_has(X86_FEATURE_DCA)) return dca_enabled_in_bios(pdev); - dev_err(&pdev->dev, "boot cpu doesn't have X86_FEATURE_DCA\n"); + dev_dbg(&pdev->dev, "boot cpu doesn't have X86_FEATURE_DCA\n"); return 0; } -- cgit v1.2.2 From 228c4f5cfbf1cda411d9aa7204a612a63c89b1e8 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 19 Nov 2009 17:07:10 -0700 Subject: ioat3: dca and raid operations are incompatible RAID operations cause a system hang on platforms with DCA (Direct-Cache-Access) enabled. So turn off RAID capabilities in this case. Signed-off-by: Dan Williams --- drivers/dma/ioat/dca.c | 2 +- drivers/dma/ioat/dma_v3.c | 6 ++++++ drivers/dma/ioat/hw.h | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) (limited to 'drivers/dma') diff --git a/drivers/dma/ioat/dca.c b/drivers/dma/ioat/dca.c index 14f0a782df44..abd9038e06b1 100644 --- a/drivers/dma/ioat/dca.c +++ b/drivers/dma/ioat/dca.c @@ -103,7 +103,7 @@ static int dca_enabled_in_bios(struct pci_dev *pdev) return res; } -static int system_has_dca_enabled(struct pci_dev *pdev) +int system_has_dca_enabled(struct pci_dev *pdev) { if (boot_cpu_has(X86_FEATURE_DCA)) return dca_enabled_in_bios(pdev); diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index 35d1e33afd5b..4f305f61f878 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -1117,6 +1117,7 @@ static int __devinit ioat3_dma_self_test(struct ioatdma_device *device) int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) { struct pci_dev *pdev = device->pdev; + int dca_en = system_has_dca_enabled(pdev); struct dma_device *dma; struct dma_chan *c; struct ioat_chan_common *chan; @@ -1137,6 +1138,11 @@ int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) dma->device_prep_dma_interrupt = ioat3_prep_interrupt_lock; cap = readl(device->reg_base + IOAT_DMA_CAP_OFFSET); + + /* dca is incompatible with raid operations */ + if (dca_en && (cap & (IOAT_CAP_XOR|IOAT_CAP_PQ))) + cap &= ~(IOAT_CAP_XOR|IOAT_CAP_PQ); + if (cap & IOAT_CAP_XOR) { is_raid_device = true; dma->max_xor = 8; diff --git a/drivers/dma/ioat/hw.h b/drivers/dma/ioat/hw.h index 99afb12bd409..60e675455b6a 100644 --- a/drivers/dma/ioat/hw.h +++ b/drivers/dma/ioat/hw.h @@ -39,6 +39,8 @@ #define IOAT_VER_3_0 0x30 /* Version 3.0 */ #define IOAT_VER_3_2 0x32 /* Version 3.2 */ +int system_has_dca_enabled(struct pci_dev *pdev); + struct ioat_dma_descriptor { uint32_t size; union { -- cgit v1.2.2 From 6f82b83b7a56bc6e9dd6d7b93531dde6027c5309 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 19 Nov 2009 17:07:57 -0700 Subject: ioat2,3: disable asynchronous error notifications Error interrupts and error completions may cause channel hangs, so poll the channel status register after a timeout. Signed-off-by: Dan Williams --- drivers/dma/ioat/registers.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/ioat/registers.h b/drivers/dma/ioat/registers.h index 63038e18ab03..f015ec196700 100644 --- a/drivers/dma/ioat/registers.h +++ b/drivers/dma/ioat/registers.h @@ -92,9 +92,7 @@ #define IOAT_CHANCTRL_ERR_COMPLETION_EN 0x0004 #define IOAT_CHANCTRL_INT_REARM 0x0001 #define IOAT_CHANCTRL_RUN (IOAT_CHANCTRL_INT_REARM |\ - IOAT_CHANCTRL_ERR_COMPLETION_EN |\ - IOAT_CHANCTRL_ANY_ERR_ABORT_EN |\ - IOAT_CHANCTRL_ERR_INT_EN) + IOAT_CHANCTRL_ANY_ERR_ABORT_EN) #define IOAT_DMA_COMP_OFFSET 0x02 /* 16-bit DMA channel compatibility */ #define IOAT_DMA_COMP_V1 0x0001 /* Compatibility with DMA version 1 */ -- cgit v1.2.2 From de581b65f6fe78168affa552c3bd15b8c80ed614 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 19 Nov 2009 17:08:45 -0700 Subject: ioat3: specify valid address for disabled-Q or disabled-P Although disabled, hardware still checks address validity, so duplicate the known address. Signed-off-by: Dan Williams --- drivers/dma/ioat/dma_v3.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index 4f305f61f878..fad2e9fba490 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -736,10 +736,16 @@ ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, unsigned int src_cnt, const unsigned char *scf, size_t len, unsigned long flags) { + /* specify valid address for disabled result */ + if (flags & DMA_PREP_PQ_DISABLE_P) + dst[0] = dst[1]; + if (flags & DMA_PREP_PQ_DISABLE_Q) + dst[1] = dst[0]; + /* handle the single source multiply case from the raid6 * recovery path */ - if (unlikely((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1)) { + if ((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1) { dma_addr_t single_source[2]; unsigned char single_source_coef[2]; @@ -761,6 +767,12 @@ ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, unsigned int src_cnt, const unsigned char *scf, size_t len, enum sum_check_flags *pqres, unsigned long flags) { + /* specify valid address for disabled result */ + if (flags & DMA_PREP_PQ_DISABLE_P) + pq[0] = pq[1]; + if (flags & DMA_PREP_PQ_DISABLE_Q) + pq[1] = pq[0]; + /* the cleanup routine only sets bits on validate failure, it * does not clear bits on validate success... so clear it here */ @@ -778,9 +790,9 @@ ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src, dma_addr_t pq[2]; memset(scf, 0, src_cnt); - flags |= DMA_PREP_PQ_DISABLE_Q; pq[0] = dst; - pq[1] = ~0; + flags |= DMA_PREP_PQ_DISABLE_Q; + pq[1] = dst; /* specify valid address for disabled result */ return __ioat3_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len, flags); @@ -800,9 +812,9 @@ ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src, *result = 0; memset(scf, 0, src_cnt); - flags |= DMA_PREP_PQ_DISABLE_Q; pq[0] = src[0]; - pq[1] = ~0; + flags |= DMA_PREP_PQ_DISABLE_Q; + pq[1] = pq[0]; /* specify valid address for disabled result */ return __ioat3_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1, scf, len, flags); -- cgit v1.2.2 From b57014def9afc2bd8a62299d2f51b77dad5ae0c7 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 19 Nov 2009 17:10:07 -0700 Subject: ioat2,3: report all uncorrectable errors Modify is_ioat_bug() to catch all errors that are uncorrectable, or not currently handled. Signed-off-by: Dan Williams --- drivers/dma/ioat/dma.h | 4 +--- drivers/dma/ioat/dma_v2.c | 2 ++ drivers/dma/ioat/dma_v3.c | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index c14fdfeb7f33..45edde996480 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -297,9 +297,7 @@ static inline bool is_ioat_suspended(unsigned long status) /* channel was fatally programmed */ static inline bool is_ioat_bug(unsigned long err) { - return !!(err & (IOAT_CHANERR_SRC_ADDR_ERR|IOAT_CHANERR_DEST_ADDR_ERR| - IOAT_CHANERR_NEXT_ADDR_ERR|IOAT_CHANERR_CONTROL_ERR| - IOAT_CHANERR_LENGTH_ERR)); + return !!err; } static inline void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len, diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c index 96ffab7d37a7..8f1f7f05deaa 100644 --- a/drivers/dma/ioat/dma_v2.c +++ b/drivers/dma/ioat/dma_v2.c @@ -279,6 +279,8 @@ void ioat2_timer_event(unsigned long data) u32 chanerr; chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); + dev_err(to_dev(chan), "%s: Channel halted (%x)\n", + __func__, chanerr); BUG_ON(is_ioat_bug(chanerr)); } diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index fad2e9fba490..252cf2d3f1da 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -378,6 +378,8 @@ static void ioat3_timer_event(unsigned long data) u32 chanerr; chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); + dev_err(to_dev(chan), "%s: Channel halted (%x)\n", + __func__, chanerr); BUG_ON(is_ioat_bug(chanerr)); } -- cgit v1.2.2 From 4499a24dec00e037da7d09caccad45e7594a9c19 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 19 Nov 2009 17:10:25 -0700 Subject: dmaengine: include xor/pq validate in device_has_all_tx_types() A channel must include these capabilities to satisfy ASYNC_TX_DISABLE_CHANNEL_SWITCH. Signed-off-by: Dan Williams --- drivers/dma/dmaengine.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'drivers/dma') diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index bd0b248de2cf..b6442f09d0fe 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -632,11 +632,15 @@ static bool device_has_all_tx_types(struct dma_device *device) #if defined(CONFIG_ASYNC_XOR) || defined(CONFIG_ASYNC_XOR_MODULE) if (!dma_has_cap(DMA_XOR, device->cap_mask)) return false; + if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask)) + return false; #endif #if defined(CONFIG_ASYNC_PQ) || defined(CONFIG_ASYNC_PQ_MODULE) if (!dma_has_cap(DMA_PQ, device->cap_mask)) return false; + if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask)) + return false; #endif return true; -- cgit v1.2.2 From 7b3cc2b1fc2066391e498f3387204908c4eced21 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 19 Nov 2009 17:10:37 -0700 Subject: async_tx: build-time toggling of async_{syndrome,xor}_val dma support ioat3.2 does not support asynchronous error notifications which makes the driver experience latencies when non-zero pq validate results are expected. Provide a mechanism for turning off async_xor_val and async_syndrome_val via Kconfig. This approach is generally useful for any driver that specifies ASYNC_TX_DISABLE_CHANNEL_SWITCH and would like to force the async_tx api to fall back to the synchronous path for certain operations. Signed-off-by: Dan Williams --- drivers/dma/Kconfig | 2 ++ drivers/dma/dmaengine.c | 6 ++++++ drivers/dma/ioat/dma_v3.c | 10 ++++++++++ 3 files changed, 18 insertions(+) (limited to 'drivers/dma') diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 5903a88351bf..b401dadad4a8 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -26,6 +26,8 @@ config INTEL_IOATDMA select DMA_ENGINE select DCA select ASYNC_TX_DISABLE_CHANNEL_SWITCH + select ASYNC_TX_DISABLE_PQ_VAL_DMA + select ASYNC_TX_DISABLE_XOR_VAL_DMA help Enable support for the Intel(R) I/OAT DMA engine present in recent Intel Xeon chipsets. diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index b6442f09d0fe..8f99354082ce 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -632,16 +632,22 @@ static bool device_has_all_tx_types(struct dma_device *device) #if defined(CONFIG_ASYNC_XOR) || defined(CONFIG_ASYNC_XOR_MODULE) if (!dma_has_cap(DMA_XOR, device->cap_mask)) return false; + + #ifndef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask)) return false; #endif + #endif #if defined(CONFIG_ASYNC_PQ) || defined(CONFIG_ASYNC_PQ_MODULE) if (!dma_has_cap(DMA_PQ, device->cap_mask)) return false; + + #ifndef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask)) return false; #endif + #endif return true; } diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index 252cf2d3f1da..189788f6351e 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -1206,6 +1206,16 @@ int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) device->timer_fn = ioat2_timer_event; } + #ifdef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA + dma_cap_clear(DMA_PQ_VAL, dma->cap_mask); + dma->device_prep_dma_pq_val = NULL; + #endif + + #ifdef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA + dma_cap_clear(DMA_XOR_VAL, dma->cap_mask); + dma->device_prep_dma_xor_val = NULL; + #endif + /* -= IOAT ver.3 workarounds =- */ /* Write CHANERRMSK_INT with 3E07h to mask out the errors * that can cause stability issues for IOAT ver.3 -- cgit v1.2.2 From 49954c1567cb0d70d28bb5512d471dc5bd4e2c3f Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 19 Nov 2009 17:11:03 -0700 Subject: ioat3: fix pq completion versus channel deallocation race The completion of a pq operation is notified with a null descriptor appended to the end of the chain. This descriptor needs to be visible to dma clients otherwise the client is precluded from ensuring all operations are quiesced before freeing channel resources, i.e. due to descriptor polling it may get the completion notification ahead of the interrupt delivered by the null descriptor. Signed-off-by: Dan Williams --- drivers/dma/ioat/dma_v3.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index 189788f6351e..42f6f10fb0cc 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -571,7 +571,7 @@ __ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result, dump_desc_dbg(ioat, compl_desc); /* we leave the channel locked to ensure in order submission */ - return &desc->txd; + return &compl_desc->txd; } static struct dma_async_tx_descriptor * @@ -730,7 +730,7 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result, dump_desc_dbg(ioat, compl_desc); /* we leave the channel locked to ensure in order submission */ - return &desc->txd; + return &compl_desc->txd; } static struct dma_async_tx_descriptor * -- cgit v1.2.2 From 56adf7e8127d601b172e180b44551ce83404348f Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Sun, 22 Nov 2009 12:10:10 -0700 Subject: shdma: fix initialization error handling 1/ Error handling code following a kzalloc should free the allocated data. 2/ Report an error when no platform data is detected Both problems fixed by moving the platform data check before the allocation, and allows a goto to be killed. Reported-by: Julia Lawall Acked-by: Julia Lawall Signed-off-by: Dan Williams --- drivers/dma/shdma.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/shdma.c b/drivers/dma/shdma.c index b3b065c4e5c1..034ecf0ace03 100644 --- a/drivers/dma/shdma.c +++ b/drivers/dma/shdma.c @@ -640,17 +640,16 @@ static int __init sh_dmae_probe(struct platform_device *pdev) #endif struct sh_dmae_device *shdev; + /* get platform data */ + if (!pdev->dev.platform_data) + return -ENODEV; + shdev = kzalloc(sizeof(struct sh_dmae_device), GFP_KERNEL); if (!shdev) { dev_err(&pdev->dev, "No enough memory\n"); - err = -ENOMEM; - goto shdev_err; + return -ENOMEM; } - /* get platform data */ - if (!pdev->dev.platform_data) - goto shdev_err; - /* platform data */ memcpy(&shdev->pdata, pdev->dev.platform_data, sizeof(struct sh_dmae_pdata)); @@ -722,7 +721,6 @@ eirq_err: rst_err: kfree(shdev); -shdev_err: return err; } -- cgit v1.2.2 From cd78809f6191485a90ea6c92c2b58900ab5c156f Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 17 Dec 2009 13:52:39 -0700 Subject: ioat3: fix p-disabled q-continuation When continuing a pq calculation the driver needs 3 extra sources. The driver can perform a 3 source calculation with a single descriptor, but needs an extended descriptor to process up to 8 sources in one operation. However, in the p-disabled case only one extra source is needed. When continuing a p-disabled operation there are occasions (i.e. 0 < src_cnt % 8 < 3) where the tail operation does not need an extended descriptor. Properly account for this fact otherwise invalid 'dmacount' values will be written to hardware usually causing the channel to halt with 'invalid descriptor' errors. Cc: Signed-off-by: Dan Williams --- drivers/dma/ioat/dma_v3.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index 42f6f10fb0cc..e58ecb29513f 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -650,9 +650,11 @@ __ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result, num_descs = ioat2_xferlen_to_descs(ioat, len); /* we need 2x the number of descriptors to cover greater than 3 - * sources + * sources (we need 1 extra source in the q-only continuation + * case and 3 extra sources in the p+q continuation case. */ - if (src_cnt > 3 || flags & DMA_PREP_CONTINUE) { + if (src_cnt + dmaf_p_disabled_continue(flags) > 3 || + (dmaf_continue(flags) && !dmaf_p_disabled_continue(flags))) { with_ext = 1; num_descs *= 2; } else -- cgit v1.2.2 From a6d52d70677e99bdb89b6921c265d0a58c22e597 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Sat, 19 Dec 2009 15:36:02 -0700 Subject: ioat2,3: put channel hardware in known state at init Put the ioat2 and ioat3 state machines in the halted state with all errors cleared. The ioat1 init path is not disturbed for stability, there are no reported ioat1 initiaization issues. Cc: Reported-by: Roland Dreier Tested-by: Roland Dreier Acked-by: Simon Horman Signed-off-by: Dan Williams --- drivers/dma/ioat/dma.c | 2 +- drivers/dma/ioat/dma.h | 18 ++++++++++++ drivers/dma/ioat/dma_v2.c | 69 +++++++++++++++++++++++++++++++++----------- drivers/dma/ioat/dma_v2.h | 2 ++ drivers/dma/ioat/dma_v3.c | 54 +++++++++++++++++++++++++--------- drivers/dma/ioat/registers.h | 1 + 6 files changed, 114 insertions(+), 32 deletions(-) (limited to 'drivers/dma') diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index c524d36d3c2e..dcc4ab78b32b 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -1032,7 +1032,7 @@ int __devinit ioat_probe(struct ioatdma_device *device) dma->dev = &pdev->dev; if (!dma->chancnt) { - dev_err(dev, "zero channels detected\n"); + dev_err(dev, "channel enumeration error\n"); goto err_setup_interrupts; } diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index 45edde996480..bbc3e78ef333 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -60,6 +60,7 @@ * @dca: direct cache access context * @intr_quirk: interrupt setup quirk (for ioat_v1 devices) * @enumerate_channels: hw version specific channel enumeration + * @reset_hw: hw version specific channel (re)initialization * @cleanup_tasklet: select between the v2 and v3 cleanup routines * @timer_fn: select between the v2 and v3 timer watchdog routines * @self_test: hardware version specific self test for each supported op type @@ -78,6 +79,7 @@ struct ioatdma_device { struct dca_provider *dca; void (*intr_quirk)(struct ioatdma_device *device); int (*enumerate_channels)(struct ioatdma_device *device); + int (*reset_hw)(struct ioat_chan_common *chan); void (*cleanup_tasklet)(unsigned long data); void (*timer_fn)(unsigned long data); int (*self_test)(struct ioatdma_device *device); @@ -264,6 +266,22 @@ static inline void ioat_suspend(struct ioat_chan_common *chan) writeb(IOAT_CHANCMD_SUSPEND, chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); } +static inline void ioat_reset(struct ioat_chan_common *chan) +{ + u8 ver = chan->device->version; + + writeb(IOAT_CHANCMD_RESET, chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); +} + +static inline bool ioat_reset_pending(struct ioat_chan_common *chan) +{ + u8 ver = chan->device->version; + u8 cmd; + + cmd = readb(chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); + return (cmd & IOAT_CHANCMD_RESET) == IOAT_CHANCMD_RESET; +} + static inline void ioat_set_chainaddr(struct ioat_dma_chan *ioat, u64 addr) { struct ioat_chan_common *chan = &ioat->base; diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c index 8f1f7f05deaa..5f7a500e18d0 100644 --- a/drivers/dma/ioat/dma_v2.c +++ b/drivers/dma/ioat/dma_v2.c @@ -239,20 +239,50 @@ void __ioat2_restart_chan(struct ioat2_dma_chan *ioat) __ioat2_start_null_desc(ioat); } -static void ioat2_restart_channel(struct ioat2_dma_chan *ioat) +int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo) { - struct ioat_chan_common *chan = &ioat->base; - unsigned long phys_complete; + unsigned long end = jiffies + tmo; + int err = 0; u32 status; status = ioat_chansts(chan); if (is_ioat_active(status) || is_ioat_idle(status)) ioat_suspend(chan); while (is_ioat_active(status) || is_ioat_idle(status)) { + if (end && time_after(jiffies, end)) { + err = -ETIMEDOUT; + break; + } status = ioat_chansts(chan); cpu_relax(); } + return err; +} + +int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo) +{ + unsigned long end = jiffies + tmo; + int err = 0; + + ioat_reset(chan); + while (ioat_reset_pending(chan)) { + if (end && time_after(jiffies, end)) { + err = -ETIMEDOUT; + break; + } + cpu_relax(); + } + + return err; +} + +static void ioat2_restart_channel(struct ioat2_dma_chan *ioat) +{ + struct ioat_chan_common *chan = &ioat->base; + unsigned long phys_complete; + + ioat2_quiesce(chan, 0); if (ioat_cleanup_preamble(chan, &phys_complete)) __cleanup(ioat, phys_complete); @@ -318,6 +348,19 @@ void ioat2_timer_event(unsigned long data) spin_unlock_bh(&chan->cleanup_lock); } +static int ioat2_reset_hw(struct ioat_chan_common *chan) +{ + /* throw away whatever the channel was doing and get it initialized */ + u32 chanerr; + + ioat2_quiesce(chan, msecs_to_jiffies(100)); + + chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); + writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); + + return ioat2_reset_sync(chan, msecs_to_jiffies(200)); +} + /** * ioat2_enumerate_channels - find and initialize the device's channels * @device: the device to be enumerated @@ -360,6 +403,10 @@ int ioat2_enumerate_channels(struct ioatdma_device *device) (unsigned long) ioat); ioat->xfercap_log = xfercap_log; spin_lock_init(&ioat->ring_lock); + if (device->reset_hw(&ioat->base)) { + i = 0; + break; + } } dma->chancnt = i; return i; @@ -467,7 +514,6 @@ int ioat2_alloc_chan_resources(struct dma_chan *c) struct ioat2_dma_chan *ioat = to_ioat2_chan(c); struct ioat_chan_common *chan = &ioat->base; struct ioat_ring_ent **ring; - u32 chanerr; int order; /* have we already been set up? */ @@ -477,12 +523,6 @@ int ioat2_alloc_chan_resources(struct dma_chan *c) /* Setup register to interrupt and write completion status on error */ writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET); - chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); - if (chanerr) { - dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr); - writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); - } - /* allocate a completion writeback area */ /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ chan->completion = pci_pool_alloc(chan->device->completion_pool, @@ -746,13 +786,7 @@ void ioat2_free_chan_resources(struct dma_chan *c) tasklet_disable(&chan->cleanup_task); del_timer_sync(&chan->timer); device->cleanup_tasklet((unsigned long) ioat); - - /* Delay 100ms after reset to allow internal DMA logic to quiesce - * before removing DMA descriptor resources. - */ - writeb(IOAT_CHANCMD_RESET, - chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version)); - mdelay(100); + device->reset_hw(chan); spin_lock_bh(&ioat->ring_lock); descs = ioat2_ring_space(ioat); @@ -839,6 +873,7 @@ int __devinit ioat2_dma_probe(struct ioatdma_device *device, int dca) int err; device->enumerate_channels = ioat2_enumerate_channels; + device->reset_hw = ioat2_reset_hw; device->cleanup_tasklet = ioat2_cleanup_tasklet; device->timer_fn = ioat2_timer_event; device->self_test = ioat_dma_self_test; diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h index 1d849ef74d5f..3afad8da43cc 100644 --- a/drivers/dma/ioat/dma_v2.h +++ b/drivers/dma/ioat/dma_v2.h @@ -185,6 +185,8 @@ bool reshape_ring(struct ioat2_dma_chan *ioat, int order); void __ioat2_issue_pending(struct ioat2_dma_chan *ioat); void ioat2_cleanup_tasklet(unsigned long data); void ioat2_timer_event(unsigned long data); +int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo); +int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo); extern struct kobj_type ioat2_ktype; extern struct kmem_cache *ioat2_cache; #endif /* IOATDMA_V2_H */ diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index e58ecb29513f..9908c9e94b2d 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -1130,6 +1130,45 @@ static int __devinit ioat3_dma_self_test(struct ioatdma_device *device) return 0; } +static int ioat3_reset_hw(struct ioat_chan_common *chan) +{ + /* throw away whatever the channel was doing and get it + * initialized, with ioat3 specific workarounds + */ + struct ioatdma_device *device = chan->device; + struct pci_dev *pdev = device->pdev; + u32 chanerr; + u16 dev_id; + int err; + + ioat2_quiesce(chan, msecs_to_jiffies(100)); + + chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); + writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); + + /* -= IOAT ver.3 workarounds =- */ + /* Write CHANERRMSK_INT with 3E07h to mask out the errors + * that can cause stability issues for IOAT ver.3, and clear any + * pending errors + */ + pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07); + err = pci_read_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, &chanerr); + if (err) { + dev_err(&pdev->dev, "channel error register unreachable\n"); + return err; + } + pci_write_config_dword(pdev, IOAT_PCI_CHANERR_INT_OFFSET, chanerr); + + /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit + * (workaround for spurious config parity error after restart) + */ + pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id); + if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) + pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10); + + return ioat2_reset_sync(chan, msecs_to_jiffies(200)); +} + int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) { struct pci_dev *pdev = device->pdev; @@ -1139,10 +1178,10 @@ int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) struct ioat_chan_common *chan; bool is_raid_device = false; int err; - u16 dev_id; u32 cap; device->enumerate_channels = ioat2_enumerate_channels; + device->reset_hw = ioat3_reset_hw; device->self_test = ioat3_dma_self_test; dma = &device->common; dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock; @@ -1218,19 +1257,6 @@ int __devinit ioat3_dma_probe(struct ioatdma_device *device, int dca) dma->device_prep_dma_xor_val = NULL; #endif - /* -= IOAT ver.3 workarounds =- */ - /* Write CHANERRMSK_INT with 3E07h to mask out the errors - * that can cause stability issues for IOAT ver.3 - */ - pci_write_config_dword(pdev, IOAT_PCI_CHANERRMASK_INT_OFFSET, 0x3e07); - - /* Clear DMAUNCERRSTS Cfg-Reg Parity Error status bit - * (workaround for spurious config parity error after restart) - */ - pci_read_config_word(pdev, IOAT_PCI_DEVICE_ID_OFFSET, &dev_id); - if (dev_id == PCI_DEVICE_ID_INTEL_IOAT_TBG0) - pci_write_config_dword(pdev, IOAT_PCI_DMAUNCERRSTS_OFFSET, 0x10); - err = ioat_probe(device); if (err) return err; diff --git a/drivers/dma/ioat/registers.h b/drivers/dma/ioat/registers.h index f015ec196700..e8ae63baf588 100644 --- a/drivers/dma/ioat/registers.h +++ b/drivers/dma/ioat/registers.h @@ -27,6 +27,7 @@ #define IOAT_PCI_DEVICE_ID_OFFSET 0x02 #define IOAT_PCI_DMAUNCERRSTS_OFFSET 0x148 +#define IOAT_PCI_CHANERR_INT_OFFSET 0x180 #define IOAT_PCI_CHANERRMASK_INT_OFFSET 0x184 /* MMIO Device Registers */ -- cgit v1.2.2