aboutsummaryrefslogtreecommitdiffstats
path: root/drivers
diff options
context:
space:
mode:
authorSuresh Siddha <suresh.b.siddha@intel.com>2008-07-10 14:16:42 -0400
committerIngo Molnar <mingo@elte.hu>2008-07-12 02:44:52 -0400
commitfe962e90cb17a8426e144dee970e77ed789d98ee (patch)
treec7b3343df9bf58e047333758a89c78f6615fb97b /drivers
parentcf1337f0447e5be8e66daa944f0ea3bcac2b6179 (diff)
x64, x2apic/intr-remap: Queued invalidation infrastructure (part of VT-d)
Queued invalidation (part of Intel Virtualization Technology for Directed I/O architecture) infrastructure. This will be used for invalidating the interrupt entry cache in the case of Interrupt-remapping and IOTLB invalidation in the case of DMA-remapping. Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com> Cc: akpm@linux-foundation.org Cc: arjan@linux.intel.com Cc: andi@firstfloor.org Cc: ebiederm@xmission.com Cc: jbarnes@virtuousgeek.org Cc: steiner@sgi.com Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'drivers')
-rw-r--r--drivers/pci/dmar.c150
-rw-r--r--drivers/pci/intel-iommu.c7
-rw-r--r--drivers/pci/intel-iommu.h61
3 files changed, 211 insertions, 7 deletions
diff --git a/drivers/pci/dmar.c b/drivers/pci/dmar.c
index 127764cfbe27..aba151ca6d26 100644
--- a/drivers/pci/dmar.c
+++ b/drivers/pci/dmar.c
@@ -28,6 +28,7 @@
28 28
29#include <linux/pci.h> 29#include <linux/pci.h>
30#include <linux/dmar.h> 30#include <linux/dmar.h>
31#include <linux/timer.h>
31#include "iova.h" 32#include "iova.h"
32#include "intel-iommu.h" 33#include "intel-iommu.h"
33 34
@@ -509,3 +510,152 @@ void free_iommu(struct intel_iommu *iommu)
509 iounmap(iommu->reg); 510 iounmap(iommu->reg);
510 kfree(iommu); 511 kfree(iommu);
511} 512}
513
514/*
515 * Reclaim all the submitted descriptors which have completed its work.
516 */
517static inline void reclaim_free_desc(struct q_inval *qi)
518{
519 while (qi->desc_status[qi->free_tail] == QI_DONE) {
520 qi->desc_status[qi->free_tail] = QI_FREE;
521 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
522 qi->free_cnt++;
523 }
524}
525
526/*
527 * Submit the queued invalidation descriptor to the remapping
528 * hardware unit and wait for its completion.
529 */
530void qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
531{
532 struct q_inval *qi = iommu->qi;
533 struct qi_desc *hw, wait_desc;
534 int wait_index, index;
535 unsigned long flags;
536
537 if (!qi)
538 return;
539
540 hw = qi->desc;
541
542 spin_lock(&qi->q_lock);
543 while (qi->free_cnt < 3) {
544 spin_unlock(&qi->q_lock);
545 cpu_relax();
546 spin_lock(&qi->q_lock);
547 }
548
549 index = qi->free_head;
550 wait_index = (index + 1) % QI_LENGTH;
551
552 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
553
554 hw[index] = *desc;
555
556 wait_desc.low = QI_IWD_STATUS_DATA(2) | QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
557 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
558
559 hw[wait_index] = wait_desc;
560
561 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
562 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
563
564 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
565 qi->free_cnt -= 2;
566
567 spin_lock_irqsave(&iommu->register_lock, flags);
568 /*
569 * update the HW tail register indicating the presence of
570 * new descriptors.
571 */
572 writel(qi->free_head << 4, iommu->reg + DMAR_IQT_REG);
573 spin_unlock_irqrestore(&iommu->register_lock, flags);
574
575 while (qi->desc_status[wait_index] != QI_DONE) {
576 spin_unlock(&qi->q_lock);
577 cpu_relax();
578 spin_lock(&qi->q_lock);
579 }
580
581 qi->desc_status[index] = QI_DONE;
582
583 reclaim_free_desc(qi);
584 spin_unlock(&qi->q_lock);
585}
586
587/*
588 * Flush the global interrupt entry cache.
589 */
590void qi_global_iec(struct intel_iommu *iommu)
591{
592 struct qi_desc desc;
593
594 desc.low = QI_IEC_TYPE;
595 desc.high = 0;
596
597 qi_submit_sync(&desc, iommu);
598}
599
600/*
601 * Enable Queued Invalidation interface. This is a must to support
602 * interrupt-remapping. Also used by DMA-remapping, which replaces
603 * register based IOTLB invalidation.
604 */
605int dmar_enable_qi(struct intel_iommu *iommu)
606{
607 u32 cmd, sts;
608 unsigned long flags;
609 struct q_inval *qi;
610
611 if (!ecap_qis(iommu->ecap))
612 return -ENOENT;
613
614 /*
615 * queued invalidation is already setup and enabled.
616 */
617 if (iommu->qi)
618 return 0;
619
620 iommu->qi = kmalloc(sizeof(*qi), GFP_KERNEL);
621 if (!iommu->qi)
622 return -ENOMEM;
623
624 qi = iommu->qi;
625
626 qi->desc = (void *)(get_zeroed_page(GFP_KERNEL));
627 if (!qi->desc) {
628 kfree(qi);
629 iommu->qi = 0;
630 return -ENOMEM;
631 }
632
633 qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_KERNEL);
634 if (!qi->desc_status) {
635 free_page((unsigned long) qi->desc);
636 kfree(qi);
637 iommu->qi = 0;
638 return -ENOMEM;
639 }
640
641 qi->free_head = qi->free_tail = 0;
642 qi->free_cnt = QI_LENGTH;
643
644 spin_lock_init(&qi->q_lock);
645
646 spin_lock_irqsave(&iommu->register_lock, flags);
647 /* write zero to the tail reg */
648 writel(0, iommu->reg + DMAR_IQT_REG);
649
650 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
651
652 cmd = iommu->gcmd | DMA_GCMD_QIE;
653 iommu->gcmd |= DMA_GCMD_QIE;
654 writel(cmd, iommu->reg + DMAR_GCMD_REG);
655
656 /* Make sure hardware complete it */
657 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
658 spin_unlock_irqrestore(&iommu->register_lock, flags);
659
660 return 0;
661}
diff --git a/drivers/pci/intel-iommu.c b/drivers/pci/intel-iommu.c
index fb701d9dd8c0..347bf2e47168 100644
--- a/drivers/pci/intel-iommu.c
+++ b/drivers/pci/intel-iommu.c
@@ -181,13 +181,6 @@ void free_iova_mem(struct iova *iova)
181 kmem_cache_free(iommu_iova_cache, iova); 181 kmem_cache_free(iommu_iova_cache, iova);
182} 182}
183 183
184static inline void __iommu_flush_cache(
185 struct intel_iommu *iommu, void *addr, int size)
186{
187 if (!ecap_coherent(iommu->ecap))
188 clflush_cache_range(addr, size);
189}
190
191/* Gets context entry for a given bus and devfn */ 184/* Gets context entry for a given bus and devfn */
192static struct context_entry * device_to_context_entry(struct intel_iommu *iommu, 185static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
193 u8 bus, u8 devfn) 186 u8 bus, u8 devfn)
diff --git a/drivers/pci/intel-iommu.h b/drivers/pci/intel-iommu.h
index 3a650e8cba33..2983ce895353 100644
--- a/drivers/pci/intel-iommu.h
+++ b/drivers/pci/intel-iommu.h
@@ -27,6 +27,7 @@
27#include <linux/sysdev.h> 27#include <linux/sysdev.h>
28#include "iova.h" 28#include "iova.h"
29#include <linux/io.h> 29#include <linux/io.h>
30#include <asm/cacheflush.h>
30#include "dma_remapping.h" 31#include "dma_remapping.h"
31 32
32/* 33/*
@@ -51,6 +52,10 @@
51#define DMAR_PLMLIMIT_REG 0x6c /* PMRR low limit */ 52#define DMAR_PLMLIMIT_REG 0x6c /* PMRR low limit */
52#define DMAR_PHMBASE_REG 0x70 /* pmrr high base addr */ 53#define DMAR_PHMBASE_REG 0x70 /* pmrr high base addr */
53#define DMAR_PHMLIMIT_REG 0x78 /* pmrr high limit */ 54#define DMAR_PHMLIMIT_REG 0x78 /* pmrr high limit */
55#define DMAR_IQH_REG 0x80 /* Invalidation queue head register */
56#define DMAR_IQT_REG 0x88 /* Invalidation queue tail register */
57#define DMAR_IQA_REG 0x90 /* Invalidation queue addr register */
58#define DMAR_ICS_REG 0x98 /* Invalidation complete status register */
54 59
55#define OFFSET_STRIDE (9) 60#define OFFSET_STRIDE (9)
56/* 61/*
@@ -114,6 +119,7 @@ static inline void dmar_writeq(void __iomem *addr, u64 val)
114#define ecap_max_iotlb_offset(e) \ 119#define ecap_max_iotlb_offset(e) \
115 (ecap_iotlb_offset(e) + ecap_niotlb_iunits(e) * 16) 120 (ecap_iotlb_offset(e) + ecap_niotlb_iunits(e) * 16)
116#define ecap_coherent(e) ((e) & 0x1) 121#define ecap_coherent(e) ((e) & 0x1)
122#define ecap_qis(e) ((e) & 0x2)
117#define ecap_eim_support(e) ((e >> 4) & 0x1) 123#define ecap_eim_support(e) ((e >> 4) & 0x1)
118#define ecap_ir_support(e) ((e >> 3) & 0x1) 124#define ecap_ir_support(e) ((e >> 3) & 0x1)
119 125
@@ -131,6 +137,17 @@ static inline void dmar_writeq(void __iomem *addr, u64 val)
131#define DMA_TLB_IH_NONLEAF (((u64)1) << 6) 137#define DMA_TLB_IH_NONLEAF (((u64)1) << 6)
132#define DMA_TLB_MAX_SIZE (0x3f) 138#define DMA_TLB_MAX_SIZE (0x3f)
133 139
140/* INVALID_DESC */
141#define DMA_ID_TLB_GLOBAL_FLUSH (((u64)1) << 3)
142#define DMA_ID_TLB_DSI_FLUSH (((u64)2) << 3)
143#define DMA_ID_TLB_PSI_FLUSH (((u64)3) << 3)
144#define DMA_ID_TLB_READ_DRAIN (((u64)1) << 7)
145#define DMA_ID_TLB_WRITE_DRAIN (((u64)1) << 6)
146#define DMA_ID_TLB_DID(id) (((u64)((id & 0xffff) << 16)))
147#define DMA_ID_TLB_IH_NONLEAF (((u64)1) << 6)
148#define DMA_ID_TLB_ADDR(addr) (addr)
149#define DMA_ID_TLB_ADDR_MASK(mask) (mask)
150
134/* PMEN_REG */ 151/* PMEN_REG */
135#define DMA_PMEN_EPM (((u32)1)<<31) 152#define DMA_PMEN_EPM (((u32)1)<<31)
136#define DMA_PMEN_PRS (((u32)1)<<0) 153#define DMA_PMEN_PRS (((u32)1)<<0)
@@ -140,6 +157,7 @@ static inline void dmar_writeq(void __iomem *addr, u64 val)
140#define DMA_GCMD_SRTP (((u32)1) << 30) 157#define DMA_GCMD_SRTP (((u32)1) << 30)
141#define DMA_GCMD_SFL (((u32)1) << 29) 158#define DMA_GCMD_SFL (((u32)1) << 29)
142#define DMA_GCMD_EAFL (((u32)1) << 28) 159#define DMA_GCMD_EAFL (((u32)1) << 28)
160#define DMA_GCMD_QIE (((u32)1) << 26)
143#define DMA_GCMD_WBF (((u32)1) << 27) 161#define DMA_GCMD_WBF (((u32)1) << 27)
144 162
145/* GSTS_REG */ 163/* GSTS_REG */
@@ -147,6 +165,7 @@ static inline void dmar_writeq(void __iomem *addr, u64 val)
147#define DMA_GSTS_RTPS (((u32)1) << 30) 165#define DMA_GSTS_RTPS (((u32)1) << 30)
148#define DMA_GSTS_FLS (((u32)1) << 29) 166#define DMA_GSTS_FLS (((u32)1) << 29)
149#define DMA_GSTS_AFLS (((u32)1) << 28) 167#define DMA_GSTS_AFLS (((u32)1) << 28)
168#define DMA_GSTS_QIES (((u32)1) << 26)
150#define DMA_GSTS_WBFS (((u32)1) << 27) 169#define DMA_GSTS_WBFS (((u32)1) << 27)
151 170
152/* CCMD_REG */ 171/* CCMD_REG */
@@ -192,6 +211,40 @@ static inline void dmar_writeq(void __iomem *addr, u64 val)
192 }\ 211 }\
193} 212}
194 213
214#define QI_LENGTH 256 /* queue length */
215
216enum {
217 QI_FREE,
218 QI_IN_USE,
219 QI_DONE
220};
221
222#define QI_CC_TYPE 0x1
223#define QI_IOTLB_TYPE 0x2
224#define QI_DIOTLB_TYPE 0x3
225#define QI_IEC_TYPE 0x4
226#define QI_IWD_TYPE 0x5
227
228#define QI_IEC_SELECTIVE (((u64)1) << 4)
229#define QI_IEC_IIDEX(idx) (((u64)(idx & 0xffff) << 32))
230#define QI_IEC_IM(m) (((u64)(m & 0x1f) << 27))
231
232#define QI_IWD_STATUS_DATA(d) (((u64)d) << 32)
233#define QI_IWD_STATUS_WRITE (((u64)1) << 5)
234
235struct qi_desc {
236 u64 low, high;
237};
238
239struct q_inval {
240 spinlock_t q_lock;
241 struct qi_desc *desc; /* invalidation queue */
242 int *desc_status; /* desc status */
243 int free_head; /* first free entry */
244 int free_tail; /* last free entry */
245 int free_cnt;
246};
247
195struct intel_iommu { 248struct intel_iommu {
196 void __iomem *reg; /* Pointer to hardware regs, virtual addr */ 249 void __iomem *reg; /* Pointer to hardware regs, virtual addr */
197 u64 cap; 250 u64 cap;
@@ -212,8 +265,16 @@ struct intel_iommu {
212 struct msi_msg saved_msg; 265 struct msi_msg saved_msg;
213 struct sys_device sysdev; 266 struct sys_device sysdev;
214#endif 267#endif
268 struct q_inval *qi; /* Queued invalidation info */
215}; 269};
216 270
271static inline void __iommu_flush_cache(
272 struct intel_iommu *iommu, void *addr, int size)
273{
274 if (!ecap_coherent(iommu->ecap))
275 clflush_cache_range(addr, size);
276}
277
217extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev); 278extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev);
218 279
219extern int alloc_iommu(struct dmar_drhd_unit *drhd); 280extern int alloc_iommu(struct dmar_drhd_unit *drhd);