aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/iommu
diff options
context:
space:
mode:
authorOhad Ben-Cohen <ohad@wizery.com>2011-06-10 14:42:27 -0400
committerJoerg Roedel <joerg.roedel@amd.com>2011-06-21 04:49:30 -0400
commit166e9278a3f98bab29ebb3d685a81cfb11b98be0 (patch)
treef8f3e8a28c5d96d9053567d6a9ef8e04e7b298dd /drivers/iommu
parent29b68415e335ba9e0eb6057f9405aa4d9c23efe4 (diff)
x86/ia64: intel-iommu: move to drivers/iommu/
This should ease finding similarities with different platforms, with the intention of solving problems once in a generic framework which everyone can use. Note: to move intel-iommu.c, the declaration of pci_find_upstream_pcie_bridge() has to move from drivers/pci/pci.h to include/linux/pci.h. This is handled in this patch, too. As suggested, also drop DMAR's EXPERIMENTAL tag while we're at it. Compile-tested on x86_64. Signed-off-by: Ohad Ben-Cohen <ohad@wizery.com> Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>
Diffstat (limited to 'drivers/iommu')
-rw-r--r--drivers/iommu/Kconfig49
-rw-r--r--drivers/iommu/Makefile2
-rw-r--r--drivers/iommu/dmar.c1461
-rw-r--r--drivers/iommu/intel-iommu.c4016
-rw-r--r--drivers/iommu/intr_remapping.c797
-rw-r--r--drivers/iommu/intr_remapping.h17
-rw-r--r--drivers/iommu/iova.c435
7 files changed, 6777 insertions, 0 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 9246c5bf25af..e2a5f141ae2d 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -46,3 +46,52 @@ config AMD_IOMMU_STATS
46 statistics about whats happening in the driver and exports that 46 statistics about whats happening in the driver and exports that
47 information to userspace via debugfs. 47 information to userspace via debugfs.
48 If unsure, say N. 48 If unsure, say N.
49
50# Intel IOMMU support
51config DMAR
52 bool "Support for DMA Remapping Devices"
53 depends on PCI_MSI && ACPI && (X86 || IA64_GENERIC)
54 select IOMMU_API
55 help
56 DMA remapping (DMAR) devices support enables independent address
57 translations for Direct Memory Access (DMA) from devices.
58 These DMA remapping devices are reported via ACPI tables
59 and include PCI device scope covered by these DMA
60 remapping devices.
61
62config DMAR_DEFAULT_ON
63 def_bool y
64 prompt "Enable DMA Remapping Devices by default"
65 depends on DMAR
66 help
67 Selecting this option will enable a DMAR device at boot time if
68 one is found. If this option is not selected, DMAR support can
69 be enabled by passing intel_iommu=on to the kernel.
70
71config DMAR_BROKEN_GFX_WA
72 bool "Workaround broken graphics drivers (going away soon)"
73 depends on DMAR && BROKEN && X86
74 ---help---
75 Current Graphics drivers tend to use physical address
76 for DMA and avoid using DMA APIs. Setting this config
77 option permits the IOMMU driver to set a unity map for
78 all the OS-visible memory. Hence the driver can continue
79 to use physical addresses for DMA, at least until this
80 option is removed in the 2.6.32 kernel.
81
82config DMAR_FLOPPY_WA
83 def_bool y
84 depends on DMAR && X86
85 ---help---
86 Floppy disk drivers are known to bypass DMA API calls
87 thereby failing to work when IOMMU is enabled. This
88 workaround will setup a 1:1 mapping for the first
89 16MiB to make floppy (an ISA device) work.
90
91config INTR_REMAP
92 bool "Support for Interrupt Remapping (EXPERIMENTAL)"
93 depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI && EXPERIMENTAL
94 ---help---
95 Supports Interrupt remapping for IO-APIC and MSI devices.
96 To use x2apic mode in the CPU's which support x2APIC enhancements or
97 to support platforms with CPU's having > 8 bit APIC ID, say Y.
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 4237eaf84609..49e9c0f46bd5 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -1,3 +1,5 @@
1obj-$(CONFIG_IOMMU_API) += iommu.o 1obj-$(CONFIG_IOMMU_API) += iommu.o
2obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o 2obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o
3obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o 3obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o
4obj-$(CONFIG_DMAR) += dmar.o iova.o intel-iommu.o
5obj-$(CONFIG_INTR_REMAP) += dmar.o intr_remapping.o
diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c
new file mode 100644
index 000000000000..3dc9befa5aec
--- /dev/null
+++ b/drivers/iommu/dmar.c
@@ -0,0 +1,1461 @@
1/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 *
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
25 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
27 */
28
29#include <linux/pci.h>
30#include <linux/dmar.h>
31#include <linux/iova.h>
32#include <linux/intel-iommu.h>
33#include <linux/timer.h>
34#include <linux/irq.h>
35#include <linux/interrupt.h>
36#include <linux/tboot.h>
37#include <linux/dmi.h>
38#include <linux/slab.h>
39#include <asm/iommu_table.h>
40
41#define PREFIX "DMAR: "
42
43/* No locks are needed as DMA remapping hardware unit
44 * list is constructed at boot time and hotplug of
45 * these units are not supported by the architecture.
46 */
47LIST_HEAD(dmar_drhd_units);
48
49static struct acpi_table_header * __initdata dmar_tbl;
50static acpi_size dmar_tbl_size;
51
52static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
53{
54 /*
55 * add INCLUDE_ALL at the tail, so scan the list will find it at
56 * the very end.
57 */
58 if (drhd->include_all)
59 list_add_tail(&drhd->list, &dmar_drhd_units);
60 else
61 list_add(&drhd->list, &dmar_drhd_units);
62}
63
64static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
65 struct pci_dev **dev, u16 segment)
66{
67 struct pci_bus *bus;
68 struct pci_dev *pdev = NULL;
69 struct acpi_dmar_pci_path *path;
70 int count;
71
72 bus = pci_find_bus(segment, scope->bus);
73 path = (struct acpi_dmar_pci_path *)(scope + 1);
74 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
75 / sizeof(struct acpi_dmar_pci_path);
76
77 while (count) {
78 if (pdev)
79 pci_dev_put(pdev);
80 /*
81 * Some BIOSes list non-exist devices in DMAR table, just
82 * ignore it
83 */
84 if (!bus) {
85 printk(KERN_WARNING
86 PREFIX "Device scope bus [%d] not found\n",
87 scope->bus);
88 break;
89 }
90 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
91 if (!pdev) {
92 printk(KERN_WARNING PREFIX
93 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
94 segment, bus->number, path->dev, path->fn);
95 break;
96 }
97 path ++;
98 count --;
99 bus = pdev->subordinate;
100 }
101 if (!pdev) {
102 printk(KERN_WARNING PREFIX
103 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
104 segment, scope->bus, path->dev, path->fn);
105 *dev = NULL;
106 return 0;
107 }
108 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
109 pdev->subordinate) || (scope->entry_type == \
110 ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
111 pci_dev_put(pdev);
112 printk(KERN_WARNING PREFIX
113 "Device scope type does not match for %s\n",
114 pci_name(pdev));
115 return -EINVAL;
116 }
117 *dev = pdev;
118 return 0;
119}
120
121static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
122 struct pci_dev ***devices, u16 segment)
123{
124 struct acpi_dmar_device_scope *scope;
125 void * tmp = start;
126 int index;
127 int ret;
128
129 *cnt = 0;
130 while (start < end) {
131 scope = start;
132 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
133 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
134 (*cnt)++;
135 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
136 printk(KERN_WARNING PREFIX
137 "Unsupported device scope\n");
138 }
139 start += scope->length;
140 }
141 if (*cnt == 0)
142 return 0;
143
144 *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
145 if (!*devices)
146 return -ENOMEM;
147
148 start = tmp;
149 index = 0;
150 while (start < end) {
151 scope = start;
152 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
153 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
154 ret = dmar_parse_one_dev_scope(scope,
155 &(*devices)[index], segment);
156 if (ret) {
157 kfree(*devices);
158 return ret;
159 }
160 index ++;
161 }
162 start += scope->length;
163 }
164
165 return 0;
166}
167
168/**
169 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
170 * structure which uniquely represent one DMA remapping hardware unit
171 * present in the platform
172 */
173static int __init
174dmar_parse_one_drhd(struct acpi_dmar_header *header)
175{
176 struct acpi_dmar_hardware_unit *drhd;
177 struct dmar_drhd_unit *dmaru;
178 int ret = 0;
179
180 drhd = (struct acpi_dmar_hardware_unit *)header;
181 dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
182 if (!dmaru)
183 return -ENOMEM;
184
185 dmaru->hdr = header;
186 dmaru->reg_base_addr = drhd->address;
187 dmaru->segment = drhd->segment;
188 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
189
190 ret = alloc_iommu(dmaru);
191 if (ret) {
192 kfree(dmaru);
193 return ret;
194 }
195 dmar_register_drhd_unit(dmaru);
196 return 0;
197}
198
199static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
200{
201 struct acpi_dmar_hardware_unit *drhd;
202 int ret = 0;
203
204 drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
205
206 if (dmaru->include_all)
207 return 0;
208
209 ret = dmar_parse_dev_scope((void *)(drhd + 1),
210 ((void *)drhd) + drhd->header.length,
211 &dmaru->devices_cnt, &dmaru->devices,
212 drhd->segment);
213 if (ret) {
214 list_del(&dmaru->list);
215 kfree(dmaru);
216 }
217 return ret;
218}
219
220#ifdef CONFIG_DMAR
221LIST_HEAD(dmar_rmrr_units);
222
223static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
224{
225 list_add(&rmrr->list, &dmar_rmrr_units);
226}
227
228
229static int __init
230dmar_parse_one_rmrr(struct acpi_dmar_header *header)
231{
232 struct acpi_dmar_reserved_memory *rmrr;
233 struct dmar_rmrr_unit *rmrru;
234
235 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
236 if (!rmrru)
237 return -ENOMEM;
238
239 rmrru->hdr = header;
240 rmrr = (struct acpi_dmar_reserved_memory *)header;
241 rmrru->base_address = rmrr->base_address;
242 rmrru->end_address = rmrr->end_address;
243
244 dmar_register_rmrr_unit(rmrru);
245 return 0;
246}
247
248static int __init
249rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
250{
251 struct acpi_dmar_reserved_memory *rmrr;
252 int ret;
253
254 rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
255 ret = dmar_parse_dev_scope((void *)(rmrr + 1),
256 ((void *)rmrr) + rmrr->header.length,
257 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
258
259 if (ret || (rmrru->devices_cnt == 0)) {
260 list_del(&rmrru->list);
261 kfree(rmrru);
262 }
263 return ret;
264}
265
266static LIST_HEAD(dmar_atsr_units);
267
268static int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
269{
270 struct acpi_dmar_atsr *atsr;
271 struct dmar_atsr_unit *atsru;
272
273 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
274 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
275 if (!atsru)
276 return -ENOMEM;
277
278 atsru->hdr = hdr;
279 atsru->include_all = atsr->flags & 0x1;
280
281 list_add(&atsru->list, &dmar_atsr_units);
282
283 return 0;
284}
285
286static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
287{
288 int rc;
289 struct acpi_dmar_atsr *atsr;
290
291 if (atsru->include_all)
292 return 0;
293
294 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
295 rc = dmar_parse_dev_scope((void *)(atsr + 1),
296 (void *)atsr + atsr->header.length,
297 &atsru->devices_cnt, &atsru->devices,
298 atsr->segment);
299 if (rc || !atsru->devices_cnt) {
300 list_del(&atsru->list);
301 kfree(atsru);
302 }
303
304 return rc;
305}
306
307int dmar_find_matched_atsr_unit(struct pci_dev *dev)
308{
309 int i;
310 struct pci_bus *bus;
311 struct acpi_dmar_atsr *atsr;
312 struct dmar_atsr_unit *atsru;
313
314 dev = pci_physfn(dev);
315
316 list_for_each_entry(atsru, &dmar_atsr_units, list) {
317 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
318 if (atsr->segment == pci_domain_nr(dev->bus))
319 goto found;
320 }
321
322 return 0;
323
324found:
325 for (bus = dev->bus; bus; bus = bus->parent) {
326 struct pci_dev *bridge = bus->self;
327
328 if (!bridge || !pci_is_pcie(bridge) ||
329 bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
330 return 0;
331
332 if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
333 for (i = 0; i < atsru->devices_cnt; i++)
334 if (atsru->devices[i] == bridge)
335 return 1;
336 break;
337 }
338 }
339
340 if (atsru->include_all)
341 return 1;
342
343 return 0;
344}
345#endif
346
347#ifdef CONFIG_ACPI_NUMA
348static int __init
349dmar_parse_one_rhsa(struct acpi_dmar_header *header)
350{
351 struct acpi_dmar_rhsa *rhsa;
352 struct dmar_drhd_unit *drhd;
353
354 rhsa = (struct acpi_dmar_rhsa *)header;
355 for_each_drhd_unit(drhd) {
356 if (drhd->reg_base_addr == rhsa->base_address) {
357 int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
358
359 if (!node_online(node))
360 node = -1;
361 drhd->iommu->node = node;
362 return 0;
363 }
364 }
365 WARN_TAINT(
366 1, TAINT_FIRMWARE_WORKAROUND,
367 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
368 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
369 drhd->reg_base_addr,
370 dmi_get_system_info(DMI_BIOS_VENDOR),
371 dmi_get_system_info(DMI_BIOS_VERSION),
372 dmi_get_system_info(DMI_PRODUCT_VERSION));
373
374 return 0;
375}
376#endif
377
378static void __init
379dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
380{
381 struct acpi_dmar_hardware_unit *drhd;
382 struct acpi_dmar_reserved_memory *rmrr;
383 struct acpi_dmar_atsr *atsr;
384 struct acpi_dmar_rhsa *rhsa;
385
386 switch (header->type) {
387 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
388 drhd = container_of(header, struct acpi_dmar_hardware_unit,
389 header);
390 printk (KERN_INFO PREFIX
391 "DRHD base: %#016Lx flags: %#x\n",
392 (unsigned long long)drhd->address, drhd->flags);
393 break;
394 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
395 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
396 header);
397 printk (KERN_INFO PREFIX
398 "RMRR base: %#016Lx end: %#016Lx\n",
399 (unsigned long long)rmrr->base_address,
400 (unsigned long long)rmrr->end_address);
401 break;
402 case ACPI_DMAR_TYPE_ATSR:
403 atsr = container_of(header, struct acpi_dmar_atsr, header);
404 printk(KERN_INFO PREFIX "ATSR flags: %#x\n", atsr->flags);
405 break;
406 case ACPI_DMAR_HARDWARE_AFFINITY:
407 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
408 printk(KERN_INFO PREFIX "RHSA base: %#016Lx proximity domain: %#x\n",
409 (unsigned long long)rhsa->base_address,
410 rhsa->proximity_domain);
411 break;
412 }
413}
414
415/**
416 * dmar_table_detect - checks to see if the platform supports DMAR devices
417 */
418static int __init dmar_table_detect(void)
419{
420 acpi_status status = AE_OK;
421
422 /* if we could find DMAR table, then there are DMAR devices */
423 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
424 (struct acpi_table_header **)&dmar_tbl,
425 &dmar_tbl_size);
426
427 if (ACPI_SUCCESS(status) && !dmar_tbl) {
428 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
429 status = AE_NOT_FOUND;
430 }
431
432 return (ACPI_SUCCESS(status) ? 1 : 0);
433}
434
435/**
436 * parse_dmar_table - parses the DMA reporting table
437 */
438static int __init
439parse_dmar_table(void)
440{
441 struct acpi_table_dmar *dmar;
442 struct acpi_dmar_header *entry_header;
443 int ret = 0;
444
445 /*
446 * Do it again, earlier dmar_tbl mapping could be mapped with
447 * fixed map.
448 */
449 dmar_table_detect();
450
451 /*
452 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
453 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
454 */
455 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
456
457 dmar = (struct acpi_table_dmar *)dmar_tbl;
458 if (!dmar)
459 return -ENODEV;
460
461 if (dmar->width < PAGE_SHIFT - 1) {
462 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
463 return -EINVAL;
464 }
465
466 printk (KERN_INFO PREFIX "Host address width %d\n",
467 dmar->width + 1);
468
469 entry_header = (struct acpi_dmar_header *)(dmar + 1);
470 while (((unsigned long)entry_header) <
471 (((unsigned long)dmar) + dmar_tbl->length)) {
472 /* Avoid looping forever on bad ACPI tables */
473 if (entry_header->length == 0) {
474 printk(KERN_WARNING PREFIX
475 "Invalid 0-length structure\n");
476 ret = -EINVAL;
477 break;
478 }
479
480 dmar_table_print_dmar_entry(entry_header);
481
482 switch (entry_header->type) {
483 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
484 ret = dmar_parse_one_drhd(entry_header);
485 break;
486 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
487#ifdef CONFIG_DMAR
488 ret = dmar_parse_one_rmrr(entry_header);
489#endif
490 break;
491 case ACPI_DMAR_TYPE_ATSR:
492#ifdef CONFIG_DMAR
493 ret = dmar_parse_one_atsr(entry_header);
494#endif
495 break;
496 case ACPI_DMAR_HARDWARE_AFFINITY:
497#ifdef CONFIG_ACPI_NUMA
498 ret = dmar_parse_one_rhsa(entry_header);
499#endif
500 break;
501 default:
502 printk(KERN_WARNING PREFIX
503 "Unknown DMAR structure type %d\n",
504 entry_header->type);
505 ret = 0; /* for forward compatibility */
506 break;
507 }
508 if (ret)
509 break;
510
511 entry_header = ((void *)entry_header + entry_header->length);
512 }
513 return ret;
514}
515
516static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
517 struct pci_dev *dev)
518{
519 int index;
520
521 while (dev) {
522 for (index = 0; index < cnt; index++)
523 if (dev == devices[index])
524 return 1;
525
526 /* Check our parent */
527 dev = dev->bus->self;
528 }
529
530 return 0;
531}
532
533struct dmar_drhd_unit *
534dmar_find_matched_drhd_unit(struct pci_dev *dev)
535{
536 struct dmar_drhd_unit *dmaru = NULL;
537 struct acpi_dmar_hardware_unit *drhd;
538
539 dev = pci_physfn(dev);
540
541 list_for_each_entry(dmaru, &dmar_drhd_units, list) {
542 drhd = container_of(dmaru->hdr,
543 struct acpi_dmar_hardware_unit,
544 header);
545
546 if (dmaru->include_all &&
547 drhd->segment == pci_domain_nr(dev->bus))
548 return dmaru;
549
550 if (dmar_pci_device_match(dmaru->devices,
551 dmaru->devices_cnt, dev))
552 return dmaru;
553 }
554
555 return NULL;
556}
557
558int __init dmar_dev_scope_init(void)
559{
560 struct dmar_drhd_unit *drhd, *drhd_n;
561 int ret = -ENODEV;
562
563 list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
564 ret = dmar_parse_dev(drhd);
565 if (ret)
566 return ret;
567 }
568
569#ifdef CONFIG_DMAR
570 {
571 struct dmar_rmrr_unit *rmrr, *rmrr_n;
572 struct dmar_atsr_unit *atsr, *atsr_n;
573
574 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
575 ret = rmrr_parse_dev(rmrr);
576 if (ret)
577 return ret;
578 }
579
580 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
581 ret = atsr_parse_dev(atsr);
582 if (ret)
583 return ret;
584 }
585 }
586#endif
587
588 return ret;
589}
590
591
592int __init dmar_table_init(void)
593{
594 static int dmar_table_initialized;
595 int ret;
596
597 if (dmar_table_initialized)
598 return 0;
599
600 dmar_table_initialized = 1;
601
602 ret = parse_dmar_table();
603 if (ret) {
604 if (ret != -ENODEV)
605 printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
606 return ret;
607 }
608
609 if (list_empty(&dmar_drhd_units)) {
610 printk(KERN_INFO PREFIX "No DMAR devices found\n");
611 return -ENODEV;
612 }
613
614#ifdef CONFIG_DMAR
615 if (list_empty(&dmar_rmrr_units))
616 printk(KERN_INFO PREFIX "No RMRR found\n");
617
618 if (list_empty(&dmar_atsr_units))
619 printk(KERN_INFO PREFIX "No ATSR found\n");
620#endif
621
622 return 0;
623}
624
625static void warn_invalid_dmar(u64 addr, const char *message)
626{
627 WARN_TAINT_ONCE(
628 1, TAINT_FIRMWARE_WORKAROUND,
629 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
630 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
631 addr, message,
632 dmi_get_system_info(DMI_BIOS_VENDOR),
633 dmi_get_system_info(DMI_BIOS_VERSION),
634 dmi_get_system_info(DMI_PRODUCT_VERSION));
635}
636
637int __init check_zero_address(void)
638{
639 struct acpi_table_dmar *dmar;
640 struct acpi_dmar_header *entry_header;
641 struct acpi_dmar_hardware_unit *drhd;
642
643 dmar = (struct acpi_table_dmar *)dmar_tbl;
644 entry_header = (struct acpi_dmar_header *)(dmar + 1);
645
646 while (((unsigned long)entry_header) <
647 (((unsigned long)dmar) + dmar_tbl->length)) {
648 /* Avoid looping forever on bad ACPI tables */
649 if (entry_header->length == 0) {
650 printk(KERN_WARNING PREFIX
651 "Invalid 0-length structure\n");
652 return 0;
653 }
654
655 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
656 void __iomem *addr;
657 u64 cap, ecap;
658
659 drhd = (void *)entry_header;
660 if (!drhd->address) {
661 warn_invalid_dmar(0, "");
662 goto failed;
663 }
664
665 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
666 if (!addr ) {
667 printk("IOMMU: can't validate: %llx\n", drhd->address);
668 goto failed;
669 }
670 cap = dmar_readq(addr + DMAR_CAP_REG);
671 ecap = dmar_readq(addr + DMAR_ECAP_REG);
672 early_iounmap(addr, VTD_PAGE_SIZE);
673 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
674 warn_invalid_dmar(drhd->address,
675 " returns all ones");
676 goto failed;
677 }
678 }
679
680 entry_header = ((void *)entry_header + entry_header->length);
681 }
682 return 1;
683
684failed:
685#ifdef CONFIG_DMAR
686 dmar_disabled = 1;
687#endif
688 return 0;
689}
690
691int __init detect_intel_iommu(void)
692{
693 int ret;
694
695 ret = dmar_table_detect();
696 if (ret)
697 ret = check_zero_address();
698 {
699#ifdef CONFIG_INTR_REMAP
700 struct acpi_table_dmar *dmar;
701
702 dmar = (struct acpi_table_dmar *) dmar_tbl;
703 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
704 printk(KERN_INFO
705 "Queued invalidation will be enabled to support "
706 "x2apic and Intr-remapping.\n");
707#endif
708#ifdef CONFIG_DMAR
709 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
710 iommu_detected = 1;
711 /* Make sure ACS will be enabled */
712 pci_request_acs();
713 }
714#endif
715#ifdef CONFIG_X86
716 if (ret)
717 x86_init.iommu.iommu_init = intel_iommu_init;
718#endif
719 }
720 early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
721 dmar_tbl = NULL;
722
723 return ret ? 1 : -ENODEV;
724}
725
726
727int alloc_iommu(struct dmar_drhd_unit *drhd)
728{
729 struct intel_iommu *iommu;
730 int map_size;
731 u32 ver;
732 static int iommu_allocated = 0;
733 int agaw = 0;
734 int msagaw = 0;
735
736 if (!drhd->reg_base_addr) {
737 warn_invalid_dmar(0, "");
738 return -EINVAL;
739 }
740
741 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
742 if (!iommu)
743 return -ENOMEM;
744
745 iommu->seq_id = iommu_allocated++;
746 sprintf (iommu->name, "dmar%d", iommu->seq_id);
747
748 iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
749 if (!iommu->reg) {
750 printk(KERN_ERR "IOMMU: can't map the region\n");
751 goto error;
752 }
753 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
754 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
755
756 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
757 warn_invalid_dmar(drhd->reg_base_addr, " returns all ones");
758 goto err_unmap;
759 }
760
761#ifdef CONFIG_DMAR
762 agaw = iommu_calculate_agaw(iommu);
763 if (agaw < 0) {
764 printk(KERN_ERR
765 "Cannot get a valid agaw for iommu (seq_id = %d)\n",
766 iommu->seq_id);
767 goto err_unmap;
768 }
769 msagaw = iommu_calculate_max_sagaw(iommu);
770 if (msagaw < 0) {
771 printk(KERN_ERR
772 "Cannot get a valid max agaw for iommu (seq_id = %d)\n",
773 iommu->seq_id);
774 goto err_unmap;
775 }
776#endif
777 iommu->agaw = agaw;
778 iommu->msagaw = msagaw;
779
780 iommu->node = -1;
781
782 /* the registers might be more than one page */
783 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
784 cap_max_fault_reg_offset(iommu->cap));
785 map_size = VTD_PAGE_ALIGN(map_size);
786 if (map_size > VTD_PAGE_SIZE) {
787 iounmap(iommu->reg);
788 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
789 if (!iommu->reg) {
790 printk(KERN_ERR "IOMMU: can't map the region\n");
791 goto error;
792 }
793 }
794
795 ver = readl(iommu->reg + DMAR_VER_REG);
796 pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
797 iommu->seq_id,
798 (unsigned long long)drhd->reg_base_addr,
799 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
800 (unsigned long long)iommu->cap,
801 (unsigned long long)iommu->ecap);
802
803 spin_lock_init(&iommu->register_lock);
804
805 drhd->iommu = iommu;
806 return 0;
807
808 err_unmap:
809 iounmap(iommu->reg);
810 error:
811 kfree(iommu);
812 return -1;
813}
814
815void free_iommu(struct intel_iommu *iommu)
816{
817 if (!iommu)
818 return;
819
820#ifdef CONFIG_DMAR
821 free_dmar_iommu(iommu);
822#endif
823
824 if (iommu->reg)
825 iounmap(iommu->reg);
826 kfree(iommu);
827}
828
829/*
830 * Reclaim all the submitted descriptors which have completed its work.
831 */
832static inline void reclaim_free_desc(struct q_inval *qi)
833{
834 while (qi->desc_status[qi->free_tail] == QI_DONE ||
835 qi->desc_status[qi->free_tail] == QI_ABORT) {
836 qi->desc_status[qi->free_tail] = QI_FREE;
837 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
838 qi->free_cnt++;
839 }
840}
841
842static int qi_check_fault(struct intel_iommu *iommu, int index)
843{
844 u32 fault;
845 int head, tail;
846 struct q_inval *qi = iommu->qi;
847 int wait_index = (index + 1) % QI_LENGTH;
848
849 if (qi->desc_status[wait_index] == QI_ABORT)
850 return -EAGAIN;
851
852 fault = readl(iommu->reg + DMAR_FSTS_REG);
853
854 /*
855 * If IQE happens, the head points to the descriptor associated
856 * with the error. No new descriptors are fetched until the IQE
857 * is cleared.
858 */
859 if (fault & DMA_FSTS_IQE) {
860 head = readl(iommu->reg + DMAR_IQH_REG);
861 if ((head >> DMAR_IQ_SHIFT) == index) {
862 printk(KERN_ERR "VT-d detected invalid descriptor: "
863 "low=%llx, high=%llx\n",
864 (unsigned long long)qi->desc[index].low,
865 (unsigned long long)qi->desc[index].high);
866 memcpy(&qi->desc[index], &qi->desc[wait_index],
867 sizeof(struct qi_desc));
868 __iommu_flush_cache(iommu, &qi->desc[index],
869 sizeof(struct qi_desc));
870 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
871 return -EINVAL;
872 }
873 }
874
875 /*
876 * If ITE happens, all pending wait_desc commands are aborted.
877 * No new descriptors are fetched until the ITE is cleared.
878 */
879 if (fault & DMA_FSTS_ITE) {
880 head = readl(iommu->reg + DMAR_IQH_REG);
881 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
882 head |= 1;
883 tail = readl(iommu->reg + DMAR_IQT_REG);
884 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
885
886 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
887
888 do {
889 if (qi->desc_status[head] == QI_IN_USE)
890 qi->desc_status[head] = QI_ABORT;
891 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
892 } while (head != tail);
893
894 if (qi->desc_status[wait_index] == QI_ABORT)
895 return -EAGAIN;
896 }
897
898 if (fault & DMA_FSTS_ICE)
899 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
900
901 return 0;
902}
903
904/*
905 * Submit the queued invalidation descriptor to the remapping
906 * hardware unit and wait for its completion.
907 */
908int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
909{
910 int rc;
911 struct q_inval *qi = iommu->qi;
912 struct qi_desc *hw, wait_desc;
913 int wait_index, index;
914 unsigned long flags;
915
916 if (!qi)
917 return 0;
918
919 hw = qi->desc;
920
921restart:
922 rc = 0;
923
924 spin_lock_irqsave(&qi->q_lock, flags);
925 while (qi->free_cnt < 3) {
926 spin_unlock_irqrestore(&qi->q_lock, flags);
927 cpu_relax();
928 spin_lock_irqsave(&qi->q_lock, flags);
929 }
930
931 index = qi->free_head;
932 wait_index = (index + 1) % QI_LENGTH;
933
934 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
935
936 hw[index] = *desc;
937
938 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
939 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
940 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
941
942 hw[wait_index] = wait_desc;
943
944 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
945 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
946
947 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
948 qi->free_cnt -= 2;
949
950 /*
951 * update the HW tail register indicating the presence of
952 * new descriptors.
953 */
954 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
955
956 while (qi->desc_status[wait_index] != QI_DONE) {
957 /*
958 * We will leave the interrupts disabled, to prevent interrupt
959 * context to queue another cmd while a cmd is already submitted
960 * and waiting for completion on this cpu. This is to avoid
961 * a deadlock where the interrupt context can wait indefinitely
962 * for free slots in the queue.
963 */
964 rc = qi_check_fault(iommu, index);
965 if (rc)
966 break;
967
968 spin_unlock(&qi->q_lock);
969 cpu_relax();
970 spin_lock(&qi->q_lock);
971 }
972
973 qi->desc_status[index] = QI_DONE;
974
975 reclaim_free_desc(qi);
976 spin_unlock_irqrestore(&qi->q_lock, flags);
977
978 if (rc == -EAGAIN)
979 goto restart;
980
981 return rc;
982}
983
984/*
985 * Flush the global interrupt entry cache.
986 */
987void qi_global_iec(struct intel_iommu *iommu)
988{
989 struct qi_desc desc;
990
991 desc.low = QI_IEC_TYPE;
992 desc.high = 0;
993
994 /* should never fail */
995 qi_submit_sync(&desc, iommu);
996}
997
998void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
999 u64 type)
1000{
1001 struct qi_desc desc;
1002
1003 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1004 | QI_CC_GRAN(type) | QI_CC_TYPE;
1005 desc.high = 0;
1006
1007 qi_submit_sync(&desc, iommu);
1008}
1009
1010void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1011 unsigned int size_order, u64 type)
1012{
1013 u8 dw = 0, dr = 0;
1014
1015 struct qi_desc desc;
1016 int ih = 0;
1017
1018 if (cap_write_drain(iommu->cap))
1019 dw = 1;
1020
1021 if (cap_read_drain(iommu->cap))
1022 dr = 1;
1023
1024 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1025 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1026 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1027 | QI_IOTLB_AM(size_order);
1028
1029 qi_submit_sync(&desc, iommu);
1030}
1031
1032void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1033 u64 addr, unsigned mask)
1034{
1035 struct qi_desc desc;
1036
1037 if (mask) {
1038 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1039 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1040 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1041 } else
1042 desc.high = QI_DEV_IOTLB_ADDR(addr);
1043
1044 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1045 qdep = 0;
1046
1047 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1048 QI_DIOTLB_TYPE;
1049
1050 qi_submit_sync(&desc, iommu);
1051}
1052
1053/*
1054 * Disable Queued Invalidation interface.
1055 */
1056void dmar_disable_qi(struct intel_iommu *iommu)
1057{
1058 unsigned long flags;
1059 u32 sts;
1060 cycles_t start_time = get_cycles();
1061
1062 if (!ecap_qis(iommu->ecap))
1063 return;
1064
1065 spin_lock_irqsave(&iommu->register_lock, flags);
1066
1067 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
1068 if (!(sts & DMA_GSTS_QIES))
1069 goto end;
1070
1071 /*
1072 * Give a chance to HW to complete the pending invalidation requests.
1073 */
1074 while ((readl(iommu->reg + DMAR_IQT_REG) !=
1075 readl(iommu->reg + DMAR_IQH_REG)) &&
1076 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1077 cpu_relax();
1078
1079 iommu->gcmd &= ~DMA_GCMD_QIE;
1080 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1081
1082 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1083 !(sts & DMA_GSTS_QIES), sts);
1084end:
1085 spin_unlock_irqrestore(&iommu->register_lock, flags);
1086}
1087
1088/*
1089 * Enable queued invalidation.
1090 */
1091static void __dmar_enable_qi(struct intel_iommu *iommu)
1092{
1093 u32 sts;
1094 unsigned long flags;
1095 struct q_inval *qi = iommu->qi;
1096
1097 qi->free_head = qi->free_tail = 0;
1098 qi->free_cnt = QI_LENGTH;
1099
1100 spin_lock_irqsave(&iommu->register_lock, flags);
1101
1102 /* write zero to the tail reg */
1103 writel(0, iommu->reg + DMAR_IQT_REG);
1104
1105 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1106
1107 iommu->gcmd |= DMA_GCMD_QIE;
1108 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1109
1110 /* Make sure hardware complete it */
1111 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1112
1113 spin_unlock_irqrestore(&iommu->register_lock, flags);
1114}
1115
1116/*
1117 * Enable Queued Invalidation interface. This is a must to support
1118 * interrupt-remapping. Also used by DMA-remapping, which replaces
1119 * register based IOTLB invalidation.
1120 */
1121int dmar_enable_qi(struct intel_iommu *iommu)
1122{
1123 struct q_inval *qi;
1124 struct page *desc_page;
1125
1126 if (!ecap_qis(iommu->ecap))
1127 return -ENOENT;
1128
1129 /*
1130 * queued invalidation is already setup and enabled.
1131 */
1132 if (iommu->qi)
1133 return 0;
1134
1135 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1136 if (!iommu->qi)
1137 return -ENOMEM;
1138
1139 qi = iommu->qi;
1140
1141
1142 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1143 if (!desc_page) {
1144 kfree(qi);
1145 iommu->qi = 0;
1146 return -ENOMEM;
1147 }
1148
1149 qi->desc = page_address(desc_page);
1150
1151 qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1152 if (!qi->desc_status) {
1153 free_page((unsigned long) qi->desc);
1154 kfree(qi);
1155 iommu->qi = 0;
1156 return -ENOMEM;
1157 }
1158
1159 qi->free_head = qi->free_tail = 0;
1160 qi->free_cnt = QI_LENGTH;
1161
1162 spin_lock_init(&qi->q_lock);
1163
1164 __dmar_enable_qi(iommu);
1165
1166 return 0;
1167}
1168
1169/* iommu interrupt handling. Most stuff are MSI-like. */
1170
1171enum faulttype {
1172 DMA_REMAP,
1173 INTR_REMAP,
1174 UNKNOWN,
1175};
1176
1177static const char *dma_remap_fault_reasons[] =
1178{
1179 "Software",
1180 "Present bit in root entry is clear",
1181 "Present bit in context entry is clear",
1182 "Invalid context entry",
1183 "Access beyond MGAW",
1184 "PTE Write access is not set",
1185 "PTE Read access is not set",
1186 "Next page table ptr is invalid",
1187 "Root table address invalid",
1188 "Context table ptr is invalid",
1189 "non-zero reserved fields in RTP",
1190 "non-zero reserved fields in CTP",
1191 "non-zero reserved fields in PTE",
1192};
1193
1194static const char *intr_remap_fault_reasons[] =
1195{
1196 "Detected reserved fields in the decoded interrupt-remapped request",
1197 "Interrupt index exceeded the interrupt-remapping table size",
1198 "Present field in the IRTE entry is clear",
1199 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1200 "Detected reserved fields in the IRTE entry",
1201 "Blocked a compatibility format interrupt request",
1202 "Blocked an interrupt request due to source-id verification failure",
1203};
1204
1205#define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
1206
1207const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1208{
1209 if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
1210 ARRAY_SIZE(intr_remap_fault_reasons))) {
1211 *fault_type = INTR_REMAP;
1212 return intr_remap_fault_reasons[fault_reason - 0x20];
1213 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1214 *fault_type = DMA_REMAP;
1215 return dma_remap_fault_reasons[fault_reason];
1216 } else {
1217 *fault_type = UNKNOWN;
1218 return "Unknown";
1219 }
1220}
1221
1222void dmar_msi_unmask(struct irq_data *data)
1223{
1224 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1225 unsigned long flag;
1226
1227 /* unmask it */
1228 spin_lock_irqsave(&iommu->register_lock, flag);
1229 writel(0, iommu->reg + DMAR_FECTL_REG);
1230 /* Read a reg to force flush the post write */
1231 readl(iommu->reg + DMAR_FECTL_REG);
1232 spin_unlock_irqrestore(&iommu->register_lock, flag);
1233}
1234
1235void dmar_msi_mask(struct irq_data *data)
1236{
1237 unsigned long flag;
1238 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1239
1240 /* mask it */
1241 spin_lock_irqsave(&iommu->register_lock, flag);
1242 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1243 /* Read a reg to force flush the post write */
1244 readl(iommu->reg + DMAR_FECTL_REG);
1245 spin_unlock_irqrestore(&iommu->register_lock, flag);
1246}
1247
1248void dmar_msi_write(int irq, struct msi_msg *msg)
1249{
1250 struct intel_iommu *iommu = irq_get_handler_data(irq);
1251 unsigned long flag;
1252
1253 spin_lock_irqsave(&iommu->register_lock, flag);
1254 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1255 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1256 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1257 spin_unlock_irqrestore(&iommu->register_lock, flag);
1258}
1259
1260void dmar_msi_read(int irq, struct msi_msg *msg)
1261{
1262 struct intel_iommu *iommu = irq_get_handler_data(irq);
1263 unsigned long flag;
1264
1265 spin_lock_irqsave(&iommu->register_lock, flag);
1266 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1267 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1268 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1269 spin_unlock_irqrestore(&iommu->register_lock, flag);
1270}
1271
1272static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1273 u8 fault_reason, u16 source_id, unsigned long long addr)
1274{
1275 const char *reason;
1276 int fault_type;
1277
1278 reason = dmar_get_fault_reason(fault_reason, &fault_type);
1279
1280 if (fault_type == INTR_REMAP)
1281 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
1282 "fault index %llx\n"
1283 "INTR-REMAP:[fault reason %02d] %s\n",
1284 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1285 PCI_FUNC(source_id & 0xFF), addr >> 48,
1286 fault_reason, reason);
1287 else
1288 printk(KERN_ERR
1289 "DMAR:[%s] Request device [%02x:%02x.%d] "
1290 "fault addr %llx \n"
1291 "DMAR:[fault reason %02d] %s\n",
1292 (type ? "DMA Read" : "DMA Write"),
1293 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1294 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1295 return 0;
1296}
1297
1298#define PRIMARY_FAULT_REG_LEN (16)
1299irqreturn_t dmar_fault(int irq, void *dev_id)
1300{
1301 struct intel_iommu *iommu = dev_id;
1302 int reg, fault_index;
1303 u32 fault_status;
1304 unsigned long flag;
1305
1306 spin_lock_irqsave(&iommu->register_lock, flag);
1307 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1308 if (fault_status)
1309 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1310 fault_status);
1311
1312 /* TBD: ignore advanced fault log currently */
1313 if (!(fault_status & DMA_FSTS_PPF))
1314 goto clear_rest;
1315
1316 fault_index = dma_fsts_fault_record_index(fault_status);
1317 reg = cap_fault_reg_offset(iommu->cap);
1318 while (1) {
1319 u8 fault_reason;
1320 u16 source_id;
1321 u64 guest_addr;
1322 int type;
1323 u32 data;
1324
1325 /* highest 32 bits */
1326 data = readl(iommu->reg + reg +
1327 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1328 if (!(data & DMA_FRCD_F))
1329 break;
1330
1331 fault_reason = dma_frcd_fault_reason(data);
1332 type = dma_frcd_type(data);
1333
1334 data = readl(iommu->reg + reg +
1335 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1336 source_id = dma_frcd_source_id(data);
1337
1338 guest_addr = dmar_readq(iommu->reg + reg +
1339 fault_index * PRIMARY_FAULT_REG_LEN);
1340 guest_addr = dma_frcd_page_addr(guest_addr);
1341 /* clear the fault */
1342 writel(DMA_FRCD_F, iommu->reg + reg +
1343 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1344
1345 spin_unlock_irqrestore(&iommu->register_lock, flag);
1346
1347 dmar_fault_do_one(iommu, type, fault_reason,
1348 source_id, guest_addr);
1349
1350 fault_index++;
1351 if (fault_index >= cap_num_fault_regs(iommu->cap))
1352 fault_index = 0;
1353 spin_lock_irqsave(&iommu->register_lock, flag);
1354 }
1355clear_rest:
1356 /* clear all the other faults */
1357 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1358 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1359
1360 spin_unlock_irqrestore(&iommu->register_lock, flag);
1361 return IRQ_HANDLED;
1362}
1363
1364int dmar_set_interrupt(struct intel_iommu *iommu)
1365{
1366 int irq, ret;
1367
1368 /*
1369 * Check if the fault interrupt is already initialized.
1370 */
1371 if (iommu->irq)
1372 return 0;
1373
1374 irq = create_irq();
1375 if (!irq) {
1376 printk(KERN_ERR "IOMMU: no free vectors\n");
1377 return -EINVAL;
1378 }
1379
1380 irq_set_handler_data(irq, iommu);
1381 iommu->irq = irq;
1382
1383 ret = arch_setup_dmar_msi(irq);
1384 if (ret) {
1385 irq_set_handler_data(irq, NULL);
1386 iommu->irq = 0;
1387 destroy_irq(irq);
1388 return ret;
1389 }
1390
1391 ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1392 if (ret)
1393 printk(KERN_ERR "IOMMU: can't request irq\n");
1394 return ret;
1395}
1396
1397int __init enable_drhd_fault_handling(void)
1398{
1399 struct dmar_drhd_unit *drhd;
1400
1401 /*
1402 * Enable fault control interrupt.
1403 */
1404 for_each_drhd_unit(drhd) {
1405 int ret;
1406 struct intel_iommu *iommu = drhd->iommu;
1407 ret = dmar_set_interrupt(iommu);
1408
1409 if (ret) {
1410 printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1411 " interrupt, ret %d\n",
1412 (unsigned long long)drhd->reg_base_addr, ret);
1413 return -1;
1414 }
1415
1416 /*
1417 * Clear any previous faults.
1418 */
1419 dmar_fault(iommu->irq, iommu);
1420 }
1421
1422 return 0;
1423}
1424
1425/*
1426 * Re-enable Queued Invalidation interface.
1427 */
1428int dmar_reenable_qi(struct intel_iommu *iommu)
1429{
1430 if (!ecap_qis(iommu->ecap))
1431 return -ENOENT;
1432
1433 if (!iommu->qi)
1434 return -ENOENT;
1435
1436 /*
1437 * First disable queued invalidation.
1438 */
1439 dmar_disable_qi(iommu);
1440 /*
1441 * Then enable queued invalidation again. Since there is no pending
1442 * invalidation requests now, it's safe to re-enable queued
1443 * invalidation.
1444 */
1445 __dmar_enable_qi(iommu);
1446
1447 return 0;
1448}
1449
1450/*
1451 * Check interrupt remapping support in DMAR table description.
1452 */
1453int __init dmar_ir_support(void)
1454{
1455 struct acpi_table_dmar *dmar;
1456 dmar = (struct acpi_table_dmar *)dmar_tbl;
1457 if (!dmar)
1458 return 0;
1459 return dmar->flags & 0x1;
1460}
1461IOMMU_INIT_POST(detect_intel_iommu);
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
new file mode 100644
index 000000000000..c621c98c99da
--- /dev/null
+++ b/drivers/iommu/intel-iommu.c
@@ -0,0 +1,4016 @@
1/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
22 */
23
24#include <linux/init.h>
25#include <linux/bitmap.h>
26#include <linux/debugfs.h>
27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
35#include <linux/timer.h>
36#include <linux/iova.h>
37#include <linux/iommu.h>
38#include <linux/intel-iommu.h>
39#include <linux/syscore_ops.h>
40#include <linux/tboot.h>
41#include <linux/dmi.h>
42#include <linux/pci-ats.h>
43#include <asm/cacheflush.h>
44#include <asm/iommu.h>
45
46#define ROOT_SIZE VTD_PAGE_SIZE
47#define CONTEXT_SIZE VTD_PAGE_SIZE
48
49#define IS_BRIDGE_HOST_DEVICE(pdev) \
50 ((pdev->class >> 8) == PCI_CLASS_BRIDGE_HOST)
51#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
52#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
53#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
54
55#define IOAPIC_RANGE_START (0xfee00000)
56#define IOAPIC_RANGE_END (0xfeefffff)
57#define IOVA_START_ADDR (0x1000)
58
59#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
60
61#define MAX_AGAW_WIDTH 64
62
63#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
64#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
65
66/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
67 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
68#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
69 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
70#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
71
72#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
73#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
74#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
75
76/* page table handling */
77#define LEVEL_STRIDE (9)
78#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
79
80static inline int agaw_to_level(int agaw)
81{
82 return agaw + 2;
83}
84
85static inline int agaw_to_width(int agaw)
86{
87 return 30 + agaw * LEVEL_STRIDE;
88}
89
90static inline int width_to_agaw(int width)
91{
92 return (width - 30) / LEVEL_STRIDE;
93}
94
95static inline unsigned int level_to_offset_bits(int level)
96{
97 return (level - 1) * LEVEL_STRIDE;
98}
99
100static inline int pfn_level_offset(unsigned long pfn, int level)
101{
102 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
103}
104
105static inline unsigned long level_mask(int level)
106{
107 return -1UL << level_to_offset_bits(level);
108}
109
110static inline unsigned long level_size(int level)
111{
112 return 1UL << level_to_offset_bits(level);
113}
114
115static inline unsigned long align_to_level(unsigned long pfn, int level)
116{
117 return (pfn + level_size(level) - 1) & level_mask(level);
118}
119
120static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
121{
122 return 1 << ((lvl - 1) * LEVEL_STRIDE);
123}
124
125/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
126 are never going to work. */
127static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
128{
129 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
130}
131
132static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
133{
134 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
135}
136static inline unsigned long page_to_dma_pfn(struct page *pg)
137{
138 return mm_to_dma_pfn(page_to_pfn(pg));
139}
140static inline unsigned long virt_to_dma_pfn(void *p)
141{
142 return page_to_dma_pfn(virt_to_page(p));
143}
144
145/* global iommu list, set NULL for ignored DMAR units */
146static struct intel_iommu **g_iommus;
147
148static void __init check_tylersburg_isoch(void);
149static int rwbf_quirk;
150
151/*
152 * set to 1 to panic kernel if can't successfully enable VT-d
153 * (used when kernel is launched w/ TXT)
154 */
155static int force_on = 0;
156
157/*
158 * 0: Present
159 * 1-11: Reserved
160 * 12-63: Context Ptr (12 - (haw-1))
161 * 64-127: Reserved
162 */
163struct root_entry {
164 u64 val;
165 u64 rsvd1;
166};
167#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
168static inline bool root_present(struct root_entry *root)
169{
170 return (root->val & 1);
171}
172static inline void set_root_present(struct root_entry *root)
173{
174 root->val |= 1;
175}
176static inline void set_root_value(struct root_entry *root, unsigned long value)
177{
178 root->val |= value & VTD_PAGE_MASK;
179}
180
181static inline struct context_entry *
182get_context_addr_from_root(struct root_entry *root)
183{
184 return (struct context_entry *)
185 (root_present(root)?phys_to_virt(
186 root->val & VTD_PAGE_MASK) :
187 NULL);
188}
189
190/*
191 * low 64 bits:
192 * 0: present
193 * 1: fault processing disable
194 * 2-3: translation type
195 * 12-63: address space root
196 * high 64 bits:
197 * 0-2: address width
198 * 3-6: aval
199 * 8-23: domain id
200 */
201struct context_entry {
202 u64 lo;
203 u64 hi;
204};
205
206static inline bool context_present(struct context_entry *context)
207{
208 return (context->lo & 1);
209}
210static inline void context_set_present(struct context_entry *context)
211{
212 context->lo |= 1;
213}
214
215static inline void context_set_fault_enable(struct context_entry *context)
216{
217 context->lo &= (((u64)-1) << 2) | 1;
218}
219
220static inline void context_set_translation_type(struct context_entry *context,
221 unsigned long value)
222{
223 context->lo &= (((u64)-1) << 4) | 3;
224 context->lo |= (value & 3) << 2;
225}
226
227static inline void context_set_address_root(struct context_entry *context,
228 unsigned long value)
229{
230 context->lo |= value & VTD_PAGE_MASK;
231}
232
233static inline void context_set_address_width(struct context_entry *context,
234 unsigned long value)
235{
236 context->hi |= value & 7;
237}
238
239static inline void context_set_domain_id(struct context_entry *context,
240 unsigned long value)
241{
242 context->hi |= (value & ((1 << 16) - 1)) << 8;
243}
244
245static inline void context_clear_entry(struct context_entry *context)
246{
247 context->lo = 0;
248 context->hi = 0;
249}
250
251/*
252 * 0: readable
253 * 1: writable
254 * 2-6: reserved
255 * 7: super page
256 * 8-10: available
257 * 11: snoop behavior
258 * 12-63: Host physcial address
259 */
260struct dma_pte {
261 u64 val;
262};
263
264static inline void dma_clear_pte(struct dma_pte *pte)
265{
266 pte->val = 0;
267}
268
269static inline void dma_set_pte_readable(struct dma_pte *pte)
270{
271 pte->val |= DMA_PTE_READ;
272}
273
274static inline void dma_set_pte_writable(struct dma_pte *pte)
275{
276 pte->val |= DMA_PTE_WRITE;
277}
278
279static inline void dma_set_pte_snp(struct dma_pte *pte)
280{
281 pte->val |= DMA_PTE_SNP;
282}
283
284static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
285{
286 pte->val = (pte->val & ~3) | (prot & 3);
287}
288
289static inline u64 dma_pte_addr(struct dma_pte *pte)
290{
291#ifdef CONFIG_64BIT
292 return pte->val & VTD_PAGE_MASK;
293#else
294 /* Must have a full atomic 64-bit read */
295 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
296#endif
297}
298
299static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
300{
301 pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
302}
303
304static inline bool dma_pte_present(struct dma_pte *pte)
305{
306 return (pte->val & 3) != 0;
307}
308
309static inline int first_pte_in_page(struct dma_pte *pte)
310{
311 return !((unsigned long)pte & ~VTD_PAGE_MASK);
312}
313
314/*
315 * This domain is a statically identity mapping domain.
316 * 1. This domain creats a static 1:1 mapping to all usable memory.
317 * 2. It maps to each iommu if successful.
318 * 3. Each iommu mapps to this domain if successful.
319 */
320static struct dmar_domain *si_domain;
321static int hw_pass_through = 1;
322
323/* devices under the same p2p bridge are owned in one domain */
324#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
325
326/* domain represents a virtual machine, more than one devices
327 * across iommus may be owned in one domain, e.g. kvm guest.
328 */
329#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
330
331/* si_domain contains mulitple devices */
332#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
333
334struct dmar_domain {
335 int id; /* domain id */
336 int nid; /* node id */
337 unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/
338
339 struct list_head devices; /* all devices' list */
340 struct iova_domain iovad; /* iova's that belong to this domain */
341
342 struct dma_pte *pgd; /* virtual address */
343 int gaw; /* max guest address width */
344
345 /* adjusted guest address width, 0 is level 2 30-bit */
346 int agaw;
347
348 int flags; /* flags to find out type of domain */
349
350 int iommu_coherency;/* indicate coherency of iommu access */
351 int iommu_snooping; /* indicate snooping control feature*/
352 int iommu_count; /* reference count of iommu */
353 int iommu_superpage;/* Level of superpages supported:
354 0 == 4KiB (no superpages), 1 == 2MiB,
355 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
356 spinlock_t iommu_lock; /* protect iommu set in domain */
357 u64 max_addr; /* maximum mapped address */
358};
359
360/* PCI domain-device relationship */
361struct device_domain_info {
362 struct list_head link; /* link to domain siblings */
363 struct list_head global; /* link to global list */
364 int segment; /* PCI domain */
365 u8 bus; /* PCI bus number */
366 u8 devfn; /* PCI devfn number */
367 struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
368 struct intel_iommu *iommu; /* IOMMU used by this device */
369 struct dmar_domain *domain; /* pointer to domain */
370};
371
372static void flush_unmaps_timeout(unsigned long data);
373
374DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
375
376#define HIGH_WATER_MARK 250
377struct deferred_flush_tables {
378 int next;
379 struct iova *iova[HIGH_WATER_MARK];
380 struct dmar_domain *domain[HIGH_WATER_MARK];
381};
382
383static struct deferred_flush_tables *deferred_flush;
384
385/* bitmap for indexing intel_iommus */
386static int g_num_of_iommus;
387
388static DEFINE_SPINLOCK(async_umap_flush_lock);
389static LIST_HEAD(unmaps_to_do);
390
391static int timer_on;
392static long list_size;
393
394static void domain_remove_dev_info(struct dmar_domain *domain);
395
396#ifdef CONFIG_DMAR_DEFAULT_ON
397int dmar_disabled = 0;
398#else
399int dmar_disabled = 1;
400#endif /*CONFIG_DMAR_DEFAULT_ON*/
401
402static int dmar_map_gfx = 1;
403static int dmar_forcedac;
404static int intel_iommu_strict;
405static int intel_iommu_superpage = 1;
406
407#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
408static DEFINE_SPINLOCK(device_domain_lock);
409static LIST_HEAD(device_domain_list);
410
411static struct iommu_ops intel_iommu_ops;
412
413static int __init intel_iommu_setup(char *str)
414{
415 if (!str)
416 return -EINVAL;
417 while (*str) {
418 if (!strncmp(str, "on", 2)) {
419 dmar_disabled = 0;
420 printk(KERN_INFO "Intel-IOMMU: enabled\n");
421 } else if (!strncmp(str, "off", 3)) {
422 dmar_disabled = 1;
423 printk(KERN_INFO "Intel-IOMMU: disabled\n");
424 } else if (!strncmp(str, "igfx_off", 8)) {
425 dmar_map_gfx = 0;
426 printk(KERN_INFO
427 "Intel-IOMMU: disable GFX device mapping\n");
428 } else if (!strncmp(str, "forcedac", 8)) {
429 printk(KERN_INFO
430 "Intel-IOMMU: Forcing DAC for PCI devices\n");
431 dmar_forcedac = 1;
432 } else if (!strncmp(str, "strict", 6)) {
433 printk(KERN_INFO
434 "Intel-IOMMU: disable batched IOTLB flush\n");
435 intel_iommu_strict = 1;
436 } else if (!strncmp(str, "sp_off", 6)) {
437 printk(KERN_INFO
438 "Intel-IOMMU: disable supported super page\n");
439 intel_iommu_superpage = 0;
440 }
441
442 str += strcspn(str, ",");
443 while (*str == ',')
444 str++;
445 }
446 return 0;
447}
448__setup("intel_iommu=", intel_iommu_setup);
449
450static struct kmem_cache *iommu_domain_cache;
451static struct kmem_cache *iommu_devinfo_cache;
452static struct kmem_cache *iommu_iova_cache;
453
454static inline void *alloc_pgtable_page(int node)
455{
456 struct page *page;
457 void *vaddr = NULL;
458
459 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
460 if (page)
461 vaddr = page_address(page);
462 return vaddr;
463}
464
465static inline void free_pgtable_page(void *vaddr)
466{
467 free_page((unsigned long)vaddr);
468}
469
470static inline void *alloc_domain_mem(void)
471{
472 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
473}
474
475static void free_domain_mem(void *vaddr)
476{
477 kmem_cache_free(iommu_domain_cache, vaddr);
478}
479
480static inline void * alloc_devinfo_mem(void)
481{
482 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
483}
484
485static inline void free_devinfo_mem(void *vaddr)
486{
487 kmem_cache_free(iommu_devinfo_cache, vaddr);
488}
489
490struct iova *alloc_iova_mem(void)
491{
492 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
493}
494
495void free_iova_mem(struct iova *iova)
496{
497 kmem_cache_free(iommu_iova_cache, iova);
498}
499
500
501static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
502{
503 unsigned long sagaw;
504 int agaw = -1;
505
506 sagaw = cap_sagaw(iommu->cap);
507 for (agaw = width_to_agaw(max_gaw);
508 agaw >= 0; agaw--) {
509 if (test_bit(agaw, &sagaw))
510 break;
511 }
512
513 return agaw;
514}
515
516/*
517 * Calculate max SAGAW for each iommu.
518 */
519int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
520{
521 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
522}
523
524/*
525 * calculate agaw for each iommu.
526 * "SAGAW" may be different across iommus, use a default agaw, and
527 * get a supported less agaw for iommus that don't support the default agaw.
528 */
529int iommu_calculate_agaw(struct intel_iommu *iommu)
530{
531 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
532}
533
534/* This functionin only returns single iommu in a domain */
535static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
536{
537 int iommu_id;
538
539 /* si_domain and vm domain should not get here. */
540 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
541 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
542
543 iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
544 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
545 return NULL;
546
547 return g_iommus[iommu_id];
548}
549
550static void domain_update_iommu_coherency(struct dmar_domain *domain)
551{
552 int i;
553
554 domain->iommu_coherency = 1;
555
556 for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
557 if (!ecap_coherent(g_iommus[i]->ecap)) {
558 domain->iommu_coherency = 0;
559 break;
560 }
561 }
562}
563
564static void domain_update_iommu_snooping(struct dmar_domain *domain)
565{
566 int i;
567
568 domain->iommu_snooping = 1;
569
570 for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
571 if (!ecap_sc_support(g_iommus[i]->ecap)) {
572 domain->iommu_snooping = 0;
573 break;
574 }
575 }
576}
577
578static void domain_update_iommu_superpage(struct dmar_domain *domain)
579{
580 int i, mask = 0xf;
581
582 if (!intel_iommu_superpage) {
583 domain->iommu_superpage = 0;
584 return;
585 }
586
587 domain->iommu_superpage = 4; /* 1TiB */
588
589 for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
590 mask |= cap_super_page_val(g_iommus[i]->cap);
591 if (!mask) {
592 break;
593 }
594 }
595 domain->iommu_superpage = fls(mask);
596}
597
598/* Some capabilities may be different across iommus */
599static void domain_update_iommu_cap(struct dmar_domain *domain)
600{
601 domain_update_iommu_coherency(domain);
602 domain_update_iommu_snooping(domain);
603 domain_update_iommu_superpage(domain);
604}
605
606static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
607{
608 struct dmar_drhd_unit *drhd = NULL;
609 int i;
610
611 for_each_drhd_unit(drhd) {
612 if (drhd->ignored)
613 continue;
614 if (segment != drhd->segment)
615 continue;
616
617 for (i = 0; i < drhd->devices_cnt; i++) {
618 if (drhd->devices[i] &&
619 drhd->devices[i]->bus->number == bus &&
620 drhd->devices[i]->devfn == devfn)
621 return drhd->iommu;
622 if (drhd->devices[i] &&
623 drhd->devices[i]->subordinate &&
624 drhd->devices[i]->subordinate->number <= bus &&
625 drhd->devices[i]->subordinate->subordinate >= bus)
626 return drhd->iommu;
627 }
628
629 if (drhd->include_all)
630 return drhd->iommu;
631 }
632
633 return NULL;
634}
635
636static void domain_flush_cache(struct dmar_domain *domain,
637 void *addr, int size)
638{
639 if (!domain->iommu_coherency)
640 clflush_cache_range(addr, size);
641}
642
643/* Gets context entry for a given bus and devfn */
644static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
645 u8 bus, u8 devfn)
646{
647 struct root_entry *root;
648 struct context_entry *context;
649 unsigned long phy_addr;
650 unsigned long flags;
651
652 spin_lock_irqsave(&iommu->lock, flags);
653 root = &iommu->root_entry[bus];
654 context = get_context_addr_from_root(root);
655 if (!context) {
656 context = (struct context_entry *)
657 alloc_pgtable_page(iommu->node);
658 if (!context) {
659 spin_unlock_irqrestore(&iommu->lock, flags);
660 return NULL;
661 }
662 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
663 phy_addr = virt_to_phys((void *)context);
664 set_root_value(root, phy_addr);
665 set_root_present(root);
666 __iommu_flush_cache(iommu, root, sizeof(*root));
667 }
668 spin_unlock_irqrestore(&iommu->lock, flags);
669 return &context[devfn];
670}
671
672static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
673{
674 struct root_entry *root;
675 struct context_entry *context;
676 int ret;
677 unsigned long flags;
678
679 spin_lock_irqsave(&iommu->lock, flags);
680 root = &iommu->root_entry[bus];
681 context = get_context_addr_from_root(root);
682 if (!context) {
683 ret = 0;
684 goto out;
685 }
686 ret = context_present(&context[devfn]);
687out:
688 spin_unlock_irqrestore(&iommu->lock, flags);
689 return ret;
690}
691
692static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
693{
694 struct root_entry *root;
695 struct context_entry *context;
696 unsigned long flags;
697
698 spin_lock_irqsave(&iommu->lock, flags);
699 root = &iommu->root_entry[bus];
700 context = get_context_addr_from_root(root);
701 if (context) {
702 context_clear_entry(&context[devfn]);
703 __iommu_flush_cache(iommu, &context[devfn], \
704 sizeof(*context));
705 }
706 spin_unlock_irqrestore(&iommu->lock, flags);
707}
708
709static void free_context_table(struct intel_iommu *iommu)
710{
711 struct root_entry *root;
712 int i;
713 unsigned long flags;
714 struct context_entry *context;
715
716 spin_lock_irqsave(&iommu->lock, flags);
717 if (!iommu->root_entry) {
718 goto out;
719 }
720 for (i = 0; i < ROOT_ENTRY_NR; i++) {
721 root = &iommu->root_entry[i];
722 context = get_context_addr_from_root(root);
723 if (context)
724 free_pgtable_page(context);
725 }
726 free_pgtable_page(iommu->root_entry);
727 iommu->root_entry = NULL;
728out:
729 spin_unlock_irqrestore(&iommu->lock, flags);
730}
731
732static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
733 unsigned long pfn, int large_level)
734{
735 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
736 struct dma_pte *parent, *pte = NULL;
737 int level = agaw_to_level(domain->agaw);
738 int offset, target_level;
739
740 BUG_ON(!domain->pgd);
741 BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
742 parent = domain->pgd;
743
744 /* Search pte */
745 if (!large_level)
746 target_level = 1;
747 else
748 target_level = large_level;
749
750 while (level > 0) {
751 void *tmp_page;
752
753 offset = pfn_level_offset(pfn, level);
754 pte = &parent[offset];
755 if (!large_level && (pte->val & DMA_PTE_LARGE_PAGE))
756 break;
757 if (level == target_level)
758 break;
759
760 if (!dma_pte_present(pte)) {
761 uint64_t pteval;
762
763 tmp_page = alloc_pgtable_page(domain->nid);
764
765 if (!tmp_page)
766 return NULL;
767
768 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
769 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
770 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
771 /* Someone else set it while we were thinking; use theirs. */
772 free_pgtable_page(tmp_page);
773 } else {
774 dma_pte_addr(pte);
775 domain_flush_cache(domain, pte, sizeof(*pte));
776 }
777 }
778 parent = phys_to_virt(dma_pte_addr(pte));
779 level--;
780 }
781
782 return pte;
783}
784
785
786/* return address's pte at specific level */
787static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
788 unsigned long pfn,
789 int level, int *large_page)
790{
791 struct dma_pte *parent, *pte = NULL;
792 int total = agaw_to_level(domain->agaw);
793 int offset;
794
795 parent = domain->pgd;
796 while (level <= total) {
797 offset = pfn_level_offset(pfn, total);
798 pte = &parent[offset];
799 if (level == total)
800 return pte;
801
802 if (!dma_pte_present(pte)) {
803 *large_page = total;
804 break;
805 }
806
807 if (pte->val & DMA_PTE_LARGE_PAGE) {
808 *large_page = total;
809 return pte;
810 }
811
812 parent = phys_to_virt(dma_pte_addr(pte));
813 total--;
814 }
815 return NULL;
816}
817
818/* clear last level pte, a tlb flush should be followed */
819static void dma_pte_clear_range(struct dmar_domain *domain,
820 unsigned long start_pfn,
821 unsigned long last_pfn)
822{
823 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
824 unsigned int large_page = 1;
825 struct dma_pte *first_pte, *pte;
826
827 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
828 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
829 BUG_ON(start_pfn > last_pfn);
830
831 /* we don't need lock here; nobody else touches the iova range */
832 do {
833 large_page = 1;
834 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
835 if (!pte) {
836 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
837 continue;
838 }
839 do {
840 dma_clear_pte(pte);
841 start_pfn += lvl_to_nr_pages(large_page);
842 pte++;
843 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
844
845 domain_flush_cache(domain, first_pte,
846 (void *)pte - (void *)first_pte);
847
848 } while (start_pfn && start_pfn <= last_pfn);
849}
850
851/* free page table pages. last level pte should already be cleared */
852static void dma_pte_free_pagetable(struct dmar_domain *domain,
853 unsigned long start_pfn,
854 unsigned long last_pfn)
855{
856 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
857 struct dma_pte *first_pte, *pte;
858 int total = agaw_to_level(domain->agaw);
859 int level;
860 unsigned long tmp;
861 int large_page = 2;
862
863 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
864 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
865 BUG_ON(start_pfn > last_pfn);
866
867 /* We don't need lock here; nobody else touches the iova range */
868 level = 2;
869 while (level <= total) {
870 tmp = align_to_level(start_pfn, level);
871
872 /* If we can't even clear one PTE at this level, we're done */
873 if (tmp + level_size(level) - 1 > last_pfn)
874 return;
875
876 do {
877 large_page = level;
878 first_pte = pte = dma_pfn_level_pte(domain, tmp, level, &large_page);
879 if (large_page > level)
880 level = large_page + 1;
881 if (!pte) {
882 tmp = align_to_level(tmp + 1, level + 1);
883 continue;
884 }
885 do {
886 if (dma_pte_present(pte)) {
887 free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
888 dma_clear_pte(pte);
889 }
890 pte++;
891 tmp += level_size(level);
892 } while (!first_pte_in_page(pte) &&
893 tmp + level_size(level) - 1 <= last_pfn);
894
895 domain_flush_cache(domain, first_pte,
896 (void *)pte - (void *)first_pte);
897
898 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
899 level++;
900 }
901 /* free pgd */
902 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
903 free_pgtable_page(domain->pgd);
904 domain->pgd = NULL;
905 }
906}
907
908/* iommu handling */
909static int iommu_alloc_root_entry(struct intel_iommu *iommu)
910{
911 struct root_entry *root;
912 unsigned long flags;
913
914 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
915 if (!root)
916 return -ENOMEM;
917
918 __iommu_flush_cache(iommu, root, ROOT_SIZE);
919
920 spin_lock_irqsave(&iommu->lock, flags);
921 iommu->root_entry = root;
922 spin_unlock_irqrestore(&iommu->lock, flags);
923
924 return 0;
925}
926
927static void iommu_set_root_entry(struct intel_iommu *iommu)
928{
929 void *addr;
930 u32 sts;
931 unsigned long flag;
932
933 addr = iommu->root_entry;
934
935 spin_lock_irqsave(&iommu->register_lock, flag);
936 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
937
938 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
939
940 /* Make sure hardware complete it */
941 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
942 readl, (sts & DMA_GSTS_RTPS), sts);
943
944 spin_unlock_irqrestore(&iommu->register_lock, flag);
945}
946
947static void iommu_flush_write_buffer(struct intel_iommu *iommu)
948{
949 u32 val;
950 unsigned long flag;
951
952 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
953 return;
954
955 spin_lock_irqsave(&iommu->register_lock, flag);
956 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
957
958 /* Make sure hardware complete it */
959 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
960 readl, (!(val & DMA_GSTS_WBFS)), val);
961
962 spin_unlock_irqrestore(&iommu->register_lock, flag);
963}
964
965/* return value determine if we need a write buffer flush */
966static void __iommu_flush_context(struct intel_iommu *iommu,
967 u16 did, u16 source_id, u8 function_mask,
968 u64 type)
969{
970 u64 val = 0;
971 unsigned long flag;
972
973 switch (type) {
974 case DMA_CCMD_GLOBAL_INVL:
975 val = DMA_CCMD_GLOBAL_INVL;
976 break;
977 case DMA_CCMD_DOMAIN_INVL:
978 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
979 break;
980 case DMA_CCMD_DEVICE_INVL:
981 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
982 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
983 break;
984 default:
985 BUG();
986 }
987 val |= DMA_CCMD_ICC;
988
989 spin_lock_irqsave(&iommu->register_lock, flag);
990 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
991
992 /* Make sure hardware complete it */
993 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
994 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
995
996 spin_unlock_irqrestore(&iommu->register_lock, flag);
997}
998
999/* return value determine if we need a write buffer flush */
1000static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1001 u64 addr, unsigned int size_order, u64 type)
1002{
1003 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1004 u64 val = 0, val_iva = 0;
1005 unsigned long flag;
1006
1007 switch (type) {
1008 case DMA_TLB_GLOBAL_FLUSH:
1009 /* global flush doesn't need set IVA_REG */
1010 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1011 break;
1012 case DMA_TLB_DSI_FLUSH:
1013 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1014 break;
1015 case DMA_TLB_PSI_FLUSH:
1016 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1017 /* Note: always flush non-leaf currently */
1018 val_iva = size_order | addr;
1019 break;
1020 default:
1021 BUG();
1022 }
1023 /* Note: set drain read/write */
1024#if 0
1025 /*
1026 * This is probably to be super secure.. Looks like we can
1027 * ignore it without any impact.
1028 */
1029 if (cap_read_drain(iommu->cap))
1030 val |= DMA_TLB_READ_DRAIN;
1031#endif
1032 if (cap_write_drain(iommu->cap))
1033 val |= DMA_TLB_WRITE_DRAIN;
1034
1035 spin_lock_irqsave(&iommu->register_lock, flag);
1036 /* Note: Only uses first TLB reg currently */
1037 if (val_iva)
1038 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1039 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1040
1041 /* Make sure hardware complete it */
1042 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1043 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1044
1045 spin_unlock_irqrestore(&iommu->register_lock, flag);
1046
1047 /* check IOTLB invalidation granularity */
1048 if (DMA_TLB_IAIG(val) == 0)
1049 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1050 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1051 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1052 (unsigned long long)DMA_TLB_IIRG(type),
1053 (unsigned long long)DMA_TLB_IAIG(val));
1054}
1055
1056static struct device_domain_info *iommu_support_dev_iotlb(
1057 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1058{
1059 int found = 0;
1060 unsigned long flags;
1061 struct device_domain_info *info;
1062 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1063
1064 if (!ecap_dev_iotlb_support(iommu->ecap))
1065 return NULL;
1066
1067 if (!iommu->qi)
1068 return NULL;
1069
1070 spin_lock_irqsave(&device_domain_lock, flags);
1071 list_for_each_entry(info, &domain->devices, link)
1072 if (info->bus == bus && info->devfn == devfn) {
1073 found = 1;
1074 break;
1075 }
1076 spin_unlock_irqrestore(&device_domain_lock, flags);
1077
1078 if (!found || !info->dev)
1079 return NULL;
1080
1081 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1082 return NULL;
1083
1084 if (!dmar_find_matched_atsr_unit(info->dev))
1085 return NULL;
1086
1087 info->iommu = iommu;
1088
1089 return info;
1090}
1091
1092static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1093{
1094 if (!info)
1095 return;
1096
1097 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1098}
1099
1100static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1101{
1102 if (!info->dev || !pci_ats_enabled(info->dev))
1103 return;
1104
1105 pci_disable_ats(info->dev);
1106}
1107
1108static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1109 u64 addr, unsigned mask)
1110{
1111 u16 sid, qdep;
1112 unsigned long flags;
1113 struct device_domain_info *info;
1114
1115 spin_lock_irqsave(&device_domain_lock, flags);
1116 list_for_each_entry(info, &domain->devices, link) {
1117 if (!info->dev || !pci_ats_enabled(info->dev))
1118 continue;
1119
1120 sid = info->bus << 8 | info->devfn;
1121 qdep = pci_ats_queue_depth(info->dev);
1122 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1123 }
1124 spin_unlock_irqrestore(&device_domain_lock, flags);
1125}
1126
1127static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1128 unsigned long pfn, unsigned int pages, int map)
1129{
1130 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1131 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1132
1133 BUG_ON(pages == 0);
1134
1135 /*
1136 * Fallback to domain selective flush if no PSI support or the size is
1137 * too big.
1138 * PSI requires page size to be 2 ^ x, and the base address is naturally
1139 * aligned to the size
1140 */
1141 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1142 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1143 DMA_TLB_DSI_FLUSH);
1144 else
1145 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1146 DMA_TLB_PSI_FLUSH);
1147
1148 /*
1149 * In caching mode, changes of pages from non-present to present require
1150 * flush. However, device IOTLB doesn't need to be flushed in this case.
1151 */
1152 if (!cap_caching_mode(iommu->cap) || !map)
1153 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1154}
1155
1156static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1157{
1158 u32 pmen;
1159 unsigned long flags;
1160
1161 spin_lock_irqsave(&iommu->register_lock, flags);
1162 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1163 pmen &= ~DMA_PMEN_EPM;
1164 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1165
1166 /* wait for the protected region status bit to clear */
1167 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1168 readl, !(pmen & DMA_PMEN_PRS), pmen);
1169
1170 spin_unlock_irqrestore(&iommu->register_lock, flags);
1171}
1172
1173static int iommu_enable_translation(struct intel_iommu *iommu)
1174{
1175 u32 sts;
1176 unsigned long flags;
1177
1178 spin_lock_irqsave(&iommu->register_lock, flags);
1179 iommu->gcmd |= DMA_GCMD_TE;
1180 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1181
1182 /* Make sure hardware complete it */
1183 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1184 readl, (sts & DMA_GSTS_TES), sts);
1185
1186 spin_unlock_irqrestore(&iommu->register_lock, flags);
1187 return 0;
1188}
1189
1190static int iommu_disable_translation(struct intel_iommu *iommu)
1191{
1192 u32 sts;
1193 unsigned long flag;
1194
1195 spin_lock_irqsave(&iommu->register_lock, flag);
1196 iommu->gcmd &= ~DMA_GCMD_TE;
1197 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1198
1199 /* Make sure hardware complete it */
1200 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1201 readl, (!(sts & DMA_GSTS_TES)), sts);
1202
1203 spin_unlock_irqrestore(&iommu->register_lock, flag);
1204 return 0;
1205}
1206
1207
1208static int iommu_init_domains(struct intel_iommu *iommu)
1209{
1210 unsigned long ndomains;
1211 unsigned long nlongs;
1212
1213 ndomains = cap_ndoms(iommu->cap);
1214 pr_debug("IOMMU %d: Number of Domains supportd <%ld>\n", iommu->seq_id,
1215 ndomains);
1216 nlongs = BITS_TO_LONGS(ndomains);
1217
1218 spin_lock_init(&iommu->lock);
1219
1220 /* TBD: there might be 64K domains,
1221 * consider other allocation for future chip
1222 */
1223 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1224 if (!iommu->domain_ids) {
1225 printk(KERN_ERR "Allocating domain id array failed\n");
1226 return -ENOMEM;
1227 }
1228 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1229 GFP_KERNEL);
1230 if (!iommu->domains) {
1231 printk(KERN_ERR "Allocating domain array failed\n");
1232 return -ENOMEM;
1233 }
1234
1235 /*
1236 * if Caching mode is set, then invalid translations are tagged
1237 * with domainid 0. Hence we need to pre-allocate it.
1238 */
1239 if (cap_caching_mode(iommu->cap))
1240 set_bit(0, iommu->domain_ids);
1241 return 0;
1242}
1243
1244
1245static void domain_exit(struct dmar_domain *domain);
1246static void vm_domain_exit(struct dmar_domain *domain);
1247
1248void free_dmar_iommu(struct intel_iommu *iommu)
1249{
1250 struct dmar_domain *domain;
1251 int i;
1252 unsigned long flags;
1253
1254 if ((iommu->domains) && (iommu->domain_ids)) {
1255 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1256 domain = iommu->domains[i];
1257 clear_bit(i, iommu->domain_ids);
1258
1259 spin_lock_irqsave(&domain->iommu_lock, flags);
1260 if (--domain->iommu_count == 0) {
1261 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1262 vm_domain_exit(domain);
1263 else
1264 domain_exit(domain);
1265 }
1266 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1267 }
1268 }
1269
1270 if (iommu->gcmd & DMA_GCMD_TE)
1271 iommu_disable_translation(iommu);
1272
1273 if (iommu->irq) {
1274 irq_set_handler_data(iommu->irq, NULL);
1275 /* This will mask the irq */
1276 free_irq(iommu->irq, iommu);
1277 destroy_irq(iommu->irq);
1278 }
1279
1280 kfree(iommu->domains);
1281 kfree(iommu->domain_ids);
1282
1283 g_iommus[iommu->seq_id] = NULL;
1284
1285 /* if all iommus are freed, free g_iommus */
1286 for (i = 0; i < g_num_of_iommus; i++) {
1287 if (g_iommus[i])
1288 break;
1289 }
1290
1291 if (i == g_num_of_iommus)
1292 kfree(g_iommus);
1293
1294 /* free context mapping */
1295 free_context_table(iommu);
1296}
1297
1298static struct dmar_domain *alloc_domain(void)
1299{
1300 struct dmar_domain *domain;
1301
1302 domain = alloc_domain_mem();
1303 if (!domain)
1304 return NULL;
1305
1306 domain->nid = -1;
1307 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1308 domain->flags = 0;
1309
1310 return domain;
1311}
1312
1313static int iommu_attach_domain(struct dmar_domain *domain,
1314 struct intel_iommu *iommu)
1315{
1316 int num;
1317 unsigned long ndomains;
1318 unsigned long flags;
1319
1320 ndomains = cap_ndoms(iommu->cap);
1321
1322 spin_lock_irqsave(&iommu->lock, flags);
1323
1324 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1325 if (num >= ndomains) {
1326 spin_unlock_irqrestore(&iommu->lock, flags);
1327 printk(KERN_ERR "IOMMU: no free domain ids\n");
1328 return -ENOMEM;
1329 }
1330
1331 domain->id = num;
1332 set_bit(num, iommu->domain_ids);
1333 set_bit(iommu->seq_id, &domain->iommu_bmp);
1334 iommu->domains[num] = domain;
1335 spin_unlock_irqrestore(&iommu->lock, flags);
1336
1337 return 0;
1338}
1339
1340static void iommu_detach_domain(struct dmar_domain *domain,
1341 struct intel_iommu *iommu)
1342{
1343 unsigned long flags;
1344 int num, ndomains;
1345 int found = 0;
1346
1347 spin_lock_irqsave(&iommu->lock, flags);
1348 ndomains = cap_ndoms(iommu->cap);
1349 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1350 if (iommu->domains[num] == domain) {
1351 found = 1;
1352 break;
1353 }
1354 }
1355
1356 if (found) {
1357 clear_bit(num, iommu->domain_ids);
1358 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1359 iommu->domains[num] = NULL;
1360 }
1361 spin_unlock_irqrestore(&iommu->lock, flags);
1362}
1363
1364static struct iova_domain reserved_iova_list;
1365static struct lock_class_key reserved_rbtree_key;
1366
1367static int dmar_init_reserved_ranges(void)
1368{
1369 struct pci_dev *pdev = NULL;
1370 struct iova *iova;
1371 int i;
1372
1373 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1374
1375 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1376 &reserved_rbtree_key);
1377
1378 /* IOAPIC ranges shouldn't be accessed by DMA */
1379 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1380 IOVA_PFN(IOAPIC_RANGE_END));
1381 if (!iova) {
1382 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1383 return -ENODEV;
1384 }
1385
1386 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1387 for_each_pci_dev(pdev) {
1388 struct resource *r;
1389
1390 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1391 r = &pdev->resource[i];
1392 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1393 continue;
1394 iova = reserve_iova(&reserved_iova_list,
1395 IOVA_PFN(r->start),
1396 IOVA_PFN(r->end));
1397 if (!iova) {
1398 printk(KERN_ERR "Reserve iova failed\n");
1399 return -ENODEV;
1400 }
1401 }
1402 }
1403 return 0;
1404}
1405
1406static void domain_reserve_special_ranges(struct dmar_domain *domain)
1407{
1408 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1409}
1410
1411static inline int guestwidth_to_adjustwidth(int gaw)
1412{
1413 int agaw;
1414 int r = (gaw - 12) % 9;
1415
1416 if (r == 0)
1417 agaw = gaw;
1418 else
1419 agaw = gaw + 9 - r;
1420 if (agaw > 64)
1421 agaw = 64;
1422 return agaw;
1423}
1424
1425static int domain_init(struct dmar_domain *domain, int guest_width)
1426{
1427 struct intel_iommu *iommu;
1428 int adjust_width, agaw;
1429 unsigned long sagaw;
1430
1431 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1432 spin_lock_init(&domain->iommu_lock);
1433
1434 domain_reserve_special_ranges(domain);
1435
1436 /* calculate AGAW */
1437 iommu = domain_get_iommu(domain);
1438 if (guest_width > cap_mgaw(iommu->cap))
1439 guest_width = cap_mgaw(iommu->cap);
1440 domain->gaw = guest_width;
1441 adjust_width = guestwidth_to_adjustwidth(guest_width);
1442 agaw = width_to_agaw(adjust_width);
1443 sagaw = cap_sagaw(iommu->cap);
1444 if (!test_bit(agaw, &sagaw)) {
1445 /* hardware doesn't support it, choose a bigger one */
1446 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1447 agaw = find_next_bit(&sagaw, 5, agaw);
1448 if (agaw >= 5)
1449 return -ENODEV;
1450 }
1451 domain->agaw = agaw;
1452 INIT_LIST_HEAD(&domain->devices);
1453
1454 if (ecap_coherent(iommu->ecap))
1455 domain->iommu_coherency = 1;
1456 else
1457 domain->iommu_coherency = 0;
1458
1459 if (ecap_sc_support(iommu->ecap))
1460 domain->iommu_snooping = 1;
1461 else
1462 domain->iommu_snooping = 0;
1463
1464 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1465 domain->iommu_count = 1;
1466 domain->nid = iommu->node;
1467
1468 /* always allocate the top pgd */
1469 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1470 if (!domain->pgd)
1471 return -ENOMEM;
1472 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1473 return 0;
1474}
1475
1476static void domain_exit(struct dmar_domain *domain)
1477{
1478 struct dmar_drhd_unit *drhd;
1479 struct intel_iommu *iommu;
1480
1481 /* Domain 0 is reserved, so dont process it */
1482 if (!domain)
1483 return;
1484
1485 /* Flush any lazy unmaps that may reference this domain */
1486 if (!intel_iommu_strict)
1487 flush_unmaps_timeout(0);
1488
1489 domain_remove_dev_info(domain);
1490 /* destroy iovas */
1491 put_iova_domain(&domain->iovad);
1492
1493 /* clear ptes */
1494 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1495
1496 /* free page tables */
1497 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1498
1499 for_each_active_iommu(iommu, drhd)
1500 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1501 iommu_detach_domain(domain, iommu);
1502
1503 free_domain_mem(domain);
1504}
1505
1506static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1507 u8 bus, u8 devfn, int translation)
1508{
1509 struct context_entry *context;
1510 unsigned long flags;
1511 struct intel_iommu *iommu;
1512 struct dma_pte *pgd;
1513 unsigned long num;
1514 unsigned long ndomains;
1515 int id;
1516 int agaw;
1517 struct device_domain_info *info = NULL;
1518
1519 pr_debug("Set context mapping for %02x:%02x.%d\n",
1520 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1521
1522 BUG_ON(!domain->pgd);
1523 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1524 translation != CONTEXT_TT_MULTI_LEVEL);
1525
1526 iommu = device_to_iommu(segment, bus, devfn);
1527 if (!iommu)
1528 return -ENODEV;
1529
1530 context = device_to_context_entry(iommu, bus, devfn);
1531 if (!context)
1532 return -ENOMEM;
1533 spin_lock_irqsave(&iommu->lock, flags);
1534 if (context_present(context)) {
1535 spin_unlock_irqrestore(&iommu->lock, flags);
1536 return 0;
1537 }
1538
1539 id = domain->id;
1540 pgd = domain->pgd;
1541
1542 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1543 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1544 int found = 0;
1545
1546 /* find an available domain id for this device in iommu */
1547 ndomains = cap_ndoms(iommu->cap);
1548 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1549 if (iommu->domains[num] == domain) {
1550 id = num;
1551 found = 1;
1552 break;
1553 }
1554 }
1555
1556 if (found == 0) {
1557 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1558 if (num >= ndomains) {
1559 spin_unlock_irqrestore(&iommu->lock, flags);
1560 printk(KERN_ERR "IOMMU: no free domain ids\n");
1561 return -EFAULT;
1562 }
1563
1564 set_bit(num, iommu->domain_ids);
1565 iommu->domains[num] = domain;
1566 id = num;
1567 }
1568
1569 /* Skip top levels of page tables for
1570 * iommu which has less agaw than default.
1571 * Unnecessary for PT mode.
1572 */
1573 if (translation != CONTEXT_TT_PASS_THROUGH) {
1574 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1575 pgd = phys_to_virt(dma_pte_addr(pgd));
1576 if (!dma_pte_present(pgd)) {
1577 spin_unlock_irqrestore(&iommu->lock, flags);
1578 return -ENOMEM;
1579 }
1580 }
1581 }
1582 }
1583
1584 context_set_domain_id(context, id);
1585
1586 if (translation != CONTEXT_TT_PASS_THROUGH) {
1587 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1588 translation = info ? CONTEXT_TT_DEV_IOTLB :
1589 CONTEXT_TT_MULTI_LEVEL;
1590 }
1591 /*
1592 * In pass through mode, AW must be programmed to indicate the largest
1593 * AGAW value supported by hardware. And ASR is ignored by hardware.
1594 */
1595 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1596 context_set_address_width(context, iommu->msagaw);
1597 else {
1598 context_set_address_root(context, virt_to_phys(pgd));
1599 context_set_address_width(context, iommu->agaw);
1600 }
1601
1602 context_set_translation_type(context, translation);
1603 context_set_fault_enable(context);
1604 context_set_present(context);
1605 domain_flush_cache(domain, context, sizeof(*context));
1606
1607 /*
1608 * It's a non-present to present mapping. If hardware doesn't cache
1609 * non-present entry we only need to flush the write-buffer. If the
1610 * _does_ cache non-present entries, then it does so in the special
1611 * domain #0, which we have to flush:
1612 */
1613 if (cap_caching_mode(iommu->cap)) {
1614 iommu->flush.flush_context(iommu, 0,
1615 (((u16)bus) << 8) | devfn,
1616 DMA_CCMD_MASK_NOBIT,
1617 DMA_CCMD_DEVICE_INVL);
1618 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
1619 } else {
1620 iommu_flush_write_buffer(iommu);
1621 }
1622 iommu_enable_dev_iotlb(info);
1623 spin_unlock_irqrestore(&iommu->lock, flags);
1624
1625 spin_lock_irqsave(&domain->iommu_lock, flags);
1626 if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1627 domain->iommu_count++;
1628 if (domain->iommu_count == 1)
1629 domain->nid = iommu->node;
1630 domain_update_iommu_cap(domain);
1631 }
1632 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1633 return 0;
1634}
1635
1636static int
1637domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1638 int translation)
1639{
1640 int ret;
1641 struct pci_dev *tmp, *parent;
1642
1643 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
1644 pdev->bus->number, pdev->devfn,
1645 translation);
1646 if (ret)
1647 return ret;
1648
1649 /* dependent device mapping */
1650 tmp = pci_find_upstream_pcie_bridge(pdev);
1651 if (!tmp)
1652 return 0;
1653 /* Secondary interface's bus number and devfn 0 */
1654 parent = pdev->bus->self;
1655 while (parent != tmp) {
1656 ret = domain_context_mapping_one(domain,
1657 pci_domain_nr(parent->bus),
1658 parent->bus->number,
1659 parent->devfn, translation);
1660 if (ret)
1661 return ret;
1662 parent = parent->bus->self;
1663 }
1664 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
1665 return domain_context_mapping_one(domain,
1666 pci_domain_nr(tmp->subordinate),
1667 tmp->subordinate->number, 0,
1668 translation);
1669 else /* this is a legacy PCI bridge */
1670 return domain_context_mapping_one(domain,
1671 pci_domain_nr(tmp->bus),
1672 tmp->bus->number,
1673 tmp->devfn,
1674 translation);
1675}
1676
1677static int domain_context_mapped(struct pci_dev *pdev)
1678{
1679 int ret;
1680 struct pci_dev *tmp, *parent;
1681 struct intel_iommu *iommu;
1682
1683 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1684 pdev->devfn);
1685 if (!iommu)
1686 return -ENODEV;
1687
1688 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
1689 if (!ret)
1690 return ret;
1691 /* dependent device mapping */
1692 tmp = pci_find_upstream_pcie_bridge(pdev);
1693 if (!tmp)
1694 return ret;
1695 /* Secondary interface's bus number and devfn 0 */
1696 parent = pdev->bus->self;
1697 while (parent != tmp) {
1698 ret = device_context_mapped(iommu, parent->bus->number,
1699 parent->devfn);
1700 if (!ret)
1701 return ret;
1702 parent = parent->bus->self;
1703 }
1704 if (pci_is_pcie(tmp))
1705 return device_context_mapped(iommu, tmp->subordinate->number,
1706 0);
1707 else
1708 return device_context_mapped(iommu, tmp->bus->number,
1709 tmp->devfn);
1710}
1711
1712/* Returns a number of VTD pages, but aligned to MM page size */
1713static inline unsigned long aligned_nrpages(unsigned long host_addr,
1714 size_t size)
1715{
1716 host_addr &= ~PAGE_MASK;
1717 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1718}
1719
1720/* Return largest possible superpage level for a given mapping */
1721static inline int hardware_largepage_caps(struct dmar_domain *domain,
1722 unsigned long iov_pfn,
1723 unsigned long phy_pfn,
1724 unsigned long pages)
1725{
1726 int support, level = 1;
1727 unsigned long pfnmerge;
1728
1729 support = domain->iommu_superpage;
1730
1731 /* To use a large page, the virtual *and* physical addresses
1732 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1733 of them will mean we have to use smaller pages. So just
1734 merge them and check both at once. */
1735 pfnmerge = iov_pfn | phy_pfn;
1736
1737 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1738 pages >>= VTD_STRIDE_SHIFT;
1739 if (!pages)
1740 break;
1741 pfnmerge >>= VTD_STRIDE_SHIFT;
1742 level++;
1743 support--;
1744 }
1745 return level;
1746}
1747
1748static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1749 struct scatterlist *sg, unsigned long phys_pfn,
1750 unsigned long nr_pages, int prot)
1751{
1752 struct dma_pte *first_pte = NULL, *pte = NULL;
1753 phys_addr_t uninitialized_var(pteval);
1754 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1755 unsigned long sg_res;
1756 unsigned int largepage_lvl = 0;
1757 unsigned long lvl_pages = 0;
1758
1759 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1760
1761 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1762 return -EINVAL;
1763
1764 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1765
1766 if (sg)
1767 sg_res = 0;
1768 else {
1769 sg_res = nr_pages + 1;
1770 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1771 }
1772
1773 while (nr_pages > 0) {
1774 uint64_t tmp;
1775
1776 if (!sg_res) {
1777 sg_res = aligned_nrpages(sg->offset, sg->length);
1778 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1779 sg->dma_length = sg->length;
1780 pteval = page_to_phys(sg_page(sg)) | prot;
1781 phys_pfn = pteval >> VTD_PAGE_SHIFT;
1782 }
1783
1784 if (!pte) {
1785 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1786
1787 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
1788 if (!pte)
1789 return -ENOMEM;
1790 /* It is large page*/
1791 if (largepage_lvl > 1)
1792 pteval |= DMA_PTE_LARGE_PAGE;
1793 else
1794 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
1795
1796 }
1797 /* We don't need lock here, nobody else
1798 * touches the iova range
1799 */
1800 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
1801 if (tmp) {
1802 static int dumps = 5;
1803 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1804 iov_pfn, tmp, (unsigned long long)pteval);
1805 if (dumps) {
1806 dumps--;
1807 debug_dma_dump_mappings(NULL);
1808 }
1809 WARN_ON(1);
1810 }
1811
1812 lvl_pages = lvl_to_nr_pages(largepage_lvl);
1813
1814 BUG_ON(nr_pages < lvl_pages);
1815 BUG_ON(sg_res < lvl_pages);
1816
1817 nr_pages -= lvl_pages;
1818 iov_pfn += lvl_pages;
1819 phys_pfn += lvl_pages;
1820 pteval += lvl_pages * VTD_PAGE_SIZE;
1821 sg_res -= lvl_pages;
1822
1823 /* If the next PTE would be the first in a new page, then we
1824 need to flush the cache on the entries we've just written.
1825 And then we'll need to recalculate 'pte', so clear it and
1826 let it get set again in the if (!pte) block above.
1827
1828 If we're done (!nr_pages) we need to flush the cache too.
1829
1830 Also if we've been setting superpages, we may need to
1831 recalculate 'pte' and switch back to smaller pages for the
1832 end of the mapping, if the trailing size is not enough to
1833 use another superpage (i.e. sg_res < lvl_pages). */
1834 pte++;
1835 if (!nr_pages || first_pte_in_page(pte) ||
1836 (largepage_lvl > 1 && sg_res < lvl_pages)) {
1837 domain_flush_cache(domain, first_pte,
1838 (void *)pte - (void *)first_pte);
1839 pte = NULL;
1840 }
1841
1842 if (!sg_res && nr_pages)
1843 sg = sg_next(sg);
1844 }
1845 return 0;
1846}
1847
1848static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1849 struct scatterlist *sg, unsigned long nr_pages,
1850 int prot)
1851{
1852 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1853}
1854
1855static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1856 unsigned long phys_pfn, unsigned long nr_pages,
1857 int prot)
1858{
1859 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
1860}
1861
1862static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1863{
1864 if (!iommu)
1865 return;
1866
1867 clear_context_table(iommu, bus, devfn);
1868 iommu->flush.flush_context(iommu, 0, 0, 0,
1869 DMA_CCMD_GLOBAL_INVL);
1870 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1871}
1872
1873static void domain_remove_dev_info(struct dmar_domain *domain)
1874{
1875 struct device_domain_info *info;
1876 unsigned long flags;
1877 struct intel_iommu *iommu;
1878
1879 spin_lock_irqsave(&device_domain_lock, flags);
1880 while (!list_empty(&domain->devices)) {
1881 info = list_entry(domain->devices.next,
1882 struct device_domain_info, link);
1883 list_del(&info->link);
1884 list_del(&info->global);
1885 if (info->dev)
1886 info->dev->dev.archdata.iommu = NULL;
1887 spin_unlock_irqrestore(&device_domain_lock, flags);
1888
1889 iommu_disable_dev_iotlb(info);
1890 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
1891 iommu_detach_dev(iommu, info->bus, info->devfn);
1892 free_devinfo_mem(info);
1893
1894 spin_lock_irqsave(&device_domain_lock, flags);
1895 }
1896 spin_unlock_irqrestore(&device_domain_lock, flags);
1897}
1898
1899/*
1900 * find_domain
1901 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1902 */
1903static struct dmar_domain *
1904find_domain(struct pci_dev *pdev)
1905{
1906 struct device_domain_info *info;
1907
1908 /* No lock here, assumes no domain exit in normal case */
1909 info = pdev->dev.archdata.iommu;
1910 if (info)
1911 return info->domain;
1912 return NULL;
1913}
1914
1915/* domain is initialized */
1916static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1917{
1918 struct dmar_domain *domain, *found = NULL;
1919 struct intel_iommu *iommu;
1920 struct dmar_drhd_unit *drhd;
1921 struct device_domain_info *info, *tmp;
1922 struct pci_dev *dev_tmp;
1923 unsigned long flags;
1924 int bus = 0, devfn = 0;
1925 int segment;
1926 int ret;
1927
1928 domain = find_domain(pdev);
1929 if (domain)
1930 return domain;
1931
1932 segment = pci_domain_nr(pdev->bus);
1933
1934 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1935 if (dev_tmp) {
1936 if (pci_is_pcie(dev_tmp)) {
1937 bus = dev_tmp->subordinate->number;
1938 devfn = 0;
1939 } else {
1940 bus = dev_tmp->bus->number;
1941 devfn = dev_tmp->devfn;
1942 }
1943 spin_lock_irqsave(&device_domain_lock, flags);
1944 list_for_each_entry(info, &device_domain_list, global) {
1945 if (info->segment == segment &&
1946 info->bus == bus && info->devfn == devfn) {
1947 found = info->domain;
1948 break;
1949 }
1950 }
1951 spin_unlock_irqrestore(&device_domain_lock, flags);
1952 /* pcie-pci bridge already has a domain, uses it */
1953 if (found) {
1954 domain = found;
1955 goto found_domain;
1956 }
1957 }
1958
1959 domain = alloc_domain();
1960 if (!domain)
1961 goto error;
1962
1963 /* Allocate new domain for the device */
1964 drhd = dmar_find_matched_drhd_unit(pdev);
1965 if (!drhd) {
1966 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1967 pci_name(pdev));
1968 return NULL;
1969 }
1970 iommu = drhd->iommu;
1971
1972 ret = iommu_attach_domain(domain, iommu);
1973 if (ret) {
1974 free_domain_mem(domain);
1975 goto error;
1976 }
1977
1978 if (domain_init(domain, gaw)) {
1979 domain_exit(domain);
1980 goto error;
1981 }
1982
1983 /* register pcie-to-pci device */
1984 if (dev_tmp) {
1985 info = alloc_devinfo_mem();
1986 if (!info) {
1987 domain_exit(domain);
1988 goto error;
1989 }
1990 info->segment = segment;
1991 info->bus = bus;
1992 info->devfn = devfn;
1993 info->dev = NULL;
1994 info->domain = domain;
1995 /* This domain is shared by devices under p2p bridge */
1996 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
1997
1998 /* pcie-to-pci bridge already has a domain, uses it */
1999 found = NULL;
2000 spin_lock_irqsave(&device_domain_lock, flags);
2001 list_for_each_entry(tmp, &device_domain_list, global) {
2002 if (tmp->segment == segment &&
2003 tmp->bus == bus && tmp->devfn == devfn) {
2004 found = tmp->domain;
2005 break;
2006 }
2007 }
2008 if (found) {
2009 spin_unlock_irqrestore(&device_domain_lock, flags);
2010 free_devinfo_mem(info);
2011 domain_exit(domain);
2012 domain = found;
2013 } else {
2014 list_add(&info->link, &domain->devices);
2015 list_add(&info->global, &device_domain_list);
2016 spin_unlock_irqrestore(&device_domain_lock, flags);
2017 }
2018 }
2019
2020found_domain:
2021 info = alloc_devinfo_mem();
2022 if (!info)
2023 goto error;
2024 info->segment = segment;
2025 info->bus = pdev->bus->number;
2026 info->devfn = pdev->devfn;
2027 info->dev = pdev;
2028 info->domain = domain;
2029 spin_lock_irqsave(&device_domain_lock, flags);
2030 /* somebody is fast */
2031 found = find_domain(pdev);
2032 if (found != NULL) {
2033 spin_unlock_irqrestore(&device_domain_lock, flags);
2034 if (found != domain) {
2035 domain_exit(domain);
2036 domain = found;
2037 }
2038 free_devinfo_mem(info);
2039 return domain;
2040 }
2041 list_add(&info->link, &domain->devices);
2042 list_add(&info->global, &device_domain_list);
2043 pdev->dev.archdata.iommu = info;
2044 spin_unlock_irqrestore(&device_domain_lock, flags);
2045 return domain;
2046error:
2047 /* recheck it here, maybe others set it */
2048 return find_domain(pdev);
2049}
2050
2051static int iommu_identity_mapping;
2052#define IDENTMAP_ALL 1
2053#define IDENTMAP_GFX 2
2054#define IDENTMAP_AZALIA 4
2055
2056static int iommu_domain_identity_map(struct dmar_domain *domain,
2057 unsigned long long start,
2058 unsigned long long end)
2059{
2060 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2061 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2062
2063 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2064 dma_to_mm_pfn(last_vpfn))) {
2065 printk(KERN_ERR "IOMMU: reserve iova failed\n");
2066 return -ENOMEM;
2067 }
2068
2069 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2070 start, end, domain->id);
2071 /*
2072 * RMRR range might have overlap with physical memory range,
2073 * clear it first
2074 */
2075 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2076
2077 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2078 last_vpfn - first_vpfn + 1,
2079 DMA_PTE_READ|DMA_PTE_WRITE);
2080}
2081
2082static int iommu_prepare_identity_map(struct pci_dev *pdev,
2083 unsigned long long start,
2084 unsigned long long end)
2085{
2086 struct dmar_domain *domain;
2087 int ret;
2088
2089 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2090 if (!domain)
2091 return -ENOMEM;
2092
2093 /* For _hardware_ passthrough, don't bother. But for software
2094 passthrough, we do it anyway -- it may indicate a memory
2095 range which is reserved in E820, so which didn't get set
2096 up to start with in si_domain */
2097 if (domain == si_domain && hw_pass_through) {
2098 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2099 pci_name(pdev), start, end);
2100 return 0;
2101 }
2102
2103 printk(KERN_INFO
2104 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2105 pci_name(pdev), start, end);
2106
2107 if (end < start) {
2108 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2109 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2110 dmi_get_system_info(DMI_BIOS_VENDOR),
2111 dmi_get_system_info(DMI_BIOS_VERSION),
2112 dmi_get_system_info(DMI_PRODUCT_VERSION));
2113 ret = -EIO;
2114 goto error;
2115 }
2116
2117 if (end >> agaw_to_width(domain->agaw)) {
2118 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2119 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2120 agaw_to_width(domain->agaw),
2121 dmi_get_system_info(DMI_BIOS_VENDOR),
2122 dmi_get_system_info(DMI_BIOS_VERSION),
2123 dmi_get_system_info(DMI_PRODUCT_VERSION));
2124 ret = -EIO;
2125 goto error;
2126 }
2127
2128 ret = iommu_domain_identity_map(domain, start, end);
2129 if (ret)
2130 goto error;
2131
2132 /* context entry init */
2133 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
2134 if (ret)
2135 goto error;
2136
2137 return 0;
2138
2139 error:
2140 domain_exit(domain);
2141 return ret;
2142}
2143
2144static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2145 struct pci_dev *pdev)
2146{
2147 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2148 return 0;
2149 return iommu_prepare_identity_map(pdev, rmrr->base_address,
2150 rmrr->end_address);
2151}
2152
2153#ifdef CONFIG_DMAR_FLOPPY_WA
2154static inline void iommu_prepare_isa(void)
2155{
2156 struct pci_dev *pdev;
2157 int ret;
2158
2159 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2160 if (!pdev)
2161 return;
2162
2163 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2164 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
2165
2166 if (ret)
2167 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2168 "floppy might not work\n");
2169
2170}
2171#else
2172static inline void iommu_prepare_isa(void)
2173{
2174 return;
2175}
2176#endif /* !CONFIG_DMAR_FLPY_WA */
2177
2178static int md_domain_init(struct dmar_domain *domain, int guest_width);
2179
2180static int __init si_domain_work_fn(unsigned long start_pfn,
2181 unsigned long end_pfn, void *datax)
2182{
2183 int *ret = datax;
2184
2185 *ret = iommu_domain_identity_map(si_domain,
2186 (uint64_t)start_pfn << PAGE_SHIFT,
2187 (uint64_t)end_pfn << PAGE_SHIFT);
2188 return *ret;
2189
2190}
2191
2192static int __init si_domain_init(int hw)
2193{
2194 struct dmar_drhd_unit *drhd;
2195 struct intel_iommu *iommu;
2196 int nid, ret = 0;
2197
2198 si_domain = alloc_domain();
2199 if (!si_domain)
2200 return -EFAULT;
2201
2202 pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2203
2204 for_each_active_iommu(iommu, drhd) {
2205 ret = iommu_attach_domain(si_domain, iommu);
2206 if (ret) {
2207 domain_exit(si_domain);
2208 return -EFAULT;
2209 }
2210 }
2211
2212 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2213 domain_exit(si_domain);
2214 return -EFAULT;
2215 }
2216
2217 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2218
2219 if (hw)
2220 return 0;
2221
2222 for_each_online_node(nid) {
2223 work_with_active_regions(nid, si_domain_work_fn, &ret);
2224 if (ret)
2225 return ret;
2226 }
2227
2228 return 0;
2229}
2230
2231static void domain_remove_one_dev_info(struct dmar_domain *domain,
2232 struct pci_dev *pdev);
2233static int identity_mapping(struct pci_dev *pdev)
2234{
2235 struct device_domain_info *info;
2236
2237 if (likely(!iommu_identity_mapping))
2238 return 0;
2239
2240 info = pdev->dev.archdata.iommu;
2241 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2242 return (info->domain == si_domain);
2243
2244 return 0;
2245}
2246
2247static int domain_add_dev_info(struct dmar_domain *domain,
2248 struct pci_dev *pdev,
2249 int translation)
2250{
2251 struct device_domain_info *info;
2252 unsigned long flags;
2253 int ret;
2254
2255 info = alloc_devinfo_mem();
2256 if (!info)
2257 return -ENOMEM;
2258
2259 ret = domain_context_mapping(domain, pdev, translation);
2260 if (ret) {
2261 free_devinfo_mem(info);
2262 return ret;
2263 }
2264
2265 info->segment = pci_domain_nr(pdev->bus);
2266 info->bus = pdev->bus->number;
2267 info->devfn = pdev->devfn;
2268 info->dev = pdev;
2269 info->domain = domain;
2270
2271 spin_lock_irqsave(&device_domain_lock, flags);
2272 list_add(&info->link, &domain->devices);
2273 list_add(&info->global, &device_domain_list);
2274 pdev->dev.archdata.iommu = info;
2275 spin_unlock_irqrestore(&device_domain_lock, flags);
2276
2277 return 0;
2278}
2279
2280static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2281{
2282 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2283 return 1;
2284
2285 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2286 return 1;
2287
2288 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2289 return 0;
2290
2291 /*
2292 * We want to start off with all devices in the 1:1 domain, and
2293 * take them out later if we find they can't access all of memory.
2294 *
2295 * However, we can't do this for PCI devices behind bridges,
2296 * because all PCI devices behind the same bridge will end up
2297 * with the same source-id on their transactions.
2298 *
2299 * Practically speaking, we can't change things around for these
2300 * devices at run-time, because we can't be sure there'll be no
2301 * DMA transactions in flight for any of their siblings.
2302 *
2303 * So PCI devices (unless they're on the root bus) as well as
2304 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2305 * the 1:1 domain, just in _case_ one of their siblings turns out
2306 * not to be able to map all of memory.
2307 */
2308 if (!pci_is_pcie(pdev)) {
2309 if (!pci_is_root_bus(pdev->bus))
2310 return 0;
2311 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2312 return 0;
2313 } else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
2314 return 0;
2315
2316 /*
2317 * At boot time, we don't yet know if devices will be 64-bit capable.
2318 * Assume that they will -- if they turn out not to be, then we can
2319 * take them out of the 1:1 domain later.
2320 */
2321 if (!startup) {
2322 /*
2323 * If the device's dma_mask is less than the system's memory
2324 * size then this is not a candidate for identity mapping.
2325 */
2326 u64 dma_mask = pdev->dma_mask;
2327
2328 if (pdev->dev.coherent_dma_mask &&
2329 pdev->dev.coherent_dma_mask < dma_mask)
2330 dma_mask = pdev->dev.coherent_dma_mask;
2331
2332 return dma_mask >= dma_get_required_mask(&pdev->dev);
2333 }
2334
2335 return 1;
2336}
2337
2338static int __init iommu_prepare_static_identity_mapping(int hw)
2339{
2340 struct pci_dev *pdev = NULL;
2341 int ret;
2342
2343 ret = si_domain_init(hw);
2344 if (ret)
2345 return -EFAULT;
2346
2347 for_each_pci_dev(pdev) {
2348 /* Skip Host/PCI Bridge devices */
2349 if (IS_BRIDGE_HOST_DEVICE(pdev))
2350 continue;
2351 if (iommu_should_identity_map(pdev, 1)) {
2352 printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
2353 hw ? "hardware" : "software", pci_name(pdev));
2354
2355 ret = domain_add_dev_info(si_domain, pdev,
2356 hw ? CONTEXT_TT_PASS_THROUGH :
2357 CONTEXT_TT_MULTI_LEVEL);
2358 if (ret)
2359 return ret;
2360 }
2361 }
2362
2363 return 0;
2364}
2365
2366static int __init init_dmars(void)
2367{
2368 struct dmar_drhd_unit *drhd;
2369 struct dmar_rmrr_unit *rmrr;
2370 struct pci_dev *pdev;
2371 struct intel_iommu *iommu;
2372 int i, ret;
2373
2374 /*
2375 * for each drhd
2376 * allocate root
2377 * initialize and program root entry to not present
2378 * endfor
2379 */
2380 for_each_drhd_unit(drhd) {
2381 g_num_of_iommus++;
2382 /*
2383 * lock not needed as this is only incremented in the single
2384 * threaded kernel __init code path all other access are read
2385 * only
2386 */
2387 }
2388
2389 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2390 GFP_KERNEL);
2391 if (!g_iommus) {
2392 printk(KERN_ERR "Allocating global iommu array failed\n");
2393 ret = -ENOMEM;
2394 goto error;
2395 }
2396
2397 deferred_flush = kzalloc(g_num_of_iommus *
2398 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2399 if (!deferred_flush) {
2400 ret = -ENOMEM;
2401 goto error;
2402 }
2403
2404 for_each_drhd_unit(drhd) {
2405 if (drhd->ignored)
2406 continue;
2407
2408 iommu = drhd->iommu;
2409 g_iommus[iommu->seq_id] = iommu;
2410
2411 ret = iommu_init_domains(iommu);
2412 if (ret)
2413 goto error;
2414
2415 /*
2416 * TBD:
2417 * we could share the same root & context tables
2418 * among all IOMMU's. Need to Split it later.
2419 */
2420 ret = iommu_alloc_root_entry(iommu);
2421 if (ret) {
2422 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2423 goto error;
2424 }
2425 if (!ecap_pass_through(iommu->ecap))
2426 hw_pass_through = 0;
2427 }
2428
2429 /*
2430 * Start from the sane iommu hardware state.
2431 */
2432 for_each_drhd_unit(drhd) {
2433 if (drhd->ignored)
2434 continue;
2435
2436 iommu = drhd->iommu;
2437
2438 /*
2439 * If the queued invalidation is already initialized by us
2440 * (for example, while enabling interrupt-remapping) then
2441 * we got the things already rolling from a sane state.
2442 */
2443 if (iommu->qi)
2444 continue;
2445
2446 /*
2447 * Clear any previous faults.
2448 */
2449 dmar_fault(-1, iommu);
2450 /*
2451 * Disable queued invalidation if supported and already enabled
2452 * before OS handover.
2453 */
2454 dmar_disable_qi(iommu);
2455 }
2456
2457 for_each_drhd_unit(drhd) {
2458 if (drhd->ignored)
2459 continue;
2460
2461 iommu = drhd->iommu;
2462
2463 if (dmar_enable_qi(iommu)) {
2464 /*
2465 * Queued Invalidate not enabled, use Register Based
2466 * Invalidate
2467 */
2468 iommu->flush.flush_context = __iommu_flush_context;
2469 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2470 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
2471 "invalidation\n",
2472 iommu->seq_id,
2473 (unsigned long long)drhd->reg_base_addr);
2474 } else {
2475 iommu->flush.flush_context = qi_flush_context;
2476 iommu->flush.flush_iotlb = qi_flush_iotlb;
2477 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
2478 "invalidation\n",
2479 iommu->seq_id,
2480 (unsigned long long)drhd->reg_base_addr);
2481 }
2482 }
2483
2484 if (iommu_pass_through)
2485 iommu_identity_mapping |= IDENTMAP_ALL;
2486
2487#ifdef CONFIG_DMAR_BROKEN_GFX_WA
2488 iommu_identity_mapping |= IDENTMAP_GFX;
2489#endif
2490
2491 check_tylersburg_isoch();
2492
2493 /*
2494 * If pass through is not set or not enabled, setup context entries for
2495 * identity mappings for rmrr, gfx, and isa and may fall back to static
2496 * identity mapping if iommu_identity_mapping is set.
2497 */
2498 if (iommu_identity_mapping) {
2499 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2500 if (ret) {
2501 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2502 goto error;
2503 }
2504 }
2505 /*
2506 * For each rmrr
2507 * for each dev attached to rmrr
2508 * do
2509 * locate drhd for dev, alloc domain for dev
2510 * allocate free domain
2511 * allocate page table entries for rmrr
2512 * if context not allocated for bus
2513 * allocate and init context
2514 * set present in root table for this bus
2515 * init context with domain, translation etc
2516 * endfor
2517 * endfor
2518 */
2519 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2520 for_each_rmrr_units(rmrr) {
2521 for (i = 0; i < rmrr->devices_cnt; i++) {
2522 pdev = rmrr->devices[i];
2523 /*
2524 * some BIOS lists non-exist devices in DMAR
2525 * table.
2526 */
2527 if (!pdev)
2528 continue;
2529 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2530 if (ret)
2531 printk(KERN_ERR
2532 "IOMMU: mapping reserved region failed\n");
2533 }
2534 }
2535
2536 iommu_prepare_isa();
2537
2538 /*
2539 * for each drhd
2540 * enable fault log
2541 * global invalidate context cache
2542 * global invalidate iotlb
2543 * enable translation
2544 */
2545 for_each_drhd_unit(drhd) {
2546 if (drhd->ignored) {
2547 /*
2548 * we always have to disable PMRs or DMA may fail on
2549 * this device
2550 */
2551 if (force_on)
2552 iommu_disable_protect_mem_regions(drhd->iommu);
2553 continue;
2554 }
2555 iommu = drhd->iommu;
2556
2557 iommu_flush_write_buffer(iommu);
2558
2559 ret = dmar_set_interrupt(iommu);
2560 if (ret)
2561 goto error;
2562
2563 iommu_set_root_entry(iommu);
2564
2565 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2566 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2567
2568 ret = iommu_enable_translation(iommu);
2569 if (ret)
2570 goto error;
2571
2572 iommu_disable_protect_mem_regions(iommu);
2573 }
2574
2575 return 0;
2576error:
2577 for_each_drhd_unit(drhd) {
2578 if (drhd->ignored)
2579 continue;
2580 iommu = drhd->iommu;
2581 free_iommu(iommu);
2582 }
2583 kfree(g_iommus);
2584 return ret;
2585}
2586
2587/* This takes a number of _MM_ pages, not VTD pages */
2588static struct iova *intel_alloc_iova(struct device *dev,
2589 struct dmar_domain *domain,
2590 unsigned long nrpages, uint64_t dma_mask)
2591{
2592 struct pci_dev *pdev = to_pci_dev(dev);
2593 struct iova *iova = NULL;
2594
2595 /* Restrict dma_mask to the width that the iommu can handle */
2596 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2597
2598 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2599 /*
2600 * First try to allocate an io virtual address in
2601 * DMA_BIT_MASK(32) and if that fails then try allocating
2602 * from higher range
2603 */
2604 iova = alloc_iova(&domain->iovad, nrpages,
2605 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2606 if (iova)
2607 return iova;
2608 }
2609 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2610 if (unlikely(!iova)) {
2611 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2612 nrpages, pci_name(pdev));
2613 return NULL;
2614 }
2615
2616 return iova;
2617}
2618
2619static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
2620{
2621 struct dmar_domain *domain;
2622 int ret;
2623
2624 domain = get_domain_for_dev(pdev,
2625 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2626 if (!domain) {
2627 printk(KERN_ERR
2628 "Allocating domain for %s failed", pci_name(pdev));
2629 return NULL;
2630 }
2631
2632 /* make sure context mapping is ok */
2633 if (unlikely(!domain_context_mapped(pdev))) {
2634 ret = domain_context_mapping(domain, pdev,
2635 CONTEXT_TT_MULTI_LEVEL);
2636 if (ret) {
2637 printk(KERN_ERR
2638 "Domain context map for %s failed",
2639 pci_name(pdev));
2640 return NULL;
2641 }
2642 }
2643
2644 return domain;
2645}
2646
2647static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2648{
2649 struct device_domain_info *info;
2650
2651 /* No lock here, assumes no domain exit in normal case */
2652 info = dev->dev.archdata.iommu;
2653 if (likely(info))
2654 return info->domain;
2655
2656 return __get_valid_domain_for_dev(dev);
2657}
2658
2659static int iommu_dummy(struct pci_dev *pdev)
2660{
2661 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2662}
2663
2664/* Check if the pdev needs to go through non-identity map and unmap process.*/
2665static int iommu_no_mapping(struct device *dev)
2666{
2667 struct pci_dev *pdev;
2668 int found;
2669
2670 if (unlikely(dev->bus != &pci_bus_type))
2671 return 1;
2672
2673 pdev = to_pci_dev(dev);
2674 if (iommu_dummy(pdev))
2675 return 1;
2676
2677 if (!iommu_identity_mapping)
2678 return 0;
2679
2680 found = identity_mapping(pdev);
2681 if (found) {
2682 if (iommu_should_identity_map(pdev, 0))
2683 return 1;
2684 else {
2685 /*
2686 * 32 bit DMA is removed from si_domain and fall back
2687 * to non-identity mapping.
2688 */
2689 domain_remove_one_dev_info(si_domain, pdev);
2690 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2691 pci_name(pdev));
2692 return 0;
2693 }
2694 } else {
2695 /*
2696 * In case of a detached 64 bit DMA device from vm, the device
2697 * is put into si_domain for identity mapping.
2698 */
2699 if (iommu_should_identity_map(pdev, 0)) {
2700 int ret;
2701 ret = domain_add_dev_info(si_domain, pdev,
2702 hw_pass_through ?
2703 CONTEXT_TT_PASS_THROUGH :
2704 CONTEXT_TT_MULTI_LEVEL);
2705 if (!ret) {
2706 printk(KERN_INFO "64bit %s uses identity mapping\n",
2707 pci_name(pdev));
2708 return 1;
2709 }
2710 }
2711 }
2712
2713 return 0;
2714}
2715
2716static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2717 size_t size, int dir, u64 dma_mask)
2718{
2719 struct pci_dev *pdev = to_pci_dev(hwdev);
2720 struct dmar_domain *domain;
2721 phys_addr_t start_paddr;
2722 struct iova *iova;
2723 int prot = 0;
2724 int ret;
2725 struct intel_iommu *iommu;
2726 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
2727
2728 BUG_ON(dir == DMA_NONE);
2729
2730 if (iommu_no_mapping(hwdev))
2731 return paddr;
2732
2733 domain = get_valid_domain_for_dev(pdev);
2734 if (!domain)
2735 return 0;
2736
2737 iommu = domain_get_iommu(domain);
2738 size = aligned_nrpages(paddr, size);
2739
2740 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
2741 if (!iova)
2742 goto error;
2743
2744 /*
2745 * Check if DMAR supports zero-length reads on write only
2746 * mappings..
2747 */
2748 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2749 !cap_zlr(iommu->cap))
2750 prot |= DMA_PTE_READ;
2751 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2752 prot |= DMA_PTE_WRITE;
2753 /*
2754 * paddr - (paddr + size) might be partial page, we should map the whole
2755 * page. Note: if two part of one page are separately mapped, we
2756 * might have two guest_addr mapping to the same host paddr, but this
2757 * is not a big problem
2758 */
2759 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
2760 mm_to_dma_pfn(paddr_pfn), size, prot);
2761 if (ret)
2762 goto error;
2763
2764 /* it's a non-present to present mapping. Only flush if caching mode */
2765 if (cap_caching_mode(iommu->cap))
2766 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
2767 else
2768 iommu_flush_write_buffer(iommu);
2769
2770 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2771 start_paddr += paddr & ~PAGE_MASK;
2772 return start_paddr;
2773
2774error:
2775 if (iova)
2776 __free_iova(&domain->iovad, iova);
2777 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
2778 pci_name(pdev), size, (unsigned long long)paddr, dir);
2779 return 0;
2780}
2781
2782static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2783 unsigned long offset, size_t size,
2784 enum dma_data_direction dir,
2785 struct dma_attrs *attrs)
2786{
2787 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2788 dir, to_pci_dev(dev)->dma_mask);
2789}
2790
2791static void flush_unmaps(void)
2792{
2793 int i, j;
2794
2795 timer_on = 0;
2796
2797 /* just flush them all */
2798 for (i = 0; i < g_num_of_iommus; i++) {
2799 struct intel_iommu *iommu = g_iommus[i];
2800 if (!iommu)
2801 continue;
2802
2803 if (!deferred_flush[i].next)
2804 continue;
2805
2806 /* In caching mode, global flushes turn emulation expensive */
2807 if (!cap_caching_mode(iommu->cap))
2808 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2809 DMA_TLB_GLOBAL_FLUSH);
2810 for (j = 0; j < deferred_flush[i].next; j++) {
2811 unsigned long mask;
2812 struct iova *iova = deferred_flush[i].iova[j];
2813 struct dmar_domain *domain = deferred_flush[i].domain[j];
2814
2815 /* On real hardware multiple invalidations are expensive */
2816 if (cap_caching_mode(iommu->cap))
2817 iommu_flush_iotlb_psi(iommu, domain->id,
2818 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2819 else {
2820 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2821 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2822 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2823 }
2824 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
2825 }
2826 deferred_flush[i].next = 0;
2827 }
2828
2829 list_size = 0;
2830}
2831
2832static void flush_unmaps_timeout(unsigned long data)
2833{
2834 unsigned long flags;
2835
2836 spin_lock_irqsave(&async_umap_flush_lock, flags);
2837 flush_unmaps();
2838 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2839}
2840
2841static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2842{
2843 unsigned long flags;
2844 int next, iommu_id;
2845 struct intel_iommu *iommu;
2846
2847 spin_lock_irqsave(&async_umap_flush_lock, flags);
2848 if (list_size == HIGH_WATER_MARK)
2849 flush_unmaps();
2850
2851 iommu = domain_get_iommu(dom);
2852 iommu_id = iommu->seq_id;
2853
2854 next = deferred_flush[iommu_id].next;
2855 deferred_flush[iommu_id].domain[next] = dom;
2856 deferred_flush[iommu_id].iova[next] = iova;
2857 deferred_flush[iommu_id].next++;
2858
2859 if (!timer_on) {
2860 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2861 timer_on = 1;
2862 }
2863 list_size++;
2864 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2865}
2866
2867static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2868 size_t size, enum dma_data_direction dir,
2869 struct dma_attrs *attrs)
2870{
2871 struct pci_dev *pdev = to_pci_dev(dev);
2872 struct dmar_domain *domain;
2873 unsigned long start_pfn, last_pfn;
2874 struct iova *iova;
2875 struct intel_iommu *iommu;
2876
2877 if (iommu_no_mapping(dev))
2878 return;
2879
2880 domain = find_domain(pdev);
2881 BUG_ON(!domain);
2882
2883 iommu = domain_get_iommu(domain);
2884
2885 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2886 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2887 (unsigned long long)dev_addr))
2888 return;
2889
2890 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2891 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2892
2893 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2894 pci_name(pdev), start_pfn, last_pfn);
2895
2896 /* clear the whole page */
2897 dma_pte_clear_range(domain, start_pfn, last_pfn);
2898
2899 /* free page tables */
2900 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2901
2902 if (intel_iommu_strict) {
2903 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2904 last_pfn - start_pfn + 1, 0);
2905 /* free iova */
2906 __free_iova(&domain->iovad, iova);
2907 } else {
2908 add_unmap(domain, iova);
2909 /*
2910 * queue up the release of the unmap to save the 1/6th of the
2911 * cpu used up by the iotlb flush operation...
2912 */
2913 }
2914}
2915
2916static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2917 dma_addr_t *dma_handle, gfp_t flags)
2918{
2919 void *vaddr;
2920 int order;
2921
2922 size = PAGE_ALIGN(size);
2923 order = get_order(size);
2924
2925 if (!iommu_no_mapping(hwdev))
2926 flags &= ~(GFP_DMA | GFP_DMA32);
2927 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2928 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2929 flags |= GFP_DMA;
2930 else
2931 flags |= GFP_DMA32;
2932 }
2933
2934 vaddr = (void *)__get_free_pages(flags, order);
2935 if (!vaddr)
2936 return NULL;
2937 memset(vaddr, 0, size);
2938
2939 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2940 DMA_BIDIRECTIONAL,
2941 hwdev->coherent_dma_mask);
2942 if (*dma_handle)
2943 return vaddr;
2944 free_pages((unsigned long)vaddr, order);
2945 return NULL;
2946}
2947
2948static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2949 dma_addr_t dma_handle)
2950{
2951 int order;
2952
2953 size = PAGE_ALIGN(size);
2954 order = get_order(size);
2955
2956 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
2957 free_pages((unsigned long)vaddr, order);
2958}
2959
2960static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2961 int nelems, enum dma_data_direction dir,
2962 struct dma_attrs *attrs)
2963{
2964 struct pci_dev *pdev = to_pci_dev(hwdev);
2965 struct dmar_domain *domain;
2966 unsigned long start_pfn, last_pfn;
2967 struct iova *iova;
2968 struct intel_iommu *iommu;
2969
2970 if (iommu_no_mapping(hwdev))
2971 return;
2972
2973 domain = find_domain(pdev);
2974 BUG_ON(!domain);
2975
2976 iommu = domain_get_iommu(domain);
2977
2978 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2979 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
2980 (unsigned long long)sglist[0].dma_address))
2981 return;
2982
2983 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2984 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2985
2986 /* clear the whole page */
2987 dma_pte_clear_range(domain, start_pfn, last_pfn);
2988
2989 /* free page tables */
2990 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2991
2992 if (intel_iommu_strict) {
2993 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2994 last_pfn - start_pfn + 1, 0);
2995 /* free iova */
2996 __free_iova(&domain->iovad, iova);
2997 } else {
2998 add_unmap(domain, iova);
2999 /*
3000 * queue up the release of the unmap to save the 1/6th of the
3001 * cpu used up by the iotlb flush operation...
3002 */
3003 }
3004}
3005
3006static int intel_nontranslate_map_sg(struct device *hddev,
3007 struct scatterlist *sglist, int nelems, int dir)
3008{
3009 int i;
3010 struct scatterlist *sg;
3011
3012 for_each_sg(sglist, sg, nelems, i) {
3013 BUG_ON(!sg_page(sg));
3014 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3015 sg->dma_length = sg->length;
3016 }
3017 return nelems;
3018}
3019
3020static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
3021 enum dma_data_direction dir, struct dma_attrs *attrs)
3022{
3023 int i;
3024 struct pci_dev *pdev = to_pci_dev(hwdev);
3025 struct dmar_domain *domain;
3026 size_t size = 0;
3027 int prot = 0;
3028 struct iova *iova = NULL;
3029 int ret;
3030 struct scatterlist *sg;
3031 unsigned long start_vpfn;
3032 struct intel_iommu *iommu;
3033
3034 BUG_ON(dir == DMA_NONE);
3035 if (iommu_no_mapping(hwdev))
3036 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
3037
3038 domain = get_valid_domain_for_dev(pdev);
3039 if (!domain)
3040 return 0;
3041
3042 iommu = domain_get_iommu(domain);
3043
3044 for_each_sg(sglist, sg, nelems, i)
3045 size += aligned_nrpages(sg->offset, sg->length);
3046
3047 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
3048 pdev->dma_mask);
3049 if (!iova) {
3050 sglist->dma_length = 0;
3051 return 0;
3052 }
3053
3054 /*
3055 * Check if DMAR supports zero-length reads on write only
3056 * mappings..
3057 */
3058 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3059 !cap_zlr(iommu->cap))
3060 prot |= DMA_PTE_READ;
3061 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3062 prot |= DMA_PTE_WRITE;
3063
3064 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3065
3066 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3067 if (unlikely(ret)) {
3068 /* clear the page */
3069 dma_pte_clear_range(domain, start_vpfn,
3070 start_vpfn + size - 1);
3071 /* free page tables */
3072 dma_pte_free_pagetable(domain, start_vpfn,
3073 start_vpfn + size - 1);
3074 /* free iova */
3075 __free_iova(&domain->iovad, iova);
3076 return 0;
3077 }
3078
3079 /* it's a non-present to present mapping. Only flush if caching mode */
3080 if (cap_caching_mode(iommu->cap))
3081 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
3082 else
3083 iommu_flush_write_buffer(iommu);
3084
3085 return nelems;
3086}
3087
3088static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3089{
3090 return !dma_addr;
3091}
3092
3093struct dma_map_ops intel_dma_ops = {
3094 .alloc_coherent = intel_alloc_coherent,
3095 .free_coherent = intel_free_coherent,
3096 .map_sg = intel_map_sg,
3097 .unmap_sg = intel_unmap_sg,
3098 .map_page = intel_map_page,
3099 .unmap_page = intel_unmap_page,
3100 .mapping_error = intel_mapping_error,
3101};
3102
3103static inline int iommu_domain_cache_init(void)
3104{
3105 int ret = 0;
3106
3107 iommu_domain_cache = kmem_cache_create("iommu_domain",
3108 sizeof(struct dmar_domain),
3109 0,
3110 SLAB_HWCACHE_ALIGN,
3111
3112 NULL);
3113 if (!iommu_domain_cache) {
3114 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3115 ret = -ENOMEM;
3116 }
3117
3118 return ret;
3119}
3120
3121static inline int iommu_devinfo_cache_init(void)
3122{
3123 int ret = 0;
3124
3125 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3126 sizeof(struct device_domain_info),
3127 0,
3128 SLAB_HWCACHE_ALIGN,
3129 NULL);
3130 if (!iommu_devinfo_cache) {
3131 printk(KERN_ERR "Couldn't create devinfo cache\n");
3132 ret = -ENOMEM;
3133 }
3134
3135 return ret;
3136}
3137
3138static inline int iommu_iova_cache_init(void)
3139{
3140 int ret = 0;
3141
3142 iommu_iova_cache = kmem_cache_create("iommu_iova",
3143 sizeof(struct iova),
3144 0,
3145 SLAB_HWCACHE_ALIGN,
3146 NULL);
3147 if (!iommu_iova_cache) {
3148 printk(KERN_ERR "Couldn't create iova cache\n");
3149 ret = -ENOMEM;
3150 }
3151
3152 return ret;
3153}
3154
3155static int __init iommu_init_mempool(void)
3156{
3157 int ret;
3158 ret = iommu_iova_cache_init();
3159 if (ret)
3160 return ret;
3161
3162 ret = iommu_domain_cache_init();
3163 if (ret)
3164 goto domain_error;
3165
3166 ret = iommu_devinfo_cache_init();
3167 if (!ret)
3168 return ret;
3169
3170 kmem_cache_destroy(iommu_domain_cache);
3171domain_error:
3172 kmem_cache_destroy(iommu_iova_cache);
3173
3174 return -ENOMEM;
3175}
3176
3177static void __init iommu_exit_mempool(void)
3178{
3179 kmem_cache_destroy(iommu_devinfo_cache);
3180 kmem_cache_destroy(iommu_domain_cache);
3181 kmem_cache_destroy(iommu_iova_cache);
3182
3183}
3184
3185static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3186{
3187 struct dmar_drhd_unit *drhd;
3188 u32 vtbar;
3189 int rc;
3190
3191 /* We know that this device on this chipset has its own IOMMU.
3192 * If we find it under a different IOMMU, then the BIOS is lying
3193 * to us. Hope that the IOMMU for this device is actually
3194 * disabled, and it needs no translation...
3195 */
3196 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3197 if (rc) {
3198 /* "can't" happen */
3199 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3200 return;
3201 }
3202 vtbar &= 0xffff0000;
3203
3204 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3205 drhd = dmar_find_matched_drhd_unit(pdev);
3206 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3207 TAINT_FIRMWARE_WORKAROUND,
3208 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3209 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3210}
3211DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3212
3213static void __init init_no_remapping_devices(void)
3214{
3215 struct dmar_drhd_unit *drhd;
3216
3217 for_each_drhd_unit(drhd) {
3218 if (!drhd->include_all) {
3219 int i;
3220 for (i = 0; i < drhd->devices_cnt; i++)
3221 if (drhd->devices[i] != NULL)
3222 break;
3223 /* ignore DMAR unit if no pci devices exist */
3224 if (i == drhd->devices_cnt)
3225 drhd->ignored = 1;
3226 }
3227 }
3228
3229 if (dmar_map_gfx)
3230 return;
3231
3232 for_each_drhd_unit(drhd) {
3233 int i;
3234 if (drhd->ignored || drhd->include_all)
3235 continue;
3236
3237 for (i = 0; i < drhd->devices_cnt; i++)
3238 if (drhd->devices[i] &&
3239 !IS_GFX_DEVICE(drhd->devices[i]))
3240 break;
3241
3242 if (i < drhd->devices_cnt)
3243 continue;
3244
3245 /* bypass IOMMU if it is just for gfx devices */
3246 drhd->ignored = 1;
3247 for (i = 0; i < drhd->devices_cnt; i++) {
3248 if (!drhd->devices[i])
3249 continue;
3250 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3251 }
3252 }
3253}
3254
3255#ifdef CONFIG_SUSPEND
3256static int init_iommu_hw(void)
3257{
3258 struct dmar_drhd_unit *drhd;
3259 struct intel_iommu *iommu = NULL;
3260
3261 for_each_active_iommu(iommu, drhd)
3262 if (iommu->qi)
3263 dmar_reenable_qi(iommu);
3264
3265 for_each_iommu(iommu, drhd) {
3266 if (drhd->ignored) {
3267 /*
3268 * we always have to disable PMRs or DMA may fail on
3269 * this device
3270 */
3271 if (force_on)
3272 iommu_disable_protect_mem_regions(iommu);
3273 continue;
3274 }
3275
3276 iommu_flush_write_buffer(iommu);
3277
3278 iommu_set_root_entry(iommu);
3279
3280 iommu->flush.flush_context(iommu, 0, 0, 0,
3281 DMA_CCMD_GLOBAL_INVL);
3282 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3283 DMA_TLB_GLOBAL_FLUSH);
3284 if (iommu_enable_translation(iommu))
3285 return 1;
3286 iommu_disable_protect_mem_regions(iommu);
3287 }
3288
3289 return 0;
3290}
3291
3292static void iommu_flush_all(void)
3293{
3294 struct dmar_drhd_unit *drhd;
3295 struct intel_iommu *iommu;
3296
3297 for_each_active_iommu(iommu, drhd) {
3298 iommu->flush.flush_context(iommu, 0, 0, 0,
3299 DMA_CCMD_GLOBAL_INVL);
3300 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3301 DMA_TLB_GLOBAL_FLUSH);
3302 }
3303}
3304
3305static int iommu_suspend(void)
3306{
3307 struct dmar_drhd_unit *drhd;
3308 struct intel_iommu *iommu = NULL;
3309 unsigned long flag;
3310
3311 for_each_active_iommu(iommu, drhd) {
3312 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3313 GFP_ATOMIC);
3314 if (!iommu->iommu_state)
3315 goto nomem;
3316 }
3317
3318 iommu_flush_all();
3319
3320 for_each_active_iommu(iommu, drhd) {
3321 iommu_disable_translation(iommu);
3322
3323 spin_lock_irqsave(&iommu->register_lock, flag);
3324
3325 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3326 readl(iommu->reg + DMAR_FECTL_REG);
3327 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3328 readl(iommu->reg + DMAR_FEDATA_REG);
3329 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3330 readl(iommu->reg + DMAR_FEADDR_REG);
3331 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3332 readl(iommu->reg + DMAR_FEUADDR_REG);
3333
3334 spin_unlock_irqrestore(&iommu->register_lock, flag);
3335 }
3336 return 0;
3337
3338nomem:
3339 for_each_active_iommu(iommu, drhd)
3340 kfree(iommu->iommu_state);
3341
3342 return -ENOMEM;
3343}
3344
3345static void iommu_resume(void)
3346{
3347 struct dmar_drhd_unit *drhd;
3348 struct intel_iommu *iommu = NULL;
3349 unsigned long flag;
3350
3351 if (init_iommu_hw()) {
3352 if (force_on)
3353 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3354 else
3355 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3356 return;
3357 }
3358
3359 for_each_active_iommu(iommu, drhd) {
3360
3361 spin_lock_irqsave(&iommu->register_lock, flag);
3362
3363 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3364 iommu->reg + DMAR_FECTL_REG);
3365 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3366 iommu->reg + DMAR_FEDATA_REG);
3367 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3368 iommu->reg + DMAR_FEADDR_REG);
3369 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3370 iommu->reg + DMAR_FEUADDR_REG);
3371
3372 spin_unlock_irqrestore(&iommu->register_lock, flag);
3373 }
3374
3375 for_each_active_iommu(iommu, drhd)
3376 kfree(iommu->iommu_state);
3377}
3378
3379static struct syscore_ops iommu_syscore_ops = {
3380 .resume = iommu_resume,
3381 .suspend = iommu_suspend,
3382};
3383
3384static void __init init_iommu_pm_ops(void)
3385{
3386 register_syscore_ops(&iommu_syscore_ops);
3387}
3388
3389#else
3390static inline void init_iommu_pm_ops(void) {}
3391#endif /* CONFIG_PM */
3392
3393/*
3394 * Here we only respond to action of unbound device from driver.
3395 *
3396 * Added device is not attached to its DMAR domain here yet. That will happen
3397 * when mapping the device to iova.
3398 */
3399static int device_notifier(struct notifier_block *nb,
3400 unsigned long action, void *data)
3401{
3402 struct device *dev = data;
3403 struct pci_dev *pdev = to_pci_dev(dev);
3404 struct dmar_domain *domain;
3405
3406 if (iommu_no_mapping(dev))
3407 return 0;
3408
3409 domain = find_domain(pdev);
3410 if (!domain)
3411 return 0;
3412
3413 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
3414 domain_remove_one_dev_info(domain, pdev);
3415
3416 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3417 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3418 list_empty(&domain->devices))
3419 domain_exit(domain);
3420 }
3421
3422 return 0;
3423}
3424
3425static struct notifier_block device_nb = {
3426 .notifier_call = device_notifier,
3427};
3428
3429int __init intel_iommu_init(void)
3430{
3431 int ret = 0;
3432
3433 /* VT-d is required for a TXT/tboot launch, so enforce that */
3434 force_on = tboot_force_iommu();
3435
3436 if (dmar_table_init()) {
3437 if (force_on)
3438 panic("tboot: Failed to initialize DMAR table\n");
3439 return -ENODEV;
3440 }
3441
3442 if (dmar_dev_scope_init()) {
3443 if (force_on)
3444 panic("tboot: Failed to initialize DMAR device scope\n");
3445 return -ENODEV;
3446 }
3447
3448 /*
3449 * Check the need for DMA-remapping initialization now.
3450 * Above initialization will also be used by Interrupt-remapping.
3451 */
3452 if (no_iommu || dmar_disabled)
3453 return -ENODEV;
3454
3455 if (iommu_init_mempool()) {
3456 if (force_on)
3457 panic("tboot: Failed to initialize iommu memory\n");
3458 return -ENODEV;
3459 }
3460
3461 if (dmar_init_reserved_ranges()) {
3462 if (force_on)
3463 panic("tboot: Failed to reserve iommu ranges\n");
3464 return -ENODEV;
3465 }
3466
3467 init_no_remapping_devices();
3468
3469 ret = init_dmars();
3470 if (ret) {
3471 if (force_on)
3472 panic("tboot: Failed to initialize DMARs\n");
3473 printk(KERN_ERR "IOMMU: dmar init failed\n");
3474 put_iova_domain(&reserved_iova_list);
3475 iommu_exit_mempool();
3476 return ret;
3477 }
3478 printk(KERN_INFO
3479 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3480
3481 init_timer(&unmap_timer);
3482#ifdef CONFIG_SWIOTLB
3483 swiotlb = 0;
3484#endif
3485 dma_ops = &intel_dma_ops;
3486
3487 init_iommu_pm_ops();
3488
3489 register_iommu(&intel_iommu_ops);
3490
3491 bus_register_notifier(&pci_bus_type, &device_nb);
3492
3493 return 0;
3494}
3495
3496static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3497 struct pci_dev *pdev)
3498{
3499 struct pci_dev *tmp, *parent;
3500
3501 if (!iommu || !pdev)
3502 return;
3503
3504 /* dependent device detach */
3505 tmp = pci_find_upstream_pcie_bridge(pdev);
3506 /* Secondary interface's bus number and devfn 0 */
3507 if (tmp) {
3508 parent = pdev->bus->self;
3509 while (parent != tmp) {
3510 iommu_detach_dev(iommu, parent->bus->number,
3511 parent->devfn);
3512 parent = parent->bus->self;
3513 }
3514 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3515 iommu_detach_dev(iommu,
3516 tmp->subordinate->number, 0);
3517 else /* this is a legacy PCI bridge */
3518 iommu_detach_dev(iommu, tmp->bus->number,
3519 tmp->devfn);
3520 }
3521}
3522
3523static void domain_remove_one_dev_info(struct dmar_domain *domain,
3524 struct pci_dev *pdev)
3525{
3526 struct device_domain_info *info;
3527 struct intel_iommu *iommu;
3528 unsigned long flags;
3529 int found = 0;
3530 struct list_head *entry, *tmp;
3531
3532 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3533 pdev->devfn);
3534 if (!iommu)
3535 return;
3536
3537 spin_lock_irqsave(&device_domain_lock, flags);
3538 list_for_each_safe(entry, tmp, &domain->devices) {
3539 info = list_entry(entry, struct device_domain_info, link);
3540 if (info->segment == pci_domain_nr(pdev->bus) &&
3541 info->bus == pdev->bus->number &&
3542 info->devfn == pdev->devfn) {
3543 list_del(&info->link);
3544 list_del(&info->global);
3545 if (info->dev)
3546 info->dev->dev.archdata.iommu = NULL;
3547 spin_unlock_irqrestore(&device_domain_lock, flags);
3548
3549 iommu_disable_dev_iotlb(info);
3550 iommu_detach_dev(iommu, info->bus, info->devfn);
3551 iommu_detach_dependent_devices(iommu, pdev);
3552 free_devinfo_mem(info);
3553
3554 spin_lock_irqsave(&device_domain_lock, flags);
3555
3556 if (found)
3557 break;
3558 else
3559 continue;
3560 }
3561
3562 /* if there is no other devices under the same iommu
3563 * owned by this domain, clear this iommu in iommu_bmp
3564 * update iommu count and coherency
3565 */
3566 if (iommu == device_to_iommu(info->segment, info->bus,
3567 info->devfn))
3568 found = 1;
3569 }
3570
3571 if (found == 0) {
3572 unsigned long tmp_flags;
3573 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3574 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3575 domain->iommu_count--;
3576 domain_update_iommu_cap(domain);
3577 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3578
3579 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3580 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
3581 spin_lock_irqsave(&iommu->lock, tmp_flags);
3582 clear_bit(domain->id, iommu->domain_ids);
3583 iommu->domains[domain->id] = NULL;
3584 spin_unlock_irqrestore(&iommu->lock, tmp_flags);
3585 }
3586 }
3587
3588 spin_unlock_irqrestore(&device_domain_lock, flags);
3589}
3590
3591static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3592{
3593 struct device_domain_info *info;
3594 struct intel_iommu *iommu;
3595 unsigned long flags1, flags2;
3596
3597 spin_lock_irqsave(&device_domain_lock, flags1);
3598 while (!list_empty(&domain->devices)) {
3599 info = list_entry(domain->devices.next,
3600 struct device_domain_info, link);
3601 list_del(&info->link);
3602 list_del(&info->global);
3603 if (info->dev)
3604 info->dev->dev.archdata.iommu = NULL;
3605
3606 spin_unlock_irqrestore(&device_domain_lock, flags1);
3607
3608 iommu_disable_dev_iotlb(info);
3609 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
3610 iommu_detach_dev(iommu, info->bus, info->devfn);
3611 iommu_detach_dependent_devices(iommu, info->dev);
3612
3613 /* clear this iommu in iommu_bmp, update iommu count
3614 * and capabilities
3615 */
3616 spin_lock_irqsave(&domain->iommu_lock, flags2);
3617 if (test_and_clear_bit(iommu->seq_id,
3618 &domain->iommu_bmp)) {
3619 domain->iommu_count--;
3620 domain_update_iommu_cap(domain);
3621 }
3622 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3623
3624 free_devinfo_mem(info);
3625 spin_lock_irqsave(&device_domain_lock, flags1);
3626 }
3627 spin_unlock_irqrestore(&device_domain_lock, flags1);
3628}
3629
3630/* domain id for virtual machine, it won't be set in context */
3631static unsigned long vm_domid;
3632
3633static struct dmar_domain *iommu_alloc_vm_domain(void)
3634{
3635 struct dmar_domain *domain;
3636
3637 domain = alloc_domain_mem();
3638 if (!domain)
3639 return NULL;
3640
3641 domain->id = vm_domid++;
3642 domain->nid = -1;
3643 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3644 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3645
3646 return domain;
3647}
3648
3649static int md_domain_init(struct dmar_domain *domain, int guest_width)
3650{
3651 int adjust_width;
3652
3653 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
3654 spin_lock_init(&domain->iommu_lock);
3655
3656 domain_reserve_special_ranges(domain);
3657
3658 /* calculate AGAW */
3659 domain->gaw = guest_width;
3660 adjust_width = guestwidth_to_adjustwidth(guest_width);
3661 domain->agaw = width_to_agaw(adjust_width);
3662
3663 INIT_LIST_HEAD(&domain->devices);
3664
3665 domain->iommu_count = 0;
3666 domain->iommu_coherency = 0;
3667 domain->iommu_snooping = 0;
3668 domain->iommu_superpage = 0;
3669 domain->max_addr = 0;
3670 domain->nid = -1;
3671
3672 /* always allocate the top pgd */
3673 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
3674 if (!domain->pgd)
3675 return -ENOMEM;
3676 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3677 return 0;
3678}
3679
3680static void iommu_free_vm_domain(struct dmar_domain *domain)
3681{
3682 unsigned long flags;
3683 struct dmar_drhd_unit *drhd;
3684 struct intel_iommu *iommu;
3685 unsigned long i;
3686 unsigned long ndomains;
3687
3688 for_each_drhd_unit(drhd) {
3689 if (drhd->ignored)
3690 continue;
3691 iommu = drhd->iommu;
3692
3693 ndomains = cap_ndoms(iommu->cap);
3694 for_each_set_bit(i, iommu->domain_ids, ndomains) {
3695 if (iommu->domains[i] == domain) {
3696 spin_lock_irqsave(&iommu->lock, flags);
3697 clear_bit(i, iommu->domain_ids);
3698 iommu->domains[i] = NULL;
3699 spin_unlock_irqrestore(&iommu->lock, flags);
3700 break;
3701 }
3702 }
3703 }
3704}
3705
3706static void vm_domain_exit(struct dmar_domain *domain)
3707{
3708 /* Domain 0 is reserved, so dont process it */
3709 if (!domain)
3710 return;
3711
3712 vm_domain_remove_all_dev_info(domain);
3713 /* destroy iovas */
3714 put_iova_domain(&domain->iovad);
3715
3716 /* clear ptes */
3717 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3718
3719 /* free page tables */
3720 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3721
3722 iommu_free_vm_domain(domain);
3723 free_domain_mem(domain);
3724}
3725
3726static int intel_iommu_domain_init(struct iommu_domain *domain)
3727{
3728 struct dmar_domain *dmar_domain;
3729
3730 dmar_domain = iommu_alloc_vm_domain();
3731 if (!dmar_domain) {
3732 printk(KERN_ERR
3733 "intel_iommu_domain_init: dmar_domain == NULL\n");
3734 return -ENOMEM;
3735 }
3736 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3737 printk(KERN_ERR
3738 "intel_iommu_domain_init() failed\n");
3739 vm_domain_exit(dmar_domain);
3740 return -ENOMEM;
3741 }
3742 domain->priv = dmar_domain;
3743
3744 return 0;
3745}
3746
3747static void intel_iommu_domain_destroy(struct iommu_domain *domain)
3748{
3749 struct dmar_domain *dmar_domain = domain->priv;
3750
3751 domain->priv = NULL;
3752 vm_domain_exit(dmar_domain);
3753}
3754
3755static int intel_iommu_attach_device(struct iommu_domain *domain,
3756 struct device *dev)
3757{
3758 struct dmar_domain *dmar_domain = domain->priv;
3759 struct pci_dev *pdev = to_pci_dev(dev);
3760 struct intel_iommu *iommu;
3761 int addr_width;
3762
3763 /* normally pdev is not mapped */
3764 if (unlikely(domain_context_mapped(pdev))) {
3765 struct dmar_domain *old_domain;
3766
3767 old_domain = find_domain(pdev);
3768 if (old_domain) {
3769 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3770 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3771 domain_remove_one_dev_info(old_domain, pdev);
3772 else
3773 domain_remove_dev_info(old_domain);
3774 }
3775 }
3776
3777 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3778 pdev->devfn);
3779 if (!iommu)
3780 return -ENODEV;
3781
3782 /* check if this iommu agaw is sufficient for max mapped address */
3783 addr_width = agaw_to_width(iommu->agaw);
3784 if (addr_width > cap_mgaw(iommu->cap))
3785 addr_width = cap_mgaw(iommu->cap);
3786
3787 if (dmar_domain->max_addr > (1LL << addr_width)) {
3788 printk(KERN_ERR "%s: iommu width (%d) is not "
3789 "sufficient for the mapped address (%llx)\n",
3790 __func__, addr_width, dmar_domain->max_addr);
3791 return -EFAULT;
3792 }
3793 dmar_domain->gaw = addr_width;
3794
3795 /*
3796 * Knock out extra levels of page tables if necessary
3797 */
3798 while (iommu->agaw < dmar_domain->agaw) {
3799 struct dma_pte *pte;
3800
3801 pte = dmar_domain->pgd;
3802 if (dma_pte_present(pte)) {
3803 dmar_domain->pgd = (struct dma_pte *)
3804 phys_to_virt(dma_pte_addr(pte));
3805 free_pgtable_page(pte);
3806 }
3807 dmar_domain->agaw--;
3808 }
3809
3810 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
3811}
3812
3813static void intel_iommu_detach_device(struct iommu_domain *domain,
3814 struct device *dev)
3815{
3816 struct dmar_domain *dmar_domain = domain->priv;
3817 struct pci_dev *pdev = to_pci_dev(dev);
3818
3819 domain_remove_one_dev_info(dmar_domain, pdev);
3820}
3821
3822static int intel_iommu_map(struct iommu_domain *domain,
3823 unsigned long iova, phys_addr_t hpa,
3824 int gfp_order, int iommu_prot)
3825{
3826 struct dmar_domain *dmar_domain = domain->priv;
3827 u64 max_addr;
3828 int prot = 0;
3829 size_t size;
3830 int ret;
3831
3832 if (iommu_prot & IOMMU_READ)
3833 prot |= DMA_PTE_READ;
3834 if (iommu_prot & IOMMU_WRITE)
3835 prot |= DMA_PTE_WRITE;
3836 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3837 prot |= DMA_PTE_SNP;
3838
3839 size = PAGE_SIZE << gfp_order;
3840 max_addr = iova + size;
3841 if (dmar_domain->max_addr < max_addr) {
3842 u64 end;
3843
3844 /* check if minimum agaw is sufficient for mapped address */
3845 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
3846 if (end < max_addr) {
3847 printk(KERN_ERR "%s: iommu width (%d) is not "
3848 "sufficient for the mapped address (%llx)\n",
3849 __func__, dmar_domain->gaw, max_addr);
3850 return -EFAULT;
3851 }
3852 dmar_domain->max_addr = max_addr;
3853 }
3854 /* Round up size to next multiple of PAGE_SIZE, if it and
3855 the low bits of hpa would take us onto the next page */
3856 size = aligned_nrpages(hpa, size);
3857 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3858 hpa >> VTD_PAGE_SHIFT, size, prot);
3859 return ret;
3860}
3861
3862static int intel_iommu_unmap(struct iommu_domain *domain,
3863 unsigned long iova, int gfp_order)
3864{
3865 struct dmar_domain *dmar_domain = domain->priv;
3866 size_t size = PAGE_SIZE << gfp_order;
3867
3868 dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
3869 (iova + size - 1) >> VTD_PAGE_SHIFT);
3870
3871 if (dmar_domain->max_addr == iova + size)
3872 dmar_domain->max_addr = iova;
3873
3874 return gfp_order;
3875}
3876
3877static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3878 unsigned long iova)
3879{
3880 struct dmar_domain *dmar_domain = domain->priv;
3881 struct dma_pte *pte;
3882 u64 phys = 0;
3883
3884 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
3885 if (pte)
3886 phys = dma_pte_addr(pte);
3887
3888 return phys;
3889}
3890
3891static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3892 unsigned long cap)
3893{
3894 struct dmar_domain *dmar_domain = domain->priv;
3895
3896 if (cap == IOMMU_CAP_CACHE_COHERENCY)
3897 return dmar_domain->iommu_snooping;
3898 if (cap == IOMMU_CAP_INTR_REMAP)
3899 return intr_remapping_enabled;
3900
3901 return 0;
3902}
3903
3904static struct iommu_ops intel_iommu_ops = {
3905 .domain_init = intel_iommu_domain_init,
3906 .domain_destroy = intel_iommu_domain_destroy,
3907 .attach_dev = intel_iommu_attach_device,
3908 .detach_dev = intel_iommu_detach_device,
3909 .map = intel_iommu_map,
3910 .unmap = intel_iommu_unmap,
3911 .iova_to_phys = intel_iommu_iova_to_phys,
3912 .domain_has_cap = intel_iommu_domain_has_cap,
3913};
3914
3915static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3916{
3917 /*
3918 * Mobile 4 Series Chipset neglects to set RWBF capability,
3919 * but needs it:
3920 */
3921 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3922 rwbf_quirk = 1;
3923
3924 /* https://bugzilla.redhat.com/show_bug.cgi?id=538163 */
3925 if (dev->revision == 0x07) {
3926 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
3927 dmar_map_gfx = 0;
3928 }
3929}
3930
3931DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
3932
3933#define GGC 0x52
3934#define GGC_MEMORY_SIZE_MASK (0xf << 8)
3935#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
3936#define GGC_MEMORY_SIZE_1M (0x1 << 8)
3937#define GGC_MEMORY_SIZE_2M (0x3 << 8)
3938#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
3939#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
3940#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
3941#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
3942
3943static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
3944{
3945 unsigned short ggc;
3946
3947 if (pci_read_config_word(dev, GGC, &ggc))
3948 return;
3949
3950 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
3951 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
3952 dmar_map_gfx = 0;
3953 }
3954}
3955DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
3956DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
3957DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
3958DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
3959
3960/* On Tylersburg chipsets, some BIOSes have been known to enable the
3961 ISOCH DMAR unit for the Azalia sound device, but not give it any
3962 TLB entries, which causes it to deadlock. Check for that. We do
3963 this in a function called from init_dmars(), instead of in a PCI
3964 quirk, because we don't want to print the obnoxious "BIOS broken"
3965 message if VT-d is actually disabled.
3966*/
3967static void __init check_tylersburg_isoch(void)
3968{
3969 struct pci_dev *pdev;
3970 uint32_t vtisochctrl;
3971
3972 /* If there's no Azalia in the system anyway, forget it. */
3973 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
3974 if (!pdev)
3975 return;
3976 pci_dev_put(pdev);
3977
3978 /* System Management Registers. Might be hidden, in which case
3979 we can't do the sanity check. But that's OK, because the
3980 known-broken BIOSes _don't_ actually hide it, so far. */
3981 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
3982 if (!pdev)
3983 return;
3984
3985 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
3986 pci_dev_put(pdev);
3987 return;
3988 }
3989
3990 pci_dev_put(pdev);
3991
3992 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
3993 if (vtisochctrl & 1)
3994 return;
3995
3996 /* Drop all bits other than the number of TLB entries */
3997 vtisochctrl &= 0x1c;
3998
3999 /* If we have the recommended number of TLB entries (16), fine. */
4000 if (vtisochctrl == 0x10)
4001 return;
4002
4003 /* Zero TLB entries? You get to ride the short bus to school. */
4004 if (!vtisochctrl) {
4005 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4006 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4007 dmi_get_system_info(DMI_BIOS_VENDOR),
4008 dmi_get_system_info(DMI_BIOS_VERSION),
4009 dmi_get_system_info(DMI_PRODUCT_VERSION));
4010 iommu_identity_mapping |= IDENTMAP_AZALIA;
4011 return;
4012 }
4013
4014 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4015 vtisochctrl);
4016}
diff --git a/drivers/iommu/intr_remapping.c b/drivers/iommu/intr_remapping.c
new file mode 100644
index 000000000000..1a89d4a2cadf
--- /dev/null
+++ b/drivers/iommu/intr_remapping.c
@@ -0,0 +1,797 @@
1#include <linux/interrupt.h>
2#include <linux/dmar.h>
3#include <linux/spinlock.h>
4#include <linux/slab.h>
5#include <linux/jiffies.h>
6#include <linux/hpet.h>
7#include <linux/pci.h>
8#include <linux/irq.h>
9#include <asm/io_apic.h>
10#include <asm/smp.h>
11#include <asm/cpu.h>
12#include <linux/intel-iommu.h>
13#include "intr_remapping.h"
14#include <acpi/acpi.h>
15#include <asm/pci-direct.h>
16
17static struct ioapic_scope ir_ioapic[MAX_IO_APICS];
18static struct hpet_scope ir_hpet[MAX_HPET_TBS];
19static int ir_ioapic_num, ir_hpet_num;
20int intr_remapping_enabled;
21
22static int disable_intremap;
23static int disable_sourceid_checking;
24
25static __init int setup_nointremap(char *str)
26{
27 disable_intremap = 1;
28 return 0;
29}
30early_param("nointremap", setup_nointremap);
31
32static __init int setup_intremap(char *str)
33{
34 if (!str)
35 return -EINVAL;
36
37 if (!strncmp(str, "on", 2))
38 disable_intremap = 0;
39 else if (!strncmp(str, "off", 3))
40 disable_intremap = 1;
41 else if (!strncmp(str, "nosid", 5))
42 disable_sourceid_checking = 1;
43
44 return 0;
45}
46early_param("intremap", setup_intremap);
47
48static DEFINE_SPINLOCK(irq_2_ir_lock);
49
50static struct irq_2_iommu *irq_2_iommu(unsigned int irq)
51{
52 struct irq_cfg *cfg = irq_get_chip_data(irq);
53 return cfg ? &cfg->irq_2_iommu : NULL;
54}
55
56int get_irte(int irq, struct irte *entry)
57{
58 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
59 unsigned long flags;
60 int index;
61
62 if (!entry || !irq_iommu)
63 return -1;
64
65 spin_lock_irqsave(&irq_2_ir_lock, flags);
66
67 index = irq_iommu->irte_index + irq_iommu->sub_handle;
68 *entry = *(irq_iommu->iommu->ir_table->base + index);
69
70 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
71 return 0;
72}
73
74int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
75{
76 struct ir_table *table = iommu->ir_table;
77 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
78 u16 index, start_index;
79 unsigned int mask = 0;
80 unsigned long flags;
81 int i;
82
83 if (!count || !irq_iommu)
84 return -1;
85
86 /*
87 * start the IRTE search from index 0.
88 */
89 index = start_index = 0;
90
91 if (count > 1) {
92 count = __roundup_pow_of_two(count);
93 mask = ilog2(count);
94 }
95
96 if (mask > ecap_max_handle_mask(iommu->ecap)) {
97 printk(KERN_ERR
98 "Requested mask %x exceeds the max invalidation handle"
99 " mask value %Lx\n", mask,
100 ecap_max_handle_mask(iommu->ecap));
101 return -1;
102 }
103
104 spin_lock_irqsave(&irq_2_ir_lock, flags);
105 do {
106 for (i = index; i < index + count; i++)
107 if (table->base[i].present)
108 break;
109 /* empty index found */
110 if (i == index + count)
111 break;
112
113 index = (index + count) % INTR_REMAP_TABLE_ENTRIES;
114
115 if (index == start_index) {
116 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
117 printk(KERN_ERR "can't allocate an IRTE\n");
118 return -1;
119 }
120 } while (1);
121
122 for (i = index; i < index + count; i++)
123 table->base[i].present = 1;
124
125 irq_iommu->iommu = iommu;
126 irq_iommu->irte_index = index;
127 irq_iommu->sub_handle = 0;
128 irq_iommu->irte_mask = mask;
129
130 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
131
132 return index;
133}
134
135static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask)
136{
137 struct qi_desc desc;
138
139 desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask)
140 | QI_IEC_SELECTIVE;
141 desc.high = 0;
142
143 return qi_submit_sync(&desc, iommu);
144}
145
146int map_irq_to_irte_handle(int irq, u16 *sub_handle)
147{
148 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
149 unsigned long flags;
150 int index;
151
152 if (!irq_iommu)
153 return -1;
154
155 spin_lock_irqsave(&irq_2_ir_lock, flags);
156 *sub_handle = irq_iommu->sub_handle;
157 index = irq_iommu->irte_index;
158 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
159 return index;
160}
161
162int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle)
163{
164 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
165 unsigned long flags;
166
167 if (!irq_iommu)
168 return -1;
169
170 spin_lock_irqsave(&irq_2_ir_lock, flags);
171
172 irq_iommu->iommu = iommu;
173 irq_iommu->irte_index = index;
174 irq_iommu->sub_handle = subhandle;
175 irq_iommu->irte_mask = 0;
176
177 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
178
179 return 0;
180}
181
182int modify_irte(int irq, struct irte *irte_modified)
183{
184 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
185 struct intel_iommu *iommu;
186 unsigned long flags;
187 struct irte *irte;
188 int rc, index;
189
190 if (!irq_iommu)
191 return -1;
192
193 spin_lock_irqsave(&irq_2_ir_lock, flags);
194
195 iommu = irq_iommu->iommu;
196
197 index = irq_iommu->irte_index + irq_iommu->sub_handle;
198 irte = &iommu->ir_table->base[index];
199
200 set_64bit(&irte->low, irte_modified->low);
201 set_64bit(&irte->high, irte_modified->high);
202 __iommu_flush_cache(iommu, irte, sizeof(*irte));
203
204 rc = qi_flush_iec(iommu, index, 0);
205 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
206
207 return rc;
208}
209
210struct intel_iommu *map_hpet_to_ir(u8 hpet_id)
211{
212 int i;
213
214 for (i = 0; i < MAX_HPET_TBS; i++)
215 if (ir_hpet[i].id == hpet_id)
216 return ir_hpet[i].iommu;
217 return NULL;
218}
219
220struct intel_iommu *map_ioapic_to_ir(int apic)
221{
222 int i;
223
224 for (i = 0; i < MAX_IO_APICS; i++)
225 if (ir_ioapic[i].id == apic)
226 return ir_ioapic[i].iommu;
227 return NULL;
228}
229
230struct intel_iommu *map_dev_to_ir(struct pci_dev *dev)
231{
232 struct dmar_drhd_unit *drhd;
233
234 drhd = dmar_find_matched_drhd_unit(dev);
235 if (!drhd)
236 return NULL;
237
238 return drhd->iommu;
239}
240
241static int clear_entries(struct irq_2_iommu *irq_iommu)
242{
243 struct irte *start, *entry, *end;
244 struct intel_iommu *iommu;
245 int index;
246
247 if (irq_iommu->sub_handle)
248 return 0;
249
250 iommu = irq_iommu->iommu;
251 index = irq_iommu->irte_index + irq_iommu->sub_handle;
252
253 start = iommu->ir_table->base + index;
254 end = start + (1 << irq_iommu->irte_mask);
255
256 for (entry = start; entry < end; entry++) {
257 set_64bit(&entry->low, 0);
258 set_64bit(&entry->high, 0);
259 }
260
261 return qi_flush_iec(iommu, index, irq_iommu->irte_mask);
262}
263
264int free_irte(int irq)
265{
266 struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
267 unsigned long flags;
268 int rc;
269
270 if (!irq_iommu)
271 return -1;
272
273 spin_lock_irqsave(&irq_2_ir_lock, flags);
274
275 rc = clear_entries(irq_iommu);
276
277 irq_iommu->iommu = NULL;
278 irq_iommu->irte_index = 0;
279 irq_iommu->sub_handle = 0;
280 irq_iommu->irte_mask = 0;
281
282 spin_unlock_irqrestore(&irq_2_ir_lock, flags);
283
284 return rc;
285}
286
287/*
288 * source validation type
289 */
290#define SVT_NO_VERIFY 0x0 /* no verification is required */
291#define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fields */
292#define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */
293
294/*
295 * source-id qualifier
296 */
297#define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */
298#define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore
299 * the third least significant bit
300 */
301#define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore
302 * the second and third least significant bits
303 */
304#define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore
305 * the least three significant bits
306 */
307
308/*
309 * set SVT, SQ and SID fields of irte to verify
310 * source ids of interrupt requests
311 */
312static void set_irte_sid(struct irte *irte, unsigned int svt,
313 unsigned int sq, unsigned int sid)
314{
315 if (disable_sourceid_checking)
316 svt = SVT_NO_VERIFY;
317 irte->svt = svt;
318 irte->sq = sq;
319 irte->sid = sid;
320}
321
322int set_ioapic_sid(struct irte *irte, int apic)
323{
324 int i;
325 u16 sid = 0;
326
327 if (!irte)
328 return -1;
329
330 for (i = 0; i < MAX_IO_APICS; i++) {
331 if (ir_ioapic[i].id == apic) {
332 sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn;
333 break;
334 }
335 }
336
337 if (sid == 0) {
338 pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic);
339 return -1;
340 }
341
342 set_irte_sid(irte, 1, 0, sid);
343
344 return 0;
345}
346
347int set_hpet_sid(struct irte *irte, u8 id)
348{
349 int i;
350 u16 sid = 0;
351
352 if (!irte)
353 return -1;
354
355 for (i = 0; i < MAX_HPET_TBS; i++) {
356 if (ir_hpet[i].id == id) {
357 sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn;
358 break;
359 }
360 }
361
362 if (sid == 0) {
363 pr_warning("Failed to set source-id of HPET block (%d)\n", id);
364 return -1;
365 }
366
367 /*
368 * Should really use SQ_ALL_16. Some platforms are broken.
369 * While we figure out the right quirks for these broken platforms, use
370 * SQ_13_IGNORE_3 for now.
371 */
372 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid);
373
374 return 0;
375}
376
377int set_msi_sid(struct irte *irte, struct pci_dev *dev)
378{
379 struct pci_dev *bridge;
380
381 if (!irte || !dev)
382 return -1;
383
384 /* PCIe device or Root Complex integrated PCI device */
385 if (pci_is_pcie(dev) || !dev->bus->parent) {
386 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16,
387 (dev->bus->number << 8) | dev->devfn);
388 return 0;
389 }
390
391 bridge = pci_find_upstream_pcie_bridge(dev);
392 if (bridge) {
393 if (pci_is_pcie(bridge))/* this is a PCIe-to-PCI/PCIX bridge */
394 set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16,
395 (bridge->bus->number << 8) | dev->bus->number);
396 else /* this is a legacy PCI bridge */
397 set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16,
398 (bridge->bus->number << 8) | bridge->devfn);
399 }
400
401 return 0;
402}
403
404static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode)
405{
406 u64 addr;
407 u32 sts;
408 unsigned long flags;
409
410 addr = virt_to_phys((void *)iommu->ir_table->base);
411
412 spin_lock_irqsave(&iommu->register_lock, flags);
413
414 dmar_writeq(iommu->reg + DMAR_IRTA_REG,
415 (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE);
416
417 /* Set interrupt-remapping table pointer */
418 iommu->gcmd |= DMA_GCMD_SIRTP;
419 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
420
421 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
422 readl, (sts & DMA_GSTS_IRTPS), sts);
423 spin_unlock_irqrestore(&iommu->register_lock, flags);
424
425 /*
426 * global invalidation of interrupt entry cache before enabling
427 * interrupt-remapping.
428 */
429 qi_global_iec(iommu);
430
431 spin_lock_irqsave(&iommu->register_lock, flags);
432
433 /* Enable interrupt-remapping */
434 iommu->gcmd |= DMA_GCMD_IRE;
435 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
436
437 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
438 readl, (sts & DMA_GSTS_IRES), sts);
439
440 spin_unlock_irqrestore(&iommu->register_lock, flags);
441}
442
443
444static int setup_intr_remapping(struct intel_iommu *iommu, int mode)
445{
446 struct ir_table *ir_table;
447 struct page *pages;
448
449 ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table),
450 GFP_ATOMIC);
451
452 if (!iommu->ir_table)
453 return -ENOMEM;
454
455 pages = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO,
456 INTR_REMAP_PAGE_ORDER);
457
458 if (!pages) {
459 printk(KERN_ERR "failed to allocate pages of order %d\n",
460 INTR_REMAP_PAGE_ORDER);
461 kfree(iommu->ir_table);
462 return -ENOMEM;
463 }
464
465 ir_table->base = page_address(pages);
466
467 iommu_set_intr_remapping(iommu, mode);
468 return 0;
469}
470
471/*
472 * Disable Interrupt Remapping.
473 */
474static void iommu_disable_intr_remapping(struct intel_iommu *iommu)
475{
476 unsigned long flags;
477 u32 sts;
478
479 if (!ecap_ir_support(iommu->ecap))
480 return;
481
482 /*
483 * global invalidation of interrupt entry cache before disabling
484 * interrupt-remapping.
485 */
486 qi_global_iec(iommu);
487
488 spin_lock_irqsave(&iommu->register_lock, flags);
489
490 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
491 if (!(sts & DMA_GSTS_IRES))
492 goto end;
493
494 iommu->gcmd &= ~DMA_GCMD_IRE;
495 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
496
497 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
498 readl, !(sts & DMA_GSTS_IRES), sts);
499
500end:
501 spin_unlock_irqrestore(&iommu->register_lock, flags);
502}
503
504int __init intr_remapping_supported(void)
505{
506 struct dmar_drhd_unit *drhd;
507
508 if (disable_intremap)
509 return 0;
510
511 if (!dmar_ir_support())
512 return 0;
513
514 for_each_drhd_unit(drhd) {
515 struct intel_iommu *iommu = drhd->iommu;
516
517 if (!ecap_ir_support(iommu->ecap))
518 return 0;
519 }
520
521 return 1;
522}
523
524int __init enable_intr_remapping(int eim)
525{
526 struct dmar_drhd_unit *drhd;
527 int setup = 0;
528
529 if (parse_ioapics_under_ir() != 1) {
530 printk(KERN_INFO "Not enable interrupt remapping\n");
531 return -1;
532 }
533
534 for_each_drhd_unit(drhd) {
535 struct intel_iommu *iommu = drhd->iommu;
536
537 /*
538 * If the queued invalidation is already initialized,
539 * shouldn't disable it.
540 */
541 if (iommu->qi)
542 continue;
543
544 /*
545 * Clear previous faults.
546 */
547 dmar_fault(-1, iommu);
548
549 /*
550 * Disable intr remapping and queued invalidation, if already
551 * enabled prior to OS handover.
552 */
553 iommu_disable_intr_remapping(iommu);
554
555 dmar_disable_qi(iommu);
556 }
557
558 /*
559 * check for the Interrupt-remapping support
560 */
561 for_each_drhd_unit(drhd) {
562 struct intel_iommu *iommu = drhd->iommu;
563
564 if (!ecap_ir_support(iommu->ecap))
565 continue;
566
567 if (eim && !ecap_eim_support(iommu->ecap)) {
568 printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
569 " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
570 return -1;
571 }
572 }
573
574 /*
575 * Enable queued invalidation for all the DRHD's.
576 */
577 for_each_drhd_unit(drhd) {
578 int ret;
579 struct intel_iommu *iommu = drhd->iommu;
580 ret = dmar_enable_qi(iommu);
581
582 if (ret) {
583 printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
584 " invalidation, ecap %Lx, ret %d\n",
585 drhd->reg_base_addr, iommu->ecap, ret);
586 return -1;
587 }
588 }
589
590 /*
591 * Setup Interrupt-remapping for all the DRHD's now.
592 */
593 for_each_drhd_unit(drhd) {
594 struct intel_iommu *iommu = drhd->iommu;
595
596 if (!ecap_ir_support(iommu->ecap))
597 continue;
598
599 if (setup_intr_remapping(iommu, eim))
600 goto error;
601
602 setup = 1;
603 }
604
605 if (!setup)
606 goto error;
607
608 intr_remapping_enabled = 1;
609
610 return 0;
611
612error:
613 /*
614 * handle error condition gracefully here!
615 */
616 return -1;
617}
618
619static void ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope,
620 struct intel_iommu *iommu)
621{
622 struct acpi_dmar_pci_path *path;
623 u8 bus;
624 int count;
625
626 bus = scope->bus;
627 path = (struct acpi_dmar_pci_path *)(scope + 1);
628 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
629 / sizeof(struct acpi_dmar_pci_path);
630
631 while (--count > 0) {
632 /*
633 * Access PCI directly due to the PCI
634 * subsystem isn't initialized yet.
635 */
636 bus = read_pci_config_byte(bus, path->dev, path->fn,
637 PCI_SECONDARY_BUS);
638 path++;
639 }
640 ir_hpet[ir_hpet_num].bus = bus;
641 ir_hpet[ir_hpet_num].devfn = PCI_DEVFN(path->dev, path->fn);
642 ir_hpet[ir_hpet_num].iommu = iommu;
643 ir_hpet[ir_hpet_num].id = scope->enumeration_id;
644 ir_hpet_num++;
645}
646
647static void ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope,
648 struct intel_iommu *iommu)
649{
650 struct acpi_dmar_pci_path *path;
651 u8 bus;
652 int count;
653
654 bus = scope->bus;
655 path = (struct acpi_dmar_pci_path *)(scope + 1);
656 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
657 / sizeof(struct acpi_dmar_pci_path);
658
659 while (--count > 0) {
660 /*
661 * Access PCI directly due to the PCI
662 * subsystem isn't initialized yet.
663 */
664 bus = read_pci_config_byte(bus, path->dev, path->fn,
665 PCI_SECONDARY_BUS);
666 path++;
667 }
668
669 ir_ioapic[ir_ioapic_num].bus = bus;
670 ir_ioapic[ir_ioapic_num].devfn = PCI_DEVFN(path->dev, path->fn);
671 ir_ioapic[ir_ioapic_num].iommu = iommu;
672 ir_ioapic[ir_ioapic_num].id = scope->enumeration_id;
673 ir_ioapic_num++;
674}
675
676static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header,
677 struct intel_iommu *iommu)
678{
679 struct acpi_dmar_hardware_unit *drhd;
680 struct acpi_dmar_device_scope *scope;
681 void *start, *end;
682
683 drhd = (struct acpi_dmar_hardware_unit *)header;
684
685 start = (void *)(drhd + 1);
686 end = ((void *)drhd) + header->length;
687
688 while (start < end) {
689 scope = start;
690 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
691 if (ir_ioapic_num == MAX_IO_APICS) {
692 printk(KERN_WARNING "Exceeded Max IO APICS\n");
693 return -1;
694 }
695
696 printk(KERN_INFO "IOAPIC id %d under DRHD base "
697 " 0x%Lx IOMMU %d\n", scope->enumeration_id,
698 drhd->address, iommu->seq_id);
699
700 ir_parse_one_ioapic_scope(scope, iommu);
701 } else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET) {
702 if (ir_hpet_num == MAX_HPET_TBS) {
703 printk(KERN_WARNING "Exceeded Max HPET blocks\n");
704 return -1;
705 }
706
707 printk(KERN_INFO "HPET id %d under DRHD base"
708 " 0x%Lx\n", scope->enumeration_id,
709 drhd->address);
710
711 ir_parse_one_hpet_scope(scope, iommu);
712 }
713 start += scope->length;
714 }
715
716 return 0;
717}
718
719/*
720 * Finds the assocaition between IOAPIC's and its Interrupt-remapping
721 * hardware unit.
722 */
723int __init parse_ioapics_under_ir(void)
724{
725 struct dmar_drhd_unit *drhd;
726 int ir_supported = 0;
727
728 for_each_drhd_unit(drhd) {
729 struct intel_iommu *iommu = drhd->iommu;
730
731 if (ecap_ir_support(iommu->ecap)) {
732 if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu))
733 return -1;
734
735 ir_supported = 1;
736 }
737 }
738
739 if (ir_supported && ir_ioapic_num != nr_ioapics) {
740 printk(KERN_WARNING
741 "Not all IO-APIC's listed under remapping hardware\n");
742 return -1;
743 }
744
745 return ir_supported;
746}
747
748void disable_intr_remapping(void)
749{
750 struct dmar_drhd_unit *drhd;
751 struct intel_iommu *iommu = NULL;
752
753 /*
754 * Disable Interrupt-remapping for all the DRHD's now.
755 */
756 for_each_iommu(iommu, drhd) {
757 if (!ecap_ir_support(iommu->ecap))
758 continue;
759
760 iommu_disable_intr_remapping(iommu);
761 }
762}
763
764int reenable_intr_remapping(int eim)
765{
766 struct dmar_drhd_unit *drhd;
767 int setup = 0;
768 struct intel_iommu *iommu = NULL;
769
770 for_each_iommu(iommu, drhd)
771 if (iommu->qi)
772 dmar_reenable_qi(iommu);
773
774 /*
775 * Setup Interrupt-remapping for all the DRHD's now.
776 */
777 for_each_iommu(iommu, drhd) {
778 if (!ecap_ir_support(iommu->ecap))
779 continue;
780
781 /* Set up interrupt remapping for iommu.*/
782 iommu_set_intr_remapping(iommu, eim);
783 setup = 1;
784 }
785
786 if (!setup)
787 goto error;
788
789 return 0;
790
791error:
792 /*
793 * handle error condition gracefully here!
794 */
795 return -1;
796}
797
diff --git a/drivers/iommu/intr_remapping.h b/drivers/iommu/intr_remapping.h
new file mode 100644
index 000000000000..5662fecfee60
--- /dev/null
+++ b/drivers/iommu/intr_remapping.h
@@ -0,0 +1,17 @@
1#include <linux/intel-iommu.h>
2
3struct ioapic_scope {
4 struct intel_iommu *iommu;
5 unsigned int id;
6 unsigned int bus; /* PCI bus number */
7 unsigned int devfn; /* PCI devfn number */
8};
9
10struct hpet_scope {
11 struct intel_iommu *iommu;
12 u8 id;
13 unsigned int bus;
14 unsigned int devfn;
15};
16
17#define IR_X2APIC_MODE(mode) (mode ? (1 << 11) : 0)
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
new file mode 100644
index 000000000000..c5c274ab5c5a
--- /dev/null
+++ b/drivers/iommu/iova.c
@@ -0,0 +1,435 @@
1/*
2 * Copyright © 2006-2009, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
18 */
19
20#include <linux/iova.h>
21
22void
23init_iova_domain(struct iova_domain *iovad, unsigned long pfn_32bit)
24{
25 spin_lock_init(&iovad->iova_rbtree_lock);
26 iovad->rbroot = RB_ROOT;
27 iovad->cached32_node = NULL;
28 iovad->dma_32bit_pfn = pfn_32bit;
29}
30
31static struct rb_node *
32__get_cached_rbnode(struct iova_domain *iovad, unsigned long *limit_pfn)
33{
34 if ((*limit_pfn != iovad->dma_32bit_pfn) ||
35 (iovad->cached32_node == NULL))
36 return rb_last(&iovad->rbroot);
37 else {
38 struct rb_node *prev_node = rb_prev(iovad->cached32_node);
39 struct iova *curr_iova =
40 container_of(iovad->cached32_node, struct iova, node);
41 *limit_pfn = curr_iova->pfn_lo - 1;
42 return prev_node;
43 }
44}
45
46static void
47__cached_rbnode_insert_update(struct iova_domain *iovad,
48 unsigned long limit_pfn, struct iova *new)
49{
50 if (limit_pfn != iovad->dma_32bit_pfn)
51 return;
52 iovad->cached32_node = &new->node;
53}
54
55static void
56__cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free)
57{
58 struct iova *cached_iova;
59 struct rb_node *curr;
60
61 if (!iovad->cached32_node)
62 return;
63 curr = iovad->cached32_node;
64 cached_iova = container_of(curr, struct iova, node);
65
66 if (free->pfn_lo >= cached_iova->pfn_lo) {
67 struct rb_node *node = rb_next(&free->node);
68 struct iova *iova = container_of(node, struct iova, node);
69
70 /* only cache if it's below 32bit pfn */
71 if (node && iova->pfn_lo < iovad->dma_32bit_pfn)
72 iovad->cached32_node = node;
73 else
74 iovad->cached32_node = NULL;
75 }
76}
77
78/* Computes the padding size required, to make the
79 * the start address naturally aligned on its size
80 */
81static int
82iova_get_pad_size(int size, unsigned int limit_pfn)
83{
84 unsigned int pad_size = 0;
85 unsigned int order = ilog2(size);
86
87 if (order)
88 pad_size = (limit_pfn + 1) % (1 << order);
89
90 return pad_size;
91}
92
93static int __alloc_and_insert_iova_range(struct iova_domain *iovad,
94 unsigned long size, unsigned long limit_pfn,
95 struct iova *new, bool size_aligned)
96{
97 struct rb_node *prev, *curr = NULL;
98 unsigned long flags;
99 unsigned long saved_pfn;
100 unsigned int pad_size = 0;
101
102 /* Walk the tree backwards */
103 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
104 saved_pfn = limit_pfn;
105 curr = __get_cached_rbnode(iovad, &limit_pfn);
106 prev = curr;
107 while (curr) {
108 struct iova *curr_iova = container_of(curr, struct iova, node);
109
110 if (limit_pfn < curr_iova->pfn_lo)
111 goto move_left;
112 else if (limit_pfn < curr_iova->pfn_hi)
113 goto adjust_limit_pfn;
114 else {
115 if (size_aligned)
116 pad_size = iova_get_pad_size(size, limit_pfn);
117 if ((curr_iova->pfn_hi + size + pad_size) <= limit_pfn)
118 break; /* found a free slot */
119 }
120adjust_limit_pfn:
121 limit_pfn = curr_iova->pfn_lo - 1;
122move_left:
123 prev = curr;
124 curr = rb_prev(curr);
125 }
126
127 if (!curr) {
128 if (size_aligned)
129 pad_size = iova_get_pad_size(size, limit_pfn);
130 if ((IOVA_START_PFN + size + pad_size) > limit_pfn) {
131 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
132 return -ENOMEM;
133 }
134 }
135
136 /* pfn_lo will point to size aligned address if size_aligned is set */
137 new->pfn_lo = limit_pfn - (size + pad_size) + 1;
138 new->pfn_hi = new->pfn_lo + size - 1;
139
140 /* Insert the new_iova into domain rbtree by holding writer lock */
141 /* Add new node and rebalance tree. */
142 {
143 struct rb_node **entry, *parent = NULL;
144
145 /* If we have 'prev', it's a valid place to start the
146 insertion. Otherwise, start from the root. */
147 if (prev)
148 entry = &prev;
149 else
150 entry = &iovad->rbroot.rb_node;
151
152 /* Figure out where to put new node */
153 while (*entry) {
154 struct iova *this = container_of(*entry,
155 struct iova, node);
156 parent = *entry;
157
158 if (new->pfn_lo < this->pfn_lo)
159 entry = &((*entry)->rb_left);
160 else if (new->pfn_lo > this->pfn_lo)
161 entry = &((*entry)->rb_right);
162 else
163 BUG(); /* this should not happen */
164 }
165
166 /* Add new node and rebalance tree. */
167 rb_link_node(&new->node, parent, entry);
168 rb_insert_color(&new->node, &iovad->rbroot);
169 }
170 __cached_rbnode_insert_update(iovad, saved_pfn, new);
171
172 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
173
174
175 return 0;
176}
177
178static void
179iova_insert_rbtree(struct rb_root *root, struct iova *iova)
180{
181 struct rb_node **new = &(root->rb_node), *parent = NULL;
182 /* Figure out where to put new node */
183 while (*new) {
184 struct iova *this = container_of(*new, struct iova, node);
185 parent = *new;
186
187 if (iova->pfn_lo < this->pfn_lo)
188 new = &((*new)->rb_left);
189 else if (iova->pfn_lo > this->pfn_lo)
190 new = &((*new)->rb_right);
191 else
192 BUG(); /* this should not happen */
193 }
194 /* Add new node and rebalance tree. */
195 rb_link_node(&iova->node, parent, new);
196 rb_insert_color(&iova->node, root);
197}
198
199/**
200 * alloc_iova - allocates an iova
201 * @iovad - iova domain in question
202 * @size - size of page frames to allocate
203 * @limit_pfn - max limit address
204 * @size_aligned - set if size_aligned address range is required
205 * This function allocates an iova in the range limit_pfn to IOVA_START_PFN
206 * looking from limit_pfn instead from IOVA_START_PFN. If the size_aligned
207 * flag is set then the allocated address iova->pfn_lo will be naturally
208 * aligned on roundup_power_of_two(size).
209 */
210struct iova *
211alloc_iova(struct iova_domain *iovad, unsigned long size,
212 unsigned long limit_pfn,
213 bool size_aligned)
214{
215 struct iova *new_iova;
216 int ret;
217
218 new_iova = alloc_iova_mem();
219 if (!new_iova)
220 return NULL;
221
222 /* If size aligned is set then round the size to
223 * to next power of two.
224 */
225 if (size_aligned)
226 size = __roundup_pow_of_two(size);
227
228 ret = __alloc_and_insert_iova_range(iovad, size, limit_pfn,
229 new_iova, size_aligned);
230
231 if (ret) {
232 free_iova_mem(new_iova);
233 return NULL;
234 }
235
236 return new_iova;
237}
238
239/**
240 * find_iova - find's an iova for a given pfn
241 * @iovad - iova domain in question.
242 * pfn - page frame number
243 * This function finds and returns an iova belonging to the
244 * given doamin which matches the given pfn.
245 */
246struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn)
247{
248 unsigned long flags;
249 struct rb_node *node;
250
251 /* Take the lock so that no other thread is manipulating the rbtree */
252 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
253 node = iovad->rbroot.rb_node;
254 while (node) {
255 struct iova *iova = container_of(node, struct iova, node);
256
257 /* If pfn falls within iova's range, return iova */
258 if ((pfn >= iova->pfn_lo) && (pfn <= iova->pfn_hi)) {
259 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
260 /* We are not holding the lock while this iova
261 * is referenced by the caller as the same thread
262 * which called this function also calls __free_iova()
263 * and it is by desing that only one thread can possibly
264 * reference a particular iova and hence no conflict.
265 */
266 return iova;
267 }
268
269 if (pfn < iova->pfn_lo)
270 node = node->rb_left;
271 else if (pfn > iova->pfn_lo)
272 node = node->rb_right;
273 }
274
275 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
276 return NULL;
277}
278
279/**
280 * __free_iova - frees the given iova
281 * @iovad: iova domain in question.
282 * @iova: iova in question.
283 * Frees the given iova belonging to the giving domain
284 */
285void
286__free_iova(struct iova_domain *iovad, struct iova *iova)
287{
288 unsigned long flags;
289
290 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
291 __cached_rbnode_delete_update(iovad, iova);
292 rb_erase(&iova->node, &iovad->rbroot);
293 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
294 free_iova_mem(iova);
295}
296
297/**
298 * free_iova - finds and frees the iova for a given pfn
299 * @iovad: - iova domain in question.
300 * @pfn: - pfn that is allocated previously
301 * This functions finds an iova for a given pfn and then
302 * frees the iova from that domain.
303 */
304void
305free_iova(struct iova_domain *iovad, unsigned long pfn)
306{
307 struct iova *iova = find_iova(iovad, pfn);
308 if (iova)
309 __free_iova(iovad, iova);
310
311}
312
313/**
314 * put_iova_domain - destroys the iova doamin
315 * @iovad: - iova domain in question.
316 * All the iova's in that domain are destroyed.
317 */
318void put_iova_domain(struct iova_domain *iovad)
319{
320 struct rb_node *node;
321 unsigned long flags;
322
323 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
324 node = rb_first(&iovad->rbroot);
325 while (node) {
326 struct iova *iova = container_of(node, struct iova, node);
327 rb_erase(node, &iovad->rbroot);
328 free_iova_mem(iova);
329 node = rb_first(&iovad->rbroot);
330 }
331 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
332}
333
334static int
335__is_range_overlap(struct rb_node *node,
336 unsigned long pfn_lo, unsigned long pfn_hi)
337{
338 struct iova *iova = container_of(node, struct iova, node);
339
340 if ((pfn_lo <= iova->pfn_hi) && (pfn_hi >= iova->pfn_lo))
341 return 1;
342 return 0;
343}
344
345static struct iova *
346__insert_new_range(struct iova_domain *iovad,
347 unsigned long pfn_lo, unsigned long pfn_hi)
348{
349 struct iova *iova;
350
351 iova = alloc_iova_mem();
352 if (!iova)
353 return iova;
354
355 iova->pfn_hi = pfn_hi;
356 iova->pfn_lo = pfn_lo;
357 iova_insert_rbtree(&iovad->rbroot, iova);
358 return iova;
359}
360
361static void
362__adjust_overlap_range(struct iova *iova,
363 unsigned long *pfn_lo, unsigned long *pfn_hi)
364{
365 if (*pfn_lo < iova->pfn_lo)
366 iova->pfn_lo = *pfn_lo;
367 if (*pfn_hi > iova->pfn_hi)
368 *pfn_lo = iova->pfn_hi + 1;
369}
370
371/**
372 * reserve_iova - reserves an iova in the given range
373 * @iovad: - iova domain pointer
374 * @pfn_lo: - lower page frame address
375 * @pfn_hi:- higher pfn adderss
376 * This function allocates reserves the address range from pfn_lo to pfn_hi so
377 * that this address is not dished out as part of alloc_iova.
378 */
379struct iova *
380reserve_iova(struct iova_domain *iovad,
381 unsigned long pfn_lo, unsigned long pfn_hi)
382{
383 struct rb_node *node;
384 unsigned long flags;
385 struct iova *iova;
386 unsigned int overlap = 0;
387
388 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
389 for (node = rb_first(&iovad->rbroot); node; node = rb_next(node)) {
390 if (__is_range_overlap(node, pfn_lo, pfn_hi)) {
391 iova = container_of(node, struct iova, node);
392 __adjust_overlap_range(iova, &pfn_lo, &pfn_hi);
393 if ((pfn_lo >= iova->pfn_lo) &&
394 (pfn_hi <= iova->pfn_hi))
395 goto finish;
396 overlap = 1;
397
398 } else if (overlap)
399 break;
400 }
401
402 /* We are here either because this is the first reserver node
403 * or need to insert remaining non overlap addr range
404 */
405 iova = __insert_new_range(iovad, pfn_lo, pfn_hi);
406finish:
407
408 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
409 return iova;
410}
411
412/**
413 * copy_reserved_iova - copies the reserved between domains
414 * @from: - source doamin from where to copy
415 * @to: - destination domin where to copy
416 * This function copies reserved iova's from one doamin to
417 * other.
418 */
419void
420copy_reserved_iova(struct iova_domain *from, struct iova_domain *to)
421{
422 unsigned long flags;
423 struct rb_node *node;
424
425 spin_lock_irqsave(&from->iova_rbtree_lock, flags);
426 for (node = rb_first(&from->rbroot); node; node = rb_next(node)) {
427 struct iova *iova = container_of(node, struct iova, node);
428 struct iova *new_iova;
429 new_iova = reserve_iova(to, iova->pfn_lo, iova->pfn_hi);
430 if (!new_iova)
431 printk(KERN_ERR "Reserve iova range %lx@%lx failed\n",
432 iova->pfn_lo, iova->pfn_lo);
433 }
434 spin_unlock_irqrestore(&from->iova_rbtree_lock, flags);
435}
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-23 14:54:06 -0500