blob: 67b243c6c4e958d5b1eaaf5a4d2e9683592823f0 [file] [log] [blame]
Joerg Roedel1c4248c2012-09-26 12:44:35 +02001#include <linux/cpumask.h>
Joerg Roedel736baef2012-03-30 11:47:00 -07002#include <linux/kernel.h>
3#include <linux/string.h>
Joerg Roedel98f1ad22012-07-06 13:28:37 +02004#include <linux/cpumask.h>
Joerg Roedel736baef2012-03-30 11:47:00 -07005#include <linux/errno.h>
Joerg Roedel98f1ad22012-07-06 13:28:37 +02006#include <linux/msi.h>
Joerg Roedel5afba622012-09-26 12:44:38 +02007#include <linux/irq.h>
8#include <linux/pci.h>
Joerg Roedel98f1ad22012-07-06 13:28:37 +02009
10#include <asm/hw_irq.h>
11#include <asm/irq_remapping.h>
Joerg Roedel1c4248c2012-09-26 12:44:35 +020012#include <asm/processor.h>
13#include <asm/x86_init.h>
14#include <asm/apic.h>
Joerg Roedel736baef2012-03-30 11:47:00 -070015
Suresh Siddha8a8f4222012-03-30 11:47:08 -070016#include "irq_remapping.h"
Joerg Roedel736baef2012-03-30 11:47:00 -070017
Suresh Siddha95a02e92012-03-30 11:47:07 -070018int irq_remapping_enabled;
Joerg Roedel736baef2012-03-30 11:47:00 -070019
Suresh Siddha95a02e92012-03-30 11:47:07 -070020int disable_irq_remap;
Joerg Roedel736baef2012-03-30 11:47:00 -070021int disable_sourceid_checking;
22int no_x2apic_optout;
23
24static struct irq_remap_ops *remap_ops;
25
Joerg Roedel5afba622012-09-26 12:44:38 +020026static int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec);
27static int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq,
28 int index, int sub_handle);
Joerg Roedel373dd7a2012-09-26 12:44:39 +020029static int set_remapped_irq_affinity(struct irq_data *data,
30 const struct cpumask *mask,
31 bool force);
Joerg Roedel5afba622012-09-26 12:44:38 +020032
Joerg Roedel1c4248c2012-09-26 12:44:35 +020033static void irq_remapping_disable_io_apic(void)
34{
35 /*
36 * With interrupt-remapping, for now we will use virtual wire A
37 * mode, as virtual wire B is little complex (need to configure
38 * both IOAPIC RTE as well as interrupt-remapping table entry).
39 * As this gets called during crash dump, keep this simple for
40 * now.
41 */
42 if (cpu_has_apic || apic_from_smp_config())
43 disconnect_bsp_APIC(0);
44}
45
Joerg Roedel5afba622012-09-26 12:44:38 +020046static int do_setup_msi_irqs(struct pci_dev *dev, int nvec)
47{
48 int node, ret, sub_handle, index = 0;
49 unsigned int irq;
50 struct msi_desc *msidesc;
51
52 nvec = __roundup_pow_of_two(nvec);
53
54 WARN_ON(!list_is_singular(&dev->msi_list));
55 msidesc = list_entry(dev->msi_list.next, struct msi_desc, list);
56 WARN_ON(msidesc->irq);
57 WARN_ON(msidesc->msi_attrib.multiple);
58
59 node = dev_to_node(&dev->dev);
60 irq = __create_irqs(get_nr_irqs_gsi(), nvec, node);
61 if (irq == 0)
62 return -ENOSPC;
63
64 msidesc->msi_attrib.multiple = ilog2(nvec);
65 for (sub_handle = 0; sub_handle < nvec; sub_handle++) {
66 if (!sub_handle) {
67 index = msi_alloc_remapped_irq(dev, irq, nvec);
68 if (index < 0) {
69 ret = index;
70 goto error;
71 }
72 } else {
73 ret = msi_setup_remapped_irq(dev, irq + sub_handle,
74 index, sub_handle);
75 if (ret < 0)
76 goto error;
77 }
78 ret = setup_msi_irq(dev, msidesc, irq, sub_handle);
79 if (ret < 0)
80 goto error;
81 }
82 return 0;
83
84error:
85 destroy_irqs(irq, nvec);
86
87 /*
88 * Restore altered MSI descriptor fields and prevent just destroyed
89 * IRQs from tearing down again in default_teardown_msi_irqs()
90 */
91 msidesc->irq = 0;
92 msidesc->msi_attrib.multiple = 0;
93
94 return ret;
95}
96
97static int do_setup_msix_irqs(struct pci_dev *dev, int nvec)
98{
99 int node, ret, sub_handle, index = 0;
100 struct msi_desc *msidesc;
101 unsigned int irq;
102
103 node = dev_to_node(&dev->dev);
104 irq = get_nr_irqs_gsi();
105 sub_handle = 0;
106
107 list_for_each_entry(msidesc, &dev->msi_list, list) {
108
109 irq = create_irq_nr(irq, node);
110 if (irq == 0)
111 return -1;
112
113 if (sub_handle == 0)
114 ret = index = msi_alloc_remapped_irq(dev, irq, nvec);
115 else
116 ret = msi_setup_remapped_irq(dev, irq, index, sub_handle);
117
118 if (ret < 0)
119 goto error;
120
121 ret = setup_msi_irq(dev, msidesc, irq, 0);
122 if (ret < 0)
123 goto error;
124
125 sub_handle += 1;
126 irq += 1;
127 }
128
129 return 0;
130
131error:
132 destroy_irq(irq);
133 return ret;
134}
135
136static int irq_remapping_setup_msi_irqs(struct pci_dev *dev,
137 int nvec, int type)
138{
139 if (type == PCI_CAP_ID_MSI)
140 return do_setup_msi_irqs(dev, nvec);
141 else
142 return do_setup_msix_irqs(dev, nvec);
143}
144
Joerg Roedel1c4248c2012-09-26 12:44:35 +0200145static void __init irq_remapping_modify_x86_ops(void)
146{
Joerg Roedel71054d82012-09-26 12:44:37 +0200147 x86_io_apic_ops.disable = irq_remapping_disable_io_apic;
Joerg Roedel373dd7a2012-09-26 12:44:39 +0200148 x86_io_apic_ops.set_affinity = set_remapped_irq_affinity;
Joerg Roedel5afba622012-09-26 12:44:38 +0200149 x86_msi.setup_msi_irqs = irq_remapping_setup_msi_irqs;
Joerg Roedel71054d82012-09-26 12:44:37 +0200150 x86_msi.setup_hpet_msi = setup_hpet_msi_remapped;
Joerg Roedel1c4248c2012-09-26 12:44:35 +0200151}
152
Joerg Roedel736baef2012-03-30 11:47:00 -0700153static __init int setup_nointremap(char *str)
154{
Suresh Siddha95a02e92012-03-30 11:47:07 -0700155 disable_irq_remap = 1;
Joerg Roedel736baef2012-03-30 11:47:00 -0700156 return 0;
157}
158early_param("nointremap", setup_nointremap);
159
Suresh Siddha95a02e92012-03-30 11:47:07 -0700160static __init int setup_irqremap(char *str)
Joerg Roedel736baef2012-03-30 11:47:00 -0700161{
162 if (!str)
163 return -EINVAL;
164
165 while (*str) {
166 if (!strncmp(str, "on", 2))
Suresh Siddha95a02e92012-03-30 11:47:07 -0700167 disable_irq_remap = 0;
Joerg Roedel736baef2012-03-30 11:47:00 -0700168 else if (!strncmp(str, "off", 3))
Suresh Siddha95a02e92012-03-30 11:47:07 -0700169 disable_irq_remap = 1;
Joerg Roedel736baef2012-03-30 11:47:00 -0700170 else if (!strncmp(str, "nosid", 5))
171 disable_sourceid_checking = 1;
172 else if (!strncmp(str, "no_x2apic_optout", 16))
173 no_x2apic_optout = 1;
174
175 str += strcspn(str, ",");
176 while (*str == ',')
177 str++;
178 }
179
180 return 0;
181}
Suresh Siddha95a02e92012-03-30 11:47:07 -0700182early_param("intremap", setup_irqremap);
Joerg Roedel736baef2012-03-30 11:47:00 -0700183
Suresh Siddha95a02e92012-03-30 11:47:07 -0700184void __init setup_irq_remapping_ops(void)
Joerg Roedel736baef2012-03-30 11:47:00 -0700185{
186 remap_ops = &intel_irq_remap_ops;
Joerg Roedelc18d2382012-06-26 16:59:34 +0200187
188#ifdef CONFIG_AMD_IOMMU
189 if (amd_iommu_irq_ops.prepare() == 0)
190 remap_ops = &amd_iommu_irq_ops;
191#endif
Joerg Roedel736baef2012-03-30 11:47:00 -0700192}
193
Suresh Siddha95a02e92012-03-30 11:47:07 -0700194int irq_remapping_supported(void)
Joerg Roedel736baef2012-03-30 11:47:00 -0700195{
Suresh Siddha95a02e92012-03-30 11:47:07 -0700196 if (disable_irq_remap)
Joerg Roedel736baef2012-03-30 11:47:00 -0700197 return 0;
198
199 if (!remap_ops || !remap_ops->supported)
200 return 0;
201
202 return remap_ops->supported();
203}
204
Suresh Siddha95a02e92012-03-30 11:47:07 -0700205int __init irq_remapping_prepare(void)
Joerg Roedel736baef2012-03-30 11:47:00 -0700206{
Suresh Siddha95a02e92012-03-30 11:47:07 -0700207 if (!remap_ops || !remap_ops->prepare)
Joerg Roedel736baef2012-03-30 11:47:00 -0700208 return -ENODEV;
209
Suresh Siddha95a02e92012-03-30 11:47:07 -0700210 return remap_ops->prepare();
Joerg Roedel736baef2012-03-30 11:47:00 -0700211}
212
Suresh Siddha95a02e92012-03-30 11:47:07 -0700213int __init irq_remapping_enable(void)
Joerg Roedel736baef2012-03-30 11:47:00 -0700214{
Joerg Roedel1c4248c2012-09-26 12:44:35 +0200215 int ret;
216
Suresh Siddha95a02e92012-03-30 11:47:07 -0700217 if (!remap_ops || !remap_ops->enable)
Joerg Roedel736baef2012-03-30 11:47:00 -0700218 return -ENODEV;
219
Joerg Roedel1c4248c2012-09-26 12:44:35 +0200220 ret = remap_ops->enable();
221
222 if (irq_remapping_enabled)
223 irq_remapping_modify_x86_ops();
224
225 return ret;
Joerg Roedel736baef2012-03-30 11:47:00 -0700226}
Joerg Roedel4f3d8b62012-03-30 11:47:01 -0700227
Suresh Siddha95a02e92012-03-30 11:47:07 -0700228void irq_remapping_disable(void)
Joerg Roedel4f3d8b62012-03-30 11:47:01 -0700229{
Joerg Roedel70733e02012-09-26 12:44:33 +0200230 if (!irq_remapping_enabled ||
231 !remap_ops ||
232 !remap_ops->disable)
Joerg Roedel4f3d8b62012-03-30 11:47:01 -0700233 return;
234
Suresh Siddha95a02e92012-03-30 11:47:07 -0700235 remap_ops->disable();
Joerg Roedel4f3d8b62012-03-30 11:47:01 -0700236}
237
Suresh Siddha95a02e92012-03-30 11:47:07 -0700238int irq_remapping_reenable(int mode)
Joerg Roedel4f3d8b62012-03-30 11:47:01 -0700239{
Joerg Roedel70733e02012-09-26 12:44:33 +0200240 if (!irq_remapping_enabled ||
241 !remap_ops ||
242 !remap_ops->reenable)
Joerg Roedel4f3d8b62012-03-30 11:47:01 -0700243 return 0;
244
Suresh Siddha95a02e92012-03-30 11:47:07 -0700245 return remap_ops->reenable(mode);
Joerg Roedel4f3d8b62012-03-30 11:47:01 -0700246}
247
Suresh Siddha95a02e92012-03-30 11:47:07 -0700248int __init irq_remap_enable_fault_handling(void)
Joerg Roedel4f3d8b62012-03-30 11:47:01 -0700249{
Joerg Roedel70733e02012-09-26 12:44:33 +0200250 if (!irq_remapping_enabled)
251 return 0;
252
Joerg Roedel4f3d8b62012-03-30 11:47:01 -0700253 if (!remap_ops || !remap_ops->enable_faulting)
254 return -ENODEV;
255
256 return remap_ops->enable_faulting();
257}
Joerg Roedel0c3f1732012-03-30 11:47:02 -0700258
Suresh Siddha95a02e92012-03-30 11:47:07 -0700259int setup_ioapic_remapped_entry(int irq,
260 struct IO_APIC_route_entry *entry,
261 unsigned int destination, int vector,
262 struct io_apic_irq_attr *attr)
Joerg Roedel0c3f1732012-03-30 11:47:02 -0700263{
264 if (!remap_ops || !remap_ops->setup_ioapic_entry)
265 return -ENODEV;
266
267 return remap_ops->setup_ioapic_entry(irq, entry, destination,
268 vector, attr);
269}
Joerg Roedel4c1bad62012-03-30 11:47:03 -0700270
Suresh Siddha95a02e92012-03-30 11:47:07 -0700271int set_remapped_irq_affinity(struct irq_data *data, const struct cpumask *mask,
272 bool force)
Joerg Roedel4c1bad62012-03-30 11:47:03 -0700273{
Suresh Siddha7eb9ae02012-06-14 18:28:49 -0700274 if (!config_enabled(CONFIG_SMP) || !remap_ops ||
275 !remap_ops->set_affinity)
Joerg Roedel4c1bad62012-03-30 11:47:03 -0700276 return 0;
277
278 return remap_ops->set_affinity(data, mask, force);
279}
Joerg Roedel9d619f62012-03-30 11:47:04 -0700280
Suresh Siddha95a02e92012-03-30 11:47:07 -0700281void free_remapped_irq(int irq)
Joerg Roedel9d619f62012-03-30 11:47:04 -0700282{
283 if (!remap_ops || !remap_ops->free_irq)
284 return;
285
286 remap_ops->free_irq(irq);
287}
Joerg Roedel5e2b9302012-03-30 11:47:05 -0700288
Suresh Siddha95a02e92012-03-30 11:47:07 -0700289void compose_remapped_msi_msg(struct pci_dev *pdev,
290 unsigned int irq, unsigned int dest,
291 struct msi_msg *msg, u8 hpet_id)
Joerg Roedel5e2b9302012-03-30 11:47:05 -0700292{
293 if (!remap_ops || !remap_ops->compose_msi_msg)
294 return;
295
296 remap_ops->compose_msi_msg(pdev, irq, dest, msg, hpet_id);
297}
298
Joerg Roedel5afba622012-09-26 12:44:38 +0200299static int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec)
Joerg Roedel5e2b9302012-03-30 11:47:05 -0700300{
301 if (!remap_ops || !remap_ops->msi_alloc_irq)
302 return -ENODEV;
303
304 return remap_ops->msi_alloc_irq(pdev, irq, nvec);
305}
306
Joerg Roedel5afba622012-09-26 12:44:38 +0200307static int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq,
308 int index, int sub_handle)
Joerg Roedel5e2b9302012-03-30 11:47:05 -0700309{
310 if (!remap_ops || !remap_ops->msi_setup_irq)
311 return -ENODEV;
312
313 return remap_ops->msi_setup_irq(pdev, irq, index, sub_handle);
314}
315
Suresh Siddha95a02e92012-03-30 11:47:07 -0700316int setup_hpet_msi_remapped(unsigned int irq, unsigned int id)
Joerg Roedel5e2b9302012-03-30 11:47:05 -0700317{
318 if (!remap_ops || !remap_ops->setup_hpet_msi)
319 return -ENODEV;
320
321 return remap_ops->setup_hpet_msi(irq, id);
322}