1 | /* |
2 | * Marvell Armada 370 and Armada XP SoC IRQ handling |
3 | * |
4 | * Copyright (C) 2012 Marvell |
5 | * |
6 | * Lior Amsalem <alior@marvell.com> |
7 | * Gregory CLEMENT <gregory.clement@free-electrons.com> |
8 | * Thomas Petazzoni <thomas.petazzoni@free-electrons.com> |
9 | * Ben Dooks <ben.dooks@codethink.co.uk> |
10 | * |
11 | * This file is licensed under the terms of the GNU General Public |
12 | * License version 2. This program is licensed "as is" without any |
13 | * warranty of any kind, whether express or implied. |
14 | */ |
15 | |
16 | #include <linux/kernel.h> |
17 | #include <linux/module.h> |
18 | #include <linux/init.h> |
19 | #include <linux/irq.h> |
20 | #include <linux/interrupt.h> |
21 | #include <linux/irqchip.h> |
22 | #include <linux/irqchip/chained_irq.h> |
23 | #include <linux/cpu.h> |
24 | #include <linux/io.h> |
25 | #include <linux/of_address.h> |
26 | #include <linux/of_irq.h> |
27 | #include <linux/of_pci.h> |
28 | #include <linux/irqdomain.h> |
29 | #include <linux/slab.h> |
30 | #include <linux/syscore_ops.h> |
31 | #include <linux/msi.h> |
32 | #include <asm/mach/arch.h> |
33 | #include <asm/exception.h> |
34 | #include <asm/smp_plat.h> |
35 | #include <asm/mach/irq.h> |
36 | |
37 | /* |
38 | * Overall diagram of the Armada XP interrupt controller: |
39 | * |
40 | * To CPU 0 To CPU 1 |
41 | * |
42 | * /\ /\ |
43 | * || || |
44 | * +---------------+ +---------------+ |
45 | * | | | | |
46 | * | per-CPU | | per-CPU | |
47 | * | mask/unmask | | mask/unmask | |
48 | * | CPU0 | | CPU1 | |
49 | * | | | | |
50 | * +---------------+ +---------------+ |
51 | * /\ /\ |
52 | * || || |
53 | * \\_______________________// |
54 | * || |
55 | * +-------------------+ |
56 | * | | |
57 | * | Global interrupt | |
58 | * | mask/unmask | |
59 | * | | |
60 | * +-------------------+ |
61 | * /\ |
62 | * || |
63 | * interrupt from |
64 | * device |
65 | * |
66 | * The "global interrupt mask/unmask" is modified using the |
67 | * ARMADA_370_XP_INT_SET_ENABLE_OFFS and |
68 | * ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS registers, which are relative |
69 | * to "main_int_base". |
70 | * |
71 | * The "per-CPU mask/unmask" is modified using the |
72 | * ARMADA_370_XP_INT_SET_MASK_OFFS and |
73 | * ARMADA_370_XP_INT_CLEAR_MASK_OFFS registers, which are relative to |
74 | * "per_cpu_int_base". This base address points to a special address, |
75 | * which automatically accesses the registers of the current CPU. |
76 | * |
77 | * The per-CPU mask/unmask can also be adjusted using the global |
78 | * per-interrupt ARMADA_370_XP_INT_SOURCE_CTL register, which we use |
79 | * to configure interrupt affinity. |
80 | * |
81 | * Due to this model, all interrupts need to be mask/unmasked at two |
82 | * different levels: at the global level and at the per-CPU level. |
83 | * |
84 | * This driver takes the following approach to deal with this: |
85 | * |
86 | * - For global interrupts: |
87 | * |
88 | * At ->map() time, a global interrupt is unmasked at the per-CPU |
89 | * mask/unmask level. It is therefore unmasked at this level for |
90 | * the current CPU, running the ->map() code. This allows to have |
91 | * the interrupt unmasked at this level in non-SMP |
92 | * configurations. In SMP configurations, the ->set_affinity() |
93 | * callback is called, which using the |
94 | * ARMADA_370_XP_INT_SOURCE_CTL() readjusts the per-CPU mask/unmask |
95 | * for the interrupt. |
96 | * |
97 | * The ->mask() and ->unmask() operations only mask/unmask the |
98 | * interrupt at the "global" level. |
99 | * |
100 | * So, a global interrupt is enabled at the per-CPU level as soon |
101 | * as it is mapped. At run time, the masking/unmasking takes place |
102 | * at the global level. |
103 | * |
104 | * - For per-CPU interrupts |
105 | * |
106 | * At ->map() time, a per-CPU interrupt is unmasked at the global |
107 | * mask/unmask level. |
108 | * |
109 | * The ->mask() and ->unmask() operations mask/unmask the interrupt |
110 | * at the per-CPU level. |
111 | * |
112 | * So, a per-CPU interrupt is enabled at the global level as soon |
113 | * as it is mapped. At run time, the masking/unmasking takes place |
114 | * at the per-CPU level. |
115 | */ |
116 | |
117 | /* Registers relative to main_int_base */ |
118 | #define ARMADA_370_XP_INT_CONTROL (0x00) |
119 | #define ARMADA_370_XP_SW_TRIG_INT_OFFS (0x04) |
120 | #define ARMADA_370_XP_INT_SET_ENABLE_OFFS (0x30) |
121 | #define ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS (0x34) |
122 | #define ARMADA_370_XP_INT_SOURCE_CTL(irq) (0x100 + irq*4) |
123 | #define ARMADA_370_XP_INT_SOURCE_CPU_MASK 0xF |
124 | #define ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << cpuid) |
125 | |
126 | /* Registers relative to per_cpu_int_base */ |
127 | #define ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS (0x08) |
128 | #define ARMADA_370_XP_IN_DRBEL_MSK_OFFS (0x0c) |
129 | #define ARMADA_375_PPI_CAUSE (0x10) |
130 | #define ARMADA_370_XP_CPU_INTACK_OFFS (0x44) |
131 | #define ARMADA_370_XP_INT_SET_MASK_OFFS (0x48) |
132 | #define ARMADA_370_XP_INT_CLEAR_MASK_OFFS (0x4C) |
133 | #define ARMADA_370_XP_INT_FABRIC_MASK_OFFS (0x54) |
134 | #define ARMADA_370_XP_INT_CAUSE_PERF(cpu) (1 << cpu) |
135 | |
136 | #define ARMADA_370_XP_MAX_PER_CPU_IRQS (28) |
137 | |
138 | #define IPI_DOORBELL_START (0) |
139 | #define IPI_DOORBELL_END (8) |
140 | #define IPI_DOORBELL_MASK 0xFF |
141 | #define PCI_MSI_DOORBELL_START (16) |
142 | #define PCI_MSI_DOORBELL_NR (16) |
143 | #define PCI_MSI_DOORBELL_END (32) |
144 | #define PCI_MSI_DOORBELL_MASK 0xFFFF0000 |
145 | |
146 | static void __iomem *per_cpu_int_base; |
147 | static void __iomem *main_int_base; |
148 | static struct irq_domain *armada_370_xp_mpic_domain; |
149 | static u32 doorbell_mask_reg; |
150 | static int parent_irq; |
151 | #ifdef CONFIG_PCI_MSI |
152 | static struct irq_domain *armada_370_xp_msi_domain; |
153 | static struct irq_domain *armada_370_xp_msi_inner_domain; |
154 | static DECLARE_BITMAP(msi_used, PCI_MSI_DOORBELL_NR); |
155 | static DEFINE_MUTEX(msi_used_lock); |
156 | static phys_addr_t msi_doorbell_addr; |
157 | #endif |
158 | |
159 | static inline bool is_percpu_irq(irq_hw_number_t irq) |
160 | { |
161 | if (irq <= ARMADA_370_XP_MAX_PER_CPU_IRQS) |
162 | return true; |
163 | |
164 | return false; |
165 | } |
166 | |
167 | /* |
168 | * In SMP mode: |
169 | * For shared global interrupts, mask/unmask global enable bit |
170 | * For CPU interrupts, mask/unmask the calling CPU's bit |
171 | */ |
172 | static void armada_370_xp_irq_mask(struct irq_data *d) |
173 | { |
174 | irq_hw_number_t hwirq = irqd_to_hwirq(d); |
175 | |
176 | if (!is_percpu_irq(irq: hwirq)) |
177 | writel(val: hwirq, addr: main_int_base + |
178 | ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS); |
179 | else |
180 | writel(val: hwirq, addr: per_cpu_int_base + |
181 | ARMADA_370_XP_INT_SET_MASK_OFFS); |
182 | } |
183 | |
184 | static void armada_370_xp_irq_unmask(struct irq_data *d) |
185 | { |
186 | irq_hw_number_t hwirq = irqd_to_hwirq(d); |
187 | |
188 | if (!is_percpu_irq(irq: hwirq)) |
189 | writel(val: hwirq, addr: main_int_base + |
190 | ARMADA_370_XP_INT_SET_ENABLE_OFFS); |
191 | else |
192 | writel(val: hwirq, addr: per_cpu_int_base + |
193 | ARMADA_370_XP_INT_CLEAR_MASK_OFFS); |
194 | } |
195 | |
196 | #ifdef CONFIG_PCI_MSI |
197 | |
198 | static struct irq_chip armada_370_xp_msi_irq_chip = { |
199 | .name = "MPIC MSI" , |
200 | .irq_mask = pci_msi_mask_irq, |
201 | .irq_unmask = pci_msi_unmask_irq, |
202 | }; |
203 | |
204 | static struct msi_domain_info armada_370_xp_msi_domain_info = { |
205 | .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | |
206 | MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX), |
207 | .chip = &armada_370_xp_msi_irq_chip, |
208 | }; |
209 | |
210 | static void armada_370_xp_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) |
211 | { |
212 | unsigned int cpu = cpumask_first(srcp: irq_data_get_effective_affinity_mask(d: data)); |
213 | |
214 | msg->address_lo = lower_32_bits(msi_doorbell_addr); |
215 | msg->address_hi = upper_32_bits(msi_doorbell_addr); |
216 | msg->data = BIT(cpu + 8) | (data->hwirq + PCI_MSI_DOORBELL_START); |
217 | } |
218 | |
219 | static int armada_370_xp_msi_set_affinity(struct irq_data *irq_data, |
220 | const struct cpumask *mask, bool force) |
221 | { |
222 | unsigned int cpu; |
223 | |
224 | if (!force) |
225 | cpu = cpumask_any_and(mask, cpu_online_mask); |
226 | else |
227 | cpu = cpumask_first(srcp: mask); |
228 | |
229 | if (cpu >= nr_cpu_ids) |
230 | return -EINVAL; |
231 | |
232 | irq_data_update_effective_affinity(d: irq_data, cpumask_of(cpu)); |
233 | |
234 | return IRQ_SET_MASK_OK; |
235 | } |
236 | |
237 | static struct irq_chip armada_370_xp_msi_bottom_irq_chip = { |
238 | .name = "MPIC MSI" , |
239 | .irq_compose_msi_msg = armada_370_xp_compose_msi_msg, |
240 | .irq_set_affinity = armada_370_xp_msi_set_affinity, |
241 | }; |
242 | |
243 | static int armada_370_xp_msi_alloc(struct irq_domain *domain, unsigned int virq, |
244 | unsigned int nr_irqs, void *args) |
245 | { |
246 | int hwirq, i; |
247 | |
248 | mutex_lock(&msi_used_lock); |
249 | hwirq = bitmap_find_free_region(bitmap: msi_used, PCI_MSI_DOORBELL_NR, |
250 | order_base_2(nr_irqs)); |
251 | mutex_unlock(lock: &msi_used_lock); |
252 | |
253 | if (hwirq < 0) |
254 | return -ENOSPC; |
255 | |
256 | for (i = 0; i < nr_irqs; i++) { |
257 | irq_domain_set_info(domain, virq: virq + i, hwirq: hwirq + i, |
258 | chip: &armada_370_xp_msi_bottom_irq_chip, |
259 | chip_data: domain->host_data, handler: handle_simple_irq, |
260 | NULL, NULL); |
261 | } |
262 | |
263 | return 0; |
264 | } |
265 | |
266 | static void armada_370_xp_msi_free(struct irq_domain *domain, |
267 | unsigned int virq, unsigned int nr_irqs) |
268 | { |
269 | struct irq_data *d = irq_domain_get_irq_data(domain, virq); |
270 | |
271 | mutex_lock(&msi_used_lock); |
272 | bitmap_release_region(bitmap: msi_used, pos: d->hwirq, order_base_2(nr_irqs)); |
273 | mutex_unlock(lock: &msi_used_lock); |
274 | } |
275 | |
276 | static const struct irq_domain_ops armada_370_xp_msi_domain_ops = { |
277 | .alloc = armada_370_xp_msi_alloc, |
278 | .free = armada_370_xp_msi_free, |
279 | }; |
280 | |
281 | static void armada_370_xp_msi_reenable_percpu(void) |
282 | { |
283 | u32 reg; |
284 | |
285 | /* Enable MSI doorbell mask and combined cpu local interrupt */ |
286 | reg = readl(addr: per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS) |
287 | | PCI_MSI_DOORBELL_MASK; |
288 | writel(val: reg, addr: per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); |
289 | /* Unmask local doorbell interrupt */ |
290 | writel(val: 1, addr: per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); |
291 | } |
292 | |
293 | static int armada_370_xp_msi_init(struct device_node *node, |
294 | phys_addr_t main_int_phys_base) |
295 | { |
296 | msi_doorbell_addr = main_int_phys_base + |
297 | ARMADA_370_XP_SW_TRIG_INT_OFFS; |
298 | |
299 | armada_370_xp_msi_inner_domain = |
300 | irq_domain_add_linear(NULL, PCI_MSI_DOORBELL_NR, |
301 | ops: &armada_370_xp_msi_domain_ops, NULL); |
302 | if (!armada_370_xp_msi_inner_domain) |
303 | return -ENOMEM; |
304 | |
305 | armada_370_xp_msi_domain = |
306 | pci_msi_create_irq_domain(fwnode: of_node_to_fwnode(node), |
307 | info: &armada_370_xp_msi_domain_info, |
308 | parent: armada_370_xp_msi_inner_domain); |
309 | if (!armada_370_xp_msi_domain) { |
310 | irq_domain_remove(host: armada_370_xp_msi_inner_domain); |
311 | return -ENOMEM; |
312 | } |
313 | |
314 | armada_370_xp_msi_reenable_percpu(); |
315 | |
316 | return 0; |
317 | } |
318 | #else |
319 | static void armada_370_xp_msi_reenable_percpu(void) {} |
320 | |
321 | static inline int armada_370_xp_msi_init(struct device_node *node, |
322 | phys_addr_t main_int_phys_base) |
323 | { |
324 | return 0; |
325 | } |
326 | #endif |
327 | |
328 | static void armada_xp_mpic_perf_init(void) |
329 | { |
330 | unsigned long cpuid; |
331 | |
332 | /* |
333 | * This Performance Counter Overflow interrupt is specific for |
334 | * Armada 370 and XP. It is not available on Armada 375, 38x and 39x. |
335 | */ |
336 | if (!of_machine_is_compatible(compat: "marvell,armada-370-xp" )) |
337 | return; |
338 | |
339 | cpuid = cpu_logical_map(smp_processor_id()); |
340 | |
341 | /* Enable Performance Counter Overflow interrupts */ |
342 | writel(ARMADA_370_XP_INT_CAUSE_PERF(cpuid), |
343 | addr: per_cpu_int_base + ARMADA_370_XP_INT_FABRIC_MASK_OFFS); |
344 | } |
345 | |
346 | #ifdef CONFIG_SMP |
347 | static struct irq_domain *ipi_domain; |
348 | |
349 | static void armada_370_xp_ipi_mask(struct irq_data *d) |
350 | { |
351 | u32 reg; |
352 | reg = readl(addr: per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); |
353 | reg &= ~BIT(d->hwirq); |
354 | writel(val: reg, addr: per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); |
355 | } |
356 | |
357 | static void armada_370_xp_ipi_unmask(struct irq_data *d) |
358 | { |
359 | u32 reg; |
360 | reg = readl(addr: per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); |
361 | reg |= BIT(d->hwirq); |
362 | writel(val: reg, addr: per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); |
363 | } |
364 | |
365 | static void armada_370_xp_ipi_send_mask(struct irq_data *d, |
366 | const struct cpumask *mask) |
367 | { |
368 | unsigned long map = 0; |
369 | int cpu; |
370 | |
371 | /* Convert our logical CPU mask into a physical one. */ |
372 | for_each_cpu(cpu, mask) |
373 | map |= 1 << cpu_logical_map(cpu); |
374 | |
375 | /* |
376 | * Ensure that stores to Normal memory are visible to the |
377 | * other CPUs before issuing the IPI. |
378 | */ |
379 | dsb(); |
380 | |
381 | /* submit softirq */ |
382 | writel(val: (map << 8) | d->hwirq, addr: main_int_base + |
383 | ARMADA_370_XP_SW_TRIG_INT_OFFS); |
384 | } |
385 | |
386 | static void armada_370_xp_ipi_ack(struct irq_data *d) |
387 | { |
388 | writel(val: ~BIT(d->hwirq), addr: per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); |
389 | } |
390 | |
391 | static struct irq_chip ipi_irqchip = { |
392 | .name = "IPI" , |
393 | .irq_ack = armada_370_xp_ipi_ack, |
394 | .irq_mask = armada_370_xp_ipi_mask, |
395 | .irq_unmask = armada_370_xp_ipi_unmask, |
396 | .ipi_send_mask = armada_370_xp_ipi_send_mask, |
397 | }; |
398 | |
399 | static int armada_370_xp_ipi_alloc(struct irq_domain *d, |
400 | unsigned int virq, |
401 | unsigned int nr_irqs, void *args) |
402 | { |
403 | int i; |
404 | |
405 | for (i = 0; i < nr_irqs; i++) { |
406 | irq_set_percpu_devid(irq: virq + i); |
407 | irq_domain_set_info(domain: d, virq: virq + i, hwirq: i, chip: &ipi_irqchip, |
408 | chip_data: d->host_data, |
409 | handler: handle_percpu_devid_irq, |
410 | NULL, NULL); |
411 | } |
412 | |
413 | return 0; |
414 | } |
415 | |
416 | static void armada_370_xp_ipi_free(struct irq_domain *d, |
417 | unsigned int virq, |
418 | unsigned int nr_irqs) |
419 | { |
420 | /* Not freeing IPIs */ |
421 | } |
422 | |
423 | static const struct irq_domain_ops ipi_domain_ops = { |
424 | .alloc = armada_370_xp_ipi_alloc, |
425 | .free = armada_370_xp_ipi_free, |
426 | }; |
427 | |
428 | static void ipi_resume(void) |
429 | { |
430 | int i; |
431 | |
432 | for (i = 0; i < IPI_DOORBELL_END; i++) { |
433 | int irq; |
434 | |
435 | irq = irq_find_mapping(domain: ipi_domain, hwirq: i); |
436 | if (irq <= 0) |
437 | continue; |
438 | if (irq_percpu_is_enabled(irq)) { |
439 | struct irq_data *d; |
440 | d = irq_domain_get_irq_data(domain: ipi_domain, virq: irq); |
441 | armada_370_xp_ipi_unmask(d); |
442 | } |
443 | } |
444 | } |
445 | |
446 | static __init void armada_xp_ipi_init(struct device_node *node) |
447 | { |
448 | int base_ipi; |
449 | |
450 | ipi_domain = irq_domain_create_linear(fwnode: of_node_to_fwnode(node), |
451 | IPI_DOORBELL_END, |
452 | ops: &ipi_domain_ops, NULL); |
453 | if (WARN_ON(!ipi_domain)) |
454 | return; |
455 | |
456 | irq_domain_update_bus_token(domain: ipi_domain, bus_token: DOMAIN_BUS_IPI); |
457 | base_ipi = irq_domain_alloc_irqs(domain: ipi_domain, IPI_DOORBELL_END, NUMA_NO_NODE, NULL); |
458 | if (WARN_ON(!base_ipi)) |
459 | return; |
460 | |
461 | set_smp_ipi_range(base_ipi, IPI_DOORBELL_END); |
462 | } |
463 | |
464 | static DEFINE_RAW_SPINLOCK(irq_controller_lock); |
465 | |
466 | static int armada_xp_set_affinity(struct irq_data *d, |
467 | const struct cpumask *mask_val, bool force) |
468 | { |
469 | irq_hw_number_t hwirq = irqd_to_hwirq(d); |
470 | unsigned long reg, mask; |
471 | int cpu; |
472 | |
473 | /* Select a single core from the affinity mask which is online */ |
474 | cpu = cpumask_any_and(mask_val, cpu_online_mask); |
475 | mask = 1UL << cpu_logical_map(cpu); |
476 | |
477 | raw_spin_lock(&irq_controller_lock); |
478 | reg = readl(addr: main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq)); |
479 | reg = (reg & (~ARMADA_370_XP_INT_SOURCE_CPU_MASK)) | mask; |
480 | writel(val: reg, addr: main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq)); |
481 | raw_spin_unlock(&irq_controller_lock); |
482 | |
483 | irq_data_update_effective_affinity(d, cpumask_of(cpu)); |
484 | |
485 | return IRQ_SET_MASK_OK; |
486 | } |
487 | |
488 | static void armada_xp_mpic_smp_cpu_init(void) |
489 | { |
490 | u32 control; |
491 | int nr_irqs, i; |
492 | |
493 | control = readl(addr: main_int_base + ARMADA_370_XP_INT_CONTROL); |
494 | nr_irqs = (control >> 2) & 0x3ff; |
495 | |
496 | for (i = 0; i < nr_irqs; i++) |
497 | writel(val: i, addr: per_cpu_int_base + ARMADA_370_XP_INT_SET_MASK_OFFS); |
498 | |
499 | /* Disable all IPIs */ |
500 | writel(val: 0, addr: per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); |
501 | |
502 | /* Clear pending IPIs */ |
503 | writel(val: 0, addr: per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); |
504 | |
505 | /* Unmask IPI interrupt */ |
506 | writel(val: 0, addr: per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); |
507 | } |
508 | |
509 | static void armada_xp_mpic_reenable_percpu(void) |
510 | { |
511 | unsigned int irq; |
512 | |
513 | /* Re-enable per-CPU interrupts that were enabled before suspend */ |
514 | for (irq = 0; irq < ARMADA_370_XP_MAX_PER_CPU_IRQS; irq++) { |
515 | struct irq_data *data; |
516 | int virq; |
517 | |
518 | virq = irq_linear_revmap(domain: armada_370_xp_mpic_domain, hwirq: irq); |
519 | if (virq == 0) |
520 | continue; |
521 | |
522 | data = irq_get_irq_data(irq: virq); |
523 | |
524 | if (!irq_percpu_is_enabled(irq: virq)) |
525 | continue; |
526 | |
527 | armada_370_xp_irq_unmask(d: data); |
528 | } |
529 | |
530 | ipi_resume(); |
531 | |
532 | armada_370_xp_msi_reenable_percpu(); |
533 | } |
534 | |
535 | static int armada_xp_mpic_starting_cpu(unsigned int cpu) |
536 | { |
537 | armada_xp_mpic_perf_init(); |
538 | armada_xp_mpic_smp_cpu_init(); |
539 | armada_xp_mpic_reenable_percpu(); |
540 | return 0; |
541 | } |
542 | |
543 | static int mpic_cascaded_starting_cpu(unsigned int cpu) |
544 | { |
545 | armada_xp_mpic_perf_init(); |
546 | armada_xp_mpic_reenable_percpu(); |
547 | enable_percpu_irq(irq: parent_irq, type: IRQ_TYPE_NONE); |
548 | return 0; |
549 | } |
550 | #else |
551 | static void armada_xp_mpic_smp_cpu_init(void) {} |
552 | static void ipi_resume(void) {} |
553 | #endif |
554 | |
555 | static struct irq_chip armada_370_xp_irq_chip = { |
556 | .name = "MPIC" , |
557 | .irq_mask = armada_370_xp_irq_mask, |
558 | .irq_mask_ack = armada_370_xp_irq_mask, |
559 | .irq_unmask = armada_370_xp_irq_unmask, |
560 | #ifdef CONFIG_SMP |
561 | .irq_set_affinity = armada_xp_set_affinity, |
562 | #endif |
563 | .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, |
564 | }; |
565 | |
566 | static int armada_370_xp_mpic_irq_map(struct irq_domain *h, |
567 | unsigned int virq, irq_hw_number_t hw) |
568 | { |
569 | armada_370_xp_irq_mask(d: irq_get_irq_data(irq: virq)); |
570 | if (!is_percpu_irq(irq: hw)) |
571 | writel(val: hw, addr: per_cpu_int_base + |
572 | ARMADA_370_XP_INT_CLEAR_MASK_OFFS); |
573 | else |
574 | writel(val: hw, addr: main_int_base + ARMADA_370_XP_INT_SET_ENABLE_OFFS); |
575 | irq_set_status_flags(irq: virq, set: IRQ_LEVEL); |
576 | |
577 | if (is_percpu_irq(irq: hw)) { |
578 | irq_set_percpu_devid(irq: virq); |
579 | irq_set_chip_and_handler(irq: virq, chip: &armada_370_xp_irq_chip, |
580 | handle: handle_percpu_devid_irq); |
581 | } else { |
582 | irq_set_chip_and_handler(irq: virq, chip: &armada_370_xp_irq_chip, |
583 | handle: handle_level_irq); |
584 | irqd_set_single_target(d: irq_desc_get_irq_data(desc: irq_to_desc(irq: virq))); |
585 | } |
586 | irq_set_probe(irq: virq); |
587 | |
588 | return 0; |
589 | } |
590 | |
591 | static const struct irq_domain_ops armada_370_xp_mpic_irq_ops = { |
592 | .map = armada_370_xp_mpic_irq_map, |
593 | .xlate = irq_domain_xlate_onecell, |
594 | }; |
595 | |
596 | #ifdef CONFIG_PCI_MSI |
597 | static void armada_370_xp_handle_msi_irq(struct pt_regs *regs, bool is_chained) |
598 | { |
599 | u32 msimask, msinr; |
600 | |
601 | msimask = readl_relaxed(per_cpu_int_base + |
602 | ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS) |
603 | & PCI_MSI_DOORBELL_MASK; |
604 | |
605 | writel(val: ~msimask, addr: per_cpu_int_base + |
606 | ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); |
607 | |
608 | for (msinr = PCI_MSI_DOORBELL_START; |
609 | msinr < PCI_MSI_DOORBELL_END; msinr++) { |
610 | unsigned int irq; |
611 | |
612 | if (!(msimask & BIT(msinr))) |
613 | continue; |
614 | |
615 | irq = msinr - PCI_MSI_DOORBELL_START; |
616 | |
617 | generic_handle_domain_irq(domain: armada_370_xp_msi_inner_domain, hwirq: irq); |
618 | } |
619 | } |
620 | #else |
621 | static void armada_370_xp_handle_msi_irq(struct pt_regs *r, bool b) {} |
622 | #endif |
623 | |
624 | static void armada_370_xp_mpic_handle_cascade_irq(struct irq_desc *desc) |
625 | { |
626 | struct irq_chip *chip = irq_desc_get_chip(desc); |
627 | unsigned long irqmap, irqn, irqsrc, cpuid; |
628 | |
629 | chained_irq_enter(chip, desc); |
630 | |
631 | irqmap = readl_relaxed(per_cpu_int_base + ARMADA_375_PPI_CAUSE); |
632 | cpuid = cpu_logical_map(smp_processor_id()); |
633 | |
634 | for_each_set_bit(irqn, &irqmap, BITS_PER_LONG) { |
635 | irqsrc = readl_relaxed(main_int_base + |
636 | ARMADA_370_XP_INT_SOURCE_CTL(irqn)); |
637 | |
638 | /* Check if the interrupt is not masked on current CPU. |
639 | * Test IRQ (0-1) and FIQ (8-9) mask bits. |
640 | */ |
641 | if (!(irqsrc & ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid))) |
642 | continue; |
643 | |
644 | if (irqn == 1) { |
645 | armada_370_xp_handle_msi_irq(NULL, is_chained: true); |
646 | continue; |
647 | } |
648 | |
649 | generic_handle_domain_irq(domain: armada_370_xp_mpic_domain, hwirq: irqn); |
650 | } |
651 | |
652 | chained_irq_exit(chip, desc); |
653 | } |
654 | |
655 | static void __exception_irq_entry |
656 | armada_370_xp_handle_irq(struct pt_regs *regs) |
657 | { |
658 | u32 irqstat, irqnr; |
659 | |
660 | do { |
661 | irqstat = readl_relaxed(per_cpu_int_base + |
662 | ARMADA_370_XP_CPU_INTACK_OFFS); |
663 | irqnr = irqstat & 0x3FF; |
664 | |
665 | if (irqnr > 1022) |
666 | break; |
667 | |
668 | if (irqnr > 1) { |
669 | generic_handle_domain_irq(armada_370_xp_mpic_domain, |
670 | irqnr); |
671 | continue; |
672 | } |
673 | |
674 | /* MSI handling */ |
675 | if (irqnr == 1) |
676 | armada_370_xp_handle_msi_irq(regs, false); |
677 | |
678 | #ifdef CONFIG_SMP |
679 | /* IPI Handling */ |
680 | if (irqnr == 0) { |
681 | unsigned long ipimask; |
682 | int ipi; |
683 | |
684 | ipimask = readl_relaxed(per_cpu_int_base + |
685 | ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS) |
686 | & IPI_DOORBELL_MASK; |
687 | |
688 | for_each_set_bit(ipi, &ipimask, IPI_DOORBELL_END) |
689 | generic_handle_domain_irq(ipi_domain, ipi); |
690 | } |
691 | #endif |
692 | |
693 | } while (1); |
694 | } |
695 | |
696 | static int armada_370_xp_mpic_suspend(void) |
697 | { |
698 | doorbell_mask_reg = readl(addr: per_cpu_int_base + |
699 | ARMADA_370_XP_IN_DRBEL_MSK_OFFS); |
700 | return 0; |
701 | } |
702 | |
703 | static void armada_370_xp_mpic_resume(void) |
704 | { |
705 | int nirqs; |
706 | irq_hw_number_t irq; |
707 | |
708 | /* Re-enable interrupts */ |
709 | nirqs = (readl(addr: main_int_base + ARMADA_370_XP_INT_CONTROL) >> 2) & 0x3ff; |
710 | for (irq = 0; irq < nirqs; irq++) { |
711 | struct irq_data *data; |
712 | int virq; |
713 | |
714 | virq = irq_linear_revmap(domain: armada_370_xp_mpic_domain, hwirq: irq); |
715 | if (virq == 0) |
716 | continue; |
717 | |
718 | data = irq_get_irq_data(irq: virq); |
719 | |
720 | if (!is_percpu_irq(irq)) { |
721 | /* Non per-CPU interrupts */ |
722 | writel(val: irq, addr: per_cpu_int_base + |
723 | ARMADA_370_XP_INT_CLEAR_MASK_OFFS); |
724 | if (!irqd_irq_disabled(d: data)) |
725 | armada_370_xp_irq_unmask(d: data); |
726 | } else { |
727 | /* Per-CPU interrupts */ |
728 | writel(val: irq, addr: main_int_base + |
729 | ARMADA_370_XP_INT_SET_ENABLE_OFFS); |
730 | |
731 | /* |
732 | * Re-enable on the current CPU, |
733 | * armada_xp_mpic_reenable_percpu() will take |
734 | * care of secondary CPUs when they come up. |
735 | */ |
736 | if (irq_percpu_is_enabled(irq: virq)) |
737 | armada_370_xp_irq_unmask(d: data); |
738 | } |
739 | } |
740 | |
741 | /* Reconfigure doorbells for IPIs and MSIs */ |
742 | writel(val: doorbell_mask_reg, |
743 | addr: per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); |
744 | if (doorbell_mask_reg & IPI_DOORBELL_MASK) |
745 | writel(val: 0, addr: per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); |
746 | if (doorbell_mask_reg & PCI_MSI_DOORBELL_MASK) |
747 | writel(val: 1, addr: per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); |
748 | |
749 | ipi_resume(); |
750 | } |
751 | |
752 | static struct syscore_ops armada_370_xp_mpic_syscore_ops = { |
753 | .suspend = armada_370_xp_mpic_suspend, |
754 | .resume = armada_370_xp_mpic_resume, |
755 | }; |
756 | |
757 | static int __init armada_370_xp_mpic_of_init(struct device_node *node, |
758 | struct device_node *parent) |
759 | { |
760 | struct resource main_int_res, per_cpu_int_res; |
761 | int nr_irqs, i; |
762 | u32 control; |
763 | |
764 | BUG_ON(of_address_to_resource(node, 0, &main_int_res)); |
765 | BUG_ON(of_address_to_resource(node, 1, &per_cpu_int_res)); |
766 | |
767 | BUG_ON(!request_mem_region(main_int_res.start, |
768 | resource_size(&main_int_res), |
769 | node->full_name)); |
770 | BUG_ON(!request_mem_region(per_cpu_int_res.start, |
771 | resource_size(&per_cpu_int_res), |
772 | node->full_name)); |
773 | |
774 | main_int_base = ioremap(offset: main_int_res.start, |
775 | size: resource_size(res: &main_int_res)); |
776 | BUG_ON(!main_int_base); |
777 | |
778 | per_cpu_int_base = ioremap(offset: per_cpu_int_res.start, |
779 | size: resource_size(res: &per_cpu_int_res)); |
780 | BUG_ON(!per_cpu_int_base); |
781 | |
782 | control = readl(addr: main_int_base + ARMADA_370_XP_INT_CONTROL); |
783 | nr_irqs = (control >> 2) & 0x3ff; |
784 | |
785 | for (i = 0; i < nr_irqs; i++) |
786 | writel(val: i, addr: main_int_base + ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS); |
787 | |
788 | armada_370_xp_mpic_domain = |
789 | irq_domain_add_linear(of_node: node, size: nr_irqs, |
790 | ops: &armada_370_xp_mpic_irq_ops, NULL); |
791 | BUG_ON(!armada_370_xp_mpic_domain); |
792 | irq_domain_update_bus_token(domain: armada_370_xp_mpic_domain, bus_token: DOMAIN_BUS_WIRED); |
793 | |
794 | /* Setup for the boot CPU */ |
795 | armada_xp_mpic_perf_init(); |
796 | armada_xp_mpic_smp_cpu_init(); |
797 | |
798 | armada_370_xp_msi_init(node, main_int_phys_base: main_int_res.start); |
799 | |
800 | parent_irq = irq_of_parse_and_map(node, index: 0); |
801 | if (parent_irq <= 0) { |
802 | irq_set_default_host(host: armada_370_xp_mpic_domain); |
803 | set_handle_irq(armada_370_xp_handle_irq); |
804 | #ifdef CONFIG_SMP |
805 | armada_xp_ipi_init(node); |
806 | cpuhp_setup_state_nocalls(state: CPUHP_AP_IRQ_ARMADA_XP_STARTING, |
807 | name: "irqchip/armada/ipi:starting" , |
808 | startup: armada_xp_mpic_starting_cpu, NULL); |
809 | #endif |
810 | } else { |
811 | #ifdef CONFIG_SMP |
812 | cpuhp_setup_state_nocalls(state: CPUHP_AP_IRQ_ARMADA_XP_STARTING, |
813 | name: "irqchip/armada/cascade:starting" , |
814 | startup: mpic_cascaded_starting_cpu, NULL); |
815 | #endif |
816 | irq_set_chained_handler(irq: parent_irq, |
817 | handle: armada_370_xp_mpic_handle_cascade_irq); |
818 | } |
819 | |
820 | register_syscore_ops(ops: &armada_370_xp_mpic_syscore_ops); |
821 | |
822 | return 0; |
823 | } |
824 | |
825 | IRQCHIP_DECLARE(armada_370_xp_mpic, "marvell,mpic" , armada_370_xp_mpic_of_init); |
826 | |