1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2016,2017 ARM Limited, All Rights Reserved.
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7#include <linux/interrupt.h>
8#include <linux/irq.h>
9#include <linux/irqdomain.h>
10#include <linux/msi.h>
11#include <linux/sched.h>
12
13#include <linux/irqchip/arm-gic-v4.h>
14
15/*
16 * WARNING: The blurb below assumes that you understand the
17 * intricacies of GICv3, GICv4, and how a guest's view of a GICv3 gets
18 * translated into GICv4 commands. So it effectively targets at most
19 * two individuals. You know who you are.
20 *
21 * The core GICv4 code is designed to *avoid* exposing too much of the
22 * core GIC code (that would in turn leak into the hypervisor code),
23 * and instead provide a hypervisor agnostic interface to the HW (of
24 * course, the astute reader will quickly realize that hypervisor
25 * agnostic actually means KVM-specific - what were you thinking?).
26 *
27 * In order to achieve a modicum of isolation, we try to hide most of
28 * the GICv4 "stuff" behind normal irqchip operations:
29 *
30 * - Any guest-visible VLPI is backed by a Linux interrupt (and a
31 * physical LPI which gets unmapped when the guest maps the
32 * VLPI). This allows the same DevID/EventID pair to be either
33 * mapped to the LPI (host) or the VLPI (guest). Note that this is
34 * exclusive, and you cannot have both.
35 *
36 * - Enabling/disabling a VLPI is done by issuing mask/unmask calls.
37 *
38 * - Guest INT/CLEAR commands are implemented through
39 * irq_set_irqchip_state().
40 *
41 * - The *bizarre* stuff (mapping/unmapping an interrupt to a VLPI, or
42 * issuing an INV after changing a priority) gets shoved into the
43 * irq_set_vcpu_affinity() method. While this is quite horrible
44 * (let's face it, this is the irqchip version of an ioctl), it
45 * confines the crap to a single location. And map/unmap really is
46 * about setting the affinity of a VLPI to a vcpu, so only INV is
47 * majorly out of place. So there.
48 *
49 * A number of commands are simply not provided by this interface, as
50 * they do not make direct sense. For example, MAPD is purely local to
51 * the virtual ITS (because it references a virtual device, and the
52 * physical ITS is still very much in charge of the physical
53 * device). Same goes for things like MAPC (the physical ITS deals
54 * with the actual vPE affinity, and not the braindead concept of
55 * collection). SYNC is not provided either, as each and every command
56 * is followed by a VSYNC. This could be relaxed in the future, should
57 * this be seen as a bottleneck (yes, this means *never*).
58 *
59 * But handling VLPIs is only one side of the job of the GICv4
60 * code. The other (darker) side is to take care of the doorbell
61 * interrupts which are delivered when a VLPI targeting a non-running
62 * vcpu is being made pending.
63 *
64 * The choice made here is that each vcpu (VPE in old northern GICv4
65 * dialect) gets a single doorbell LPI, no matter how many interrupts
66 * are targeting it. This has a nice property, which is that the
67 * interrupt becomes a handle for the VPE, and that the hypervisor
68 * code can manipulate it through the normal interrupt API:
69 *
70 * - VMs (or rather the VM abstraction that matters to the GIC)
71 * contain an irq domain where each interrupt maps to a VPE. In
72 * turn, this domain sits on top of the normal LPI allocator, and a
73 * specially crafted irq_chip implementation.
74 *
75 * - mask/unmask do what is expected on the doorbell interrupt.
76 *
77 * - irq_set_affinity is used to move a VPE from one redistributor to
78 * another.
79 *
80 * - irq_set_vcpu_affinity once again gets hijacked for the purpose of
81 * creating a new sub-API, namely scheduling/descheduling a VPE
82 * (which involves programming GICR_V{PROP,PEND}BASER) and
83 * performing INVALL operations.
84 */
85
86static struct irq_domain *gic_domain;
87static const struct irq_domain_ops *vpe_domain_ops;
88static const struct irq_domain_ops *sgi_domain_ops;
89
90#ifdef CONFIG_ARM64
91#include <asm/cpufeature.h>
92
93bool gic_cpuif_has_vsgi(void)
94{
95 unsigned long fld, reg = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
96
97 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_EL1_GIC_SHIFT);
98
99 return fld >= 0x3;
100}
101#else
102bool gic_cpuif_has_vsgi(void)
103{
104 return false;
105}
106#endif
107
108static bool has_v4_1(void)
109{
110 return !!sgi_domain_ops;
111}
112
113static bool has_v4_1_sgi(void)
114{
115 return has_v4_1() && gic_cpuif_has_vsgi();
116}
117
118static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx)
119{
120 char *name;
121 int sgi_base;
122
123 if (!has_v4_1_sgi())
124 return 0;
125
126 name = kasprintf(GFP_KERNEL, fmt: "GICv4-sgi-%d", task_pid_nr(current));
127 if (!name)
128 goto err;
129
130 vpe->fwnode = irq_domain_alloc_named_id_fwnode(name, id: idx);
131 if (!vpe->fwnode)
132 goto err;
133
134 kfree(objp: name);
135 name = NULL;
136
137 vpe->sgi_domain = irq_domain_create_linear(fwnode: vpe->fwnode, size: 16,
138 ops: sgi_domain_ops, host_data: vpe);
139 if (!vpe->sgi_domain)
140 goto err;
141
142 sgi_base = irq_domain_alloc_irqs(domain: vpe->sgi_domain, nr_irqs: 16, NUMA_NO_NODE, arg: vpe);
143 if (sgi_base <= 0)
144 goto err;
145
146 return 0;
147
148err:
149 if (vpe->sgi_domain)
150 irq_domain_remove(host: vpe->sgi_domain);
151 if (vpe->fwnode)
152 irq_domain_free_fwnode(fwnode: vpe->fwnode);
153 kfree(objp: name);
154 return -ENOMEM;
155}
156
157int its_alloc_vcpu_irqs(struct its_vm *vm)
158{
159 int vpe_base_irq, i;
160
161 vm->fwnode = irq_domain_alloc_named_id_fwnode(name: "GICv4-vpe",
162 id: task_pid_nr(current));
163 if (!vm->fwnode)
164 goto err;
165
166 vm->domain = irq_domain_create_hierarchy(parent: gic_domain, flags: 0, size: vm->nr_vpes,
167 fwnode: vm->fwnode, ops: vpe_domain_ops,
168 host_data: vm);
169 if (!vm->domain)
170 goto err;
171
172 for (i = 0; i < vm->nr_vpes; i++) {
173 vm->vpes[i]->its_vm = vm;
174 vm->vpes[i]->idai = true;
175 }
176
177 vpe_base_irq = irq_domain_alloc_irqs(domain: vm->domain, nr_irqs: vm->nr_vpes,
178 NUMA_NO_NODE, arg: vm);
179 if (vpe_base_irq <= 0)
180 goto err;
181
182 for (i = 0; i < vm->nr_vpes; i++) {
183 int ret;
184 vm->vpes[i]->irq = vpe_base_irq + i;
185 ret = its_alloc_vcpu_sgis(vpe: vm->vpes[i], idx: i);
186 if (ret)
187 goto err;
188 }
189
190 return 0;
191
192err:
193 if (vm->domain)
194 irq_domain_remove(host: vm->domain);
195 if (vm->fwnode)
196 irq_domain_free_fwnode(fwnode: vm->fwnode);
197
198 return -ENOMEM;
199}
200
201static void its_free_sgi_irqs(struct its_vm *vm)
202{
203 int i;
204
205 if (!has_v4_1_sgi())
206 return;
207
208 for (i = 0; i < vm->nr_vpes; i++) {
209 unsigned int irq = irq_find_mapping(domain: vm->vpes[i]->sgi_domain, hwirq: 0);
210
211 if (WARN_ON(!irq))
212 continue;
213
214 irq_domain_free_irqs(virq: irq, nr_irqs: 16);
215 irq_domain_remove(host: vm->vpes[i]->sgi_domain);
216 irq_domain_free_fwnode(fwnode: vm->vpes[i]->fwnode);
217 }
218}
219
220void its_free_vcpu_irqs(struct its_vm *vm)
221{
222 its_free_sgi_irqs(vm);
223 irq_domain_free_irqs(virq: vm->vpes[0]->irq, nr_irqs: vm->nr_vpes);
224 irq_domain_remove(host: vm->domain);
225 irq_domain_free_fwnode(fwnode: vm->fwnode);
226}
227
228static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info)
229{
230 return irq_set_vcpu_affinity(irq: vpe->irq, vcpu_info: info);
231}
232
233int its_make_vpe_non_resident(struct its_vpe *vpe, bool db)
234{
235 struct irq_desc *desc = irq_to_desc(irq: vpe->irq);
236 struct its_cmd_info info = { };
237 int ret;
238
239 WARN_ON(preemptible());
240
241 info.cmd_type = DESCHEDULE_VPE;
242 if (has_v4_1()) {
243 /* GICv4.1 can directly deal with doorbells */
244 info.req_db = db;
245 } else {
246 /* Undo the nested disable_irq() calls... */
247 while (db && irqd_irq_disabled(d: &desc->irq_data))
248 enable_irq(irq: vpe->irq);
249 }
250
251 ret = its_send_vpe_cmd(vpe, info: &info);
252 if (!ret)
253 vpe->resident = false;
254
255 vpe->ready = false;
256
257 return ret;
258}
259
260int its_make_vpe_resident(struct its_vpe *vpe, bool g0en, bool g1en)
261{
262 struct its_cmd_info info = { };
263 int ret;
264
265 WARN_ON(preemptible());
266
267 info.cmd_type = SCHEDULE_VPE;
268 if (has_v4_1()) {
269 info.g0en = g0en;
270 info.g1en = g1en;
271 } else {
272 /* Disabled the doorbell, as we're about to enter the guest */
273 disable_irq_nosync(irq: vpe->irq);
274 }
275
276 ret = its_send_vpe_cmd(vpe, info: &info);
277 if (!ret)
278 vpe->resident = true;
279
280 return ret;
281}
282
283int its_commit_vpe(struct its_vpe *vpe)
284{
285 struct its_cmd_info info = {
286 .cmd_type = COMMIT_VPE,
287 };
288 int ret;
289
290 WARN_ON(preemptible());
291
292 ret = its_send_vpe_cmd(vpe, info: &info);
293 if (!ret)
294 vpe->ready = true;
295
296 return ret;
297}
298
299
300int its_invall_vpe(struct its_vpe *vpe)
301{
302 struct its_cmd_info info = {
303 .cmd_type = INVALL_VPE,
304 };
305
306 return its_send_vpe_cmd(vpe, info: &info);
307}
308
309int its_map_vlpi(int irq, struct its_vlpi_map *map)
310{
311 struct its_cmd_info info = {
312 .cmd_type = MAP_VLPI,
313 {
314 .map = map,
315 },
316 };
317 int ret;
318
319 /*
320 * The host will never see that interrupt firing again, so it
321 * is vital that we don't do any lazy masking.
322 */
323 irq_set_status_flags(irq, set: IRQ_DISABLE_UNLAZY);
324
325 ret = irq_set_vcpu_affinity(irq, vcpu_info: &info);
326 if (ret)
327 irq_clear_status_flags(irq, clr: IRQ_DISABLE_UNLAZY);
328
329 return ret;
330}
331
332int its_get_vlpi(int irq, struct its_vlpi_map *map)
333{
334 struct its_cmd_info info = {
335 .cmd_type = GET_VLPI,
336 {
337 .map = map,
338 },
339 };
340
341 return irq_set_vcpu_affinity(irq, vcpu_info: &info);
342}
343
344int its_unmap_vlpi(int irq)
345{
346 irq_clear_status_flags(irq, clr: IRQ_DISABLE_UNLAZY);
347 return irq_set_vcpu_affinity(irq, NULL);
348}
349
350int its_prop_update_vlpi(int irq, u8 config, bool inv)
351{
352 struct its_cmd_info info = {
353 .cmd_type = inv ? PROP_UPDATE_AND_INV_VLPI : PROP_UPDATE_VLPI,
354 {
355 .config = config,
356 },
357 };
358
359 return irq_set_vcpu_affinity(irq, vcpu_info: &info);
360}
361
362int its_prop_update_vsgi(int irq, u8 priority, bool group)
363{
364 struct its_cmd_info info = {
365 .cmd_type = PROP_UPDATE_VSGI,
366 {
367 .priority = priority,
368 .group = group,
369 },
370 };
371
372 return irq_set_vcpu_affinity(irq, vcpu_info: &info);
373}
374
375int its_init_v4(struct irq_domain *domain,
376 const struct irq_domain_ops *vpe_ops,
377 const struct irq_domain_ops *sgi_ops)
378{
379 if (domain) {
380 pr_info("ITS: Enabling GICv4 support\n");
381 gic_domain = domain;
382 vpe_domain_ops = vpe_ops;
383 sgi_domain_ops = sgi_ops;
384 return 0;
385 }
386
387 pr_err("ITS: No GICv4 VPE domain allocated\n");
388 return -ENODEV;
389}
390

source code of linux/drivers/irqchip/irq-gic-v4.c