Warning: That file was not part of the compilation database. It may have many parsing errors.

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * VFIO PCI NVIDIA Whitherspoon GPU support a.k.a. NVLink2.
4 *
5 * Copyright (C) 2018 IBM Corp. All rights reserved.
6 * Author: Alexey Kardashevskiy <aik@ozlabs.ru>
7 *
8 * Register an on-GPU RAM region for cacheable access.
9 *
10 * Derived from original vfio_pci_igd.c:
11 * Copyright (C) 2016 Red Hat, Inc. All rights reserved.
12 * Author: Alex Williamson <alex.williamson@redhat.com>
13 */
14
15#include <linux/io.h>
16#include <linux/pci.h>
17#include <linux/uaccess.h>
18#include <linux/vfio.h>
19#include <linux/sched/mm.h>
20#include <linux/mmu_context.h>
21#include <asm/kvm_ppc.h>
22#include "vfio_pci_private.h"
23
24#define CREATE_TRACE_POINTS
25#include "trace.h"
26
27EXPORT_TRACEPOINT_SYMBOL_GPL(vfio_pci_nvgpu_mmap_fault);
28EXPORT_TRACEPOINT_SYMBOL_GPL(vfio_pci_nvgpu_mmap);
29EXPORT_TRACEPOINT_SYMBOL_GPL(vfio_pci_npu2_mmap);
30
31struct vfio_pci_nvgpu_data {
32 unsigned long gpu_hpa; /* GPU RAM physical address */
33 unsigned long gpu_tgt; /* TGT address of corresponding GPU RAM */
34 unsigned long useraddr; /* GPU RAM userspace address */
35 unsigned long size; /* Size of the GPU RAM window (usually 128GB) */
36 struct mm_struct *mm;
37 struct mm_iommu_table_group_mem_t *mem; /* Pre-registered RAM descr. */
38 struct pci_dev *gpdev;
39 struct notifier_block group_notifier;
40};
41
42static size_t vfio_pci_nvgpu_rw(struct vfio_pci_device *vdev,
43 char __user *buf, size_t count, loff_t *ppos, bool iswrite)
44{
45 unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - VFIO_PCI_NUM_REGIONS;
46 struct vfio_pci_nvgpu_data *data = vdev->region[i].data;
47 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
48 loff_t posaligned = pos & PAGE_MASK, posoff = pos & ~PAGE_MASK;
49 size_t sizealigned;
50 void __iomem *ptr;
51
52 if (pos >= vdev->region[i].size)
53 return -EINVAL;
54
55 count = min(count, (size_t)(vdev->region[i].size - pos));
56
57 /*
58 * We map only a bit of GPU RAM for a short time instead of mapping it
59 * for the guest lifetime as:
60 *
61 * 1) we do not know GPU RAM size, only aperture which is 4-8 times
62 * bigger than actual RAM size (16/32GB RAM vs. 128GB aperture);
63 * 2) mapping GPU RAM allows CPU to prefetch and if this happens
64 * before NVLink bridge is reset (which fences GPU RAM),
65 * hardware management interrupts (HMI) might happen, this
66 * will freeze NVLink bridge.
67 *
68 * This is not fast path anyway.
69 */
70 sizealigned = _ALIGN_UP(posoff + count, PAGE_SIZE);
71 ptr = ioremap_cache(data->gpu_hpa + posaligned, sizealigned);
72 if (!ptr)
73 return -EFAULT;
74
75 if (iswrite) {
76 if (copy_from_user(ptr + posoff, buf, count))
77 count = -EFAULT;
78 else
79 *ppos += count;
80 } else {
81 if (copy_to_user(buf, ptr + posoff, count))
82 count = -EFAULT;
83 else
84 *ppos += count;
85 }
86
87 iounmap(ptr);
88
89 return count;
90}
91
92static void vfio_pci_nvgpu_release(struct vfio_pci_device *vdev,
93 struct vfio_pci_region *region)
94{
95 struct vfio_pci_nvgpu_data *data = region->data;
96 long ret;
97
98 /* If there were any mappings at all... */
99 if (data->mm) {
100 ret = mm_iommu_put(data->mm, data->mem);
101 WARN_ON(ret);
102
103 mmdrop(data->mm);
104 }
105
106 vfio_unregister_notifier(&data->gpdev->dev, VFIO_GROUP_NOTIFY,
107 &data->group_notifier);
108
109 pnv_npu2_unmap_lpar_dev(data->gpdev);
110
111 kfree(data);
112}
113
114static vm_fault_t vfio_pci_nvgpu_mmap_fault(struct vm_fault *vmf)
115{
116 vm_fault_t ret;
117 struct vm_area_struct *vma = vmf->vma;
118 struct vfio_pci_region *region = vma->vm_private_data;
119 struct vfio_pci_nvgpu_data *data = region->data;
120 unsigned long vmf_off = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
121 unsigned long nv2pg = data->gpu_hpa >> PAGE_SHIFT;
122 unsigned long vm_pgoff = vma->vm_pgoff &
123 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
124 unsigned long pfn = nv2pg + vm_pgoff + vmf_off;
125
126 ret = vmf_insert_pfn(vma, vmf->address, pfn);
127 trace_vfio_pci_nvgpu_mmap_fault(data->gpdev, pfn << PAGE_SHIFT,
128 vmf->address, ret);
129
130 return ret;
131}
132
133static const struct vm_operations_struct vfio_pci_nvgpu_mmap_vmops = {
134 .fault = vfio_pci_nvgpu_mmap_fault,
135};
136
137static int vfio_pci_nvgpu_mmap(struct vfio_pci_device *vdev,
138 struct vfio_pci_region *region, struct vm_area_struct *vma)
139{
140 int ret;
141 struct vfio_pci_nvgpu_data *data = region->data;
142
143 if (data->useraddr)
144 return -EPERM;
145
146 if (vma->vm_end - vma->vm_start > data->size)
147 return -EINVAL;
148
149 vma->vm_private_data = region;
150 vma->vm_flags |= VM_PFNMAP;
151 vma->vm_ops = &vfio_pci_nvgpu_mmap_vmops;
152
153 /*
154 * Calling mm_iommu_newdev() here once as the region is not
155 * registered yet and therefore right initialization will happen now.
156 * Other places will use mm_iommu_find() which returns
157 * registered @mem and does not go gup().
158 */
159 data->useraddr = vma->vm_start;
160 data->mm = current->mm;
161
162 atomic_inc(&data->mm->mm_count);
163 ret = (int) mm_iommu_newdev(data->mm, data->useraddr,
164 (vma->vm_end - vma->vm_start) >> PAGE_SHIFT,
165 data->gpu_hpa, &data->mem);
166
167 trace_vfio_pci_nvgpu_mmap(vdev->pdev, data->gpu_hpa, data->useraddr,
168 vma->vm_end - vma->vm_start, ret);
169
170 return ret;
171}
172
173static int vfio_pci_nvgpu_add_capability(struct vfio_pci_device *vdev,
174 struct vfio_pci_region *region, struct vfio_info_cap *caps)
175{
176 struct vfio_pci_nvgpu_data *data = region->data;
177 struct vfio_region_info_cap_nvlink2_ssatgt cap = {
178 .header.id = VFIO_REGION_INFO_CAP_NVLINK2_SSATGT,
179 .header.version = 1,
180 .tgt = data->gpu_tgt
181 };
182
183 return vfio_info_add_capability(caps, &cap.header, sizeof(cap));
184}
185
186static const struct vfio_pci_regops vfio_pci_nvgpu_regops = {
187 .rw = vfio_pci_nvgpu_rw,
188 .release = vfio_pci_nvgpu_release,
189 .mmap = vfio_pci_nvgpu_mmap,
190 .add_capability = vfio_pci_nvgpu_add_capability,
191};
192
193static int vfio_pci_nvgpu_group_notifier(struct notifier_block *nb,
194 unsigned long action, void *opaque)
195{
196 struct kvm *kvm = opaque;
197 struct vfio_pci_nvgpu_data *data = container_of(nb,
198 struct vfio_pci_nvgpu_data,
199 group_notifier);
200
201 if (action == VFIO_GROUP_NOTIFY_SET_KVM && kvm &&
202 pnv_npu2_map_lpar_dev(data->gpdev,
203 kvm->arch.lpid, MSR_DR | MSR_PR))
204 return NOTIFY_BAD;
205
206 return NOTIFY_OK;
207}
208
209int vfio_pci_nvdia_v100_nvlink2_init(struct vfio_pci_device *vdev)
210{
211 int ret;
212 u64 reg[2];
213 u64 tgt = 0;
214 struct device_node *npu_node, *mem_node;
215 struct pci_dev *npu_dev;
216 struct vfio_pci_nvgpu_data *data;
217 uint32_t mem_phandle = 0;
218 unsigned long events = VFIO_GROUP_NOTIFY_SET_KVM;
219
220 /*
221 * PCI config space does not tell us about NVLink presense but
222 * platform does, use this.
223 */
224 npu_dev = pnv_pci_get_npu_dev(vdev->pdev, 0);
225 if (!npu_dev)
226 return -ENODEV;
227
228 npu_node = pci_device_to_OF_node(npu_dev);
229 if (!npu_node)
230 return -EINVAL;
231
232 if (of_property_read_u32(npu_node, "memory-region", &mem_phandle))
233 return -EINVAL;
234
235 mem_node = of_find_node_by_phandle(mem_phandle);
236 if (!mem_node)
237 return -EINVAL;
238
239 if (of_property_read_variable_u64_array(mem_node, "reg", reg,
240 ARRAY_SIZE(reg), ARRAY_SIZE(reg)) !=
241 ARRAY_SIZE(reg))
242 return -EINVAL;
243
244 if (of_property_read_u64(npu_node, "ibm,device-tgt-addr", &tgt)) {
245 dev_warn(&vdev->pdev->dev, "No ibm,device-tgt-addr found\n");
246 return -EFAULT;
247 }
248
249 data = kzalloc(sizeof(*data), GFP_KERNEL);
250 if (!data)
251 return -ENOMEM;
252
253 data->gpu_hpa = reg[0];
254 data->gpu_tgt = tgt;
255 data->size = reg[1];
256
257 dev_dbg(&vdev->pdev->dev, "%lx..%lx\n", data->gpu_hpa,
258 data->gpu_hpa + data->size - 1);
259
260 data->gpdev = vdev->pdev;
261 data->group_notifier.notifier_call = vfio_pci_nvgpu_group_notifier;
262
263 ret = vfio_register_notifier(&data->gpdev->dev, VFIO_GROUP_NOTIFY,
264 &events, &data->group_notifier);
265 if (ret)
266 goto free_exit;
267
268 /*
269 * We have just set KVM, we do not need the listener anymore.
270 * Also, keeping it registered means that if more than one GPU is
271 * assigned, we will get several similar notifiers notifying about
272 * the same device again which does not help with anything.
273 */
274 vfio_unregister_notifier(&data->gpdev->dev, VFIO_GROUP_NOTIFY,
275 &data->group_notifier);
276
277 ret = vfio_pci_register_dev_region(vdev,
278 PCI_VENDOR_ID_NVIDIA | VFIO_REGION_TYPE_PCI_VENDOR_TYPE,
279 VFIO_REGION_SUBTYPE_NVIDIA_NVLINK2_RAM,
280 &vfio_pci_nvgpu_regops,
281 data->size,
282 VFIO_REGION_INFO_FLAG_READ |
283 VFIO_REGION_INFO_FLAG_WRITE |
284 VFIO_REGION_INFO_FLAG_MMAP,
285 data);
286 if (ret)
287 goto free_exit;
288
289 return 0;
290free_exit:
291 kfree(data);
292
293 return ret;
294}
295
296/*
297 * IBM NPU2 bridge
298 */
299struct vfio_pci_npu2_data {
300 void *base; /* ATSD register virtual address, for emulated access */
301 unsigned long mmio_atsd; /* ATSD physical address */
302 unsigned long gpu_tgt; /* TGT address of corresponding GPU RAM */
303 unsigned int link_speed; /* The link speed from DT's ibm,nvlink-speed */
304};
305
306static size_t vfio_pci_npu2_rw(struct vfio_pci_device *vdev,
307 char __user *buf, size_t count, loff_t *ppos, bool iswrite)
308{
309 unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - VFIO_PCI_NUM_REGIONS;
310 struct vfio_pci_npu2_data *data = vdev->region[i].data;
311 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
312
313 if (pos >= vdev->region[i].size)
314 return -EINVAL;
315
316 count = min(count, (size_t)(vdev->region[i].size - pos));
317
318 if (iswrite) {
319 if (copy_from_user(data->base + pos, buf, count))
320 return -EFAULT;
321 } else {
322 if (copy_to_user(buf, data->base + pos, count))
323 return -EFAULT;
324 }
325 *ppos += count;
326
327 return count;
328}
329
330static int vfio_pci_npu2_mmap(struct vfio_pci_device *vdev,
331 struct vfio_pci_region *region, struct vm_area_struct *vma)
332{
333 int ret;
334 struct vfio_pci_npu2_data *data = region->data;
335 unsigned long req_len = vma->vm_end - vma->vm_start;
336
337 if (req_len != PAGE_SIZE)
338 return -EINVAL;
339
340 vma->vm_flags |= VM_PFNMAP;
341 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
342
343 ret = remap_pfn_range(vma, vma->vm_start, data->mmio_atsd >> PAGE_SHIFT,
344 req_len, vma->vm_page_prot);
345 trace_vfio_pci_npu2_mmap(vdev->pdev, data->mmio_atsd, vma->vm_start,
346 vma->vm_end - vma->vm_start, ret);
347
348 return ret;
349}
350
351static void vfio_pci_npu2_release(struct vfio_pci_device *vdev,
352 struct vfio_pci_region *region)
353{
354 struct vfio_pci_npu2_data *data = region->data;
355
356 memunmap(data->base);
357 kfree(data);
358}
359
360static int vfio_pci_npu2_add_capability(struct vfio_pci_device *vdev,
361 struct vfio_pci_region *region, struct vfio_info_cap *caps)
362{
363 struct vfio_pci_npu2_data *data = region->data;
364 struct vfio_region_info_cap_nvlink2_ssatgt captgt = {
365 .header.id = VFIO_REGION_INFO_CAP_NVLINK2_SSATGT,
366 .header.version = 1,
367 .tgt = data->gpu_tgt
368 };
369 struct vfio_region_info_cap_nvlink2_lnkspd capspd = {
370 .header.id = VFIO_REGION_INFO_CAP_NVLINK2_LNKSPD,
371 .header.version = 1,
372 .link_speed = data->link_speed
373 };
374 int ret;
375
376 ret = vfio_info_add_capability(caps, &captgt.header, sizeof(captgt));
377 if (ret)
378 return ret;
379
380 return vfio_info_add_capability(caps, &capspd.header, sizeof(capspd));
381}
382
383static const struct vfio_pci_regops vfio_pci_npu2_regops = {
384 .rw = vfio_pci_npu2_rw,
385 .mmap = vfio_pci_npu2_mmap,
386 .release = vfio_pci_npu2_release,
387 .add_capability = vfio_pci_npu2_add_capability,
388};
389
390int vfio_pci_ibm_npu2_init(struct vfio_pci_device *vdev)
391{
392 int ret;
393 struct vfio_pci_npu2_data *data;
394 struct device_node *nvlink_dn;
395 u32 nvlink_index = 0;
396 struct pci_dev *npdev = vdev->pdev;
397 struct device_node *npu_node = pci_device_to_OF_node(npdev);
398 struct pci_controller *hose = pci_bus_to_host(npdev->bus);
399 u64 mmio_atsd = 0;
400 u64 tgt = 0;
401 u32 link_speed = 0xff;
402
403 /*
404 * PCI config space does not tell us about NVLink presense but
405 * platform does, use this.
406 */
407 if (!pnv_pci_get_gpu_dev(vdev->pdev))
408 return -ENODEV;
409
410 /*
411 * NPU2 normally has 8 ATSD registers (for concurrency) and 6 links
412 * so we can allocate one register per link, using nvlink index as
413 * a key.
414 * There is always at least one ATSD register so as long as at least
415 * NVLink bridge #0 is passed to the guest, ATSD will be available.
416 */
417 nvlink_dn = of_parse_phandle(npdev->dev.of_node, "ibm,nvlink", 0);
418 if (WARN_ON(of_property_read_u32(nvlink_dn, "ibm,npu-link-index",
419 &nvlink_index)))
420 return -ENODEV;
421
422 if (of_property_read_u64_index(hose->dn, "ibm,mmio-atsd", nvlink_index,
423 &mmio_atsd)) {
424 dev_warn(&vdev->pdev->dev, "No available ATSD found\n");
425 mmio_atsd = 0;
426 }
427
428 if (of_property_read_u64(npu_node, "ibm,device-tgt-addr", &tgt)) {
429 dev_warn(&vdev->pdev->dev, "No ibm,device-tgt-addr found\n");
430 return -EFAULT;
431 }
432
433 if (of_property_read_u32(npu_node, "ibm,nvlink-speed", &link_speed)) {
434 dev_warn(&vdev->pdev->dev, "No ibm,nvlink-speed found\n");
435 return -EFAULT;
436 }
437
438 data = kzalloc(sizeof(*data), GFP_KERNEL);
439 if (!data)
440 return -ENOMEM;
441
442 data->mmio_atsd = mmio_atsd;
443 data->gpu_tgt = tgt;
444 data->link_speed = link_speed;
445 if (data->mmio_atsd) {
446 data->base = memremap(data->mmio_atsd, SZ_64K, MEMREMAP_WT);
447 if (!data->base) {
448 ret = -ENOMEM;
449 goto free_exit;
450 }
451 }
452
453 /*
454 * We want to expose the capability even if this specific NVLink
455 * did not get its own ATSD register because capabilities
456 * belong to VFIO regions and normally there will be ATSD register
457 * assigned to the NVLink bridge.
458 */
459 ret = vfio_pci_register_dev_region(vdev,
460 PCI_VENDOR_ID_IBM |
461 VFIO_REGION_TYPE_PCI_VENDOR_TYPE,
462 VFIO_REGION_SUBTYPE_IBM_NVLINK2_ATSD,
463 &vfio_pci_npu2_regops,
464 data->mmio_atsd ? PAGE_SIZE : 0,
465 VFIO_REGION_INFO_FLAG_READ |
466 VFIO_REGION_INFO_FLAG_WRITE |
467 VFIO_REGION_INFO_FLAG_MMAP,
468 data);
469 if (ret)
470 goto free_exit;
471
472 return 0;
473
474free_exit:
475 kfree(data);
476
477 return ret;
478}
479

Warning: That file was not part of the compilation database. It may have many parsing errors.