1/*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include <linux/printk.h>
24#include <linux/device.h>
25#include <linux/slab.h>
26#include <linux/pci.h>
27#include <linux/amd-iommu.h>
28#include "kfd_priv.h"
29#include "kfd_dbgmgr.h"
30#include "kfd_topology.h"
31#include "kfd_iommu.h"
32
33static const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
34 AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
35 AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
36
37/** kfd_iommu_check_device - Check whether IOMMU is available for device
38 */
39int kfd_iommu_check_device(struct kfd_dev *kfd)
40{
41 struct amd_iommu_device_info iommu_info;
42 int err;
43
44 if (!kfd->device_info->needs_iommu_device)
45 return -ENODEV;
46
47 iommu_info.flags = 0;
48 err = amd_iommu_device_info(kfd->pdev, &iommu_info);
49 if (err)
50 return err;
51
52 if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags)
53 return -ENODEV;
54
55 return 0;
56}
57
58/** kfd_iommu_device_init - Initialize IOMMU for device
59 */
60int kfd_iommu_device_init(struct kfd_dev *kfd)
61{
62 struct amd_iommu_device_info iommu_info;
63 unsigned int pasid_limit;
64 int err;
65 struct kfd_topology_device *top_dev;
66
67 top_dev = kfd_topology_device_by_id(kfd->id);
68
69 /*
70 * Overwrite ATS capability according to needs_iommu_device to fix
71 * potential missing corresponding bit in CRAT of BIOS.
72 */
73 if (!kfd->device_info->needs_iommu_device) {
74 top_dev->node_props.capability &= ~HSA_CAP_ATS_PRESENT;
75 return 0;
76 }
77
78 top_dev->node_props.capability |= HSA_CAP_ATS_PRESENT;
79
80 iommu_info.flags = 0;
81 err = amd_iommu_device_info(kfd->pdev, &iommu_info);
82 if (err < 0) {
83 dev_err(kfd_device,
84 "error getting iommu info. is the iommu enabled?\n");
85 return -ENODEV;
86 }
87
88 if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags) {
89 dev_err(kfd_device,
90 "error required iommu flags ats %i, pri %i, pasid %i\n",
91 (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0,
92 (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0,
93 (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP)
94 != 0);
95 return -ENODEV;
96 }
97
98 pasid_limit = min_t(unsigned int,
99 (unsigned int)(1 << kfd->device_info->max_pasid_bits),
100 iommu_info.max_pasids);
101
102 if (!kfd_set_pasid_limit(pasid_limit)) {
103 dev_err(kfd_device, "error setting pasid limit\n");
104 return -EBUSY;
105 }
106
107 return 0;
108}
109
110/** kfd_iommu_bind_process_to_device - Have the IOMMU bind a process
111 *
112 * Binds the given process to the given device using its PASID. This
113 * enables IOMMUv2 address translation for the process on the device.
114 *
115 * This function assumes that the process mutex is held.
116 */
117int kfd_iommu_bind_process_to_device(struct kfd_process_device *pdd)
118{
119 struct kfd_dev *dev = pdd->dev;
120 struct kfd_process *p = pdd->process;
121 int err;
122
123 if (!dev->device_info->needs_iommu_device || pdd->bound == PDD_BOUND)
124 return 0;
125
126 if (unlikely(pdd->bound == PDD_BOUND_SUSPENDED)) {
127 pr_err("Binding PDD_BOUND_SUSPENDED pdd is unexpected!\n");
128 return -EINVAL;
129 }
130
131 err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
132 if (!err)
133 pdd->bound = PDD_BOUND;
134
135 return err;
136}
137
138/** kfd_iommu_unbind_process - Unbind process from all devices
139 *
140 * This removes all IOMMU device bindings of the process. To be used
141 * before process termination.
142 */
143void kfd_iommu_unbind_process(struct kfd_process *p)
144{
145 struct kfd_process_device *pdd;
146
147 list_for_each_entry(pdd, &p->per_device_data, per_device_list)
148 if (pdd->bound == PDD_BOUND)
149 amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
150}
151
152/* Callback for process shutdown invoked by the IOMMU driver */
153static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, int pasid)
154{
155 struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
156 struct kfd_process *p;
157 struct kfd_process_device *pdd;
158
159 if (!dev)
160 return;
161
162 /*
163 * Look for the process that matches the pasid. If there is no such
164 * process, we either released it in amdkfd's own notifier, or there
165 * is a bug. Unfortunately, there is no way to tell...
166 */
167 p = kfd_lookup_process_by_pasid(pasid);
168 if (!p)
169 return;
170
171 pr_debug("Unbinding process %d from IOMMU\n", pasid);
172
173 mutex_lock(kfd_get_dbgmgr_mutex());
174
175 if (dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
176 if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
177 kfd_dbgmgr_destroy(dev->dbgmgr);
178 dev->dbgmgr = NULL;
179 }
180 }
181
182 mutex_unlock(kfd_get_dbgmgr_mutex());
183
184 mutex_lock(&p->mutex);
185
186 pdd = kfd_get_process_device_data(dev, p);
187 if (pdd)
188 /* For GPU relying on IOMMU, we need to dequeue here
189 * when PASID is still bound.
190 */
191 kfd_process_dequeue_from_device(pdd);
192
193 mutex_unlock(&p->mutex);
194
195 kfd_unref_process(p);
196}
197
198/* This function called by IOMMU driver on PPR failure */
199static int iommu_invalid_ppr_cb(struct pci_dev *pdev, int pasid,
200 unsigned long address, u16 flags)
201{
202 struct kfd_dev *dev;
203
204 dev_warn_ratelimited(kfd_device,
205 "Invalid PPR device %x:%x.%x pasid %d address 0x%lX flags 0x%X",
206 PCI_BUS_NUM(pdev->devfn),
207 PCI_SLOT(pdev->devfn),
208 PCI_FUNC(pdev->devfn),
209 pasid,
210 address,
211 flags);
212
213 dev = kfd_device_by_pci_dev(pdev);
214 if (!WARN_ON(!dev))
215 kfd_signal_iommu_event(dev, pasid, address,
216 flags & PPR_FAULT_WRITE, flags & PPR_FAULT_EXEC);
217
218 return AMD_IOMMU_INV_PRI_RSP_INVALID;
219}
220
221/*
222 * Bind processes do the device that have been temporarily unbound
223 * (PDD_BOUND_SUSPENDED) in kfd_unbind_processes_from_device.
224 */
225static int kfd_bind_processes_to_device(struct kfd_dev *kfd)
226{
227 struct kfd_process_device *pdd;
228 struct kfd_process *p;
229 unsigned int temp;
230 int err = 0;
231
232 int idx = srcu_read_lock(&kfd_processes_srcu);
233
234 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
235 mutex_lock(&p->mutex);
236 pdd = kfd_get_process_device_data(kfd, p);
237
238 if (WARN_ON(!pdd) || pdd->bound != PDD_BOUND_SUSPENDED) {
239 mutex_unlock(&p->mutex);
240 continue;
241 }
242
243 err = amd_iommu_bind_pasid(kfd->pdev, p->pasid,
244 p->lead_thread);
245 if (err < 0) {
246 pr_err("Unexpected pasid %d binding failure\n",
247 p->pasid);
248 mutex_unlock(&p->mutex);
249 break;
250 }
251
252 pdd->bound = PDD_BOUND;
253 mutex_unlock(&p->mutex);
254 }
255
256 srcu_read_unlock(&kfd_processes_srcu, idx);
257
258 return err;
259}
260
261/*
262 * Mark currently bound processes as PDD_BOUND_SUSPENDED. These
263 * processes will be restored to PDD_BOUND state in
264 * kfd_bind_processes_to_device.
265 */
266static void kfd_unbind_processes_from_device(struct kfd_dev *kfd)
267{
268 struct kfd_process_device *pdd;
269 struct kfd_process *p;
270 unsigned int temp;
271
272 int idx = srcu_read_lock(&kfd_processes_srcu);
273
274 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
275 mutex_lock(&p->mutex);
276 pdd = kfd_get_process_device_data(kfd, p);
277
278 if (WARN_ON(!pdd)) {
279 mutex_unlock(&p->mutex);
280 continue;
281 }
282
283 if (pdd->bound == PDD_BOUND)
284 pdd->bound = PDD_BOUND_SUSPENDED;
285 mutex_unlock(&p->mutex);
286 }
287
288 srcu_read_unlock(&kfd_processes_srcu, idx);
289}
290
291/** kfd_iommu_suspend - Prepare IOMMU for suspend
292 *
293 * This unbinds processes from the device and disables the IOMMU for
294 * the device.
295 */
296void kfd_iommu_suspend(struct kfd_dev *kfd)
297{
298 if (!kfd->device_info->needs_iommu_device)
299 return;
300
301 kfd_unbind_processes_from_device(kfd);
302
303 amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
304 amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
305 amd_iommu_free_device(kfd->pdev);
306}
307
308/** kfd_iommu_resume - Restore IOMMU after resume
309 *
310 * This reinitializes the IOMMU for the device and re-binds previously
311 * suspended processes to the device.
312 */
313int kfd_iommu_resume(struct kfd_dev *kfd)
314{
315 unsigned int pasid_limit;
316 int err;
317
318 if (!kfd->device_info->needs_iommu_device)
319 return 0;
320
321 pasid_limit = kfd_get_pasid_limit();
322
323 err = amd_iommu_init_device(kfd->pdev, pasid_limit);
324 if (err)
325 return -ENXIO;
326
327 amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
328 iommu_pasid_shutdown_callback);
329 amd_iommu_set_invalid_ppr_cb(kfd->pdev,
330 iommu_invalid_ppr_cb);
331
332 err = kfd_bind_processes_to_device(kfd);
333 if (err) {
334 amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
335 amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
336 amd_iommu_free_device(kfd->pdev);
337 return err;
338 }
339
340 return 0;
341}
342
343extern bool amd_iommu_pc_supported(void);
344extern u8 amd_iommu_pc_get_max_banks(u16 devid);
345extern u8 amd_iommu_pc_get_max_counters(u16 devid);
346
347/** kfd_iommu_add_perf_counters - Add IOMMU performance counters to topology
348 */
349int kfd_iommu_add_perf_counters(struct kfd_topology_device *kdev)
350{
351 struct kfd_perf_properties *props;
352
353 if (!(kdev->node_props.capability & HSA_CAP_ATS_PRESENT))
354 return 0;
355
356 if (!amd_iommu_pc_supported())
357 return 0;
358
359 props = kfd_alloc_struct(props);
360 if (!props)
361 return -ENOMEM;
362 strcpy(props->block_name, "iommu");
363 props->max_concurrent = amd_iommu_pc_get_max_banks(0) *
364 amd_iommu_pc_get_max_counters(0); /* assume one iommu */
365 list_add_tail(&props->list, &kdev->perf_props);
366
367 return 0;
368}
369