1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | |
3 | #include <linux/virtio_pci_modern.h> |
4 | #include <linux/module.h> |
5 | #include <linux/pci.h> |
6 | #include <linux/delay.h> |
7 | |
8 | /* |
9 | * vp_modern_map_capability - map a part of virtio pci capability |
10 | * @mdev: the modern virtio-pci device |
11 | * @off: offset of the capability |
12 | * @minlen: minimal length of the capability |
13 | * @align: align requirement |
14 | * @start: start from the capability |
15 | * @size: map size |
16 | * @len: the length that is actually mapped |
17 | * @pa: physical address of the capability |
18 | * |
19 | * Returns the io address of for the part of the capability |
20 | */ |
21 | static void __iomem * |
22 | vp_modern_map_capability(struct virtio_pci_modern_device *mdev, int off, |
23 | size_t minlen, u32 align, u32 start, u32 size, |
24 | size_t *len, resource_size_t *pa) |
25 | { |
26 | struct pci_dev *dev = mdev->pci_dev; |
27 | u8 bar; |
28 | u32 offset, length; |
29 | void __iomem *p; |
30 | |
31 | pci_read_config_byte(dev, where: off + offsetof(struct virtio_pci_cap, |
32 | bar), |
33 | val: &bar); |
34 | pci_read_config_dword(dev, where: off + offsetof(struct virtio_pci_cap, offset), |
35 | val: &offset); |
36 | pci_read_config_dword(dev, where: off + offsetof(struct virtio_pci_cap, length), |
37 | val: &length); |
38 | |
39 | /* Check if the BAR may have changed since we requested the region. */ |
40 | if (bar >= PCI_STD_NUM_BARS || !(mdev->modern_bars & (1 << bar))) { |
41 | dev_err(&dev->dev, |
42 | "virtio_pci: bar unexpectedly changed to %u\n" , bar); |
43 | return NULL; |
44 | } |
45 | |
46 | if (length <= start) { |
47 | dev_err(&dev->dev, |
48 | "virtio_pci: bad capability len %u (>%u expected)\n" , |
49 | length, start); |
50 | return NULL; |
51 | } |
52 | |
53 | if (length - start < minlen) { |
54 | dev_err(&dev->dev, |
55 | "virtio_pci: bad capability len %u (>=%zu expected)\n" , |
56 | length, minlen); |
57 | return NULL; |
58 | } |
59 | |
60 | length -= start; |
61 | |
62 | if (start + offset < offset) { |
63 | dev_err(&dev->dev, |
64 | "virtio_pci: map wrap-around %u+%u\n" , |
65 | start, offset); |
66 | return NULL; |
67 | } |
68 | |
69 | offset += start; |
70 | |
71 | if (offset & (align - 1)) { |
72 | dev_err(&dev->dev, |
73 | "virtio_pci: offset %u not aligned to %u\n" , |
74 | offset, align); |
75 | return NULL; |
76 | } |
77 | |
78 | if (length > size) |
79 | length = size; |
80 | |
81 | if (len) |
82 | *len = length; |
83 | |
84 | if (minlen + offset < minlen || |
85 | minlen + offset > pci_resource_len(dev, bar)) { |
86 | dev_err(&dev->dev, |
87 | "virtio_pci: map virtio %zu@%u " |
88 | "out of range on bar %i length %lu\n" , |
89 | minlen, offset, |
90 | bar, (unsigned long)pci_resource_len(dev, bar)); |
91 | return NULL; |
92 | } |
93 | |
94 | p = pci_iomap_range(dev, bar, offset, maxlen: length); |
95 | if (!p) |
96 | dev_err(&dev->dev, |
97 | "virtio_pci: unable to map virtio %u@%u on bar %i\n" , |
98 | length, offset, bar); |
99 | else if (pa) |
100 | *pa = pci_resource_start(dev, bar) + offset; |
101 | |
102 | return p; |
103 | } |
104 | |
105 | /** |
106 | * virtio_pci_find_capability - walk capabilities to find device info. |
107 | * @dev: the pci device |
108 | * @cfg_type: the VIRTIO_PCI_CAP_* value we seek |
109 | * @ioresource_types: IORESOURCE_MEM and/or IORESOURCE_IO. |
110 | * @bars: the bitmask of BARs |
111 | * |
112 | * Returns offset of the capability, or 0. |
113 | */ |
114 | static inline int virtio_pci_find_capability(struct pci_dev *dev, u8 cfg_type, |
115 | u32 ioresource_types, int *bars) |
116 | { |
117 | int pos; |
118 | |
119 | for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); |
120 | pos > 0; |
121 | pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) { |
122 | u8 type, bar; |
123 | pci_read_config_byte(dev, where: pos + offsetof(struct virtio_pci_cap, |
124 | cfg_type), |
125 | val: &type); |
126 | pci_read_config_byte(dev, where: pos + offsetof(struct virtio_pci_cap, |
127 | bar), |
128 | val: &bar); |
129 | |
130 | /* Ignore structures with reserved BAR values */ |
131 | if (bar >= PCI_STD_NUM_BARS) |
132 | continue; |
133 | |
134 | if (type == cfg_type) { |
135 | if (pci_resource_len(dev, bar) && |
136 | pci_resource_flags(dev, bar) & ioresource_types) { |
137 | *bars |= (1 << bar); |
138 | return pos; |
139 | } |
140 | } |
141 | } |
142 | return 0; |
143 | } |
144 | |
145 | /* This is part of the ABI. Don't screw with it. */ |
146 | static inline void check_offsets(void) |
147 | { |
148 | /* Note: disk space was harmed in compilation of this function. */ |
149 | BUILD_BUG_ON(VIRTIO_PCI_CAP_VNDR != |
150 | offsetof(struct virtio_pci_cap, cap_vndr)); |
151 | BUILD_BUG_ON(VIRTIO_PCI_CAP_NEXT != |
152 | offsetof(struct virtio_pci_cap, cap_next)); |
153 | BUILD_BUG_ON(VIRTIO_PCI_CAP_LEN != |
154 | offsetof(struct virtio_pci_cap, cap_len)); |
155 | BUILD_BUG_ON(VIRTIO_PCI_CAP_CFG_TYPE != |
156 | offsetof(struct virtio_pci_cap, cfg_type)); |
157 | BUILD_BUG_ON(VIRTIO_PCI_CAP_BAR != |
158 | offsetof(struct virtio_pci_cap, bar)); |
159 | BUILD_BUG_ON(VIRTIO_PCI_CAP_OFFSET != |
160 | offsetof(struct virtio_pci_cap, offset)); |
161 | BUILD_BUG_ON(VIRTIO_PCI_CAP_LENGTH != |
162 | offsetof(struct virtio_pci_cap, length)); |
163 | BUILD_BUG_ON(VIRTIO_PCI_NOTIFY_CAP_MULT != |
164 | offsetof(struct virtio_pci_notify_cap, |
165 | notify_off_multiplier)); |
166 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_DFSELECT != |
167 | offsetof(struct virtio_pci_common_cfg, |
168 | device_feature_select)); |
169 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_DF != |
170 | offsetof(struct virtio_pci_common_cfg, device_feature)); |
171 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_GFSELECT != |
172 | offsetof(struct virtio_pci_common_cfg, |
173 | guest_feature_select)); |
174 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_GF != |
175 | offsetof(struct virtio_pci_common_cfg, guest_feature)); |
176 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_MSIX != |
177 | offsetof(struct virtio_pci_common_cfg, msix_config)); |
178 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_NUMQ != |
179 | offsetof(struct virtio_pci_common_cfg, num_queues)); |
180 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_STATUS != |
181 | offsetof(struct virtio_pci_common_cfg, device_status)); |
182 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_CFGGENERATION != |
183 | offsetof(struct virtio_pci_common_cfg, config_generation)); |
184 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SELECT != |
185 | offsetof(struct virtio_pci_common_cfg, queue_select)); |
186 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_SIZE != |
187 | offsetof(struct virtio_pci_common_cfg, queue_size)); |
188 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_MSIX != |
189 | offsetof(struct virtio_pci_common_cfg, queue_msix_vector)); |
190 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_ENABLE != |
191 | offsetof(struct virtio_pci_common_cfg, queue_enable)); |
192 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_NOFF != |
193 | offsetof(struct virtio_pci_common_cfg, queue_notify_off)); |
194 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCLO != |
195 | offsetof(struct virtio_pci_common_cfg, queue_desc_lo)); |
196 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_DESCHI != |
197 | offsetof(struct virtio_pci_common_cfg, queue_desc_hi)); |
198 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILLO != |
199 | offsetof(struct virtio_pci_common_cfg, queue_avail_lo)); |
200 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_AVAILHI != |
201 | offsetof(struct virtio_pci_common_cfg, queue_avail_hi)); |
202 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDLO != |
203 | offsetof(struct virtio_pci_common_cfg, queue_used_lo)); |
204 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_USEDHI != |
205 | offsetof(struct virtio_pci_common_cfg, queue_used_hi)); |
206 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_NDATA != |
207 | offsetof(struct virtio_pci_modern_common_cfg, queue_notify_data)); |
208 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_Q_RESET != |
209 | offsetof(struct virtio_pci_modern_common_cfg, queue_reset)); |
210 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_ADM_Q_IDX != |
211 | offsetof(struct virtio_pci_modern_common_cfg, admin_queue_index)); |
212 | BUILD_BUG_ON(VIRTIO_PCI_COMMON_ADM_Q_NUM != |
213 | offsetof(struct virtio_pci_modern_common_cfg, admin_queue_num)); |
214 | } |
215 | |
216 | /* |
217 | * vp_modern_probe: probe the modern virtio pci device, note that the |
218 | * caller is required to enable PCI device before calling this function. |
219 | * @mdev: the modern virtio-pci device |
220 | * |
221 | * Return 0 on succeed otherwise fail |
222 | */ |
223 | int vp_modern_probe(struct virtio_pci_modern_device *mdev) |
224 | { |
225 | struct pci_dev *pci_dev = mdev->pci_dev; |
226 | int err, common, isr, notify, device; |
227 | u32 notify_length; |
228 | u32 notify_offset; |
229 | int devid; |
230 | |
231 | check_offsets(); |
232 | |
233 | if (mdev->device_id_check) { |
234 | devid = mdev->device_id_check(pci_dev); |
235 | if (devid < 0) |
236 | return devid; |
237 | mdev->id.device = devid; |
238 | } else { |
239 | /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */ |
240 | if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f) |
241 | return -ENODEV; |
242 | |
243 | if (pci_dev->device < 0x1040) { |
244 | /* Transitional devices: use the PCI subsystem device id as |
245 | * virtio device id, same as legacy driver always did. |
246 | */ |
247 | mdev->id.device = pci_dev->subsystem_device; |
248 | } else { |
249 | /* Modern devices: simply use PCI device id, but start from 0x1040. */ |
250 | mdev->id.device = pci_dev->device - 0x1040; |
251 | } |
252 | } |
253 | mdev->id.vendor = pci_dev->subsystem_vendor; |
254 | |
255 | /* check for a common config: if not, use legacy mode (bar 0). */ |
256 | common = virtio_pci_find_capability(dev: pci_dev, VIRTIO_PCI_CAP_COMMON_CFG, |
257 | IORESOURCE_IO | IORESOURCE_MEM, |
258 | bars: &mdev->modern_bars); |
259 | if (!common) { |
260 | dev_info(&pci_dev->dev, |
261 | "virtio_pci: leaving for legacy driver\n" ); |
262 | return -ENODEV; |
263 | } |
264 | |
265 | /* If common is there, these should be too... */ |
266 | isr = virtio_pci_find_capability(dev: pci_dev, VIRTIO_PCI_CAP_ISR_CFG, |
267 | IORESOURCE_IO | IORESOURCE_MEM, |
268 | bars: &mdev->modern_bars); |
269 | notify = virtio_pci_find_capability(dev: pci_dev, VIRTIO_PCI_CAP_NOTIFY_CFG, |
270 | IORESOURCE_IO | IORESOURCE_MEM, |
271 | bars: &mdev->modern_bars); |
272 | if (!isr || !notify) { |
273 | dev_err(&pci_dev->dev, |
274 | "virtio_pci: missing capabilities %i/%i/%i\n" , |
275 | common, isr, notify); |
276 | return -EINVAL; |
277 | } |
278 | |
279 | err = dma_set_mask_and_coherent(dev: &pci_dev->dev, |
280 | mask: mdev->dma_mask ? : DMA_BIT_MASK(64)); |
281 | if (err) |
282 | err = dma_set_mask_and_coherent(dev: &pci_dev->dev, |
283 | DMA_BIT_MASK(32)); |
284 | if (err) |
285 | dev_warn(&pci_dev->dev, "Failed to enable 64-bit or 32-bit DMA. Trying to continue, but this might not work.\n" ); |
286 | |
287 | /* Device capability is only mandatory for devices that have |
288 | * device-specific configuration. |
289 | */ |
290 | device = virtio_pci_find_capability(dev: pci_dev, VIRTIO_PCI_CAP_DEVICE_CFG, |
291 | IORESOURCE_IO | IORESOURCE_MEM, |
292 | bars: &mdev->modern_bars); |
293 | |
294 | err = pci_request_selected_regions(pci_dev, mdev->modern_bars, |
295 | "virtio-pci-modern" ); |
296 | if (err) |
297 | return err; |
298 | |
299 | err = -EINVAL; |
300 | mdev->common = vp_modern_map_capability(mdev, off: common, |
301 | minlen: sizeof(struct virtio_pci_common_cfg), align: 4, start: 0, |
302 | offsetofend(struct virtio_pci_modern_common_cfg, |
303 | admin_queue_num), |
304 | len: &mdev->common_len, NULL); |
305 | if (!mdev->common) |
306 | goto err_map_common; |
307 | mdev->isr = vp_modern_map_capability(mdev, off: isr, minlen: sizeof(u8), align: 1, |
308 | start: 0, size: 1, |
309 | NULL, NULL); |
310 | if (!mdev->isr) |
311 | goto err_map_isr; |
312 | |
313 | /* Read notify_off_multiplier from config space. */ |
314 | pci_read_config_dword(dev: pci_dev, |
315 | where: notify + offsetof(struct virtio_pci_notify_cap, |
316 | notify_off_multiplier), |
317 | val: &mdev->notify_offset_multiplier); |
318 | /* Read notify length and offset from config space. */ |
319 | pci_read_config_dword(dev: pci_dev, |
320 | where: notify + offsetof(struct virtio_pci_notify_cap, |
321 | cap.length), |
322 | val: ¬ify_length); |
323 | |
324 | pci_read_config_dword(dev: pci_dev, |
325 | where: notify + offsetof(struct virtio_pci_notify_cap, |
326 | cap.offset), |
327 | val: ¬ify_offset); |
328 | |
329 | /* We don't know how many VQs we'll map, ahead of the time. |
330 | * If notify length is small, map it all now. |
331 | * Otherwise, map each VQ individually later. |
332 | */ |
333 | if ((u64)notify_length + (notify_offset % PAGE_SIZE) <= PAGE_SIZE) { |
334 | mdev->notify_base = vp_modern_map_capability(mdev, off: notify, |
335 | minlen: 2, align: 2, |
336 | start: 0, size: notify_length, |
337 | len: &mdev->notify_len, |
338 | pa: &mdev->notify_pa); |
339 | if (!mdev->notify_base) |
340 | goto err_map_notify; |
341 | } else { |
342 | mdev->notify_map_cap = notify; |
343 | } |
344 | |
345 | /* Again, we don't know how much we should map, but PAGE_SIZE |
346 | * is more than enough for all existing devices. |
347 | */ |
348 | if (device) { |
349 | mdev->device = vp_modern_map_capability(mdev, off: device, minlen: 0, align: 4, |
350 | start: 0, PAGE_SIZE, |
351 | len: &mdev->device_len, |
352 | NULL); |
353 | if (!mdev->device) |
354 | goto err_map_device; |
355 | } |
356 | |
357 | return 0; |
358 | |
359 | err_map_device: |
360 | if (mdev->notify_base) |
361 | pci_iounmap(dev: pci_dev, mdev->notify_base); |
362 | err_map_notify: |
363 | pci_iounmap(dev: pci_dev, mdev->isr); |
364 | err_map_isr: |
365 | pci_iounmap(dev: pci_dev, mdev->common); |
366 | err_map_common: |
367 | pci_release_selected_regions(pci_dev, mdev->modern_bars); |
368 | return err; |
369 | } |
370 | EXPORT_SYMBOL_GPL(vp_modern_probe); |
371 | |
372 | /* |
373 | * vp_modern_remove: remove and cleanup the modern virtio pci device |
374 | * @mdev: the modern virtio-pci device |
375 | */ |
376 | void vp_modern_remove(struct virtio_pci_modern_device *mdev) |
377 | { |
378 | struct pci_dev *pci_dev = mdev->pci_dev; |
379 | |
380 | if (mdev->device) |
381 | pci_iounmap(dev: pci_dev, mdev->device); |
382 | if (mdev->notify_base) |
383 | pci_iounmap(dev: pci_dev, mdev->notify_base); |
384 | pci_iounmap(dev: pci_dev, mdev->isr); |
385 | pci_iounmap(dev: pci_dev, mdev->common); |
386 | pci_release_selected_regions(pci_dev, mdev->modern_bars); |
387 | } |
388 | EXPORT_SYMBOL_GPL(vp_modern_remove); |
389 | |
390 | /* |
391 | * vp_modern_get_features - get features from device |
392 | * @mdev: the modern virtio-pci device |
393 | * |
394 | * Returns the features read from the device |
395 | */ |
396 | u64 vp_modern_get_features(struct virtio_pci_modern_device *mdev) |
397 | { |
398 | struct virtio_pci_common_cfg __iomem *cfg = mdev->common; |
399 | |
400 | u64 features; |
401 | |
402 | vp_iowrite32(value: 0, addr: &cfg->device_feature_select); |
403 | features = vp_ioread32(addr: &cfg->device_feature); |
404 | vp_iowrite32(value: 1, addr: &cfg->device_feature_select); |
405 | features |= ((u64)vp_ioread32(addr: &cfg->device_feature) << 32); |
406 | |
407 | return features; |
408 | } |
409 | EXPORT_SYMBOL_GPL(vp_modern_get_features); |
410 | |
411 | /* |
412 | * vp_modern_get_driver_features - get driver features from device |
413 | * @mdev: the modern virtio-pci device |
414 | * |
415 | * Returns the driver features read from the device |
416 | */ |
417 | u64 vp_modern_get_driver_features(struct virtio_pci_modern_device *mdev) |
418 | { |
419 | struct virtio_pci_common_cfg __iomem *cfg = mdev->common; |
420 | |
421 | u64 features; |
422 | |
423 | vp_iowrite32(value: 0, addr: &cfg->guest_feature_select); |
424 | features = vp_ioread32(addr: &cfg->guest_feature); |
425 | vp_iowrite32(value: 1, addr: &cfg->guest_feature_select); |
426 | features |= ((u64)vp_ioread32(addr: &cfg->guest_feature) << 32); |
427 | |
428 | return features; |
429 | } |
430 | EXPORT_SYMBOL_GPL(vp_modern_get_driver_features); |
431 | |
432 | /* |
433 | * vp_modern_set_features - set features to device |
434 | * @mdev: the modern virtio-pci device |
435 | * @features: the features set to device |
436 | */ |
437 | void vp_modern_set_features(struct virtio_pci_modern_device *mdev, |
438 | u64 features) |
439 | { |
440 | struct virtio_pci_common_cfg __iomem *cfg = mdev->common; |
441 | |
442 | vp_iowrite32(value: 0, addr: &cfg->guest_feature_select); |
443 | vp_iowrite32(value: (u32)features, addr: &cfg->guest_feature); |
444 | vp_iowrite32(value: 1, addr: &cfg->guest_feature_select); |
445 | vp_iowrite32(value: features >> 32, addr: &cfg->guest_feature); |
446 | } |
447 | EXPORT_SYMBOL_GPL(vp_modern_set_features); |
448 | |
449 | /* |
450 | * vp_modern_generation - get the device genreation |
451 | * @mdev: the modern virtio-pci device |
452 | * |
453 | * Returns the genreation read from device |
454 | */ |
455 | u32 vp_modern_generation(struct virtio_pci_modern_device *mdev) |
456 | { |
457 | struct virtio_pci_common_cfg __iomem *cfg = mdev->common; |
458 | |
459 | return vp_ioread8(addr: &cfg->config_generation); |
460 | } |
461 | EXPORT_SYMBOL_GPL(vp_modern_generation); |
462 | |
463 | /* |
464 | * vp_modern_get_status - get the device status |
465 | * @mdev: the modern virtio-pci device |
466 | * |
467 | * Returns the status read from device |
468 | */ |
469 | u8 vp_modern_get_status(struct virtio_pci_modern_device *mdev) |
470 | { |
471 | struct virtio_pci_common_cfg __iomem *cfg = mdev->common; |
472 | |
473 | return vp_ioread8(addr: &cfg->device_status); |
474 | } |
475 | EXPORT_SYMBOL_GPL(vp_modern_get_status); |
476 | |
477 | /* |
478 | * vp_modern_set_status - set status to device |
479 | * @mdev: the modern virtio-pci device |
480 | * @status: the status set to device |
481 | */ |
482 | void vp_modern_set_status(struct virtio_pci_modern_device *mdev, |
483 | u8 status) |
484 | { |
485 | struct virtio_pci_common_cfg __iomem *cfg = mdev->common; |
486 | |
487 | /* |
488 | * Per memory-barriers.txt, wmb() is not needed to guarantee |
489 | * that the cache coherent memory writes have completed |
490 | * before writing to the MMIO region. |
491 | */ |
492 | vp_iowrite8(value: status, addr: &cfg->device_status); |
493 | } |
494 | EXPORT_SYMBOL_GPL(vp_modern_set_status); |
495 | |
496 | /* |
497 | * vp_modern_get_queue_reset - get the queue reset status |
498 | * @mdev: the modern virtio-pci device |
499 | * @index: queue index |
500 | */ |
501 | int vp_modern_get_queue_reset(struct virtio_pci_modern_device *mdev, u16 index) |
502 | { |
503 | struct virtio_pci_modern_common_cfg __iomem *cfg; |
504 | |
505 | cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common; |
506 | |
507 | vp_iowrite16(value: index, addr: &cfg->cfg.queue_select); |
508 | return vp_ioread16(addr: &cfg->queue_reset); |
509 | } |
510 | EXPORT_SYMBOL_GPL(vp_modern_get_queue_reset); |
511 | |
512 | /* |
513 | * vp_modern_set_queue_reset - reset the queue |
514 | * @mdev: the modern virtio-pci device |
515 | * @index: queue index |
516 | */ |
517 | void vp_modern_set_queue_reset(struct virtio_pci_modern_device *mdev, u16 index) |
518 | { |
519 | struct virtio_pci_modern_common_cfg __iomem *cfg; |
520 | |
521 | cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common; |
522 | |
523 | vp_iowrite16(value: index, addr: &cfg->cfg.queue_select); |
524 | vp_iowrite16(value: 1, addr: &cfg->queue_reset); |
525 | |
526 | while (vp_ioread16(addr: &cfg->queue_reset)) |
527 | msleep(msecs: 1); |
528 | |
529 | while (vp_ioread16(addr: &cfg->cfg.queue_enable)) |
530 | msleep(msecs: 1); |
531 | } |
532 | EXPORT_SYMBOL_GPL(vp_modern_set_queue_reset); |
533 | |
534 | /* |
535 | * vp_modern_queue_vector - set the MSIX vector for a specific virtqueue |
536 | * @mdev: the modern virtio-pci device |
537 | * @index: queue index |
538 | * @vector: the config vector |
539 | * |
540 | * Returns the config vector read from the device |
541 | */ |
542 | u16 vp_modern_queue_vector(struct virtio_pci_modern_device *mdev, |
543 | u16 index, u16 vector) |
544 | { |
545 | struct virtio_pci_common_cfg __iomem *cfg = mdev->common; |
546 | |
547 | vp_iowrite16(value: index, addr: &cfg->queue_select); |
548 | vp_iowrite16(value: vector, addr: &cfg->queue_msix_vector); |
549 | /* Flush the write out to device */ |
550 | return vp_ioread16(addr: &cfg->queue_msix_vector); |
551 | } |
552 | EXPORT_SYMBOL_GPL(vp_modern_queue_vector); |
553 | |
554 | /* |
555 | * vp_modern_config_vector - set the vector for config interrupt |
556 | * @mdev: the modern virtio-pci device |
557 | * @vector: the config vector |
558 | * |
559 | * Returns the config vector read from the device |
560 | */ |
561 | u16 vp_modern_config_vector(struct virtio_pci_modern_device *mdev, |
562 | u16 vector) |
563 | { |
564 | struct virtio_pci_common_cfg __iomem *cfg = mdev->common; |
565 | |
566 | /* Setup the vector used for configuration events */ |
567 | vp_iowrite16(value: vector, addr: &cfg->msix_config); |
568 | /* Verify we had enough resources to assign the vector */ |
569 | /* Will also flush the write out to device */ |
570 | return vp_ioread16(addr: &cfg->msix_config); |
571 | } |
572 | EXPORT_SYMBOL_GPL(vp_modern_config_vector); |
573 | |
574 | /* |
575 | * vp_modern_queue_address - set the virtqueue address |
576 | * @mdev: the modern virtio-pci device |
577 | * @index: the queue index |
578 | * @desc_addr: address of the descriptor area |
579 | * @driver_addr: address of the driver area |
580 | * @device_addr: address of the device area |
581 | */ |
582 | void vp_modern_queue_address(struct virtio_pci_modern_device *mdev, |
583 | u16 index, u64 desc_addr, u64 driver_addr, |
584 | u64 device_addr) |
585 | { |
586 | struct virtio_pci_common_cfg __iomem *cfg = mdev->common; |
587 | |
588 | vp_iowrite16(value: index, addr: &cfg->queue_select); |
589 | |
590 | vp_iowrite64_twopart(val: desc_addr, lo: &cfg->queue_desc_lo, |
591 | hi: &cfg->queue_desc_hi); |
592 | vp_iowrite64_twopart(val: driver_addr, lo: &cfg->queue_avail_lo, |
593 | hi: &cfg->queue_avail_hi); |
594 | vp_iowrite64_twopart(val: device_addr, lo: &cfg->queue_used_lo, |
595 | hi: &cfg->queue_used_hi); |
596 | } |
597 | EXPORT_SYMBOL_GPL(vp_modern_queue_address); |
598 | |
599 | /* |
600 | * vp_modern_set_queue_enable - enable a virtqueue |
601 | * @mdev: the modern virtio-pci device |
602 | * @index: the queue index |
603 | * @enable: whether the virtqueue is enable or not |
604 | */ |
605 | void vp_modern_set_queue_enable(struct virtio_pci_modern_device *mdev, |
606 | u16 index, bool enable) |
607 | { |
608 | vp_iowrite16(value: index, addr: &mdev->common->queue_select); |
609 | vp_iowrite16(value: enable, addr: &mdev->common->queue_enable); |
610 | } |
611 | EXPORT_SYMBOL_GPL(vp_modern_set_queue_enable); |
612 | |
613 | /* |
614 | * vp_modern_get_queue_enable - enable a virtqueue |
615 | * @mdev: the modern virtio-pci device |
616 | * @index: the queue index |
617 | * |
618 | * Returns whether a virtqueue is enabled or not |
619 | */ |
620 | bool vp_modern_get_queue_enable(struct virtio_pci_modern_device *mdev, |
621 | u16 index) |
622 | { |
623 | vp_iowrite16(value: index, addr: &mdev->common->queue_select); |
624 | |
625 | return vp_ioread16(addr: &mdev->common->queue_enable); |
626 | } |
627 | EXPORT_SYMBOL_GPL(vp_modern_get_queue_enable); |
628 | |
629 | /* |
630 | * vp_modern_set_queue_size - set size for a virtqueue |
631 | * @mdev: the modern virtio-pci device |
632 | * @index: the queue index |
633 | * @size: the size of the virtqueue |
634 | */ |
635 | void vp_modern_set_queue_size(struct virtio_pci_modern_device *mdev, |
636 | u16 index, u16 size) |
637 | { |
638 | vp_iowrite16(value: index, addr: &mdev->common->queue_select); |
639 | vp_iowrite16(value: size, addr: &mdev->common->queue_size); |
640 | |
641 | } |
642 | EXPORT_SYMBOL_GPL(vp_modern_set_queue_size); |
643 | |
644 | /* |
645 | * vp_modern_get_queue_size - get size for a virtqueue |
646 | * @mdev: the modern virtio-pci device |
647 | * @index: the queue index |
648 | * |
649 | * Returns the size of the virtqueue |
650 | */ |
651 | u16 vp_modern_get_queue_size(struct virtio_pci_modern_device *mdev, |
652 | u16 index) |
653 | { |
654 | vp_iowrite16(value: index, addr: &mdev->common->queue_select); |
655 | |
656 | return vp_ioread16(addr: &mdev->common->queue_size); |
657 | |
658 | } |
659 | EXPORT_SYMBOL_GPL(vp_modern_get_queue_size); |
660 | |
661 | /* |
662 | * vp_modern_get_num_queues - get the number of virtqueues |
663 | * @mdev: the modern virtio-pci device |
664 | * |
665 | * Returns the number of virtqueues |
666 | */ |
667 | u16 vp_modern_get_num_queues(struct virtio_pci_modern_device *mdev) |
668 | { |
669 | return vp_ioread16(addr: &mdev->common->num_queues); |
670 | } |
671 | EXPORT_SYMBOL_GPL(vp_modern_get_num_queues); |
672 | |
673 | /* |
674 | * vp_modern_get_queue_notify_off - get notification offset for a virtqueue |
675 | * @mdev: the modern virtio-pci device |
676 | * @index: the queue index |
677 | * |
678 | * Returns the notification offset for a virtqueue |
679 | */ |
680 | static u16 vp_modern_get_queue_notify_off(struct virtio_pci_modern_device *mdev, |
681 | u16 index) |
682 | { |
683 | vp_iowrite16(value: index, addr: &mdev->common->queue_select); |
684 | |
685 | return vp_ioread16(addr: &mdev->common->queue_notify_off); |
686 | } |
687 | |
688 | /* |
689 | * vp_modern_map_vq_notify - map notification area for a |
690 | * specific virtqueue |
691 | * @mdev: the modern virtio-pci device |
692 | * @index: the queue index |
693 | * @pa: the pointer to the physical address of the nofity area |
694 | * |
695 | * Returns the address of the notification area |
696 | */ |
697 | void __iomem *vp_modern_map_vq_notify(struct virtio_pci_modern_device *mdev, |
698 | u16 index, resource_size_t *pa) |
699 | { |
700 | u16 off = vp_modern_get_queue_notify_off(mdev, index); |
701 | |
702 | if (mdev->notify_base) { |
703 | /* offset should not wrap */ |
704 | if ((u64)off * mdev->notify_offset_multiplier + 2 |
705 | > mdev->notify_len) { |
706 | dev_warn(&mdev->pci_dev->dev, |
707 | "bad notification offset %u (x %u) " |
708 | "for queue %u > %zd" , |
709 | off, mdev->notify_offset_multiplier, |
710 | index, mdev->notify_len); |
711 | return NULL; |
712 | } |
713 | if (pa) |
714 | *pa = mdev->notify_pa + |
715 | off * mdev->notify_offset_multiplier; |
716 | return mdev->notify_base + off * mdev->notify_offset_multiplier; |
717 | } else { |
718 | return vp_modern_map_capability(mdev, |
719 | off: mdev->notify_map_cap, minlen: 2, align: 2, |
720 | start: off * mdev->notify_offset_multiplier, size: 2, |
721 | NULL, pa); |
722 | } |
723 | } |
724 | EXPORT_SYMBOL_GPL(vp_modern_map_vq_notify); |
725 | |
726 | u16 vp_modern_avq_num(struct virtio_pci_modern_device *mdev) |
727 | { |
728 | struct virtio_pci_modern_common_cfg __iomem *cfg; |
729 | |
730 | cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common; |
731 | return vp_ioread16(addr: &cfg->admin_queue_num); |
732 | } |
733 | EXPORT_SYMBOL_GPL(vp_modern_avq_num); |
734 | |
735 | u16 vp_modern_avq_index(struct virtio_pci_modern_device *mdev) |
736 | { |
737 | struct virtio_pci_modern_common_cfg __iomem *cfg; |
738 | |
739 | cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common; |
740 | return vp_ioread16(addr: &cfg->admin_queue_index); |
741 | } |
742 | EXPORT_SYMBOL_GPL(vp_modern_avq_index); |
743 | |
744 | MODULE_VERSION("0.1" ); |
745 | MODULE_DESCRIPTION("Modern Virtio PCI Device" ); |
746 | MODULE_AUTHOR("Jason Wang <jasowang@redhat.com>" ); |
747 | MODULE_LICENSE("GPL" ); |
748 | |