1 | /* |
2 | * This file is provided under a dual BSD/GPLv2 license. When using or |
3 | * redistributing this file, you may do so under either license. |
4 | * |
5 | * GPL LICENSE SUMMARY |
6 | * |
7 | * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. |
8 | * Copyright (C) 2016 T-Platforms. All Rights Reserved. |
9 | * |
10 | * This program is free software; you can redistribute it and/or modify |
11 | * it under the terms of version 2 of the GNU General Public License as |
12 | * published by the Free Software Foundation. |
13 | * |
14 | * BSD LICENSE |
15 | * |
16 | * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. |
17 | * Copyright (C) 2016 T-Platforms. All Rights Reserved. |
18 | * |
19 | * Redistribution and use in source and binary forms, with or without |
20 | * modification, are permitted provided that the following conditions |
21 | * are met: |
22 | * |
23 | * * Redistributions of source code must retain the above copyright |
24 | * notice, this list of conditions and the following disclaimer. |
25 | * * Redistributions in binary form must reproduce the above copy |
26 | * notice, this list of conditions and the following disclaimer in |
27 | * the documentation and/or other materials provided with the |
28 | * distribution. |
29 | * * Neither the name of AMD Corporation nor the names of its |
30 | * contributors may be used to endorse or promote products derived |
31 | * from this software without specific prior written permission. |
32 | * |
33 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
34 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
35 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
36 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
37 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
38 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
39 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
40 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
41 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
42 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
43 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
44 | * |
45 | * AMD PCIe NTB Linux driver |
46 | * |
47 | * Contact Information: |
48 | * Xiangliang Yu <Xiangliang.Yu@amd.com> |
49 | */ |
50 | |
51 | #include <linux/debugfs.h> |
52 | #include <linux/delay.h> |
53 | #include <linux/init.h> |
54 | #include <linux/interrupt.h> |
55 | #include <linux/module.h> |
56 | #include <linux/acpi.h> |
57 | #include <linux/pci.h> |
58 | #include <linux/random.h> |
59 | #include <linux/slab.h> |
60 | #include <linux/ntb.h> |
61 | |
62 | #include "ntb_hw_amd.h" |
63 | |
64 | #define NTB_NAME "ntb_hw_amd" |
65 | #define NTB_DESC "AMD(R) PCI-E Non-Transparent Bridge Driver" |
66 | #define NTB_VER "1.0" |
67 | |
68 | MODULE_DESCRIPTION(NTB_DESC); |
69 | MODULE_VERSION(NTB_VER); |
70 | MODULE_LICENSE("Dual BSD/GPL" ); |
71 | MODULE_AUTHOR("AMD Inc." ); |
72 | |
73 | static const struct file_operations amd_ntb_debugfs_info; |
74 | static struct dentry *debugfs_dir; |
75 | |
76 | static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx) |
77 | { |
78 | if (idx < 0 || idx > ndev->mw_count) |
79 | return -EINVAL; |
80 | |
81 | return ndev->dev_data->mw_idx << idx; |
82 | } |
83 | |
84 | static int amd_ntb_mw_count(struct ntb_dev *ntb, int pidx) |
85 | { |
86 | if (pidx != NTB_DEF_PEER_IDX) |
87 | return -EINVAL; |
88 | |
89 | return ntb_ndev(ntb)->mw_count; |
90 | } |
91 | |
92 | static int amd_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx, |
93 | resource_size_t *addr_align, |
94 | resource_size_t *size_align, |
95 | resource_size_t *size_max) |
96 | { |
97 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); |
98 | int bar; |
99 | |
100 | if (pidx != NTB_DEF_PEER_IDX) |
101 | return -EINVAL; |
102 | |
103 | bar = ndev_mw_to_bar(ndev, idx); |
104 | if (bar < 0) |
105 | return bar; |
106 | |
107 | if (addr_align) |
108 | *addr_align = SZ_4K; |
109 | |
110 | if (size_align) |
111 | *size_align = 1; |
112 | |
113 | if (size_max) |
114 | *size_max = pci_resource_len(ndev->ntb.pdev, bar); |
115 | |
116 | return 0; |
117 | } |
118 | |
119 | static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, |
120 | dma_addr_t addr, resource_size_t size) |
121 | { |
122 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); |
123 | unsigned long xlat_reg, limit_reg = 0; |
124 | resource_size_t mw_size; |
125 | void __iomem *mmio, *peer_mmio; |
126 | u64 base_addr, limit, reg_val; |
127 | int bar; |
128 | |
129 | if (pidx != NTB_DEF_PEER_IDX) |
130 | return -EINVAL; |
131 | |
132 | bar = ndev_mw_to_bar(ndev, idx); |
133 | if (bar < 0) |
134 | return bar; |
135 | |
136 | mw_size = pci_resource_len(ntb->pdev, bar); |
137 | |
138 | /* make sure the range fits in the usable mw size */ |
139 | if (size > mw_size) |
140 | return -EINVAL; |
141 | |
142 | mmio = ndev->self_mmio; |
143 | peer_mmio = ndev->peer_mmio; |
144 | |
145 | base_addr = pci_resource_start(ntb->pdev, bar); |
146 | |
147 | if (bar != 1) { |
148 | xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 2); |
149 | limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 2); |
150 | |
151 | /* Set the limit if supported */ |
152 | limit = size; |
153 | |
154 | /* set and verify setting the translation address */ |
155 | write64(val: addr, addr: peer_mmio + xlat_reg); |
156 | reg_val = read64(addr: peer_mmio + xlat_reg); |
157 | if (reg_val != addr) { |
158 | write64(val: 0, addr: peer_mmio + xlat_reg); |
159 | return -EIO; |
160 | } |
161 | |
162 | /* set and verify setting the limit */ |
163 | write64(val: limit, addr: peer_mmio + limit_reg); |
164 | reg_val = read64(addr: peer_mmio + limit_reg); |
165 | if (reg_val != limit) { |
166 | write64(val: base_addr, addr: mmio + limit_reg); |
167 | write64(val: 0, addr: peer_mmio + xlat_reg); |
168 | return -EIO; |
169 | } |
170 | } else { |
171 | xlat_reg = AMD_BAR1XLAT_OFFSET; |
172 | limit_reg = AMD_BAR1LMT_OFFSET; |
173 | |
174 | /* Set the limit if supported */ |
175 | limit = size; |
176 | |
177 | /* set and verify setting the translation address */ |
178 | write64(val: addr, addr: peer_mmio + xlat_reg); |
179 | reg_val = read64(addr: peer_mmio + xlat_reg); |
180 | if (reg_val != addr) { |
181 | write64(val: 0, addr: peer_mmio + xlat_reg); |
182 | return -EIO; |
183 | } |
184 | |
185 | /* set and verify setting the limit */ |
186 | writel(val: limit, addr: peer_mmio + limit_reg); |
187 | reg_val = readl(addr: peer_mmio + limit_reg); |
188 | if (reg_val != limit) { |
189 | writel(val: base_addr, addr: mmio + limit_reg); |
190 | writel(val: 0, addr: peer_mmio + xlat_reg); |
191 | return -EIO; |
192 | } |
193 | } |
194 | |
195 | return 0; |
196 | } |
197 | |
198 | static int amd_ntb_get_link_status(struct amd_ntb_dev *ndev) |
199 | { |
200 | struct pci_dev *pdev = NULL; |
201 | struct pci_dev *pci_swds = NULL; |
202 | struct pci_dev *pci_swus = NULL; |
203 | u32 stat; |
204 | int rc; |
205 | |
206 | if (ndev->ntb.topo == NTB_TOPO_SEC) { |
207 | /* Locate the pointer to Downstream Switch for this device */ |
208 | pci_swds = pci_upstream_bridge(dev: ndev->ntb.pdev); |
209 | if (pci_swds) { |
210 | /* |
211 | * Locate the pointer to Upstream Switch for |
212 | * the Downstream Switch. |
213 | */ |
214 | pci_swus = pci_upstream_bridge(dev: pci_swds); |
215 | if (pci_swus) { |
216 | rc = pcie_capability_read_dword(dev: pci_swus, |
217 | PCI_EXP_LNKCTL, |
218 | val: &stat); |
219 | if (rc) |
220 | return 0; |
221 | } else { |
222 | return 0; |
223 | } |
224 | } else { |
225 | return 0; |
226 | } |
227 | } else if (ndev->ntb.topo == NTB_TOPO_PRI) { |
228 | /* |
229 | * For NTB primary, we simply read the Link Status and control |
230 | * register of the NTB device itself. |
231 | */ |
232 | pdev = ndev->ntb.pdev; |
233 | rc = pcie_capability_read_dword(dev: pdev, PCI_EXP_LNKCTL, val: &stat); |
234 | if (rc) |
235 | return 0; |
236 | } else { |
237 | /* Catch all for everything else */ |
238 | return 0; |
239 | } |
240 | |
241 | ndev->lnk_sta = stat; |
242 | |
243 | return 1; |
244 | } |
245 | |
246 | static int amd_link_is_up(struct amd_ntb_dev *ndev) |
247 | { |
248 | int ret; |
249 | |
250 | /* |
251 | * We consider the link to be up under two conditions: |
252 | * |
253 | * - When a link-up event is received. This is indicated by |
254 | * AMD_LINK_UP_EVENT set in peer_sta. |
255 | * - When driver on both sides of the link have been loaded. |
256 | * This is indicated by bit 1 being set in the peer |
257 | * SIDEINFO register. |
258 | * |
259 | * This function should return 1 when the latter of the above |
260 | * two conditions is true. |
261 | * |
262 | * Now consider the sequence of events - Link-Up event occurs, |
263 | * then the peer side driver loads. In this case, we would have |
264 | * received LINK_UP event and bit 1 of peer SIDEINFO is also |
265 | * set. What happens now if the link goes down? Bit 1 of |
266 | * peer SIDEINFO remains set, but LINK_DOWN bit is set in |
267 | * peer_sta. So we should return 0 from this function. Not only |
268 | * that, we clear bit 1 of peer SIDEINFO to 0, since the peer |
269 | * side driver did not even get a chance to clear it before |
270 | * the link went down. This can be the case of surprise link |
271 | * removal. |
272 | * |
273 | * LINK_UP event will always occur before the peer side driver |
274 | * gets loaded the very first time. So there can be a case when |
275 | * the LINK_UP event has occurred, but the peer side driver hasn't |
276 | * yet loaded. We return 0 in that case. |
277 | * |
278 | * There is also a special case when the primary side driver is |
279 | * unloaded and then loaded again. Since there is no change in |
280 | * the status of NTB secondary in this case, there is no Link-Up |
281 | * or Link-Down notification received. We recognize this condition |
282 | * with peer_sta being set to 0. |
283 | * |
284 | * If bit 1 of peer SIDEINFO register is not set, then we |
285 | * simply return 0 irrespective of the link up or down status |
286 | * set in peer_sta. |
287 | */ |
288 | ret = amd_poll_link(ndev); |
289 | if (ret) { |
290 | /* |
291 | * We need to check the below only for NTB primary. For NTB |
292 | * secondary, simply checking the result of PSIDE_INFO |
293 | * register will suffice. |
294 | */ |
295 | if (ndev->ntb.topo == NTB_TOPO_PRI) { |
296 | if ((ndev->peer_sta & AMD_LINK_UP_EVENT) || |
297 | (ndev->peer_sta == 0)) |
298 | return ret; |
299 | else if (ndev->peer_sta & AMD_LINK_DOWN_EVENT) { |
300 | /* Clear peer sideinfo register */ |
301 | amd_clear_side_info_reg(ndev, peer: true); |
302 | |
303 | return 0; |
304 | } |
305 | } else { /* NTB_TOPO_SEC */ |
306 | return ret; |
307 | } |
308 | } |
309 | |
310 | return 0; |
311 | } |
312 | |
313 | static u64 amd_ntb_link_is_up(struct ntb_dev *ntb, |
314 | enum ntb_speed *speed, |
315 | enum ntb_width *width) |
316 | { |
317 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); |
318 | int ret = 0; |
319 | |
320 | if (amd_link_is_up(ndev)) { |
321 | if (speed) |
322 | *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta); |
323 | if (width) |
324 | *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta); |
325 | |
326 | dev_dbg(&ntb->pdev->dev, "link is up.\n" ); |
327 | |
328 | ret = 1; |
329 | } else { |
330 | if (speed) |
331 | *speed = NTB_SPEED_NONE; |
332 | if (width) |
333 | *width = NTB_WIDTH_NONE; |
334 | |
335 | dev_dbg(&ntb->pdev->dev, "link is down.\n" ); |
336 | } |
337 | |
338 | return ret; |
339 | } |
340 | |
341 | static int amd_ntb_link_enable(struct ntb_dev *ntb, |
342 | enum ntb_speed max_speed, |
343 | enum ntb_width max_width) |
344 | { |
345 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); |
346 | void __iomem *mmio = ndev->self_mmio; |
347 | |
348 | /* Enable event interrupt */ |
349 | ndev->int_mask &= ~AMD_EVENT_INTMASK; |
350 | writel(val: ndev->int_mask, addr: mmio + AMD_INTMASK_OFFSET); |
351 | |
352 | if (ndev->ntb.topo == NTB_TOPO_SEC) |
353 | return -EINVAL; |
354 | dev_dbg(&ntb->pdev->dev, "Enabling Link.\n" ); |
355 | |
356 | return 0; |
357 | } |
358 | |
359 | static int amd_ntb_link_disable(struct ntb_dev *ntb) |
360 | { |
361 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); |
362 | void __iomem *mmio = ndev->self_mmio; |
363 | |
364 | /* Disable event interrupt */ |
365 | ndev->int_mask |= AMD_EVENT_INTMASK; |
366 | writel(val: ndev->int_mask, addr: mmio + AMD_INTMASK_OFFSET); |
367 | |
368 | if (ndev->ntb.topo == NTB_TOPO_SEC) |
369 | return -EINVAL; |
370 | dev_dbg(&ntb->pdev->dev, "Enabling Link.\n" ); |
371 | |
372 | return 0; |
373 | } |
374 | |
375 | static int amd_ntb_peer_mw_count(struct ntb_dev *ntb) |
376 | { |
377 | /* The same as for inbound MWs */ |
378 | return ntb_ndev(ntb)->mw_count; |
379 | } |
380 | |
381 | static int amd_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx, |
382 | phys_addr_t *base, resource_size_t *size) |
383 | { |
384 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); |
385 | int bar; |
386 | |
387 | bar = ndev_mw_to_bar(ndev, idx); |
388 | if (bar < 0) |
389 | return bar; |
390 | |
391 | if (base) |
392 | *base = pci_resource_start(ndev->ntb.pdev, bar); |
393 | |
394 | if (size) |
395 | *size = pci_resource_len(ndev->ntb.pdev, bar); |
396 | |
397 | return 0; |
398 | } |
399 | |
400 | static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb) |
401 | { |
402 | return ntb_ndev(ntb)->db_valid_mask; |
403 | } |
404 | |
405 | static int amd_ntb_db_vector_count(struct ntb_dev *ntb) |
406 | { |
407 | return ntb_ndev(ntb)->db_count; |
408 | } |
409 | |
410 | static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) |
411 | { |
412 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); |
413 | |
414 | if (db_vector < 0 || db_vector > ndev->db_count) |
415 | return 0; |
416 | |
417 | return ntb_ndev(ntb)->db_valid_mask & (1ULL << db_vector); |
418 | } |
419 | |
420 | static u64 amd_ntb_db_read(struct ntb_dev *ntb) |
421 | { |
422 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); |
423 | void __iomem *mmio = ndev->self_mmio; |
424 | |
425 | return (u64)readw(addr: mmio + AMD_DBSTAT_OFFSET); |
426 | } |
427 | |
428 | static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) |
429 | { |
430 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); |
431 | void __iomem *mmio = ndev->self_mmio; |
432 | |
433 | writew(val: (u16)db_bits, addr: mmio + AMD_DBSTAT_OFFSET); |
434 | |
435 | return 0; |
436 | } |
437 | |
438 | static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) |
439 | { |
440 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); |
441 | void __iomem *mmio = ndev->self_mmio; |
442 | unsigned long flags; |
443 | |
444 | if (db_bits & ~ndev->db_valid_mask) |
445 | return -EINVAL; |
446 | |
447 | spin_lock_irqsave(&ndev->db_mask_lock, flags); |
448 | ndev->db_mask |= db_bits; |
449 | writew(val: (u16)ndev->db_mask, addr: mmio + AMD_DBMASK_OFFSET); |
450 | spin_unlock_irqrestore(lock: &ndev->db_mask_lock, flags); |
451 | |
452 | return 0; |
453 | } |
454 | |
455 | static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) |
456 | { |
457 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); |
458 | void __iomem *mmio = ndev->self_mmio; |
459 | unsigned long flags; |
460 | |
461 | if (db_bits & ~ndev->db_valid_mask) |
462 | return -EINVAL; |
463 | |
464 | spin_lock_irqsave(&ndev->db_mask_lock, flags); |
465 | ndev->db_mask &= ~db_bits; |
466 | writew(val: (u16)ndev->db_mask, addr: mmio + AMD_DBMASK_OFFSET); |
467 | spin_unlock_irqrestore(lock: &ndev->db_mask_lock, flags); |
468 | |
469 | return 0; |
470 | } |
471 | |
472 | static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) |
473 | { |
474 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); |
475 | void __iomem *mmio = ndev->self_mmio; |
476 | |
477 | writew(val: (u16)db_bits, addr: mmio + AMD_DBREQ_OFFSET); |
478 | |
479 | return 0; |
480 | } |
481 | |
482 | static int amd_ntb_spad_count(struct ntb_dev *ntb) |
483 | { |
484 | return ntb_ndev(ntb)->spad_count; |
485 | } |
486 | |
487 | static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx) |
488 | { |
489 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); |
490 | void __iomem *mmio = ndev->self_mmio; |
491 | u32 offset; |
492 | |
493 | if (idx < 0 || idx >= ndev->spad_count) |
494 | return 0; |
495 | |
496 | offset = ndev->self_spad + (idx << 2); |
497 | return readl(addr: mmio + AMD_SPAD_OFFSET + offset); |
498 | } |
499 | |
500 | static int amd_ntb_spad_write(struct ntb_dev *ntb, |
501 | int idx, u32 val) |
502 | { |
503 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); |
504 | void __iomem *mmio = ndev->self_mmio; |
505 | u32 offset; |
506 | |
507 | if (idx < 0 || idx >= ndev->spad_count) |
508 | return -EINVAL; |
509 | |
510 | offset = ndev->self_spad + (idx << 2); |
511 | writel(val, addr: mmio + AMD_SPAD_OFFSET + offset); |
512 | |
513 | return 0; |
514 | } |
515 | |
516 | static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx) |
517 | { |
518 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); |
519 | void __iomem *mmio = ndev->self_mmio; |
520 | u32 offset; |
521 | |
522 | if (sidx < 0 || sidx >= ndev->spad_count) |
523 | return -EINVAL; |
524 | |
525 | offset = ndev->peer_spad + (sidx << 2); |
526 | return readl(addr: mmio + AMD_SPAD_OFFSET + offset); |
527 | } |
528 | |
529 | static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, |
530 | int sidx, u32 val) |
531 | { |
532 | struct amd_ntb_dev *ndev = ntb_ndev(ntb); |
533 | void __iomem *mmio = ndev->self_mmio; |
534 | u32 offset; |
535 | |
536 | if (sidx < 0 || sidx >= ndev->spad_count) |
537 | return -EINVAL; |
538 | |
539 | offset = ndev->peer_spad + (sidx << 2); |
540 | writel(val, addr: mmio + AMD_SPAD_OFFSET + offset); |
541 | |
542 | return 0; |
543 | } |
544 | |
545 | static const struct ntb_dev_ops amd_ntb_ops = { |
546 | .mw_count = amd_ntb_mw_count, |
547 | .mw_get_align = amd_ntb_mw_get_align, |
548 | .mw_set_trans = amd_ntb_mw_set_trans, |
549 | .peer_mw_count = amd_ntb_peer_mw_count, |
550 | .peer_mw_get_addr = amd_ntb_peer_mw_get_addr, |
551 | .link_is_up = amd_ntb_link_is_up, |
552 | .link_enable = amd_ntb_link_enable, |
553 | .link_disable = amd_ntb_link_disable, |
554 | .db_valid_mask = amd_ntb_db_valid_mask, |
555 | .db_vector_count = amd_ntb_db_vector_count, |
556 | .db_vector_mask = amd_ntb_db_vector_mask, |
557 | .db_read = amd_ntb_db_read, |
558 | .db_clear = amd_ntb_db_clear, |
559 | .db_set_mask = amd_ntb_db_set_mask, |
560 | .db_clear_mask = amd_ntb_db_clear_mask, |
561 | .peer_db_set = amd_ntb_peer_db_set, |
562 | .spad_count = amd_ntb_spad_count, |
563 | .spad_read = amd_ntb_spad_read, |
564 | .spad_write = amd_ntb_spad_write, |
565 | .peer_spad_read = amd_ntb_peer_spad_read, |
566 | .peer_spad_write = amd_ntb_peer_spad_write, |
567 | }; |
568 | |
569 | static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit) |
570 | { |
571 | void __iomem *mmio = ndev->self_mmio; |
572 | int reg; |
573 | |
574 | reg = readl(addr: mmio + AMD_SMUACK_OFFSET); |
575 | reg |= bit; |
576 | writel(val: reg, addr: mmio + AMD_SMUACK_OFFSET); |
577 | } |
578 | |
579 | static void amd_handle_event(struct amd_ntb_dev *ndev, int vec) |
580 | { |
581 | void __iomem *mmio = ndev->self_mmio; |
582 | struct device *dev = &ndev->ntb.pdev->dev; |
583 | u32 status; |
584 | |
585 | status = readl(addr: mmio + AMD_INTSTAT_OFFSET); |
586 | if (!(status & AMD_EVENT_INTMASK)) |
587 | return; |
588 | |
589 | dev_dbg(dev, "status = 0x%x and vec = %d\n" , status, vec); |
590 | |
591 | status &= AMD_EVENT_INTMASK; |
592 | switch (status) { |
593 | case AMD_PEER_FLUSH_EVENT: |
594 | ndev->peer_sta |= AMD_PEER_FLUSH_EVENT; |
595 | dev_info(dev, "Flush is done.\n" ); |
596 | break; |
597 | case AMD_PEER_RESET_EVENT: |
598 | case AMD_LINK_DOWN_EVENT: |
599 | ndev->peer_sta |= status; |
600 | if (status == AMD_LINK_DOWN_EVENT) |
601 | ndev->peer_sta &= ~AMD_LINK_UP_EVENT; |
602 | |
603 | amd_ack_smu(ndev, bit: status); |
604 | |
605 | /* link down first */ |
606 | ntb_link_event(ntb: &ndev->ntb); |
607 | /* polling peer status */ |
608 | schedule_delayed_work(dwork: &ndev->hb_timer, AMD_LINK_HB_TIMEOUT); |
609 | |
610 | break; |
611 | case AMD_PEER_D3_EVENT: |
612 | case AMD_PEER_PMETO_EVENT: |
613 | case AMD_LINK_UP_EVENT: |
614 | ndev->peer_sta |= status; |
615 | if (status == AMD_LINK_UP_EVENT) |
616 | ndev->peer_sta &= ~AMD_LINK_DOWN_EVENT; |
617 | else if (status == AMD_PEER_D3_EVENT) |
618 | ndev->peer_sta &= ~AMD_PEER_D0_EVENT; |
619 | |
620 | amd_ack_smu(ndev, bit: status); |
621 | |
622 | /* link down */ |
623 | ntb_link_event(ntb: &ndev->ntb); |
624 | |
625 | break; |
626 | case AMD_PEER_D0_EVENT: |
627 | mmio = ndev->peer_mmio; |
628 | status = readl(addr: mmio + AMD_PMESTAT_OFFSET); |
629 | /* check if this is WAKEUP event */ |
630 | if (status & 0x1) |
631 | dev_info(dev, "Wakeup is done.\n" ); |
632 | |
633 | ndev->peer_sta |= AMD_PEER_D0_EVENT; |
634 | ndev->peer_sta &= ~AMD_PEER_D3_EVENT; |
635 | amd_ack_smu(ndev, bit: AMD_PEER_D0_EVENT); |
636 | |
637 | /* start a timer to poll link status */ |
638 | schedule_delayed_work(dwork: &ndev->hb_timer, |
639 | AMD_LINK_HB_TIMEOUT); |
640 | break; |
641 | default: |
642 | dev_info(dev, "event status = 0x%x.\n" , status); |
643 | break; |
644 | } |
645 | |
646 | /* Clear the interrupt status */ |
647 | writel(val: status, addr: mmio + AMD_INTSTAT_OFFSET); |
648 | } |
649 | |
650 | static void amd_handle_db_event(struct amd_ntb_dev *ndev, int vec) |
651 | { |
652 | struct device *dev = &ndev->ntb.pdev->dev; |
653 | u64 status; |
654 | |
655 | status = amd_ntb_db_read(ntb: &ndev->ntb); |
656 | |
657 | dev_dbg(dev, "status = 0x%llx and vec = %d\n" , status, vec); |
658 | |
659 | /* |
660 | * Since we had reserved highest order bit of DB for signaling peer of |
661 | * a special event, this is the only status bit we should be concerned |
662 | * here now. |
663 | */ |
664 | if (status & BIT(ndev->db_last_bit)) { |
665 | ntb_db_clear(ntb: &ndev->ntb, BIT(ndev->db_last_bit)); |
666 | /* send link down event notification */ |
667 | ntb_link_event(ntb: &ndev->ntb); |
668 | |
669 | /* |
670 | * If we are here, that means the peer has signalled a special |
671 | * event which notifies that the peer driver has been |
672 | * un-loaded for some reason. Since there is a chance that the |
673 | * peer will load its driver again sometime, we schedule link |
674 | * polling routine. |
675 | */ |
676 | schedule_delayed_work(dwork: &ndev->hb_timer, AMD_LINK_HB_TIMEOUT); |
677 | } |
678 | } |
679 | |
680 | static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec) |
681 | { |
682 | dev_dbg(&ndev->ntb.pdev->dev, "vec %d\n" , vec); |
683 | |
684 | if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1)) |
685 | amd_handle_event(ndev, vec); |
686 | |
687 | if (vec < AMD_DB_CNT) { |
688 | amd_handle_db_event(ndev, vec); |
689 | ntb_db_event(ntb: &ndev->ntb, vector: vec); |
690 | } |
691 | |
692 | return IRQ_HANDLED; |
693 | } |
694 | |
695 | static irqreturn_t ndev_vec_isr(int irq, void *dev) |
696 | { |
697 | struct amd_ntb_vec *nvec = dev; |
698 | |
699 | return ndev_interrupt(ndev: nvec->ndev, vec: nvec->num); |
700 | } |
701 | |
702 | static irqreturn_t ndev_irq_isr(int irq, void *dev) |
703 | { |
704 | struct amd_ntb_dev *ndev = dev; |
705 | |
706 | return ndev_interrupt(ndev, vec: irq - ndev->ntb.pdev->irq); |
707 | } |
708 | |
709 | static int ndev_init_isr(struct amd_ntb_dev *ndev, |
710 | int msix_min, int msix_max) |
711 | { |
712 | struct pci_dev *pdev; |
713 | int rc, i, msix_count, node; |
714 | |
715 | pdev = ndev->ntb.pdev; |
716 | |
717 | node = dev_to_node(dev: &pdev->dev); |
718 | |
719 | ndev->db_mask = ndev->db_valid_mask; |
720 | |
721 | /* Try to set up msix irq */ |
722 | ndev->vec = kcalloc_node(n: msix_max, size: sizeof(*ndev->vec), |
723 | GFP_KERNEL, node); |
724 | if (!ndev->vec) |
725 | goto err_msix_vec_alloc; |
726 | |
727 | ndev->msix = kcalloc_node(n: msix_max, size: sizeof(*ndev->msix), |
728 | GFP_KERNEL, node); |
729 | if (!ndev->msix) |
730 | goto err_msix_alloc; |
731 | |
732 | for (i = 0; i < msix_max; ++i) |
733 | ndev->msix[i].entry = i; |
734 | |
735 | msix_count = pci_enable_msix_range(dev: pdev, entries: ndev->msix, |
736 | minvec: msix_min, maxvec: msix_max); |
737 | if (msix_count < 0) |
738 | goto err_msix_enable; |
739 | |
740 | /* NOTE: Disable MSIX if msix count is less than 16 because of |
741 | * hardware limitation. |
742 | */ |
743 | if (msix_count < msix_min) { |
744 | pci_disable_msix(dev: pdev); |
745 | goto err_msix_enable; |
746 | } |
747 | |
748 | for (i = 0; i < msix_count; ++i) { |
749 | ndev->vec[i].ndev = ndev; |
750 | ndev->vec[i].num = i; |
751 | rc = request_irq(irq: ndev->msix[i].vector, handler: ndev_vec_isr, flags: 0, |
752 | name: "ndev_vec_isr" , dev: &ndev->vec[i]); |
753 | if (rc) |
754 | goto err_msix_request; |
755 | } |
756 | |
757 | dev_dbg(&pdev->dev, "Using msix interrupts\n" ); |
758 | ndev->db_count = msix_min; |
759 | ndev->msix_vec_count = msix_max; |
760 | return 0; |
761 | |
762 | err_msix_request: |
763 | while (i-- > 0) |
764 | free_irq(ndev->msix[i].vector, &ndev->vec[i]); |
765 | pci_disable_msix(dev: pdev); |
766 | err_msix_enable: |
767 | kfree(objp: ndev->msix); |
768 | err_msix_alloc: |
769 | kfree(objp: ndev->vec); |
770 | err_msix_vec_alloc: |
771 | ndev->msix = NULL; |
772 | ndev->vec = NULL; |
773 | |
774 | /* Try to set up msi irq */ |
775 | rc = pci_enable_msi(dev: pdev); |
776 | if (rc) |
777 | goto err_msi_enable; |
778 | |
779 | rc = request_irq(irq: pdev->irq, handler: ndev_irq_isr, flags: 0, |
780 | name: "ndev_irq_isr" , dev: ndev); |
781 | if (rc) |
782 | goto err_msi_request; |
783 | |
784 | dev_dbg(&pdev->dev, "Using msi interrupts\n" ); |
785 | ndev->db_count = 1; |
786 | ndev->msix_vec_count = 1; |
787 | return 0; |
788 | |
789 | err_msi_request: |
790 | pci_disable_msi(dev: pdev); |
791 | err_msi_enable: |
792 | |
793 | /* Try to set up intx irq */ |
794 | pci_intx(dev: pdev, enable: 1); |
795 | |
796 | rc = request_irq(irq: pdev->irq, handler: ndev_irq_isr, IRQF_SHARED, |
797 | name: "ndev_irq_isr" , dev: ndev); |
798 | if (rc) |
799 | goto err_intx_request; |
800 | |
801 | dev_dbg(&pdev->dev, "Using intx interrupts\n" ); |
802 | ndev->db_count = 1; |
803 | ndev->msix_vec_count = 1; |
804 | return 0; |
805 | |
806 | err_intx_request: |
807 | return rc; |
808 | } |
809 | |
810 | static void ndev_deinit_isr(struct amd_ntb_dev *ndev) |
811 | { |
812 | struct pci_dev *pdev; |
813 | void __iomem *mmio = ndev->self_mmio; |
814 | int i; |
815 | |
816 | pdev = ndev->ntb.pdev; |
817 | |
818 | /* Mask all doorbell interrupts */ |
819 | ndev->db_mask = ndev->db_valid_mask; |
820 | writel(val: ndev->db_mask, addr: mmio + AMD_DBMASK_OFFSET); |
821 | |
822 | if (ndev->msix) { |
823 | i = ndev->msix_vec_count; |
824 | while (i--) |
825 | free_irq(ndev->msix[i].vector, &ndev->vec[i]); |
826 | pci_disable_msix(dev: pdev); |
827 | kfree(objp: ndev->msix); |
828 | kfree(objp: ndev->vec); |
829 | } else { |
830 | free_irq(pdev->irq, ndev); |
831 | if (pci_dev_msi_enabled(pci_dev: pdev)) |
832 | pci_disable_msi(dev: pdev); |
833 | else |
834 | pci_intx(dev: pdev, enable: 0); |
835 | } |
836 | } |
837 | |
838 | static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf, |
839 | size_t count, loff_t *offp) |
840 | { |
841 | struct amd_ntb_dev *ndev; |
842 | void __iomem *mmio; |
843 | char *buf; |
844 | size_t buf_size; |
845 | ssize_t ret, off; |
846 | union { u64 v64; u32 v32; u16 v16; } u; |
847 | |
848 | ndev = filp->private_data; |
849 | mmio = ndev->self_mmio; |
850 | |
851 | buf_size = min(count, 0x800ul); |
852 | |
853 | buf = kmalloc(size: buf_size, GFP_KERNEL); |
854 | if (!buf) |
855 | return -ENOMEM; |
856 | |
857 | off = 0; |
858 | |
859 | off += scnprintf(buf: buf + off, size: buf_size - off, |
860 | fmt: "NTB Device Information:\n" ); |
861 | |
862 | off += scnprintf(buf: buf + off, size: buf_size - off, |
863 | fmt: "Connection Topology -\t%s\n" , |
864 | ntb_topo_string(topo: ndev->ntb.topo)); |
865 | |
866 | off += scnprintf(buf: buf + off, size: buf_size - off, |
867 | fmt: "LNK STA -\t\t%#06x\n" , ndev->lnk_sta); |
868 | |
869 | if (!amd_link_is_up(ndev)) { |
870 | off += scnprintf(buf: buf + off, size: buf_size - off, |
871 | fmt: "Link Status -\t\tDown\n" ); |
872 | } else { |
873 | off += scnprintf(buf: buf + off, size: buf_size - off, |
874 | fmt: "Link Status -\t\tUp\n" ); |
875 | off += scnprintf(buf: buf + off, size: buf_size - off, |
876 | fmt: "Link Speed -\t\tPCI-E Gen %u\n" , |
877 | NTB_LNK_STA_SPEED(ndev->lnk_sta)); |
878 | off += scnprintf(buf: buf + off, size: buf_size - off, |
879 | fmt: "Link Width -\t\tx%u\n" , |
880 | NTB_LNK_STA_WIDTH(ndev->lnk_sta)); |
881 | } |
882 | |
883 | off += scnprintf(buf: buf + off, size: buf_size - off, |
884 | fmt: "Memory Window Count -\t%u\n" , ndev->mw_count); |
885 | off += scnprintf(buf: buf + off, size: buf_size - off, |
886 | fmt: "Scratchpad Count -\t%u\n" , ndev->spad_count); |
887 | off += scnprintf(buf: buf + off, size: buf_size - off, |
888 | fmt: "Doorbell Count -\t%u\n" , ndev->db_count); |
889 | off += scnprintf(buf: buf + off, size: buf_size - off, |
890 | fmt: "MSIX Vector Count -\t%u\n" , ndev->msix_vec_count); |
891 | |
892 | off += scnprintf(buf: buf + off, size: buf_size - off, |
893 | fmt: "Doorbell Valid Mask -\t%#llx\n" , ndev->db_valid_mask); |
894 | |
895 | u.v32 = readl(addr: ndev->self_mmio + AMD_DBMASK_OFFSET); |
896 | off += scnprintf(buf: buf + off, size: buf_size - off, |
897 | fmt: "Doorbell Mask -\t\t\t%#06x\n" , u.v32); |
898 | |
899 | u.v32 = readl(addr: mmio + AMD_DBSTAT_OFFSET); |
900 | off += scnprintf(buf: buf + off, size: buf_size - off, |
901 | fmt: "Doorbell Bell -\t\t\t%#06x\n" , u.v32); |
902 | |
903 | off += scnprintf(buf: buf + off, size: buf_size - off, |
904 | fmt: "\nNTB Incoming XLAT:\n" ); |
905 | |
906 | u.v64 = read64(addr: mmio + AMD_BAR1XLAT_OFFSET); |
907 | off += scnprintf(buf: buf + off, size: buf_size - off, |
908 | fmt: "XLAT1 -\t\t%#018llx\n" , u.v64); |
909 | |
910 | u.v64 = read64(addr: ndev->self_mmio + AMD_BAR23XLAT_OFFSET); |
911 | off += scnprintf(buf: buf + off, size: buf_size - off, |
912 | fmt: "XLAT23 -\t\t%#018llx\n" , u.v64); |
913 | |
914 | u.v64 = read64(addr: ndev->self_mmio + AMD_BAR45XLAT_OFFSET); |
915 | off += scnprintf(buf: buf + off, size: buf_size - off, |
916 | fmt: "XLAT45 -\t\t%#018llx\n" , u.v64); |
917 | |
918 | u.v32 = readl(addr: mmio + AMD_BAR1LMT_OFFSET); |
919 | off += scnprintf(buf: buf + off, size: buf_size - off, |
920 | fmt: "LMT1 -\t\t\t%#06x\n" , u.v32); |
921 | |
922 | u.v64 = read64(addr: ndev->self_mmio + AMD_BAR23LMT_OFFSET); |
923 | off += scnprintf(buf: buf + off, size: buf_size - off, |
924 | fmt: "LMT23 -\t\t\t%#018llx\n" , u.v64); |
925 | |
926 | u.v64 = read64(addr: ndev->self_mmio + AMD_BAR45LMT_OFFSET); |
927 | off += scnprintf(buf: buf + off, size: buf_size - off, |
928 | fmt: "LMT45 -\t\t\t%#018llx\n" , u.v64); |
929 | |
930 | ret = simple_read_from_buffer(to: ubuf, count, ppos: offp, from: buf, available: off); |
931 | kfree(objp: buf); |
932 | return ret; |
933 | } |
934 | |
935 | static void ndev_init_debugfs(struct amd_ntb_dev *ndev) |
936 | { |
937 | if (!debugfs_dir) { |
938 | ndev->debugfs_dir = NULL; |
939 | ndev->debugfs_info = NULL; |
940 | } else { |
941 | ndev->debugfs_dir = |
942 | debugfs_create_dir(name: pci_name(pdev: ndev->ntb.pdev), |
943 | parent: debugfs_dir); |
944 | ndev->debugfs_info = |
945 | debugfs_create_file(name: "info" , S_IRUSR, |
946 | parent: ndev->debugfs_dir, data: ndev, |
947 | fops: &amd_ntb_debugfs_info); |
948 | } |
949 | } |
950 | |
951 | static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev) |
952 | { |
953 | debugfs_remove_recursive(dentry: ndev->debugfs_dir); |
954 | } |
955 | |
956 | static inline void ndev_init_struct(struct amd_ntb_dev *ndev, |
957 | struct pci_dev *pdev) |
958 | { |
959 | ndev->ntb.pdev = pdev; |
960 | ndev->ntb.topo = NTB_TOPO_NONE; |
961 | ndev->ntb.ops = &amd_ntb_ops; |
962 | ndev->int_mask = AMD_EVENT_INTMASK; |
963 | spin_lock_init(&ndev->db_mask_lock); |
964 | } |
965 | |
966 | static int amd_poll_link(struct amd_ntb_dev *ndev) |
967 | { |
968 | void __iomem *mmio = ndev->peer_mmio; |
969 | u32 reg; |
970 | |
971 | reg = readl(addr: mmio + AMD_SIDEINFO_OFFSET); |
972 | reg &= AMD_SIDE_READY; |
973 | |
974 | dev_dbg(&ndev->ntb.pdev->dev, "%s: reg_val = 0x%x.\n" , __func__, reg); |
975 | |
976 | ndev->cntl_sta = reg; |
977 | |
978 | amd_ntb_get_link_status(ndev); |
979 | |
980 | return ndev->cntl_sta; |
981 | } |
982 | |
983 | static void amd_link_hb(struct work_struct *work) |
984 | { |
985 | struct amd_ntb_dev *ndev = hb_ndev(work); |
986 | |
987 | if (amd_poll_link(ndev)) |
988 | ntb_link_event(ntb: &ndev->ntb); |
989 | |
990 | if (!amd_link_is_up(ndev)) |
991 | schedule_delayed_work(dwork: &ndev->hb_timer, AMD_LINK_HB_TIMEOUT); |
992 | } |
993 | |
994 | static int amd_init_isr(struct amd_ntb_dev *ndev) |
995 | { |
996 | return ndev_init_isr(ndev, msix_min: AMD_DB_CNT, msix_max: AMD_MSIX_VECTOR_CNT); |
997 | } |
998 | |
999 | static void amd_set_side_info_reg(struct amd_ntb_dev *ndev, bool peer) |
1000 | { |
1001 | void __iomem *mmio = NULL; |
1002 | unsigned int reg; |
1003 | |
1004 | if (peer) |
1005 | mmio = ndev->peer_mmio; |
1006 | else |
1007 | mmio = ndev->self_mmio; |
1008 | |
1009 | reg = readl(addr: mmio + AMD_SIDEINFO_OFFSET); |
1010 | if (!(reg & AMD_SIDE_READY)) { |
1011 | reg |= AMD_SIDE_READY; |
1012 | writel(val: reg, addr: mmio + AMD_SIDEINFO_OFFSET); |
1013 | } |
1014 | } |
1015 | |
1016 | static void amd_clear_side_info_reg(struct amd_ntb_dev *ndev, bool peer) |
1017 | { |
1018 | void __iomem *mmio = NULL; |
1019 | unsigned int reg; |
1020 | |
1021 | if (peer) |
1022 | mmio = ndev->peer_mmio; |
1023 | else |
1024 | mmio = ndev->self_mmio; |
1025 | |
1026 | reg = readl(addr: mmio + AMD_SIDEINFO_OFFSET); |
1027 | if (reg & AMD_SIDE_READY) { |
1028 | reg &= ~AMD_SIDE_READY; |
1029 | writel(val: reg, addr: mmio + AMD_SIDEINFO_OFFSET); |
1030 | readl(addr: mmio + AMD_SIDEINFO_OFFSET); |
1031 | } |
1032 | } |
1033 | |
1034 | static void amd_init_side_info(struct amd_ntb_dev *ndev) |
1035 | { |
1036 | void __iomem *mmio = ndev->self_mmio; |
1037 | u32 ntb_ctl; |
1038 | |
1039 | amd_set_side_info_reg(ndev, peer: false); |
1040 | |
1041 | ntb_ctl = readl(addr: mmio + AMD_CNTL_OFFSET); |
1042 | ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL); |
1043 | writel(val: ntb_ctl, addr: mmio + AMD_CNTL_OFFSET); |
1044 | } |
1045 | |
1046 | static void amd_deinit_side_info(struct amd_ntb_dev *ndev) |
1047 | { |
1048 | void __iomem *mmio = ndev->self_mmio; |
1049 | u32 ntb_ctl; |
1050 | |
1051 | amd_clear_side_info_reg(ndev, peer: false); |
1052 | |
1053 | ntb_ctl = readl(addr: mmio + AMD_CNTL_OFFSET); |
1054 | ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL); |
1055 | writel(val: ntb_ctl, addr: mmio + AMD_CNTL_OFFSET); |
1056 | } |
1057 | |
1058 | static int amd_init_ntb(struct amd_ntb_dev *ndev) |
1059 | { |
1060 | void __iomem *mmio = ndev->self_mmio; |
1061 | |
1062 | ndev->mw_count = ndev->dev_data->mw_count; |
1063 | ndev->spad_count = AMD_SPADS_CNT; |
1064 | ndev->db_count = AMD_DB_CNT; |
1065 | |
1066 | switch (ndev->ntb.topo) { |
1067 | case NTB_TOPO_PRI: |
1068 | case NTB_TOPO_SEC: |
1069 | ndev->spad_count >>= 1; |
1070 | if (ndev->ntb.topo == NTB_TOPO_PRI) { |
1071 | ndev->self_spad = 0; |
1072 | ndev->peer_spad = 0x20; |
1073 | } else { |
1074 | ndev->self_spad = 0x20; |
1075 | ndev->peer_spad = 0; |
1076 | } |
1077 | |
1078 | INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb); |
1079 | schedule_delayed_work(dwork: &ndev->hb_timer, AMD_LINK_HB_TIMEOUT); |
1080 | |
1081 | break; |
1082 | default: |
1083 | dev_err(&ndev->ntb.pdev->dev, |
1084 | "AMD NTB does not support B2B mode.\n" ); |
1085 | return -EINVAL; |
1086 | } |
1087 | |
1088 | /* Mask event interrupts */ |
1089 | writel(val: ndev->int_mask, addr: mmio + AMD_INTMASK_OFFSET); |
1090 | |
1091 | return 0; |
1092 | } |
1093 | |
1094 | static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev) |
1095 | { |
1096 | void __iomem *mmio = ndev->self_mmio; |
1097 | u32 info; |
1098 | |
1099 | info = readl(addr: mmio + AMD_SIDEINFO_OFFSET); |
1100 | if (info & AMD_SIDE_MASK) |
1101 | return NTB_TOPO_SEC; |
1102 | else |
1103 | return NTB_TOPO_PRI; |
1104 | } |
1105 | |
1106 | static int amd_init_dev(struct amd_ntb_dev *ndev) |
1107 | { |
1108 | void __iomem *mmio = ndev->self_mmio; |
1109 | struct pci_dev *pdev; |
1110 | int rc = 0; |
1111 | |
1112 | pdev = ndev->ntb.pdev; |
1113 | |
1114 | ndev->ntb.topo = amd_get_topo(ndev); |
1115 | dev_dbg(&pdev->dev, "AMD NTB topo is %s\n" , |
1116 | ntb_topo_string(ndev->ntb.topo)); |
1117 | |
1118 | rc = amd_init_ntb(ndev); |
1119 | if (rc) |
1120 | return rc; |
1121 | |
1122 | rc = amd_init_isr(ndev); |
1123 | if (rc) { |
1124 | dev_err(&pdev->dev, "fail to init isr.\n" ); |
1125 | return rc; |
1126 | } |
1127 | |
1128 | ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; |
1129 | /* |
1130 | * We reserve the highest order bit of the DB register which will |
1131 | * be used to notify peer when the driver on this side is being |
1132 | * un-loaded. |
1133 | */ |
1134 | ndev->db_last_bit = |
1135 | find_last_bit(addr: (unsigned long *)&ndev->db_valid_mask, |
1136 | hweight64(ndev->db_valid_mask)); |
1137 | writew(val: (u16)~BIT(ndev->db_last_bit), addr: mmio + AMD_DBMASK_OFFSET); |
1138 | /* |
1139 | * Since now there is one less bit to account for, the DB count |
1140 | * and DB mask should be adjusted accordingly. |
1141 | */ |
1142 | ndev->db_count -= 1; |
1143 | ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; |
1144 | |
1145 | /* Enable Link-Up and Link-Down event interrupts */ |
1146 | ndev->int_mask &= ~(AMD_LINK_UP_EVENT | AMD_LINK_DOWN_EVENT); |
1147 | writel(val: ndev->int_mask, addr: mmio + AMD_INTMASK_OFFSET); |
1148 | |
1149 | return 0; |
1150 | } |
1151 | |
1152 | static void amd_deinit_dev(struct amd_ntb_dev *ndev) |
1153 | { |
1154 | cancel_delayed_work_sync(dwork: &ndev->hb_timer); |
1155 | |
1156 | ndev_deinit_isr(ndev); |
1157 | } |
1158 | |
1159 | static int amd_ntb_init_pci(struct amd_ntb_dev *ndev, |
1160 | struct pci_dev *pdev) |
1161 | { |
1162 | int rc; |
1163 | |
1164 | pci_set_drvdata(pdev, data: ndev); |
1165 | |
1166 | rc = pci_enable_device(dev: pdev); |
1167 | if (rc) |
1168 | goto err_pci_enable; |
1169 | |
1170 | rc = pci_request_regions(pdev, NTB_NAME); |
1171 | if (rc) |
1172 | goto err_pci_regions; |
1173 | |
1174 | pci_set_master(dev: pdev); |
1175 | |
1176 | rc = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64)); |
1177 | if (rc) { |
1178 | rc = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(32)); |
1179 | if (rc) |
1180 | goto err_dma_mask; |
1181 | dev_warn(&pdev->dev, "Cannot DMA highmem\n" ); |
1182 | } |
1183 | |
1184 | ndev->self_mmio = pci_iomap(dev: pdev, bar: 0, max: 0); |
1185 | if (!ndev->self_mmio) { |
1186 | rc = -EIO; |
1187 | goto err_dma_mask; |
1188 | } |
1189 | ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET; |
1190 | |
1191 | return 0; |
1192 | |
1193 | err_dma_mask: |
1194 | pci_release_regions(pdev); |
1195 | err_pci_regions: |
1196 | pci_disable_device(dev: pdev); |
1197 | err_pci_enable: |
1198 | pci_set_drvdata(pdev, NULL); |
1199 | return rc; |
1200 | } |
1201 | |
1202 | static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev) |
1203 | { |
1204 | struct pci_dev *pdev = ndev->ntb.pdev; |
1205 | |
1206 | pci_iounmap(dev: pdev, ndev->self_mmio); |
1207 | |
1208 | pci_release_regions(pdev); |
1209 | pci_disable_device(dev: pdev); |
1210 | pci_set_drvdata(pdev, NULL); |
1211 | } |
1212 | |
1213 | static int amd_ntb_pci_probe(struct pci_dev *pdev, |
1214 | const struct pci_device_id *id) |
1215 | { |
1216 | struct amd_ntb_dev *ndev; |
1217 | int rc, node; |
1218 | |
1219 | node = dev_to_node(dev: &pdev->dev); |
1220 | |
1221 | ndev = kzalloc_node(size: sizeof(*ndev), GFP_KERNEL, node); |
1222 | if (!ndev) { |
1223 | rc = -ENOMEM; |
1224 | goto err_ndev; |
1225 | } |
1226 | |
1227 | ndev->dev_data = (struct ntb_dev_data *)id->driver_data; |
1228 | |
1229 | ndev_init_struct(ndev, pdev); |
1230 | |
1231 | rc = amd_ntb_init_pci(ndev, pdev); |
1232 | if (rc) |
1233 | goto err_init_pci; |
1234 | |
1235 | rc = amd_init_dev(ndev); |
1236 | if (rc) |
1237 | goto err_init_dev; |
1238 | |
1239 | /* write side info */ |
1240 | amd_init_side_info(ndev); |
1241 | |
1242 | amd_poll_link(ndev); |
1243 | |
1244 | ndev_init_debugfs(ndev); |
1245 | |
1246 | rc = ntb_register_device(ntb: &ndev->ntb); |
1247 | if (rc) |
1248 | goto err_register; |
1249 | |
1250 | dev_info(&pdev->dev, "NTB device registered.\n" ); |
1251 | |
1252 | return 0; |
1253 | |
1254 | err_register: |
1255 | ndev_deinit_debugfs(ndev); |
1256 | amd_deinit_dev(ndev); |
1257 | err_init_dev: |
1258 | amd_ntb_deinit_pci(ndev); |
1259 | err_init_pci: |
1260 | kfree(objp: ndev); |
1261 | err_ndev: |
1262 | return rc; |
1263 | } |
1264 | |
1265 | static void amd_ntb_pci_remove(struct pci_dev *pdev) |
1266 | { |
1267 | struct amd_ntb_dev *ndev = pci_get_drvdata(pdev); |
1268 | |
1269 | /* |
1270 | * Clear the READY bit in SIDEINFO register before sending DB event |
1271 | * to the peer. This will make sure that when the peer handles the |
1272 | * DB event, it correctly reads this bit as being 0. |
1273 | */ |
1274 | amd_deinit_side_info(ndev); |
1275 | ntb_peer_db_set(ntb: &ndev->ntb, BIT_ULL(ndev->db_last_bit)); |
1276 | ntb_unregister_device(ntb: &ndev->ntb); |
1277 | ndev_deinit_debugfs(ndev); |
1278 | amd_deinit_dev(ndev); |
1279 | amd_ntb_deinit_pci(ndev); |
1280 | kfree(objp: ndev); |
1281 | } |
1282 | |
1283 | static void amd_ntb_pci_shutdown(struct pci_dev *pdev) |
1284 | { |
1285 | struct amd_ntb_dev *ndev = pci_get_drvdata(pdev); |
1286 | |
1287 | /* Send link down notification */ |
1288 | ntb_link_event(ntb: &ndev->ntb); |
1289 | |
1290 | amd_deinit_side_info(ndev); |
1291 | ntb_peer_db_set(ntb: &ndev->ntb, BIT_ULL(ndev->db_last_bit)); |
1292 | ntb_unregister_device(ntb: &ndev->ntb); |
1293 | ndev_deinit_debugfs(ndev); |
1294 | amd_deinit_dev(ndev); |
1295 | amd_ntb_deinit_pci(ndev); |
1296 | kfree(objp: ndev); |
1297 | } |
1298 | |
1299 | static const struct file_operations amd_ntb_debugfs_info = { |
1300 | .owner = THIS_MODULE, |
1301 | .open = simple_open, |
1302 | .read = ndev_debugfs_read, |
1303 | }; |
1304 | |
1305 | static const struct ntb_dev_data dev_data[] = { |
1306 | { /* for device 145b */ |
1307 | .mw_count = 3, |
1308 | .mw_idx = 1, |
1309 | }, |
1310 | { /* for device 148b */ |
1311 | .mw_count = 2, |
1312 | .mw_idx = 2, |
1313 | }, |
1314 | }; |
1315 | |
1316 | static const struct pci_device_id amd_ntb_pci_tbl[] = { |
1317 | { PCI_VDEVICE(AMD, 0x145b), (kernel_ulong_t)&dev_data[0] }, |
1318 | { PCI_VDEVICE(AMD, 0x148b), (kernel_ulong_t)&dev_data[1] }, |
1319 | { PCI_VDEVICE(AMD, 0x14c0), (kernel_ulong_t)&dev_data[1] }, |
1320 | { PCI_VDEVICE(AMD, 0x14c3), (kernel_ulong_t)&dev_data[1] }, |
1321 | { PCI_VDEVICE(HYGON, 0x145b), (kernel_ulong_t)&dev_data[0] }, |
1322 | { 0, } |
1323 | }; |
1324 | MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl); |
1325 | |
1326 | static struct pci_driver amd_ntb_pci_driver = { |
1327 | .name = KBUILD_MODNAME, |
1328 | .id_table = amd_ntb_pci_tbl, |
1329 | .probe = amd_ntb_pci_probe, |
1330 | .remove = amd_ntb_pci_remove, |
1331 | .shutdown = amd_ntb_pci_shutdown, |
1332 | }; |
1333 | |
1334 | static int __init amd_ntb_pci_driver_init(void) |
1335 | { |
1336 | int ret; |
1337 | pr_info("%s %s\n" , NTB_DESC, NTB_VER); |
1338 | |
1339 | if (debugfs_initialized()) |
1340 | debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); |
1341 | |
1342 | ret = pci_register_driver(&amd_ntb_pci_driver); |
1343 | if (ret) |
1344 | debugfs_remove_recursive(dentry: debugfs_dir); |
1345 | |
1346 | return ret; |
1347 | } |
1348 | module_init(amd_ntb_pci_driver_init); |
1349 | |
1350 | static void __exit amd_ntb_pci_driver_exit(void) |
1351 | { |
1352 | pci_unregister_driver(dev: &amd_ntb_pci_driver); |
1353 | debugfs_remove_recursive(dentry: debugfs_dir); |
1354 | } |
1355 | module_exit(amd_ntb_pci_driver_exit); |
1356 | |