1/*
2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
4 *
5 * GPL LICENSE SUMMARY
6 *
7 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
8 * Copyright (C) 2016 T-Platforms. All Rights Reserved.
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of version 2 of the GNU General Public License as
12 * published by the Free Software Foundation.
13 *
14 * BSD LICENSE
15 *
16 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
17 * Copyright (C) 2016 T-Platforms. All Rights Reserved.
18 *
19 * Redistribution and use in source and binary forms, with or without
20 * modification, are permitted provided that the following conditions
21 * are met:
22 *
23 * * Redistributions of source code must retain the above copyright
24 * notice, this list of conditions and the following disclaimer.
25 * * Redistributions in binary form must reproduce the above copy
26 * notice, this list of conditions and the following disclaimer in
27 * the documentation and/or other materials provided with the
28 * distribution.
29 * * Neither the name of AMD Corporation nor the names of its
30 * contributors may be used to endorse or promote products derived
31 * from this software without specific prior written permission.
32 *
33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
34 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
35 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
36 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
37 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
38 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
39 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
40 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
41 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
44 *
45 * AMD PCIe NTB Linux driver
46 *
47 * Contact Information:
48 * Xiangliang Yu <Xiangliang.Yu@amd.com>
49 */
50
51#include <linux/debugfs.h>
52#include <linux/delay.h>
53#include <linux/init.h>
54#include <linux/interrupt.h>
55#include <linux/module.h>
56#include <linux/acpi.h>
57#include <linux/pci.h>
58#include <linux/random.h>
59#include <linux/slab.h>
60#include <linux/ntb.h>
61
62#include "ntb_hw_amd.h"
63
64#define NTB_NAME "ntb_hw_amd"
65#define NTB_DESC "AMD(R) PCI-E Non-Transparent Bridge Driver"
66#define NTB_VER "1.0"
67
68MODULE_DESCRIPTION(NTB_DESC);
69MODULE_VERSION(NTB_VER);
70MODULE_LICENSE("Dual BSD/GPL");
71MODULE_AUTHOR("AMD Inc.");
72
73static const struct file_operations amd_ntb_debugfs_info;
74static struct dentry *debugfs_dir;
75
76static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx)
77{
78 if (idx < 0 || idx > ndev->mw_count)
79 return -EINVAL;
80
81 return ndev->dev_data->mw_idx << idx;
82}
83
84static int amd_ntb_mw_count(struct ntb_dev *ntb, int pidx)
85{
86 if (pidx != NTB_DEF_PEER_IDX)
87 return -EINVAL;
88
89 return ntb_ndev(ntb)->mw_count;
90}
91
92static int amd_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
93 resource_size_t *addr_align,
94 resource_size_t *size_align,
95 resource_size_t *size_max)
96{
97 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
98 int bar;
99
100 if (pidx != NTB_DEF_PEER_IDX)
101 return -EINVAL;
102
103 bar = ndev_mw_to_bar(ndev, idx);
104 if (bar < 0)
105 return bar;
106
107 if (addr_align)
108 *addr_align = SZ_4K;
109
110 if (size_align)
111 *size_align = 1;
112
113 if (size_max)
114 *size_max = pci_resource_len(ndev->ntb.pdev, bar);
115
116 return 0;
117}
118
119static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
120 dma_addr_t addr, resource_size_t size)
121{
122 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
123 unsigned long xlat_reg, limit_reg = 0;
124 resource_size_t mw_size;
125 void __iomem *mmio, *peer_mmio;
126 u64 base_addr, limit, reg_val;
127 int bar;
128
129 if (pidx != NTB_DEF_PEER_IDX)
130 return -EINVAL;
131
132 bar = ndev_mw_to_bar(ndev, idx);
133 if (bar < 0)
134 return bar;
135
136 mw_size = pci_resource_len(ntb->pdev, bar);
137
138 /* make sure the range fits in the usable mw size */
139 if (size > mw_size)
140 return -EINVAL;
141
142 mmio = ndev->self_mmio;
143 peer_mmio = ndev->peer_mmio;
144
145 base_addr = pci_resource_start(ntb->pdev, bar);
146
147 if (bar != 1) {
148 xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 2);
149 limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 2);
150
151 /* Set the limit if supported */
152 limit = size;
153
154 /* set and verify setting the translation address */
155 write64(val: addr, addr: peer_mmio + xlat_reg);
156 reg_val = read64(addr: peer_mmio + xlat_reg);
157 if (reg_val != addr) {
158 write64(val: 0, addr: peer_mmio + xlat_reg);
159 return -EIO;
160 }
161
162 /* set and verify setting the limit */
163 write64(val: limit, addr: peer_mmio + limit_reg);
164 reg_val = read64(addr: peer_mmio + limit_reg);
165 if (reg_val != limit) {
166 write64(val: base_addr, addr: mmio + limit_reg);
167 write64(val: 0, addr: peer_mmio + xlat_reg);
168 return -EIO;
169 }
170 } else {
171 xlat_reg = AMD_BAR1XLAT_OFFSET;
172 limit_reg = AMD_BAR1LMT_OFFSET;
173
174 /* Set the limit if supported */
175 limit = size;
176
177 /* set and verify setting the translation address */
178 write64(val: addr, addr: peer_mmio + xlat_reg);
179 reg_val = read64(addr: peer_mmio + xlat_reg);
180 if (reg_val != addr) {
181 write64(val: 0, addr: peer_mmio + xlat_reg);
182 return -EIO;
183 }
184
185 /* set and verify setting the limit */
186 writel(val: limit, addr: peer_mmio + limit_reg);
187 reg_val = readl(addr: peer_mmio + limit_reg);
188 if (reg_val != limit) {
189 writel(val: base_addr, addr: mmio + limit_reg);
190 writel(val: 0, addr: peer_mmio + xlat_reg);
191 return -EIO;
192 }
193 }
194
195 return 0;
196}
197
198static int amd_ntb_get_link_status(struct amd_ntb_dev *ndev)
199{
200 struct pci_dev *pdev = NULL;
201 struct pci_dev *pci_swds = NULL;
202 struct pci_dev *pci_swus = NULL;
203 u32 stat;
204 int rc;
205
206 if (ndev->ntb.topo == NTB_TOPO_SEC) {
207 /* Locate the pointer to Downstream Switch for this device */
208 pci_swds = pci_upstream_bridge(dev: ndev->ntb.pdev);
209 if (pci_swds) {
210 /*
211 * Locate the pointer to Upstream Switch for
212 * the Downstream Switch.
213 */
214 pci_swus = pci_upstream_bridge(dev: pci_swds);
215 if (pci_swus) {
216 rc = pcie_capability_read_dword(dev: pci_swus,
217 PCI_EXP_LNKCTL,
218 val: &stat);
219 if (rc)
220 return 0;
221 } else {
222 return 0;
223 }
224 } else {
225 return 0;
226 }
227 } else if (ndev->ntb.topo == NTB_TOPO_PRI) {
228 /*
229 * For NTB primary, we simply read the Link Status and control
230 * register of the NTB device itself.
231 */
232 pdev = ndev->ntb.pdev;
233 rc = pcie_capability_read_dword(dev: pdev, PCI_EXP_LNKCTL, val: &stat);
234 if (rc)
235 return 0;
236 } else {
237 /* Catch all for everything else */
238 return 0;
239 }
240
241 ndev->lnk_sta = stat;
242
243 return 1;
244}
245
246static int amd_link_is_up(struct amd_ntb_dev *ndev)
247{
248 int ret;
249
250 /*
251 * We consider the link to be up under two conditions:
252 *
253 * - When a link-up event is received. This is indicated by
254 * AMD_LINK_UP_EVENT set in peer_sta.
255 * - When driver on both sides of the link have been loaded.
256 * This is indicated by bit 1 being set in the peer
257 * SIDEINFO register.
258 *
259 * This function should return 1 when the latter of the above
260 * two conditions is true.
261 *
262 * Now consider the sequence of events - Link-Up event occurs,
263 * then the peer side driver loads. In this case, we would have
264 * received LINK_UP event and bit 1 of peer SIDEINFO is also
265 * set. What happens now if the link goes down? Bit 1 of
266 * peer SIDEINFO remains set, but LINK_DOWN bit is set in
267 * peer_sta. So we should return 0 from this function. Not only
268 * that, we clear bit 1 of peer SIDEINFO to 0, since the peer
269 * side driver did not even get a chance to clear it before
270 * the link went down. This can be the case of surprise link
271 * removal.
272 *
273 * LINK_UP event will always occur before the peer side driver
274 * gets loaded the very first time. So there can be a case when
275 * the LINK_UP event has occurred, but the peer side driver hasn't
276 * yet loaded. We return 0 in that case.
277 *
278 * There is also a special case when the primary side driver is
279 * unloaded and then loaded again. Since there is no change in
280 * the status of NTB secondary in this case, there is no Link-Up
281 * or Link-Down notification received. We recognize this condition
282 * with peer_sta being set to 0.
283 *
284 * If bit 1 of peer SIDEINFO register is not set, then we
285 * simply return 0 irrespective of the link up or down status
286 * set in peer_sta.
287 */
288 ret = amd_poll_link(ndev);
289 if (ret) {
290 /*
291 * We need to check the below only for NTB primary. For NTB
292 * secondary, simply checking the result of PSIDE_INFO
293 * register will suffice.
294 */
295 if (ndev->ntb.topo == NTB_TOPO_PRI) {
296 if ((ndev->peer_sta & AMD_LINK_UP_EVENT) ||
297 (ndev->peer_sta == 0))
298 return ret;
299 else if (ndev->peer_sta & AMD_LINK_DOWN_EVENT) {
300 /* Clear peer sideinfo register */
301 amd_clear_side_info_reg(ndev, peer: true);
302
303 return 0;
304 }
305 } else { /* NTB_TOPO_SEC */
306 return ret;
307 }
308 }
309
310 return 0;
311}
312
313static u64 amd_ntb_link_is_up(struct ntb_dev *ntb,
314 enum ntb_speed *speed,
315 enum ntb_width *width)
316{
317 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
318 int ret = 0;
319
320 if (amd_link_is_up(ndev)) {
321 if (speed)
322 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
323 if (width)
324 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
325
326 dev_dbg(&ntb->pdev->dev, "link is up.\n");
327
328 ret = 1;
329 } else {
330 if (speed)
331 *speed = NTB_SPEED_NONE;
332 if (width)
333 *width = NTB_WIDTH_NONE;
334
335 dev_dbg(&ntb->pdev->dev, "link is down.\n");
336 }
337
338 return ret;
339}
340
341static int amd_ntb_link_enable(struct ntb_dev *ntb,
342 enum ntb_speed max_speed,
343 enum ntb_width max_width)
344{
345 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
346 void __iomem *mmio = ndev->self_mmio;
347
348 /* Enable event interrupt */
349 ndev->int_mask &= ~AMD_EVENT_INTMASK;
350 writel(val: ndev->int_mask, addr: mmio + AMD_INTMASK_OFFSET);
351
352 if (ndev->ntb.topo == NTB_TOPO_SEC)
353 return -EINVAL;
354 dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
355
356 return 0;
357}
358
359static int amd_ntb_link_disable(struct ntb_dev *ntb)
360{
361 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
362 void __iomem *mmio = ndev->self_mmio;
363
364 /* Disable event interrupt */
365 ndev->int_mask |= AMD_EVENT_INTMASK;
366 writel(val: ndev->int_mask, addr: mmio + AMD_INTMASK_OFFSET);
367
368 if (ndev->ntb.topo == NTB_TOPO_SEC)
369 return -EINVAL;
370 dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
371
372 return 0;
373}
374
375static int amd_ntb_peer_mw_count(struct ntb_dev *ntb)
376{
377 /* The same as for inbound MWs */
378 return ntb_ndev(ntb)->mw_count;
379}
380
381static int amd_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
382 phys_addr_t *base, resource_size_t *size)
383{
384 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
385 int bar;
386
387 bar = ndev_mw_to_bar(ndev, idx);
388 if (bar < 0)
389 return bar;
390
391 if (base)
392 *base = pci_resource_start(ndev->ntb.pdev, bar);
393
394 if (size)
395 *size = pci_resource_len(ndev->ntb.pdev, bar);
396
397 return 0;
398}
399
400static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb)
401{
402 return ntb_ndev(ntb)->db_valid_mask;
403}
404
405static int amd_ntb_db_vector_count(struct ntb_dev *ntb)
406{
407 return ntb_ndev(ntb)->db_count;
408}
409
410static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
411{
412 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
413
414 if (db_vector < 0 || db_vector > ndev->db_count)
415 return 0;
416
417 return ntb_ndev(ntb)->db_valid_mask & (1ULL << db_vector);
418}
419
420static u64 amd_ntb_db_read(struct ntb_dev *ntb)
421{
422 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
423 void __iomem *mmio = ndev->self_mmio;
424
425 return (u64)readw(addr: mmio + AMD_DBSTAT_OFFSET);
426}
427
428static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
429{
430 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
431 void __iomem *mmio = ndev->self_mmio;
432
433 writew(val: (u16)db_bits, addr: mmio + AMD_DBSTAT_OFFSET);
434
435 return 0;
436}
437
438static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
439{
440 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
441 void __iomem *mmio = ndev->self_mmio;
442 unsigned long flags;
443
444 if (db_bits & ~ndev->db_valid_mask)
445 return -EINVAL;
446
447 spin_lock_irqsave(&ndev->db_mask_lock, flags);
448 ndev->db_mask |= db_bits;
449 writew(val: (u16)ndev->db_mask, addr: mmio + AMD_DBMASK_OFFSET);
450 spin_unlock_irqrestore(lock: &ndev->db_mask_lock, flags);
451
452 return 0;
453}
454
455static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
456{
457 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
458 void __iomem *mmio = ndev->self_mmio;
459 unsigned long flags;
460
461 if (db_bits & ~ndev->db_valid_mask)
462 return -EINVAL;
463
464 spin_lock_irqsave(&ndev->db_mask_lock, flags);
465 ndev->db_mask &= ~db_bits;
466 writew(val: (u16)ndev->db_mask, addr: mmio + AMD_DBMASK_OFFSET);
467 spin_unlock_irqrestore(lock: &ndev->db_mask_lock, flags);
468
469 return 0;
470}
471
472static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
473{
474 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
475 void __iomem *mmio = ndev->self_mmio;
476
477 writew(val: (u16)db_bits, addr: mmio + AMD_DBREQ_OFFSET);
478
479 return 0;
480}
481
482static int amd_ntb_spad_count(struct ntb_dev *ntb)
483{
484 return ntb_ndev(ntb)->spad_count;
485}
486
487static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx)
488{
489 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
490 void __iomem *mmio = ndev->self_mmio;
491 u32 offset;
492
493 if (idx < 0 || idx >= ndev->spad_count)
494 return 0;
495
496 offset = ndev->self_spad + (idx << 2);
497 return readl(addr: mmio + AMD_SPAD_OFFSET + offset);
498}
499
500static int amd_ntb_spad_write(struct ntb_dev *ntb,
501 int idx, u32 val)
502{
503 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
504 void __iomem *mmio = ndev->self_mmio;
505 u32 offset;
506
507 if (idx < 0 || idx >= ndev->spad_count)
508 return -EINVAL;
509
510 offset = ndev->self_spad + (idx << 2);
511 writel(val, addr: mmio + AMD_SPAD_OFFSET + offset);
512
513 return 0;
514}
515
516static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
517{
518 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
519 void __iomem *mmio = ndev->self_mmio;
520 u32 offset;
521
522 if (sidx < 0 || sidx >= ndev->spad_count)
523 return -EINVAL;
524
525 offset = ndev->peer_spad + (sidx << 2);
526 return readl(addr: mmio + AMD_SPAD_OFFSET + offset);
527}
528
529static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
530 int sidx, u32 val)
531{
532 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
533 void __iomem *mmio = ndev->self_mmio;
534 u32 offset;
535
536 if (sidx < 0 || sidx >= ndev->spad_count)
537 return -EINVAL;
538
539 offset = ndev->peer_spad + (sidx << 2);
540 writel(val, addr: mmio + AMD_SPAD_OFFSET + offset);
541
542 return 0;
543}
544
545static const struct ntb_dev_ops amd_ntb_ops = {
546 .mw_count = amd_ntb_mw_count,
547 .mw_get_align = amd_ntb_mw_get_align,
548 .mw_set_trans = amd_ntb_mw_set_trans,
549 .peer_mw_count = amd_ntb_peer_mw_count,
550 .peer_mw_get_addr = amd_ntb_peer_mw_get_addr,
551 .link_is_up = amd_ntb_link_is_up,
552 .link_enable = amd_ntb_link_enable,
553 .link_disable = amd_ntb_link_disable,
554 .db_valid_mask = amd_ntb_db_valid_mask,
555 .db_vector_count = amd_ntb_db_vector_count,
556 .db_vector_mask = amd_ntb_db_vector_mask,
557 .db_read = amd_ntb_db_read,
558 .db_clear = amd_ntb_db_clear,
559 .db_set_mask = amd_ntb_db_set_mask,
560 .db_clear_mask = amd_ntb_db_clear_mask,
561 .peer_db_set = amd_ntb_peer_db_set,
562 .spad_count = amd_ntb_spad_count,
563 .spad_read = amd_ntb_spad_read,
564 .spad_write = amd_ntb_spad_write,
565 .peer_spad_read = amd_ntb_peer_spad_read,
566 .peer_spad_write = amd_ntb_peer_spad_write,
567};
568
569static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit)
570{
571 void __iomem *mmio = ndev->self_mmio;
572 int reg;
573
574 reg = readl(addr: mmio + AMD_SMUACK_OFFSET);
575 reg |= bit;
576 writel(val: reg, addr: mmio + AMD_SMUACK_OFFSET);
577}
578
579static void amd_handle_event(struct amd_ntb_dev *ndev, int vec)
580{
581 void __iomem *mmio = ndev->self_mmio;
582 struct device *dev = &ndev->ntb.pdev->dev;
583 u32 status;
584
585 status = readl(addr: mmio + AMD_INTSTAT_OFFSET);
586 if (!(status & AMD_EVENT_INTMASK))
587 return;
588
589 dev_dbg(dev, "status = 0x%x and vec = %d\n", status, vec);
590
591 status &= AMD_EVENT_INTMASK;
592 switch (status) {
593 case AMD_PEER_FLUSH_EVENT:
594 ndev->peer_sta |= AMD_PEER_FLUSH_EVENT;
595 dev_info(dev, "Flush is done.\n");
596 break;
597 case AMD_PEER_RESET_EVENT:
598 case AMD_LINK_DOWN_EVENT:
599 ndev->peer_sta |= status;
600 if (status == AMD_LINK_DOWN_EVENT)
601 ndev->peer_sta &= ~AMD_LINK_UP_EVENT;
602
603 amd_ack_smu(ndev, bit: status);
604
605 /* link down first */
606 ntb_link_event(ntb: &ndev->ntb);
607 /* polling peer status */
608 schedule_delayed_work(dwork: &ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
609
610 break;
611 case AMD_PEER_D3_EVENT:
612 case AMD_PEER_PMETO_EVENT:
613 case AMD_LINK_UP_EVENT:
614 ndev->peer_sta |= status;
615 if (status == AMD_LINK_UP_EVENT)
616 ndev->peer_sta &= ~AMD_LINK_DOWN_EVENT;
617 else if (status == AMD_PEER_D3_EVENT)
618 ndev->peer_sta &= ~AMD_PEER_D0_EVENT;
619
620 amd_ack_smu(ndev, bit: status);
621
622 /* link down */
623 ntb_link_event(ntb: &ndev->ntb);
624
625 break;
626 case AMD_PEER_D0_EVENT:
627 mmio = ndev->peer_mmio;
628 status = readl(addr: mmio + AMD_PMESTAT_OFFSET);
629 /* check if this is WAKEUP event */
630 if (status & 0x1)
631 dev_info(dev, "Wakeup is done.\n");
632
633 ndev->peer_sta |= AMD_PEER_D0_EVENT;
634 ndev->peer_sta &= ~AMD_PEER_D3_EVENT;
635 amd_ack_smu(ndev, bit: AMD_PEER_D0_EVENT);
636
637 /* start a timer to poll link status */
638 schedule_delayed_work(dwork: &ndev->hb_timer,
639 AMD_LINK_HB_TIMEOUT);
640 break;
641 default:
642 dev_info(dev, "event status = 0x%x.\n", status);
643 break;
644 }
645
646 /* Clear the interrupt status */
647 writel(val: status, addr: mmio + AMD_INTSTAT_OFFSET);
648}
649
650static void amd_handle_db_event(struct amd_ntb_dev *ndev, int vec)
651{
652 struct device *dev = &ndev->ntb.pdev->dev;
653 u64 status;
654
655 status = amd_ntb_db_read(ntb: &ndev->ntb);
656
657 dev_dbg(dev, "status = 0x%llx and vec = %d\n", status, vec);
658
659 /*
660 * Since we had reserved highest order bit of DB for signaling peer of
661 * a special event, this is the only status bit we should be concerned
662 * here now.
663 */
664 if (status & BIT(ndev->db_last_bit)) {
665 ntb_db_clear(ntb: &ndev->ntb, BIT(ndev->db_last_bit));
666 /* send link down event notification */
667 ntb_link_event(ntb: &ndev->ntb);
668
669 /*
670 * If we are here, that means the peer has signalled a special
671 * event which notifies that the peer driver has been
672 * un-loaded for some reason. Since there is a chance that the
673 * peer will load its driver again sometime, we schedule link
674 * polling routine.
675 */
676 schedule_delayed_work(dwork: &ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
677 }
678}
679
680static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec)
681{
682 dev_dbg(&ndev->ntb.pdev->dev, "vec %d\n", vec);
683
684 if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1))
685 amd_handle_event(ndev, vec);
686
687 if (vec < AMD_DB_CNT) {
688 amd_handle_db_event(ndev, vec);
689 ntb_db_event(ntb: &ndev->ntb, vector: vec);
690 }
691
692 return IRQ_HANDLED;
693}
694
695static irqreturn_t ndev_vec_isr(int irq, void *dev)
696{
697 struct amd_ntb_vec *nvec = dev;
698
699 return ndev_interrupt(ndev: nvec->ndev, vec: nvec->num);
700}
701
702static irqreturn_t ndev_irq_isr(int irq, void *dev)
703{
704 struct amd_ntb_dev *ndev = dev;
705
706 return ndev_interrupt(ndev, vec: irq - ndev->ntb.pdev->irq);
707}
708
709static int ndev_init_isr(struct amd_ntb_dev *ndev,
710 int msix_min, int msix_max)
711{
712 struct pci_dev *pdev;
713 int rc, i, msix_count, node;
714
715 pdev = ndev->ntb.pdev;
716
717 node = dev_to_node(dev: &pdev->dev);
718
719 ndev->db_mask = ndev->db_valid_mask;
720
721 /* Try to set up msix irq */
722 ndev->vec = kcalloc_node(n: msix_max, size: sizeof(*ndev->vec),
723 GFP_KERNEL, node);
724 if (!ndev->vec)
725 goto err_msix_vec_alloc;
726
727 ndev->msix = kcalloc_node(n: msix_max, size: sizeof(*ndev->msix),
728 GFP_KERNEL, node);
729 if (!ndev->msix)
730 goto err_msix_alloc;
731
732 for (i = 0; i < msix_max; ++i)
733 ndev->msix[i].entry = i;
734
735 msix_count = pci_enable_msix_range(dev: pdev, entries: ndev->msix,
736 minvec: msix_min, maxvec: msix_max);
737 if (msix_count < 0)
738 goto err_msix_enable;
739
740 /* NOTE: Disable MSIX if msix count is less than 16 because of
741 * hardware limitation.
742 */
743 if (msix_count < msix_min) {
744 pci_disable_msix(dev: pdev);
745 goto err_msix_enable;
746 }
747
748 for (i = 0; i < msix_count; ++i) {
749 ndev->vec[i].ndev = ndev;
750 ndev->vec[i].num = i;
751 rc = request_irq(irq: ndev->msix[i].vector, handler: ndev_vec_isr, flags: 0,
752 name: "ndev_vec_isr", dev: &ndev->vec[i]);
753 if (rc)
754 goto err_msix_request;
755 }
756
757 dev_dbg(&pdev->dev, "Using msix interrupts\n");
758 ndev->db_count = msix_min;
759 ndev->msix_vec_count = msix_max;
760 return 0;
761
762err_msix_request:
763 while (i-- > 0)
764 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
765 pci_disable_msix(dev: pdev);
766err_msix_enable:
767 kfree(objp: ndev->msix);
768err_msix_alloc:
769 kfree(objp: ndev->vec);
770err_msix_vec_alloc:
771 ndev->msix = NULL;
772 ndev->vec = NULL;
773
774 /* Try to set up msi irq */
775 rc = pci_enable_msi(dev: pdev);
776 if (rc)
777 goto err_msi_enable;
778
779 rc = request_irq(irq: pdev->irq, handler: ndev_irq_isr, flags: 0,
780 name: "ndev_irq_isr", dev: ndev);
781 if (rc)
782 goto err_msi_request;
783
784 dev_dbg(&pdev->dev, "Using msi interrupts\n");
785 ndev->db_count = 1;
786 ndev->msix_vec_count = 1;
787 return 0;
788
789err_msi_request:
790 pci_disable_msi(dev: pdev);
791err_msi_enable:
792
793 /* Try to set up intx irq */
794 pci_intx(dev: pdev, enable: 1);
795
796 rc = request_irq(irq: pdev->irq, handler: ndev_irq_isr, IRQF_SHARED,
797 name: "ndev_irq_isr", dev: ndev);
798 if (rc)
799 goto err_intx_request;
800
801 dev_dbg(&pdev->dev, "Using intx interrupts\n");
802 ndev->db_count = 1;
803 ndev->msix_vec_count = 1;
804 return 0;
805
806err_intx_request:
807 return rc;
808}
809
810static void ndev_deinit_isr(struct amd_ntb_dev *ndev)
811{
812 struct pci_dev *pdev;
813 void __iomem *mmio = ndev->self_mmio;
814 int i;
815
816 pdev = ndev->ntb.pdev;
817
818 /* Mask all doorbell interrupts */
819 ndev->db_mask = ndev->db_valid_mask;
820 writel(val: ndev->db_mask, addr: mmio + AMD_DBMASK_OFFSET);
821
822 if (ndev->msix) {
823 i = ndev->msix_vec_count;
824 while (i--)
825 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
826 pci_disable_msix(dev: pdev);
827 kfree(objp: ndev->msix);
828 kfree(objp: ndev->vec);
829 } else {
830 free_irq(pdev->irq, ndev);
831 if (pci_dev_msi_enabled(pci_dev: pdev))
832 pci_disable_msi(dev: pdev);
833 else
834 pci_intx(dev: pdev, enable: 0);
835 }
836}
837
838static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
839 size_t count, loff_t *offp)
840{
841 struct amd_ntb_dev *ndev;
842 void __iomem *mmio;
843 char *buf;
844 size_t buf_size;
845 ssize_t ret, off;
846 union { u64 v64; u32 v32; u16 v16; } u;
847
848 ndev = filp->private_data;
849 mmio = ndev->self_mmio;
850
851 buf_size = min(count, 0x800ul);
852
853 buf = kmalloc(size: buf_size, GFP_KERNEL);
854 if (!buf)
855 return -ENOMEM;
856
857 off = 0;
858
859 off += scnprintf(buf: buf + off, size: buf_size - off,
860 fmt: "NTB Device Information:\n");
861
862 off += scnprintf(buf: buf + off, size: buf_size - off,
863 fmt: "Connection Topology -\t%s\n",
864 ntb_topo_string(topo: ndev->ntb.topo));
865
866 off += scnprintf(buf: buf + off, size: buf_size - off,
867 fmt: "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
868
869 if (!amd_link_is_up(ndev)) {
870 off += scnprintf(buf: buf + off, size: buf_size - off,
871 fmt: "Link Status -\t\tDown\n");
872 } else {
873 off += scnprintf(buf: buf + off, size: buf_size - off,
874 fmt: "Link Status -\t\tUp\n");
875 off += scnprintf(buf: buf + off, size: buf_size - off,
876 fmt: "Link Speed -\t\tPCI-E Gen %u\n",
877 NTB_LNK_STA_SPEED(ndev->lnk_sta));
878 off += scnprintf(buf: buf + off, size: buf_size - off,
879 fmt: "Link Width -\t\tx%u\n",
880 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
881 }
882
883 off += scnprintf(buf: buf + off, size: buf_size - off,
884 fmt: "Memory Window Count -\t%u\n", ndev->mw_count);
885 off += scnprintf(buf: buf + off, size: buf_size - off,
886 fmt: "Scratchpad Count -\t%u\n", ndev->spad_count);
887 off += scnprintf(buf: buf + off, size: buf_size - off,
888 fmt: "Doorbell Count -\t%u\n", ndev->db_count);
889 off += scnprintf(buf: buf + off, size: buf_size - off,
890 fmt: "MSIX Vector Count -\t%u\n", ndev->msix_vec_count);
891
892 off += scnprintf(buf: buf + off, size: buf_size - off,
893 fmt: "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
894
895 u.v32 = readl(addr: ndev->self_mmio + AMD_DBMASK_OFFSET);
896 off += scnprintf(buf: buf + off, size: buf_size - off,
897 fmt: "Doorbell Mask -\t\t\t%#06x\n", u.v32);
898
899 u.v32 = readl(addr: mmio + AMD_DBSTAT_OFFSET);
900 off += scnprintf(buf: buf + off, size: buf_size - off,
901 fmt: "Doorbell Bell -\t\t\t%#06x\n", u.v32);
902
903 off += scnprintf(buf: buf + off, size: buf_size - off,
904 fmt: "\nNTB Incoming XLAT:\n");
905
906 u.v64 = read64(addr: mmio + AMD_BAR1XLAT_OFFSET);
907 off += scnprintf(buf: buf + off, size: buf_size - off,
908 fmt: "XLAT1 -\t\t%#018llx\n", u.v64);
909
910 u.v64 = read64(addr: ndev->self_mmio + AMD_BAR23XLAT_OFFSET);
911 off += scnprintf(buf: buf + off, size: buf_size - off,
912 fmt: "XLAT23 -\t\t%#018llx\n", u.v64);
913
914 u.v64 = read64(addr: ndev->self_mmio + AMD_BAR45XLAT_OFFSET);
915 off += scnprintf(buf: buf + off, size: buf_size - off,
916 fmt: "XLAT45 -\t\t%#018llx\n", u.v64);
917
918 u.v32 = readl(addr: mmio + AMD_BAR1LMT_OFFSET);
919 off += scnprintf(buf: buf + off, size: buf_size - off,
920 fmt: "LMT1 -\t\t\t%#06x\n", u.v32);
921
922 u.v64 = read64(addr: ndev->self_mmio + AMD_BAR23LMT_OFFSET);
923 off += scnprintf(buf: buf + off, size: buf_size - off,
924 fmt: "LMT23 -\t\t\t%#018llx\n", u.v64);
925
926 u.v64 = read64(addr: ndev->self_mmio + AMD_BAR45LMT_OFFSET);
927 off += scnprintf(buf: buf + off, size: buf_size - off,
928 fmt: "LMT45 -\t\t\t%#018llx\n", u.v64);
929
930 ret = simple_read_from_buffer(to: ubuf, count, ppos: offp, from: buf, available: off);
931 kfree(objp: buf);
932 return ret;
933}
934
935static void ndev_init_debugfs(struct amd_ntb_dev *ndev)
936{
937 if (!debugfs_dir) {
938 ndev->debugfs_dir = NULL;
939 ndev->debugfs_info = NULL;
940 } else {
941 ndev->debugfs_dir =
942 debugfs_create_dir(name: pci_name(pdev: ndev->ntb.pdev),
943 parent: debugfs_dir);
944 ndev->debugfs_info =
945 debugfs_create_file(name: "info", S_IRUSR,
946 parent: ndev->debugfs_dir, data: ndev,
947 fops: &amd_ntb_debugfs_info);
948 }
949}
950
951static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev)
952{
953 debugfs_remove_recursive(dentry: ndev->debugfs_dir);
954}
955
956static inline void ndev_init_struct(struct amd_ntb_dev *ndev,
957 struct pci_dev *pdev)
958{
959 ndev->ntb.pdev = pdev;
960 ndev->ntb.topo = NTB_TOPO_NONE;
961 ndev->ntb.ops = &amd_ntb_ops;
962 ndev->int_mask = AMD_EVENT_INTMASK;
963 spin_lock_init(&ndev->db_mask_lock);
964}
965
966static int amd_poll_link(struct amd_ntb_dev *ndev)
967{
968 void __iomem *mmio = ndev->peer_mmio;
969 u32 reg;
970
971 reg = readl(addr: mmio + AMD_SIDEINFO_OFFSET);
972 reg &= AMD_SIDE_READY;
973
974 dev_dbg(&ndev->ntb.pdev->dev, "%s: reg_val = 0x%x.\n", __func__, reg);
975
976 ndev->cntl_sta = reg;
977
978 amd_ntb_get_link_status(ndev);
979
980 return ndev->cntl_sta;
981}
982
983static void amd_link_hb(struct work_struct *work)
984{
985 struct amd_ntb_dev *ndev = hb_ndev(work);
986
987 if (amd_poll_link(ndev))
988 ntb_link_event(ntb: &ndev->ntb);
989
990 if (!amd_link_is_up(ndev))
991 schedule_delayed_work(dwork: &ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
992}
993
994static int amd_init_isr(struct amd_ntb_dev *ndev)
995{
996 return ndev_init_isr(ndev, msix_min: AMD_DB_CNT, msix_max: AMD_MSIX_VECTOR_CNT);
997}
998
999static void amd_set_side_info_reg(struct amd_ntb_dev *ndev, bool peer)
1000{
1001 void __iomem *mmio = NULL;
1002 unsigned int reg;
1003
1004 if (peer)
1005 mmio = ndev->peer_mmio;
1006 else
1007 mmio = ndev->self_mmio;
1008
1009 reg = readl(addr: mmio + AMD_SIDEINFO_OFFSET);
1010 if (!(reg & AMD_SIDE_READY)) {
1011 reg |= AMD_SIDE_READY;
1012 writel(val: reg, addr: mmio + AMD_SIDEINFO_OFFSET);
1013 }
1014}
1015
1016static void amd_clear_side_info_reg(struct amd_ntb_dev *ndev, bool peer)
1017{
1018 void __iomem *mmio = NULL;
1019 unsigned int reg;
1020
1021 if (peer)
1022 mmio = ndev->peer_mmio;
1023 else
1024 mmio = ndev->self_mmio;
1025
1026 reg = readl(addr: mmio + AMD_SIDEINFO_OFFSET);
1027 if (reg & AMD_SIDE_READY) {
1028 reg &= ~AMD_SIDE_READY;
1029 writel(val: reg, addr: mmio + AMD_SIDEINFO_OFFSET);
1030 readl(addr: mmio + AMD_SIDEINFO_OFFSET);
1031 }
1032}
1033
1034static void amd_init_side_info(struct amd_ntb_dev *ndev)
1035{
1036 void __iomem *mmio = ndev->self_mmio;
1037 u32 ntb_ctl;
1038
1039 amd_set_side_info_reg(ndev, peer: false);
1040
1041 ntb_ctl = readl(addr: mmio + AMD_CNTL_OFFSET);
1042 ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL);
1043 writel(val: ntb_ctl, addr: mmio + AMD_CNTL_OFFSET);
1044}
1045
1046static void amd_deinit_side_info(struct amd_ntb_dev *ndev)
1047{
1048 void __iomem *mmio = ndev->self_mmio;
1049 u32 ntb_ctl;
1050
1051 amd_clear_side_info_reg(ndev, peer: false);
1052
1053 ntb_ctl = readl(addr: mmio + AMD_CNTL_OFFSET);
1054 ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL);
1055 writel(val: ntb_ctl, addr: mmio + AMD_CNTL_OFFSET);
1056}
1057
1058static int amd_init_ntb(struct amd_ntb_dev *ndev)
1059{
1060 void __iomem *mmio = ndev->self_mmio;
1061
1062 ndev->mw_count = ndev->dev_data->mw_count;
1063 ndev->spad_count = AMD_SPADS_CNT;
1064 ndev->db_count = AMD_DB_CNT;
1065
1066 switch (ndev->ntb.topo) {
1067 case NTB_TOPO_PRI:
1068 case NTB_TOPO_SEC:
1069 ndev->spad_count >>= 1;
1070 if (ndev->ntb.topo == NTB_TOPO_PRI) {
1071 ndev->self_spad = 0;
1072 ndev->peer_spad = 0x20;
1073 } else {
1074 ndev->self_spad = 0x20;
1075 ndev->peer_spad = 0;
1076 }
1077
1078 INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb);
1079 schedule_delayed_work(dwork: &ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
1080
1081 break;
1082 default:
1083 dev_err(&ndev->ntb.pdev->dev,
1084 "AMD NTB does not support B2B mode.\n");
1085 return -EINVAL;
1086 }
1087
1088 /* Mask event interrupts */
1089 writel(val: ndev->int_mask, addr: mmio + AMD_INTMASK_OFFSET);
1090
1091 return 0;
1092}
1093
1094static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev)
1095{
1096 void __iomem *mmio = ndev->self_mmio;
1097 u32 info;
1098
1099 info = readl(addr: mmio + AMD_SIDEINFO_OFFSET);
1100 if (info & AMD_SIDE_MASK)
1101 return NTB_TOPO_SEC;
1102 else
1103 return NTB_TOPO_PRI;
1104}
1105
1106static int amd_init_dev(struct amd_ntb_dev *ndev)
1107{
1108 void __iomem *mmio = ndev->self_mmio;
1109 struct pci_dev *pdev;
1110 int rc = 0;
1111
1112 pdev = ndev->ntb.pdev;
1113
1114 ndev->ntb.topo = amd_get_topo(ndev);
1115 dev_dbg(&pdev->dev, "AMD NTB topo is %s\n",
1116 ntb_topo_string(ndev->ntb.topo));
1117
1118 rc = amd_init_ntb(ndev);
1119 if (rc)
1120 return rc;
1121
1122 rc = amd_init_isr(ndev);
1123 if (rc) {
1124 dev_err(&pdev->dev, "fail to init isr.\n");
1125 return rc;
1126 }
1127
1128 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1129 /*
1130 * We reserve the highest order bit of the DB register which will
1131 * be used to notify peer when the driver on this side is being
1132 * un-loaded.
1133 */
1134 ndev->db_last_bit =
1135 find_last_bit(addr: (unsigned long *)&ndev->db_valid_mask,
1136 hweight64(ndev->db_valid_mask));
1137 writew(val: (u16)~BIT(ndev->db_last_bit), addr: mmio + AMD_DBMASK_OFFSET);
1138 /*
1139 * Since now there is one less bit to account for, the DB count
1140 * and DB mask should be adjusted accordingly.
1141 */
1142 ndev->db_count -= 1;
1143 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1144
1145 /* Enable Link-Up and Link-Down event interrupts */
1146 ndev->int_mask &= ~(AMD_LINK_UP_EVENT | AMD_LINK_DOWN_EVENT);
1147 writel(val: ndev->int_mask, addr: mmio + AMD_INTMASK_OFFSET);
1148
1149 return 0;
1150}
1151
1152static void amd_deinit_dev(struct amd_ntb_dev *ndev)
1153{
1154 cancel_delayed_work_sync(dwork: &ndev->hb_timer);
1155
1156 ndev_deinit_isr(ndev);
1157}
1158
1159static int amd_ntb_init_pci(struct amd_ntb_dev *ndev,
1160 struct pci_dev *pdev)
1161{
1162 int rc;
1163
1164 pci_set_drvdata(pdev, data: ndev);
1165
1166 rc = pci_enable_device(dev: pdev);
1167 if (rc)
1168 goto err_pci_enable;
1169
1170 rc = pci_request_regions(pdev, NTB_NAME);
1171 if (rc)
1172 goto err_pci_regions;
1173
1174 pci_set_master(dev: pdev);
1175
1176 rc = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(64));
1177 if (rc) {
1178 rc = dma_set_mask_and_coherent(dev: &pdev->dev, DMA_BIT_MASK(32));
1179 if (rc)
1180 goto err_dma_mask;
1181 dev_warn(&pdev->dev, "Cannot DMA highmem\n");
1182 }
1183
1184 ndev->self_mmio = pci_iomap(dev: pdev, bar: 0, max: 0);
1185 if (!ndev->self_mmio) {
1186 rc = -EIO;
1187 goto err_dma_mask;
1188 }
1189 ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET;
1190
1191 return 0;
1192
1193err_dma_mask:
1194 pci_release_regions(pdev);
1195err_pci_regions:
1196 pci_disable_device(dev: pdev);
1197err_pci_enable:
1198 pci_set_drvdata(pdev, NULL);
1199 return rc;
1200}
1201
1202static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev)
1203{
1204 struct pci_dev *pdev = ndev->ntb.pdev;
1205
1206 pci_iounmap(dev: pdev, ndev->self_mmio);
1207
1208 pci_release_regions(pdev);
1209 pci_disable_device(dev: pdev);
1210 pci_set_drvdata(pdev, NULL);
1211}
1212
1213static int amd_ntb_pci_probe(struct pci_dev *pdev,
1214 const struct pci_device_id *id)
1215{
1216 struct amd_ntb_dev *ndev;
1217 int rc, node;
1218
1219 node = dev_to_node(dev: &pdev->dev);
1220
1221 ndev = kzalloc_node(size: sizeof(*ndev), GFP_KERNEL, node);
1222 if (!ndev) {
1223 rc = -ENOMEM;
1224 goto err_ndev;
1225 }
1226
1227 ndev->dev_data = (struct ntb_dev_data *)id->driver_data;
1228
1229 ndev_init_struct(ndev, pdev);
1230
1231 rc = amd_ntb_init_pci(ndev, pdev);
1232 if (rc)
1233 goto err_init_pci;
1234
1235 rc = amd_init_dev(ndev);
1236 if (rc)
1237 goto err_init_dev;
1238
1239 /* write side info */
1240 amd_init_side_info(ndev);
1241
1242 amd_poll_link(ndev);
1243
1244 ndev_init_debugfs(ndev);
1245
1246 rc = ntb_register_device(ntb: &ndev->ntb);
1247 if (rc)
1248 goto err_register;
1249
1250 dev_info(&pdev->dev, "NTB device registered.\n");
1251
1252 return 0;
1253
1254err_register:
1255 ndev_deinit_debugfs(ndev);
1256 amd_deinit_dev(ndev);
1257err_init_dev:
1258 amd_ntb_deinit_pci(ndev);
1259err_init_pci:
1260 kfree(objp: ndev);
1261err_ndev:
1262 return rc;
1263}
1264
1265static void amd_ntb_pci_remove(struct pci_dev *pdev)
1266{
1267 struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
1268
1269 /*
1270 * Clear the READY bit in SIDEINFO register before sending DB event
1271 * to the peer. This will make sure that when the peer handles the
1272 * DB event, it correctly reads this bit as being 0.
1273 */
1274 amd_deinit_side_info(ndev);
1275 ntb_peer_db_set(ntb: &ndev->ntb, BIT_ULL(ndev->db_last_bit));
1276 ntb_unregister_device(ntb: &ndev->ntb);
1277 ndev_deinit_debugfs(ndev);
1278 amd_deinit_dev(ndev);
1279 amd_ntb_deinit_pci(ndev);
1280 kfree(objp: ndev);
1281}
1282
1283static void amd_ntb_pci_shutdown(struct pci_dev *pdev)
1284{
1285 struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
1286
1287 /* Send link down notification */
1288 ntb_link_event(ntb: &ndev->ntb);
1289
1290 amd_deinit_side_info(ndev);
1291 ntb_peer_db_set(ntb: &ndev->ntb, BIT_ULL(ndev->db_last_bit));
1292 ntb_unregister_device(ntb: &ndev->ntb);
1293 ndev_deinit_debugfs(ndev);
1294 amd_deinit_dev(ndev);
1295 amd_ntb_deinit_pci(ndev);
1296 kfree(objp: ndev);
1297}
1298
1299static const struct file_operations amd_ntb_debugfs_info = {
1300 .owner = THIS_MODULE,
1301 .open = simple_open,
1302 .read = ndev_debugfs_read,
1303};
1304
1305static const struct ntb_dev_data dev_data[] = {
1306 { /* for device 145b */
1307 .mw_count = 3,
1308 .mw_idx = 1,
1309 },
1310 { /* for device 148b */
1311 .mw_count = 2,
1312 .mw_idx = 2,
1313 },
1314};
1315
1316static const struct pci_device_id amd_ntb_pci_tbl[] = {
1317 { PCI_VDEVICE(AMD, 0x145b), (kernel_ulong_t)&dev_data[0] },
1318 { PCI_VDEVICE(AMD, 0x148b), (kernel_ulong_t)&dev_data[1] },
1319 { PCI_VDEVICE(AMD, 0x14c0), (kernel_ulong_t)&dev_data[1] },
1320 { PCI_VDEVICE(AMD, 0x14c3), (kernel_ulong_t)&dev_data[1] },
1321 { PCI_VDEVICE(HYGON, 0x145b), (kernel_ulong_t)&dev_data[0] },
1322 { 0, }
1323};
1324MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl);
1325
1326static struct pci_driver amd_ntb_pci_driver = {
1327 .name = KBUILD_MODNAME,
1328 .id_table = amd_ntb_pci_tbl,
1329 .probe = amd_ntb_pci_probe,
1330 .remove = amd_ntb_pci_remove,
1331 .shutdown = amd_ntb_pci_shutdown,
1332};
1333
1334static int __init amd_ntb_pci_driver_init(void)
1335{
1336 int ret;
1337 pr_info("%s %s\n", NTB_DESC, NTB_VER);
1338
1339 if (debugfs_initialized())
1340 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1341
1342 ret = pci_register_driver(&amd_ntb_pci_driver);
1343 if (ret)
1344 debugfs_remove_recursive(dentry: debugfs_dir);
1345
1346 return ret;
1347}
1348module_init(amd_ntb_pci_driver_init);
1349
1350static void __exit amd_ntb_pci_driver_exit(void)
1351{
1352 pci_unregister_driver(dev: &amd_ntb_pci_driver);
1353 debugfs_remove_recursive(dentry: debugfs_dir);
1354}
1355module_exit(amd_ntb_pci_driver_exit);
1356

source code of linux/drivers/ntb/hw/amd/ntb_hw_amd.c