1/*
2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
4 *
5 * GPL LICENSE SUMMARY
6 *
7 * Copyright(c) 2012 Intel Corporation. All rights reserved.
8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of version 2 of the GNU General Public License as
12 * published by the Free Software Foundation.
13 *
14 * BSD LICENSE
15 *
16 * Copyright(c) 2012 Intel Corporation. All rights reserved.
17 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
18 *
19 * Redistribution and use in source and binary forms, with or without
20 * modification, are permitted provided that the following conditions
21 * are met:
22 *
23 * * Redistributions of source code must retain the above copyright
24 * notice, this list of conditions and the following disclaimer.
25 * * Redistributions in binary form must reproduce the above copy
26 * notice, this list of conditions and the following disclaimer in
27 * the documentation and/or other materials provided with the
28 * distribution.
29 * * Neither the name of Intel Corporation nor the names of its
30 * contributors may be used to endorse or promote products derived
31 * from this software without specific prior written permission.
32 *
33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
34 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
35 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
36 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
37 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
38 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
39 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
40 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
41 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
44 *
45 * PCIe NTB Transport Linux driver
46 *
47 * Contact Information:
48 * Jon Mason <jon.mason@intel.com>
49 */
50#include <linux/debugfs.h>
51#include <linux/delay.h>
52#include <linux/dmaengine.h>
53#include <linux/dma-mapping.h>
54#include <linux/errno.h>
55#include <linux/export.h>
56#include <linux/interrupt.h>
57#include <linux/module.h>
58#include <linux/pci.h>
59#include <linux/slab.h>
60#include <linux/types.h>
61#include <linux/uaccess.h>
62#include "linux/ntb.h"
63#include "linux/ntb_transport.h"
64
65#define NTB_TRANSPORT_VERSION 4
66#define NTB_TRANSPORT_VER "4"
67#define NTB_TRANSPORT_NAME "ntb_transport"
68#define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB"
69#define NTB_TRANSPORT_MIN_SPADS (MW0_SZ_HIGH + 2)
70
71MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
72MODULE_VERSION(NTB_TRANSPORT_VER);
73MODULE_LICENSE("Dual BSD/GPL");
74MODULE_AUTHOR("Intel Corporation");
75
76static unsigned long max_mw_size;
77module_param(max_mw_size, ulong, 0644);
78MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");
79
80static unsigned int transport_mtu = 0x10000;
81module_param(transport_mtu, uint, 0644);
82MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets");
83
84static unsigned char max_num_clients;
85module_param(max_num_clients, byte, 0644);
86MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients");
87
88static unsigned int copy_bytes = 1024;
89module_param(copy_bytes, uint, 0644);
90MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA");
91
92static bool use_dma;
93module_param(use_dma, bool, 0644);
94MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy");
95
96static struct dentry *nt_debugfs_dir;
97
98/* Only two-ports NTB devices are supported */
99#define PIDX NTB_DEF_PEER_IDX
100
101struct ntb_queue_entry {
102 /* ntb_queue list reference */
103 struct list_head entry;
104 /* pointers to data to be transferred */
105 void *cb_data;
106 void *buf;
107 unsigned int len;
108 unsigned int flags;
109 int retries;
110 int errors;
111 unsigned int tx_index;
112 unsigned int rx_index;
113
114 struct ntb_transport_qp *qp;
115 union {
116 struct ntb_payload_header __iomem *tx_hdr;
117 struct ntb_payload_header *rx_hdr;
118 };
119};
120
121struct ntb_rx_info {
122 unsigned int entry;
123};
124
125struct ntb_transport_qp {
126 struct ntb_transport_ctx *transport;
127 struct ntb_dev *ndev;
128 void *cb_data;
129 struct dma_chan *tx_dma_chan;
130 struct dma_chan *rx_dma_chan;
131
132 bool client_ready;
133 bool link_is_up;
134 bool active;
135
136 u8 qp_num; /* Only 64 QP's are allowed. 0-63 */
137 u64 qp_bit;
138
139 struct ntb_rx_info __iomem *rx_info;
140 struct ntb_rx_info *remote_rx_info;
141
142 void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data,
143 void *data, int len);
144 struct list_head tx_free_q;
145 spinlock_t ntb_tx_free_q_lock;
146 void __iomem *tx_mw;
147 phys_addr_t tx_mw_phys;
148 size_t tx_mw_size;
149 dma_addr_t tx_mw_dma_addr;
150 unsigned int tx_index;
151 unsigned int tx_max_entry;
152 unsigned int tx_max_frame;
153
154 void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data,
155 void *data, int len);
156 struct list_head rx_post_q;
157 struct list_head rx_pend_q;
158 struct list_head rx_free_q;
159 /* ntb_rx_q_lock: synchronize access to rx_XXXX_q */
160 spinlock_t ntb_rx_q_lock;
161 void *rx_buff;
162 unsigned int rx_index;
163 unsigned int rx_max_entry;
164 unsigned int rx_max_frame;
165 unsigned int rx_alloc_entry;
166 dma_cookie_t last_cookie;
167 struct tasklet_struct rxc_db_work;
168
169 void (*event_handler)(void *data, int status);
170 struct delayed_work link_work;
171 struct work_struct link_cleanup;
172
173 struct dentry *debugfs_dir;
174 struct dentry *debugfs_stats;
175
176 /* Stats */
177 u64 rx_bytes;
178 u64 rx_pkts;
179 u64 rx_ring_empty;
180 u64 rx_err_no_buf;
181 u64 rx_err_oflow;
182 u64 rx_err_ver;
183 u64 rx_memcpy;
184 u64 rx_async;
185 u64 tx_bytes;
186 u64 tx_pkts;
187 u64 tx_ring_full;
188 u64 tx_err_no_buf;
189 u64 tx_memcpy;
190 u64 tx_async;
191};
192
193struct ntb_transport_mw {
194 phys_addr_t phys_addr;
195 resource_size_t phys_size;
196 void __iomem *vbase;
197 size_t xlat_size;
198 size_t buff_size;
199 size_t alloc_size;
200 void *alloc_addr;
201 void *virt_addr;
202 dma_addr_t dma_addr;
203};
204
205struct ntb_transport_client_dev {
206 struct list_head entry;
207 struct ntb_transport_ctx *nt;
208 struct device dev;
209};
210
211struct ntb_transport_ctx {
212 struct list_head entry;
213 struct list_head client_devs;
214
215 struct ntb_dev *ndev;
216
217 struct ntb_transport_mw *mw_vec;
218 struct ntb_transport_qp *qp_vec;
219 unsigned int mw_count;
220 unsigned int qp_count;
221 u64 qp_bitmap;
222 u64 qp_bitmap_free;
223
224 bool link_is_up;
225 struct delayed_work link_work;
226 struct work_struct link_cleanup;
227
228 struct dentry *debugfs_node_dir;
229};
230
231enum {
232 DESC_DONE_FLAG = BIT(0),
233 LINK_DOWN_FLAG = BIT(1),
234};
235
236struct ntb_payload_header {
237 unsigned int ver;
238 unsigned int len;
239 unsigned int flags;
240};
241
242enum {
243 VERSION = 0,
244 QP_LINKS,
245 NUM_QPS,
246 NUM_MWS,
247 MW0_SZ_HIGH,
248 MW0_SZ_LOW,
249};
250
251#define dev_client_dev(__dev) \
252 container_of((__dev), struct ntb_transport_client_dev, dev)
253
254#define drv_client(__drv) \
255 container_of((__drv), struct ntb_transport_client, driver)
256
257#define QP_TO_MW(nt, qp) ((qp) % nt->mw_count)
258#define NTB_QP_DEF_NUM_ENTRIES 100
259#define NTB_LINK_DOWN_TIMEOUT 10
260
261static void ntb_transport_rxc_db(unsigned long data);
262static const struct ntb_ctx_ops ntb_transport_ops;
263static struct ntb_client ntb_transport_client;
264static int ntb_async_tx_submit(struct ntb_transport_qp *qp,
265 struct ntb_queue_entry *entry);
266static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset);
267static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset);
268static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset);
269
270
271static int ntb_transport_bus_match(struct device *dev,
272 struct device_driver *drv)
273{
274 return !strncmp(dev_name(dev), drv->name, strlen(drv->name));
275}
276
277static int ntb_transport_bus_probe(struct device *dev)
278{
279 const struct ntb_transport_client *client;
280 int rc = -EINVAL;
281
282 get_device(dev);
283
284 client = drv_client(dev->driver);
285 rc = client->probe(dev);
286 if (rc)
287 put_device(dev);
288
289 return rc;
290}
291
292static int ntb_transport_bus_remove(struct device *dev)
293{
294 const struct ntb_transport_client *client;
295
296 client = drv_client(dev->driver);
297 client->remove(dev);
298
299 put_device(dev);
300
301 return 0;
302}
303
304static struct bus_type ntb_transport_bus = {
305 .name = "ntb_transport",
306 .match = ntb_transport_bus_match,
307 .probe = ntb_transport_bus_probe,
308 .remove = ntb_transport_bus_remove,
309};
310
311static LIST_HEAD(ntb_transport_list);
312
313static int ntb_bus_init(struct ntb_transport_ctx *nt)
314{
315 list_add_tail(&nt->entry, &ntb_transport_list);
316 return 0;
317}
318
319static void ntb_bus_remove(struct ntb_transport_ctx *nt)
320{
321 struct ntb_transport_client_dev *client_dev, *cd;
322
323 list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) {
324 dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n",
325 dev_name(&client_dev->dev));
326 list_del(&client_dev->entry);
327 device_unregister(&client_dev->dev);
328 }
329
330 list_del(&nt->entry);
331}
332
333static void ntb_transport_client_release(struct device *dev)
334{
335 struct ntb_transport_client_dev *client_dev;
336
337 client_dev = dev_client_dev(dev);
338 kfree(client_dev);
339}
340
341/**
342 * ntb_transport_unregister_client_dev - Unregister NTB client device
343 * @device_name: Name of NTB client device
344 *
345 * Unregister an NTB client device with the NTB transport layer
346 */
347void ntb_transport_unregister_client_dev(char *device_name)
348{
349 struct ntb_transport_client_dev *client, *cd;
350 struct ntb_transport_ctx *nt;
351
352 list_for_each_entry(nt, &ntb_transport_list, entry)
353 list_for_each_entry_safe(client, cd, &nt->client_devs, entry)
354 if (!strncmp(dev_name(&client->dev), device_name,
355 strlen(device_name))) {
356 list_del(&client->entry);
357 device_unregister(&client->dev);
358 }
359}
360EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev);
361
362/**
363 * ntb_transport_register_client_dev - Register NTB client device
364 * @device_name: Name of NTB client device
365 *
366 * Register an NTB client device with the NTB transport layer
367 */
368int ntb_transport_register_client_dev(char *device_name)
369{
370 struct ntb_transport_client_dev *client_dev;
371 struct ntb_transport_ctx *nt;
372 int node;
373 int rc, i = 0;
374
375 if (list_empty(&ntb_transport_list))
376 return -ENODEV;
377
378 list_for_each_entry(nt, &ntb_transport_list, entry) {
379 struct device *dev;
380
381 node = dev_to_node(&nt->ndev->dev);
382
383 client_dev = kzalloc_node(sizeof(*client_dev),
384 GFP_KERNEL, node);
385 if (!client_dev) {
386 rc = -ENOMEM;
387 goto err;
388 }
389
390 dev = &client_dev->dev;
391
392 /* setup and register client devices */
393 dev_set_name(dev, "%s%d", device_name, i);
394 dev->bus = &ntb_transport_bus;
395 dev->release = ntb_transport_client_release;
396 dev->parent = &nt->ndev->dev;
397
398 rc = device_register(dev);
399 if (rc) {
400 kfree(client_dev);
401 goto err;
402 }
403
404 list_add_tail(&client_dev->entry, &nt->client_devs);
405 i++;
406 }
407
408 return 0;
409
410err:
411 ntb_transport_unregister_client_dev(device_name);
412
413 return rc;
414}
415EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev);
416
417/**
418 * ntb_transport_register_client - Register NTB client driver
419 * @drv: NTB client driver to be registered
420 *
421 * Register an NTB client driver with the NTB transport layer
422 *
423 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
424 */
425int ntb_transport_register_client(struct ntb_transport_client *drv)
426{
427 drv->driver.bus = &ntb_transport_bus;
428
429 if (list_empty(&ntb_transport_list))
430 return -ENODEV;
431
432 return driver_register(&drv->driver);
433}
434EXPORT_SYMBOL_GPL(ntb_transport_register_client);
435
436/**
437 * ntb_transport_unregister_client - Unregister NTB client driver
438 * @drv: NTB client driver to be unregistered
439 *
440 * Unregister an NTB client driver with the NTB transport layer
441 *
442 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
443 */
444void ntb_transport_unregister_client(struct ntb_transport_client *drv)
445{
446 driver_unregister(&drv->driver);
447}
448EXPORT_SYMBOL_GPL(ntb_transport_unregister_client);
449
450static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
451 loff_t *offp)
452{
453 struct ntb_transport_qp *qp;
454 char *buf;
455 ssize_t ret, out_offset, out_count;
456
457 qp = filp->private_data;
458
459 if (!qp || !qp->link_is_up)
460 return 0;
461
462 out_count = 1000;
463
464 buf = kmalloc(out_count, GFP_KERNEL);
465 if (!buf)
466 return -ENOMEM;
467
468 out_offset = 0;
469 out_offset += snprintf(buf + out_offset, out_count - out_offset,
470 "\nNTB QP stats:\n\n");
471 out_offset += snprintf(buf + out_offset, out_count - out_offset,
472 "rx_bytes - \t%llu\n", qp->rx_bytes);
473 out_offset += snprintf(buf + out_offset, out_count - out_offset,
474 "rx_pkts - \t%llu\n", qp->rx_pkts);
475 out_offset += snprintf(buf + out_offset, out_count - out_offset,
476 "rx_memcpy - \t%llu\n", qp->rx_memcpy);
477 out_offset += snprintf(buf + out_offset, out_count - out_offset,
478 "rx_async - \t%llu\n", qp->rx_async);
479 out_offset += snprintf(buf + out_offset, out_count - out_offset,
480 "rx_ring_empty - %llu\n", qp->rx_ring_empty);
481 out_offset += snprintf(buf + out_offset, out_count - out_offset,
482 "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
483 out_offset += snprintf(buf + out_offset, out_count - out_offset,
484 "rx_err_oflow - \t%llu\n", qp->rx_err_oflow);
485 out_offset += snprintf(buf + out_offset, out_count - out_offset,
486 "rx_err_ver - \t%llu\n", qp->rx_err_ver);
487 out_offset += snprintf(buf + out_offset, out_count - out_offset,
488 "rx_buff - \t0x%p\n", qp->rx_buff);
489 out_offset += snprintf(buf + out_offset, out_count - out_offset,
490 "rx_index - \t%u\n", qp->rx_index);
491 out_offset += snprintf(buf + out_offset, out_count - out_offset,
492 "rx_max_entry - \t%u\n", qp->rx_max_entry);
493 out_offset += snprintf(buf + out_offset, out_count - out_offset,
494 "rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry);
495
496 out_offset += snprintf(buf + out_offset, out_count - out_offset,
497 "tx_bytes - \t%llu\n", qp->tx_bytes);
498 out_offset += snprintf(buf + out_offset, out_count - out_offset,
499 "tx_pkts - \t%llu\n", qp->tx_pkts);
500 out_offset += snprintf(buf + out_offset, out_count - out_offset,
501 "tx_memcpy - \t%llu\n", qp->tx_memcpy);
502 out_offset += snprintf(buf + out_offset, out_count - out_offset,
503 "tx_async - \t%llu\n", qp->tx_async);
504 out_offset += snprintf(buf + out_offset, out_count - out_offset,
505 "tx_ring_full - \t%llu\n", qp->tx_ring_full);
506 out_offset += snprintf(buf + out_offset, out_count - out_offset,
507 "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
508 out_offset += snprintf(buf + out_offset, out_count - out_offset,
509 "tx_mw - \t0x%p\n", qp->tx_mw);
510 out_offset += snprintf(buf + out_offset, out_count - out_offset,
511 "tx_index (H) - \t%u\n", qp->tx_index);
512 out_offset += snprintf(buf + out_offset, out_count - out_offset,
513 "RRI (T) - \t%u\n",
514 qp->remote_rx_info->entry);
515 out_offset += snprintf(buf + out_offset, out_count - out_offset,
516 "tx_max_entry - \t%u\n", qp->tx_max_entry);
517 out_offset += snprintf(buf + out_offset, out_count - out_offset,
518 "free tx - \t%u\n",
519 ntb_transport_tx_free_entry(qp));
520
521 out_offset += snprintf(buf + out_offset, out_count - out_offset,
522 "\n");
523 out_offset += snprintf(buf + out_offset, out_count - out_offset,
524 "Using TX DMA - \t%s\n",
525 qp->tx_dma_chan ? "Yes" : "No");
526 out_offset += snprintf(buf + out_offset, out_count - out_offset,
527 "Using RX DMA - \t%s\n",
528 qp->rx_dma_chan ? "Yes" : "No");
529 out_offset += snprintf(buf + out_offset, out_count - out_offset,
530 "QP Link - \t%s\n",
531 qp->link_is_up ? "Up" : "Down");
532 out_offset += snprintf(buf + out_offset, out_count - out_offset,
533 "\n");
534
535 if (out_offset > out_count)
536 out_offset = out_count;
537
538 ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
539 kfree(buf);
540 return ret;
541}
542
543static const struct file_operations ntb_qp_debugfs_stats = {
544 .owner = THIS_MODULE,
545 .open = simple_open,
546 .read = debugfs_read,
547};
548
549static void ntb_list_add(spinlock_t *lock, struct list_head *entry,
550 struct list_head *list)
551{
552 unsigned long flags;
553
554 spin_lock_irqsave(lock, flags);
555 list_add_tail(entry, list);
556 spin_unlock_irqrestore(lock, flags);
557}
558
559static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock,
560 struct list_head *list)
561{
562 struct ntb_queue_entry *entry;
563 unsigned long flags;
564
565 spin_lock_irqsave(lock, flags);
566 if (list_empty(list)) {
567 entry = NULL;
568 goto out;
569 }
570 entry = list_first_entry(list, struct ntb_queue_entry, entry);
571 list_del(&entry->entry);
572
573out:
574 spin_unlock_irqrestore(lock, flags);
575
576 return entry;
577}
578
579static struct ntb_queue_entry *ntb_list_mv(spinlock_t *lock,
580 struct list_head *list,
581 struct list_head *to_list)
582{
583 struct ntb_queue_entry *entry;
584 unsigned long flags;
585
586 spin_lock_irqsave(lock, flags);
587
588 if (list_empty(list)) {
589 entry = NULL;
590 } else {
591 entry = list_first_entry(list, struct ntb_queue_entry, entry);
592 list_move_tail(&entry->entry, to_list);
593 }
594
595 spin_unlock_irqrestore(lock, flags);
596
597 return entry;
598}
599
600static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
601 unsigned int qp_num)
602{
603 struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
604 struct ntb_transport_mw *mw;
605 struct ntb_dev *ndev = nt->ndev;
606 struct ntb_queue_entry *entry;
607 unsigned int rx_size, num_qps_mw;
608 unsigned int mw_num, mw_count, qp_count;
609 unsigned int i;
610 int node;
611
612 mw_count = nt->mw_count;
613 qp_count = nt->qp_count;
614
615 mw_num = QP_TO_MW(nt, qp_num);
616 mw = &nt->mw_vec[mw_num];
617
618 if (!mw->virt_addr)
619 return -ENOMEM;
620
621 if (mw_num < qp_count % mw_count)
622 num_qps_mw = qp_count / mw_count + 1;
623 else
624 num_qps_mw = qp_count / mw_count;
625
626 rx_size = (unsigned int)mw->xlat_size / num_qps_mw;
627 qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count);
628 rx_size -= sizeof(struct ntb_rx_info);
629
630 qp->remote_rx_info = qp->rx_buff + rx_size;
631
632 /* Due to housekeeping, there must be atleast 2 buffs */
633 qp->rx_max_frame = min(transport_mtu, rx_size / 2);
634 qp->rx_max_entry = rx_size / qp->rx_max_frame;
635 qp->rx_index = 0;
636
637 /*
638 * Checking to see if we have more entries than the default.
639 * We should add additional entries if that is the case so we
640 * can be in sync with the transport frames.
641 */
642 node = dev_to_node(&ndev->dev);
643 for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) {
644 entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node);
645 if (!entry)
646 return -ENOMEM;
647
648 entry->qp = qp;
649 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
650 &qp->rx_free_q);
651 qp->rx_alloc_entry++;
652 }
653
654 qp->remote_rx_info->entry = qp->rx_max_entry - 1;
655
656 /* setup the hdr offsets with 0's */
657 for (i = 0; i < qp->rx_max_entry; i++) {
658 void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) -
659 sizeof(struct ntb_payload_header));
660 memset(offset, 0, sizeof(struct ntb_payload_header));
661 }
662
663 qp->rx_pkts = 0;
664 qp->tx_pkts = 0;
665 qp->tx_index = 0;
666
667 return 0;
668}
669
670static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)
671{
672 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
673 struct pci_dev *pdev = nt->ndev->pdev;
674
675 if (!mw->virt_addr)
676 return;
677
678 ntb_mw_clear_trans(nt->ndev, PIDX, num_mw);
679 dma_free_coherent(&pdev->dev, mw->alloc_size,
680 mw->alloc_addr, mw->dma_addr);
681 mw->xlat_size = 0;
682 mw->buff_size = 0;
683 mw->alloc_size = 0;
684 mw->alloc_addr = NULL;
685 mw->virt_addr = NULL;
686}
687
688static int ntb_alloc_mw_buffer(struct ntb_transport_mw *mw,
689 struct device *dma_dev, size_t align)
690{
691 dma_addr_t dma_addr;
692 void *alloc_addr, *virt_addr;
693 int rc;
694
695 alloc_addr = dma_alloc_coherent(dma_dev, mw->alloc_size,
696 &dma_addr, GFP_KERNEL);
697 if (!alloc_addr) {
698 dev_err(dma_dev, "Unable to alloc MW buff of size %zu\n",
699 mw->alloc_size);
700 return -ENOMEM;
701 }
702 virt_addr = alloc_addr;
703
704 /*
705 * we must ensure that the memory address allocated is BAR size
706 * aligned in order for the XLAT register to take the value. This
707 * is a requirement of the hardware. It is recommended to setup CMA
708 * for BAR sizes equal or greater than 4MB.
709 */
710 if (!IS_ALIGNED(dma_addr, align)) {
711 if (mw->alloc_size > mw->buff_size) {
712 virt_addr = PTR_ALIGN(alloc_addr, align);
713 dma_addr = ALIGN(dma_addr, align);
714 } else {
715 rc = -ENOMEM;
716 goto err;
717 }
718 }
719
720 mw->alloc_addr = alloc_addr;
721 mw->virt_addr = virt_addr;
722 mw->dma_addr = dma_addr;
723
724 return 0;
725
726err:
727 dma_free_coherent(dma_dev, mw->alloc_size, alloc_addr, dma_addr);
728
729 return rc;
730}
731
732static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw,
733 resource_size_t size)
734{
735 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
736 struct pci_dev *pdev = nt->ndev->pdev;
737 size_t xlat_size, buff_size;
738 resource_size_t xlat_align;
739 resource_size_t xlat_align_size;
740 int rc;
741
742 if (!size)
743 return -EINVAL;
744
745 rc = ntb_mw_get_align(nt->ndev, PIDX, num_mw, &xlat_align,
746 &xlat_align_size, NULL);
747 if (rc)
748 return rc;
749
750 xlat_size = round_up(size, xlat_align_size);
751 buff_size = round_up(size, xlat_align);
752
753 /* No need to re-setup */
754 if (mw->xlat_size == xlat_size)
755 return 0;
756
757 if (mw->buff_size)
758 ntb_free_mw(nt, num_mw);
759
760 /* Alloc memory for receiving data. Must be aligned */
761 mw->xlat_size = xlat_size;
762 mw->buff_size = buff_size;
763 mw->alloc_size = buff_size;
764
765 rc = ntb_alloc_mw_buffer(mw, &pdev->dev, xlat_align);
766 if (rc) {
767 mw->alloc_size *= 2;
768 rc = ntb_alloc_mw_buffer(mw, &pdev->dev, xlat_align);
769 if (rc) {
770 dev_err(&pdev->dev,
771 "Unable to alloc aligned MW buff\n");
772 mw->xlat_size = 0;
773 mw->buff_size = 0;
774 mw->alloc_size = 0;
775 return rc;
776 }
777 }
778
779 /* Notify HW the memory location of the receive buffer */
780 rc = ntb_mw_set_trans(nt->ndev, PIDX, num_mw, mw->dma_addr,
781 mw->xlat_size);
782 if (rc) {
783 dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw);
784 ntb_free_mw(nt, num_mw);
785 return -EIO;
786 }
787
788 return 0;
789}
790
791static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp)
792{
793 qp->link_is_up = false;
794 qp->active = false;
795
796 qp->tx_index = 0;
797 qp->rx_index = 0;
798 qp->rx_bytes = 0;
799 qp->rx_pkts = 0;
800 qp->rx_ring_empty = 0;
801 qp->rx_err_no_buf = 0;
802 qp->rx_err_oflow = 0;
803 qp->rx_err_ver = 0;
804 qp->rx_memcpy = 0;
805 qp->rx_async = 0;
806 qp->tx_bytes = 0;
807 qp->tx_pkts = 0;
808 qp->tx_ring_full = 0;
809 qp->tx_err_no_buf = 0;
810 qp->tx_memcpy = 0;
811 qp->tx_async = 0;
812}
813
814static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
815{
816 struct ntb_transport_ctx *nt = qp->transport;
817 struct pci_dev *pdev = nt->ndev->pdev;
818
819 dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num);
820
821 cancel_delayed_work_sync(&qp->link_work);
822 ntb_qp_link_down_reset(qp);
823
824 if (qp->event_handler)
825 qp->event_handler(qp->cb_data, qp->link_is_up);
826}
827
828static void ntb_qp_link_cleanup_work(struct work_struct *work)
829{
830 struct ntb_transport_qp *qp = container_of(work,
831 struct ntb_transport_qp,
832 link_cleanup);
833 struct ntb_transport_ctx *nt = qp->transport;
834
835 ntb_qp_link_cleanup(qp);
836
837 if (nt->link_is_up)
838 schedule_delayed_work(&qp->link_work,
839 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
840}
841
842static void ntb_qp_link_down(struct ntb_transport_qp *qp)
843{
844 schedule_work(&qp->link_cleanup);
845}
846
847static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
848{
849 struct ntb_transport_qp *qp;
850 u64 qp_bitmap_alloc;
851 unsigned int i, count;
852
853 qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;
854
855 /* Pass along the info to any clients */
856 for (i = 0; i < nt->qp_count; i++)
857 if (qp_bitmap_alloc & BIT_ULL(i)) {
858 qp = &nt->qp_vec[i];
859 ntb_qp_link_cleanup(qp);
860 cancel_work_sync(&qp->link_cleanup);
861 cancel_delayed_work_sync(&qp->link_work);
862 }
863
864 if (!nt->link_is_up)
865 cancel_delayed_work_sync(&nt->link_work);
866
867 for (i = 0; i < nt->mw_count; i++)
868 ntb_free_mw(nt, i);
869
870 /* The scratchpad registers keep the values if the remote side
871 * goes down, blast them now to give them a sane value the next
872 * time they are accessed
873 */
874 count = ntb_spad_count(nt->ndev);
875 for (i = 0; i < count; i++)
876 ntb_spad_write(nt->ndev, i, 0);
877}
878
879static void ntb_transport_link_cleanup_work(struct work_struct *work)
880{
881 struct ntb_transport_ctx *nt =
882 container_of(work, struct ntb_transport_ctx, link_cleanup);
883
884 ntb_transport_link_cleanup(nt);
885}
886
887static void ntb_transport_event_callback(void *data)
888{
889 struct ntb_transport_ctx *nt = data;
890
891 if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1)
892 schedule_delayed_work(&nt->link_work, 0);
893 else
894 schedule_work(&nt->link_cleanup);
895}
896
897static void ntb_transport_link_work(struct work_struct *work)
898{
899 struct ntb_transport_ctx *nt =
900 container_of(work, struct ntb_transport_ctx, link_work.work);
901 struct ntb_dev *ndev = nt->ndev;
902 struct pci_dev *pdev = ndev->pdev;
903 resource_size_t size;
904 u32 val;
905 int rc = 0, i, spad;
906
907 /* send the local info, in the opposite order of the way we read it */
908 for (i = 0; i < nt->mw_count; i++) {
909 size = nt->mw_vec[i].phys_size;
910
911 if (max_mw_size && size > max_mw_size)
912 size = max_mw_size;
913
914 spad = MW0_SZ_HIGH + (i * 2);
915 ntb_peer_spad_write(ndev, PIDX, spad, upper_32_bits(size));
916
917 spad = MW0_SZ_LOW + (i * 2);
918 ntb_peer_spad_write(ndev, PIDX, spad, lower_32_bits(size));
919 }
920
921 ntb_peer_spad_write(ndev, PIDX, NUM_MWS, nt->mw_count);
922
923 ntb_peer_spad_write(ndev, PIDX, NUM_QPS, nt->qp_count);
924
925 ntb_peer_spad_write(ndev, PIDX, VERSION, NTB_TRANSPORT_VERSION);
926
927 /* Query the remote side for its info */
928 val = ntb_spad_read(ndev, VERSION);
929 dev_dbg(&pdev->dev, "Remote version = %d\n", val);
930 if (val != NTB_TRANSPORT_VERSION)
931 goto out;
932
933 val = ntb_spad_read(ndev, NUM_QPS);
934 dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val);
935 if (val != nt->qp_count)
936 goto out;
937
938 val = ntb_spad_read(ndev, NUM_MWS);
939 dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val);
940 if (val != nt->mw_count)
941 goto out;
942
943 for (i = 0; i < nt->mw_count; i++) {
944 u64 val64;
945
946 val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2));
947 val64 = (u64)val << 32;
948
949 val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2));
950 val64 |= val;
951
952 dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64);
953
954 rc = ntb_set_mw(nt, i, val64);
955 if (rc)
956 goto out1;
957 }
958
959 nt->link_is_up = true;
960
961 for (i = 0; i < nt->qp_count; i++) {
962 struct ntb_transport_qp *qp = &nt->qp_vec[i];
963
964 ntb_transport_setup_qp_mw(nt, i);
965
966 if (qp->client_ready)
967 schedule_delayed_work(&qp->link_work, 0);
968 }
969
970 return;
971
972out1:
973 for (i = 0; i < nt->mw_count; i++)
974 ntb_free_mw(nt, i);
975
976 /* if there's an actual failure, we should just bail */
977 if (rc < 0)
978 return;
979
980out:
981 if (ntb_link_is_up(ndev, NULL, NULL) == 1)
982 schedule_delayed_work(&nt->link_work,
983 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
984}
985
986static void ntb_qp_link_work(struct work_struct *work)
987{
988 struct ntb_transport_qp *qp = container_of(work,
989 struct ntb_transport_qp,
990 link_work.work);
991 struct pci_dev *pdev = qp->ndev->pdev;
992 struct ntb_transport_ctx *nt = qp->transport;
993 int val;
994
995 WARN_ON(!nt->link_is_up);
996
997 val = ntb_spad_read(nt->ndev, QP_LINKS);
998
999 ntb_peer_spad_write(nt->ndev, PIDX, QP_LINKS, val | BIT(qp->qp_num));
1000
1001 /* query remote spad for qp ready bits */
1002 dev_dbg_ratelimited(&pdev->dev, "Remote QP link status = %x\n", val);
1003
1004 /* See if the remote side is up */
1005 if (val & BIT(qp->qp_num)) {
1006 dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num);
1007 qp->link_is_up = true;
1008 qp->active = true;
1009
1010 if (qp->event_handler)
1011 qp->event_handler(qp->cb_data, qp->link_is_up);
1012
1013 if (qp->active)
1014 tasklet_schedule(&qp->rxc_db_work);
1015 } else if (nt->link_is_up)
1016 schedule_delayed_work(&qp->link_work,
1017 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
1018}
1019
1020static int ntb_transport_init_queue(struct ntb_transport_ctx *nt,
1021 unsigned int qp_num)
1022{
1023 struct ntb_transport_qp *qp;
1024 phys_addr_t mw_base;
1025 resource_size_t mw_size;
1026 unsigned int num_qps_mw, tx_size;
1027 unsigned int mw_num, mw_count, qp_count;
1028 u64 qp_offset;
1029
1030 mw_count = nt->mw_count;
1031 qp_count = nt->qp_count;
1032
1033 mw_num = QP_TO_MW(nt, qp_num);
1034
1035 qp = &nt->qp_vec[qp_num];
1036 qp->qp_num = qp_num;
1037 qp->transport = nt;
1038 qp->ndev = nt->ndev;
1039 qp->client_ready = false;
1040 qp->event_handler = NULL;
1041 ntb_qp_link_down_reset(qp);
1042
1043 if (mw_num < qp_count % mw_count)
1044 num_qps_mw = qp_count / mw_count + 1;
1045 else
1046 num_qps_mw = qp_count / mw_count;
1047
1048 mw_base = nt->mw_vec[mw_num].phys_addr;
1049 mw_size = nt->mw_vec[mw_num].phys_size;
1050
1051 if (max_mw_size && mw_size > max_mw_size)
1052 mw_size = max_mw_size;
1053
1054 tx_size = (unsigned int)mw_size / num_qps_mw;
1055 qp_offset = tx_size * (qp_num / mw_count);
1056
1057 qp->tx_mw_size = tx_size;
1058 qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset;
1059 if (!qp->tx_mw)
1060 return -EINVAL;
1061
1062 qp->tx_mw_phys = mw_base + qp_offset;
1063 if (!qp->tx_mw_phys)
1064 return -EINVAL;
1065
1066 tx_size -= sizeof(struct ntb_rx_info);
1067 qp->rx_info = qp->tx_mw + tx_size;
1068
1069 /* Due to housekeeping, there must be atleast 2 buffs */
1070 qp->tx_max_frame = min(transport_mtu, tx_size / 2);
1071 qp->tx_max_entry = tx_size / qp->tx_max_frame;
1072
1073 if (nt->debugfs_node_dir) {
1074 char debugfs_name[4];
1075
1076 snprintf(debugfs_name, 4, "qp%d", qp_num);
1077 qp->debugfs_dir = debugfs_create_dir(debugfs_name,
1078 nt->debugfs_node_dir);
1079
1080 qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR,
1081 qp->debugfs_dir, qp,
1082 &ntb_qp_debugfs_stats);
1083 } else {
1084 qp->debugfs_dir = NULL;
1085 qp->debugfs_stats = NULL;
1086 }
1087
1088 INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work);
1089 INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work);
1090
1091 spin_lock_init(&qp->ntb_rx_q_lock);
1092 spin_lock_init(&qp->ntb_tx_free_q_lock);
1093
1094 INIT_LIST_HEAD(&qp->rx_post_q);
1095 INIT_LIST_HEAD(&qp->rx_pend_q);
1096 INIT_LIST_HEAD(&qp->rx_free_q);
1097 INIT_LIST_HEAD(&qp->tx_free_q);
1098
1099 tasklet_init(&qp->rxc_db_work, ntb_transport_rxc_db,
1100 (unsigned long)qp);
1101
1102 return 0;
1103}
1104
1105static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
1106{
1107 struct ntb_transport_ctx *nt;
1108 struct ntb_transport_mw *mw;
1109 unsigned int mw_count, qp_count, spad_count, max_mw_count_for_spads;
1110 u64 qp_bitmap;
1111 int node;
1112 int rc, i;
1113
1114 mw_count = ntb_peer_mw_count(ndev);
1115
1116 if (!ndev->ops->mw_set_trans) {
1117 dev_err(&ndev->dev, "Inbound MW based NTB API is required\n");
1118 return -EINVAL;
1119 }
1120
1121 if (ntb_db_is_unsafe(ndev))
1122 dev_dbg(&ndev->dev,
1123 "doorbell is unsafe, proceed anyway...\n");
1124 if (ntb_spad_is_unsafe(ndev))
1125 dev_dbg(&ndev->dev,
1126 "scratchpad is unsafe, proceed anyway...\n");
1127
1128 if (ntb_peer_port_count(ndev) != NTB_DEF_PEER_CNT)
1129 dev_warn(&ndev->dev, "Multi-port NTB devices unsupported\n");
1130
1131 node = dev_to_node(&ndev->dev);
1132
1133 nt = kzalloc_node(sizeof(*nt), GFP_KERNEL, node);
1134 if (!nt)
1135 return -ENOMEM;
1136
1137 nt->ndev = ndev;
1138 spad_count = ntb_spad_count(ndev);
1139
1140 /* Limit the MW's based on the availability of scratchpads */
1141
1142 if (spad_count < NTB_TRANSPORT_MIN_SPADS) {
1143 nt->mw_count = 0;
1144 rc = -EINVAL;
1145 goto err;
1146 }
1147
1148 max_mw_count_for_spads = (spad_count - MW0_SZ_HIGH) / 2;
1149 nt->mw_count = min(mw_count, max_mw_count_for_spads);
1150
1151 nt->mw_vec = kcalloc_node(mw_count, sizeof(*nt->mw_vec),
1152 GFP_KERNEL, node);
1153 if (!nt->mw_vec) {
1154 rc = -ENOMEM;
1155 goto err;
1156 }
1157
1158 for (i = 0; i < mw_count; i++) {
1159 mw = &nt->mw_vec[i];
1160
1161 rc = ntb_peer_mw_get_addr(ndev, i, &mw->phys_addr,
1162 &mw->phys_size);
1163 if (rc)
1164 goto err1;
1165
1166 mw->vbase = ioremap_wc(mw->phys_addr, mw->phys_size);
1167 if (!mw->vbase) {
1168 rc = -ENOMEM;
1169 goto err1;
1170 }
1171
1172 mw->buff_size = 0;
1173 mw->xlat_size = 0;
1174 mw->virt_addr = NULL;
1175 mw->dma_addr = 0;
1176 }
1177
1178 qp_bitmap = ntb_db_valid_mask(ndev);
1179
1180 qp_count = ilog2(qp_bitmap);
1181 if (max_num_clients && max_num_clients < qp_count)
1182 qp_count = max_num_clients;
1183 else if (nt->mw_count < qp_count)
1184 qp_count = nt->mw_count;
1185
1186 qp_bitmap &= BIT_ULL(qp_count) - 1;
1187
1188 nt->qp_count = qp_count;
1189 nt->qp_bitmap = qp_bitmap;
1190 nt->qp_bitmap_free = qp_bitmap;
1191
1192 nt->qp_vec = kcalloc_node(qp_count, sizeof(*nt->qp_vec),
1193 GFP_KERNEL, node);
1194 if (!nt->qp_vec) {
1195 rc = -ENOMEM;
1196 goto err1;
1197 }
1198
1199 if (nt_debugfs_dir) {
1200 nt->debugfs_node_dir =
1201 debugfs_create_dir(pci_name(ndev->pdev),
1202 nt_debugfs_dir);
1203 }
1204
1205 for (i = 0; i < qp_count; i++) {
1206 rc = ntb_transport_init_queue(nt, i);
1207 if (rc)
1208 goto err2;
1209 }
1210
1211 INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work);
1212 INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work);
1213
1214 rc = ntb_set_ctx(ndev, nt, &ntb_transport_ops);
1215 if (rc)
1216 goto err2;
1217
1218 INIT_LIST_HEAD(&nt->client_devs);
1219 rc = ntb_bus_init(nt);
1220 if (rc)
1221 goto err3;
1222
1223 nt->link_is_up = false;
1224 ntb_link_enable(ndev, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
1225 ntb_link_event(ndev);
1226
1227 return 0;
1228
1229err3:
1230 ntb_clear_ctx(ndev);
1231err2:
1232 kfree(nt->qp_vec);
1233err1:
1234 while (i--) {
1235 mw = &nt->mw_vec[i];
1236 iounmap(mw->vbase);
1237 }
1238 kfree(nt->mw_vec);
1239err:
1240 kfree(nt);
1241 return rc;
1242}
1243
1244static void ntb_transport_free(struct ntb_client *self, struct ntb_dev *ndev)
1245{
1246 struct ntb_transport_ctx *nt = ndev->ctx;
1247 struct ntb_transport_qp *qp;
1248 u64 qp_bitmap_alloc;
1249 int i;
1250
1251 ntb_transport_link_cleanup(nt);
1252 cancel_work_sync(&nt->link_cleanup);
1253 cancel_delayed_work_sync(&nt->link_work);
1254
1255 qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;
1256
1257 /* verify that all the qp's are freed */
1258 for (i = 0; i < nt->qp_count; i++) {
1259 qp = &nt->qp_vec[i];
1260 if (qp_bitmap_alloc & BIT_ULL(i))
1261 ntb_transport_free_queue(qp);
1262 debugfs_remove_recursive(qp->debugfs_dir);
1263 }
1264
1265 ntb_link_disable(ndev);
1266 ntb_clear_ctx(ndev);
1267
1268 ntb_bus_remove(nt);
1269
1270 for (i = nt->mw_count; i--; ) {
1271 ntb_free_mw(nt, i);
1272 iounmap(nt->mw_vec[i].vbase);
1273 }
1274
1275 kfree(nt->qp_vec);
1276 kfree(nt->mw_vec);
1277 kfree(nt);
1278}
1279
1280static void ntb_complete_rxc(struct ntb_transport_qp *qp)
1281{
1282 struct ntb_queue_entry *entry;
1283 void *cb_data;
1284 unsigned int len;
1285 unsigned long irqflags;
1286
1287 spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags);
1288
1289 while (!list_empty(&qp->rx_post_q)) {
1290 entry = list_first_entry(&qp->rx_post_q,
1291 struct ntb_queue_entry, entry);
1292 if (!(entry->flags & DESC_DONE_FLAG))
1293 break;
1294
1295 entry->rx_hdr->flags = 0;
1296 iowrite32(entry->rx_index, &qp->rx_info->entry);
1297
1298 cb_data = entry->cb_data;
1299 len = entry->len;
1300
1301 list_move_tail(&entry->entry, &qp->rx_free_q);
1302
1303 spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags);
1304
1305 if (qp->rx_handler && qp->client_ready)
1306 qp->rx_handler(qp, qp->cb_data, cb_data, len);
1307
1308 spin_lock_irqsave(&qp->ntb_rx_q_lock, irqflags);
1309 }
1310
1311 spin_unlock_irqrestore(&qp->ntb_rx_q_lock, irqflags);
1312}
1313
1314static void ntb_rx_copy_callback(void *data,
1315 const struct dmaengine_result *res)
1316{
1317 struct ntb_queue_entry *entry = data;
1318
1319 /* we need to check DMA results if we are using DMA */
1320 if (res) {
1321 enum dmaengine_tx_result dma_err = res->result;
1322
1323 switch (dma_err) {
1324 case DMA_TRANS_READ_FAILED:
1325 case DMA_TRANS_WRITE_FAILED:
1326 entry->errors++;
1327 /* fall through */
1328 case DMA_TRANS_ABORTED:
1329 {
1330 struct ntb_transport_qp *qp = entry->qp;
1331 void *offset = qp->rx_buff + qp->rx_max_frame *
1332 qp->rx_index;
1333
1334 ntb_memcpy_rx(entry, offset);
1335 qp->rx_memcpy++;
1336 return;
1337 }
1338
1339 case DMA_TRANS_NOERROR:
1340 default:
1341 break;
1342 }
1343 }
1344
1345 entry->flags |= DESC_DONE_FLAG;
1346
1347 ntb_complete_rxc(entry->qp);
1348}
1349
1350static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset)
1351{
1352 void *buf = entry->buf;
1353 size_t len = entry->len;
1354
1355 memcpy(buf, offset, len);
1356
1357 /* Ensure that the data is fully copied out before clearing the flag */
1358 wmb();
1359
1360 ntb_rx_copy_callback(entry, NULL);
1361}
1362
1363static int ntb_async_rx_submit(struct ntb_queue_entry *entry, void *offset)
1364{
1365 struct dma_async_tx_descriptor *txd;
1366 struct ntb_transport_qp *qp = entry->qp;
1367 struct dma_chan *chan = qp->rx_dma_chan;
1368 struct dma_device *device;
1369 size_t pay_off, buff_off, len;
1370 struct dmaengine_unmap_data *unmap;
1371 dma_cookie_t cookie;
1372 void *buf = entry->buf;
1373
1374 len = entry->len;
1375 device = chan->device;
1376 pay_off = (size_t)offset & ~PAGE_MASK;
1377 buff_off = (size_t)buf & ~PAGE_MASK;
1378
1379 if (!is_dma_copy_aligned(device, pay_off, buff_off, len))
1380 goto err;
1381
1382 unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOWAIT);
1383 if (!unmap)
1384 goto err;
1385
1386 unmap->len = len;
1387 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(offset),
1388 pay_off, len, DMA_TO_DEVICE);
1389 if (dma_mapping_error(device->dev, unmap->addr[0]))
1390 goto err_get_unmap;
1391
1392 unmap->to_cnt = 1;
1393
1394 unmap->addr[1] = dma_map_page(device->dev, virt_to_page(buf),
1395 buff_off, len, DMA_FROM_DEVICE);
1396 if (dma_mapping_error(device->dev, unmap->addr[1]))
1397 goto err_get_unmap;
1398
1399 unmap->from_cnt = 1;
1400
1401 txd = device->device_prep_dma_memcpy(chan, unmap->addr[1],
1402 unmap->addr[0], len,
1403 DMA_PREP_INTERRUPT);
1404 if (!txd)
1405 goto err_get_unmap;
1406
1407 txd->callback_result = ntb_rx_copy_callback;
1408 txd->callback_param = entry;
1409 dma_set_unmap(txd, unmap);
1410
1411 cookie = dmaengine_submit(txd);
1412 if (dma_submit_error(cookie))
1413 goto err_set_unmap;
1414
1415 dmaengine_unmap_put(unmap);
1416
1417 qp->last_cookie = cookie;
1418
1419 qp->rx_async++;
1420
1421 return 0;
1422
1423err_set_unmap:
1424 dmaengine_unmap_put(unmap);
1425err_get_unmap:
1426 dmaengine_unmap_put(unmap);
1427err:
1428 return -ENXIO;
1429}
1430
1431static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset)
1432{
1433 struct ntb_transport_qp *qp = entry->qp;
1434 struct dma_chan *chan = qp->rx_dma_chan;
1435 int res;
1436
1437 if (!chan)
1438 goto err;
1439
1440 if (entry->len < copy_bytes)
1441 goto err;
1442
1443 res = ntb_async_rx_submit(entry, offset);
1444 if (res < 0)
1445 goto err;
1446
1447 if (!entry->retries)
1448 qp->rx_async++;
1449
1450 return;
1451
1452err:
1453 ntb_memcpy_rx(entry, offset);
1454 qp->rx_memcpy++;
1455}
1456
1457static int ntb_process_rxc(struct ntb_transport_qp *qp)
1458{
1459 struct ntb_payload_header *hdr;
1460 struct ntb_queue_entry *entry;
1461 void *offset;
1462
1463 offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index;
1464 hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header);
1465
1466 dev_dbg(&qp->ndev->pdev->dev, "qp %d: RX ver %u len %d flags %x\n",
1467 qp->qp_num, hdr->ver, hdr->len, hdr->flags);
1468
1469 if (!(hdr->flags & DESC_DONE_FLAG)) {
1470 dev_dbg(&qp->ndev->pdev->dev, "done flag not set\n");
1471 qp->rx_ring_empty++;
1472 return -EAGAIN;
1473 }
1474
1475 if (hdr->flags & LINK_DOWN_FLAG) {
1476 dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n");
1477 ntb_qp_link_down(qp);
1478 hdr->flags = 0;
1479 return -EAGAIN;
1480 }
1481
1482 if (hdr->ver != (u32)qp->rx_pkts) {
1483 dev_dbg(&qp->ndev->pdev->dev,
1484 "version mismatch, expected %llu - got %u\n",
1485 qp->rx_pkts, hdr->ver);
1486 qp->rx_err_ver++;
1487 return -EIO;
1488 }
1489
1490 entry = ntb_list_mv(&qp->ntb_rx_q_lock, &qp->rx_pend_q, &qp->rx_post_q);
1491 if (!entry) {
1492 dev_dbg(&qp->ndev->pdev->dev, "no receive buffer\n");
1493 qp->rx_err_no_buf++;
1494 return -EAGAIN;
1495 }
1496
1497 entry->rx_hdr = hdr;
1498 entry->rx_index = qp->rx_index;
1499
1500 if (hdr->len > entry->len) {
1501 dev_dbg(&qp->ndev->pdev->dev,
1502 "receive buffer overflow! Wanted %d got %d\n",
1503 hdr->len, entry->len);
1504 qp->rx_err_oflow++;
1505
1506 entry->len = -EIO;
1507 entry->flags |= DESC_DONE_FLAG;
1508
1509 ntb_complete_rxc(qp);
1510 } else {
1511 dev_dbg(&qp->ndev->pdev->dev,
1512 "RX OK index %u ver %u size %d into buf size %d\n",
1513 qp->rx_index, hdr->ver, hdr->len, entry->len);
1514
1515 qp->rx_bytes += hdr->len;
1516 qp->rx_pkts++;
1517
1518 entry->len = hdr->len;
1519
1520 ntb_async_rx(entry, offset);
1521 }
1522
1523 qp->rx_index++;
1524 qp->rx_index %= qp->rx_max_entry;
1525
1526 return 0;
1527}
1528
1529static void ntb_transport_rxc_db(unsigned long data)
1530{
1531 struct ntb_transport_qp *qp = (void *)data;
1532 int rc, i;
1533
1534 dev_dbg(&qp->ndev->pdev->dev, "%s: doorbell %d received\n",
1535 __func__, qp->qp_num);
1536
1537 /* Limit the number of packets processed in a single interrupt to
1538 * provide fairness to others
1539 */
1540 for (i = 0; i < qp->rx_max_entry; i++) {
1541 rc = ntb_process_rxc(qp);
1542 if (rc)
1543 break;
1544 }
1545
1546 if (i && qp->rx_dma_chan)
1547 dma_async_issue_pending(qp->rx_dma_chan);
1548
1549 if (i == qp->rx_max_entry) {
1550 /* there is more work to do */
1551 if (qp->active)
1552 tasklet_schedule(&qp->rxc_db_work);
1553 } else if (ntb_db_read(qp->ndev) & BIT_ULL(qp->qp_num)) {
1554 /* the doorbell bit is set: clear it */
1555 ntb_db_clear(qp->ndev, BIT_ULL(qp->qp_num));
1556 /* ntb_db_read ensures ntb_db_clear write is committed */
1557 ntb_db_read(qp->ndev);
1558
1559 /* an interrupt may have arrived between finishing
1560 * ntb_process_rxc and clearing the doorbell bit:
1561 * there might be some more work to do.
1562 */
1563 if (qp->active)
1564 tasklet_schedule(&qp->rxc_db_work);
1565 }
1566}
1567
1568static void ntb_tx_copy_callback(void *data,
1569 const struct dmaengine_result *res)
1570{
1571 struct ntb_queue_entry *entry = data;
1572 struct ntb_transport_qp *qp = entry->qp;
1573 struct ntb_payload_header __iomem *hdr = entry->tx_hdr;
1574
1575 /* we need to check DMA results if we are using DMA */
1576 if (res) {
1577 enum dmaengine_tx_result dma_err = res->result;
1578
1579 switch (dma_err) {
1580 case DMA_TRANS_READ_FAILED:
1581 case DMA_TRANS_WRITE_FAILED:
1582 entry->errors++;
1583 /* fall through */
1584 case DMA_TRANS_ABORTED:
1585 {
1586 void __iomem *offset =
1587 qp->tx_mw + qp->tx_max_frame *
1588 entry->tx_index;
1589
1590 /* resubmit via CPU */
1591 ntb_memcpy_tx(entry, offset);
1592 qp->tx_memcpy++;
1593 return;
1594 }
1595
1596 case DMA_TRANS_NOERROR:
1597 default:
1598 break;
1599 }
1600 }
1601
1602 iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags);
1603
1604 ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num));
1605
1606 /* The entry length can only be zero if the packet is intended to be a
1607 * "link down" or similar. Since no payload is being sent in these
1608 * cases, there is nothing to add to the completion queue.
1609 */
1610 if (entry->len > 0) {
1611 qp->tx_bytes += entry->len;
1612
1613 if (qp->tx_handler)
1614 qp->tx_handler(qp, qp->cb_data, entry->cb_data,
1615 entry->len);
1616 }
1617
1618 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q);
1619}
1620
1621static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset)
1622{
1623#ifdef ARCH_HAS_NOCACHE_UACCESS
1624 /*
1625 * Using non-temporal mov to improve performance on non-cached
1626 * writes, even though we aren't actually copying from user space.
1627 */
1628 __copy_from_user_inatomic_nocache(offset, entry->buf, entry->len);
1629#else
1630 memcpy_toio(offset, entry->buf, entry->len);
1631#endif
1632
1633 /* Ensure that the data is fully copied out before setting the flags */
1634 wmb();
1635
1636 ntb_tx_copy_callback(entry, NULL);
1637}
1638
1639static int ntb_async_tx_submit(struct ntb_transport_qp *qp,
1640 struct ntb_queue_entry *entry)
1641{
1642 struct dma_async_tx_descriptor *txd;
1643 struct dma_chan *chan = qp->tx_dma_chan;
1644 struct dma_device *device;
1645 size_t len = entry->len;
1646 void *buf = entry->buf;
1647 size_t dest_off, buff_off;
1648 struct dmaengine_unmap_data *unmap;
1649 dma_addr_t dest;
1650 dma_cookie_t cookie;
1651
1652 device = chan->device;
1653 dest = qp->tx_mw_dma_addr + qp->tx_max_frame * entry->tx_index;
1654 buff_off = (size_t)buf & ~PAGE_MASK;
1655 dest_off = (size_t)dest & ~PAGE_MASK;
1656
1657 if (!is_dma_copy_aligned(device, buff_off, dest_off, len))
1658 goto err;
1659
1660 unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT);
1661 if (!unmap)
1662 goto err;
1663
1664 unmap->len = len;
1665 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(buf),
1666 buff_off, len, DMA_TO_DEVICE);
1667 if (dma_mapping_error(device->dev, unmap->addr[0]))
1668 goto err_get_unmap;
1669
1670 unmap->to_cnt = 1;
1671
1672 txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0], len,
1673 DMA_PREP_INTERRUPT);
1674 if (!txd)
1675 goto err_get_unmap;
1676
1677 txd->callback_result = ntb_tx_copy_callback;
1678 txd->callback_param = entry;
1679 dma_set_unmap(txd, unmap);
1680
1681 cookie = dmaengine_submit(txd);
1682 if (dma_submit_error(cookie))
1683 goto err_set_unmap;
1684
1685 dmaengine_unmap_put(unmap);
1686
1687 dma_async_issue_pending(chan);
1688
1689 return 0;
1690err_set_unmap:
1691 dmaengine_unmap_put(unmap);
1692err_get_unmap:
1693 dmaengine_unmap_put(unmap);
1694err:
1695 return -ENXIO;
1696}
1697
1698static void ntb_async_tx(struct ntb_transport_qp *qp,
1699 struct ntb_queue_entry *entry)
1700{
1701 struct ntb_payload_header __iomem *hdr;
1702 struct dma_chan *chan = qp->tx_dma_chan;
1703 void __iomem *offset;
1704 int res;
1705
1706 entry->tx_index = qp->tx_index;
1707 offset = qp->tx_mw + qp->tx_max_frame * entry->tx_index;
1708 hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header);
1709 entry->tx_hdr = hdr;
1710
1711 iowrite32(entry->len, &hdr->len);
1712 iowrite32((u32)qp->tx_pkts, &hdr->ver);
1713
1714 if (!chan)
1715 goto err;
1716
1717 if (entry->len < copy_bytes)
1718 goto err;
1719
1720 res = ntb_async_tx_submit(qp, entry);
1721 if (res < 0)
1722 goto err;
1723
1724 if (!entry->retries)
1725 qp->tx_async++;
1726
1727 return;
1728
1729err:
1730 ntb_memcpy_tx(entry, offset);
1731 qp->tx_memcpy++;
1732}
1733
1734static int ntb_process_tx(struct ntb_transport_qp *qp,
1735 struct ntb_queue_entry *entry)
1736{
1737 if (qp->tx_index == qp->remote_rx_info->entry) {
1738 qp->tx_ring_full++;
1739 return -EAGAIN;
1740 }
1741
1742 if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) {
1743 if (qp->tx_handler)
1744 qp->tx_handler(qp, qp->cb_data, NULL, -EIO);
1745
1746 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
1747 &qp->tx_free_q);
1748 return 0;
1749 }
1750
1751 ntb_async_tx(qp, entry);
1752
1753 qp->tx_index++;
1754 qp->tx_index %= qp->tx_max_entry;
1755
1756 qp->tx_pkts++;
1757
1758 return 0;
1759}
1760
1761static void ntb_send_link_down(struct ntb_transport_qp *qp)
1762{
1763 struct pci_dev *pdev = qp->ndev->pdev;
1764 struct ntb_queue_entry *entry;
1765 int i, rc;
1766
1767 if (!qp->link_is_up)
1768 return;
1769
1770 dev_info(&pdev->dev, "qp %d: Send Link Down\n", qp->qp_num);
1771
1772 for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) {
1773 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);
1774 if (entry)
1775 break;
1776 msleep(100);
1777 }
1778
1779 if (!entry)
1780 return;
1781
1782 entry->cb_data = NULL;
1783 entry->buf = NULL;
1784 entry->len = 0;
1785 entry->flags = LINK_DOWN_FLAG;
1786
1787 rc = ntb_process_tx(qp, entry);
1788 if (rc)
1789 dev_err(&pdev->dev, "ntb: QP%d unable to send linkdown msg\n",
1790 qp->qp_num);
1791
1792 ntb_qp_link_down_reset(qp);
1793}
1794
1795static bool ntb_dma_filter_fn(struct dma_chan *chan, void *node)
1796{
1797 return dev_to_node(&chan->dev->device) == (int)(unsigned long)node;
1798}
1799
1800/**
1801 * ntb_transport_create_queue - Create a new NTB transport layer queue
1802 * @rx_handler: receive callback function
1803 * @tx_handler: transmit callback function
1804 * @event_handler: event callback function
1805 *
1806 * Create a new NTB transport layer queue and provide the queue with a callback
1807 * routine for both transmit and receive. The receive callback routine will be
1808 * used to pass up data when the transport has received it on the queue. The
1809 * transmit callback routine will be called when the transport has completed the
1810 * transmission of the data on the queue and the data is ready to be freed.
1811 *
1812 * RETURNS: pointer to newly created ntb_queue, NULL on error.
1813 */
1814struct ntb_transport_qp *
1815ntb_transport_create_queue(void *data, struct device *client_dev,
1816 const struct ntb_queue_handlers *handlers)
1817{
1818 struct ntb_dev *ndev;
1819 struct pci_dev *pdev;
1820 struct ntb_transport_ctx *nt;
1821 struct ntb_queue_entry *entry;
1822 struct ntb_transport_qp *qp;
1823 u64 qp_bit;
1824 unsigned int free_queue;
1825 dma_cap_mask_t dma_mask;
1826 int node;
1827 int i;
1828
1829 ndev = dev_ntb(client_dev->parent);
1830 pdev = ndev->pdev;
1831 nt = ndev->ctx;
1832
1833 node = dev_to_node(&ndev->dev);
1834
1835 free_queue = ffs(nt->qp_bitmap_free);
1836 if (!free_queue)
1837 goto err;
1838
1839 /* decrement free_queue to make it zero based */
1840 free_queue--;
1841
1842 qp = &nt->qp_vec[free_queue];
1843 qp_bit = BIT_ULL(qp->qp_num);
1844
1845 nt->qp_bitmap_free &= ~qp_bit;
1846
1847 qp->cb_data = data;
1848 qp->rx_handler = handlers->rx_handler;
1849 qp->tx_handler = handlers->tx_handler;
1850 qp->event_handler = handlers->event_handler;
1851
1852 dma_cap_zero(dma_mask);
1853 dma_cap_set(DMA_MEMCPY, dma_mask);
1854
1855 if (use_dma) {
1856 qp->tx_dma_chan =
1857 dma_request_channel(dma_mask, ntb_dma_filter_fn,
1858 (void *)(unsigned long)node);
1859 if (!qp->tx_dma_chan)
1860 dev_info(&pdev->dev, "Unable to allocate TX DMA channel\n");
1861
1862 qp->rx_dma_chan =
1863 dma_request_channel(dma_mask, ntb_dma_filter_fn,
1864 (void *)(unsigned long)node);
1865 if (!qp->rx_dma_chan)
1866 dev_info(&pdev->dev, "Unable to allocate RX DMA channel\n");
1867 } else {
1868 qp->tx_dma_chan = NULL;
1869 qp->rx_dma_chan = NULL;
1870 }
1871
1872 if (qp->tx_dma_chan) {
1873 qp->tx_mw_dma_addr =
1874 dma_map_resource(qp->tx_dma_chan->device->dev,
1875 qp->tx_mw_phys, qp->tx_mw_size,
1876 DMA_FROM_DEVICE, 0);
1877 if (dma_mapping_error(qp->tx_dma_chan->device->dev,
1878 qp->tx_mw_dma_addr)) {
1879 qp->tx_mw_dma_addr = 0;
1880 goto err1;
1881 }
1882 }
1883
1884 dev_dbg(&pdev->dev, "Using %s memcpy for TX\n",
1885 qp->tx_dma_chan ? "DMA" : "CPU");
1886
1887 dev_dbg(&pdev->dev, "Using %s memcpy for RX\n",
1888 qp->rx_dma_chan ? "DMA" : "CPU");
1889
1890 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
1891 entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node);
1892 if (!entry)
1893 goto err1;
1894
1895 entry->qp = qp;
1896 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry,
1897 &qp->rx_free_q);
1898 }
1899 qp->rx_alloc_entry = NTB_QP_DEF_NUM_ENTRIES;
1900
1901 for (i = 0; i < qp->tx_max_entry; i++) {
1902 entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node);
1903 if (!entry)
1904 goto err2;
1905
1906 entry->qp = qp;
1907 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
1908 &qp->tx_free_q);
1909 }
1910
1911 ntb_db_clear(qp->ndev, qp_bit);
1912 ntb_db_clear_mask(qp->ndev, qp_bit);
1913
1914 dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num);
1915
1916 return qp;
1917
1918err2:
1919 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
1920 kfree(entry);
1921err1:
1922 qp->rx_alloc_entry = 0;
1923 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q)))
1924 kfree(entry);
1925 if (qp->tx_mw_dma_addr)
1926 dma_unmap_resource(qp->tx_dma_chan->device->dev,
1927 qp->tx_mw_dma_addr, qp->tx_mw_size,
1928 DMA_FROM_DEVICE, 0);
1929 if (qp->tx_dma_chan)
1930 dma_release_channel(qp->tx_dma_chan);
1931 if (qp->rx_dma_chan)
1932 dma_release_channel(qp->rx_dma_chan);
1933 nt->qp_bitmap_free |= qp_bit;
1934err:
1935 return NULL;
1936}
1937EXPORT_SYMBOL_GPL(ntb_transport_create_queue);
1938
1939/**
1940 * ntb_transport_free_queue - Frees NTB transport queue
1941 * @qp: NTB queue to be freed
1942 *
1943 * Frees NTB transport queue
1944 */
1945void ntb_transport_free_queue(struct ntb_transport_qp *qp)
1946{
1947 struct pci_dev *pdev;
1948 struct ntb_queue_entry *entry;
1949 u64 qp_bit;
1950
1951 if (!qp)
1952 return;
1953
1954 pdev = qp->ndev->pdev;
1955
1956 qp->active = false;
1957
1958 if (qp->tx_dma_chan) {
1959 struct dma_chan *chan = qp->tx_dma_chan;
1960 /* Putting the dma_chan to NULL will force any new traffic to be
1961 * processed by the CPU instead of the DAM engine
1962 */
1963 qp->tx_dma_chan = NULL;
1964
1965 /* Try to be nice and wait for any queued DMA engine
1966 * transactions to process before smashing it with a rock
1967 */
1968 dma_sync_wait(chan, qp->last_cookie);
1969 dmaengine_terminate_all(chan);
1970
1971 dma_unmap_resource(chan->device->dev,
1972 qp->tx_mw_dma_addr, qp->tx_mw_size,
1973 DMA_FROM_DEVICE, 0);
1974
1975 dma_release_channel(chan);
1976 }
1977
1978 if (qp->rx_dma_chan) {
1979 struct dma_chan *chan = qp->rx_dma_chan;
1980 /* Putting the dma_chan to NULL will force any new traffic to be
1981 * processed by the CPU instead of the DAM engine
1982 */
1983 qp->rx_dma_chan = NULL;
1984
1985 /* Try to be nice and wait for any queued DMA engine
1986 * transactions to process before smashing it with a rock
1987 */
1988 dma_sync_wait(chan, qp->last_cookie);
1989 dmaengine_terminate_all(chan);
1990 dma_release_channel(chan);
1991 }
1992
1993 qp_bit = BIT_ULL(qp->qp_num);
1994
1995 ntb_db_set_mask(qp->ndev, qp_bit);
1996 tasklet_kill(&qp->rxc_db_work);
1997
1998 cancel_delayed_work_sync(&qp->link_work);
1999
2000 qp->cb_data = NULL;
2001 qp->rx_handler = NULL;
2002 qp->tx_handler = NULL;
2003 qp->event_handler = NULL;
2004
2005 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q)))
2006 kfree(entry);
2007
2008 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q))) {
2009 dev_warn(&pdev->dev, "Freeing item from non-empty rx_pend_q\n");
2010 kfree(entry);
2011 }
2012
2013 while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_post_q))) {
2014 dev_warn(&pdev->dev, "Freeing item from non-empty rx_post_q\n");
2015 kfree(entry);
2016 }
2017
2018 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
2019 kfree(entry);
2020
2021 qp->transport->qp_bitmap_free |= qp_bit;
2022
2023 dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num);
2024}
2025EXPORT_SYMBOL_GPL(ntb_transport_free_queue);
2026
2027/**
2028 * ntb_transport_rx_remove - Dequeues enqueued rx packet
2029 * @qp: NTB queue to be freed
2030 * @len: pointer to variable to write enqueued buffers length
2031 *
2032 * Dequeues unused buffers from receive queue. Should only be used during
2033 * shutdown of qp.
2034 *
2035 * RETURNS: NULL error value on error, or void* for success.
2036 */
2037void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len)
2038{
2039 struct ntb_queue_entry *entry;
2040 void *buf;
2041
2042 if (!qp || qp->client_ready)
2043 return NULL;
2044
2045 entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q);
2046 if (!entry)
2047 return NULL;
2048
2049 buf = entry->cb_data;
2050 *len = entry->len;
2051
2052 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_free_q);
2053
2054 return buf;
2055}
2056EXPORT_SYMBOL_GPL(ntb_transport_rx_remove);
2057
2058/**
2059 * ntb_transport_rx_enqueue - Enqueue a new NTB queue entry
2060 * @qp: NTB transport layer queue the entry is to be enqueued on
2061 * @cb: per buffer pointer for callback function to use
2062 * @data: pointer to data buffer that incoming packets will be copied into
2063 * @len: length of the data buffer
2064 *
2065 * Enqueue a new receive buffer onto the transport queue into which a NTB
2066 * payload can be received into.
2067 *
2068 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
2069 */
2070int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
2071 unsigned int len)
2072{
2073 struct ntb_queue_entry *entry;
2074
2075 if (!qp)
2076 return -EINVAL;
2077
2078 entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_free_q);
2079 if (!entry)
2080 return -ENOMEM;
2081
2082 entry->cb_data = cb;
2083 entry->buf = data;
2084 entry->len = len;
2085 entry->flags = 0;
2086 entry->retries = 0;
2087 entry->errors = 0;
2088 entry->rx_index = 0;
2089
2090 ntb_list_add(&qp->ntb_rx_q_lock, &entry->entry, &qp->rx_pend_q);
2091
2092 if (qp->active)
2093 tasklet_schedule(&qp->rxc_db_work);
2094
2095 return 0;
2096}
2097EXPORT_SYMBOL_GPL(ntb_transport_rx_enqueue);
2098
2099/**
2100 * ntb_transport_tx_enqueue - Enqueue a new NTB queue entry
2101 * @qp: NTB transport layer queue the entry is to be enqueued on
2102 * @cb: per buffer pointer for callback function to use
2103 * @data: pointer to data buffer that will be sent
2104 * @len: length of the data buffer
2105 *
2106 * Enqueue a new transmit buffer onto the transport queue from which a NTB
2107 * payload will be transmitted. This assumes that a lock is being held to
2108 * serialize access to the qp.
2109 *
2110 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
2111 */
2112int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
2113 unsigned int len)
2114{
2115 struct ntb_queue_entry *entry;
2116 int rc;
2117
2118 if (!qp || !qp->link_is_up || !len)
2119 return -EINVAL;
2120
2121 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);
2122 if (!entry) {
2123 qp->tx_err_no_buf++;
2124 return -EBUSY;
2125 }
2126
2127 entry->cb_data = cb;
2128 entry->buf = data;
2129 entry->len = len;
2130 entry->flags = 0;
2131 entry->errors = 0;
2132 entry->retries = 0;
2133 entry->tx_index = 0;
2134
2135 rc = ntb_process_tx(qp, entry);
2136 if (rc)
2137 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
2138 &qp->tx_free_q);
2139
2140 return rc;
2141}
2142EXPORT_SYMBOL_GPL(ntb_transport_tx_enqueue);
2143
2144/**
2145 * ntb_transport_link_up - Notify NTB transport of client readiness to use queue
2146 * @qp: NTB transport layer queue to be enabled
2147 *
2148 * Notify NTB transport layer of client readiness to use queue
2149 */
2150void ntb_transport_link_up(struct ntb_transport_qp *qp)
2151{
2152 if (!qp)
2153 return;
2154
2155 qp->client_ready = true;
2156
2157 if (qp->transport->link_is_up)
2158 schedule_delayed_work(&qp->link_work, 0);
2159}
2160EXPORT_SYMBOL_GPL(ntb_transport_link_up);
2161
2162/**
2163 * ntb_transport_link_down - Notify NTB transport to no longer enqueue data
2164 * @qp: NTB transport layer queue to be disabled
2165 *
2166 * Notify NTB transport layer of client's desire to no longer receive data on
2167 * transport queue specified. It is the client's responsibility to ensure all
2168 * entries on queue are purged or otherwise handled appropriately.
2169 */
2170void ntb_transport_link_down(struct ntb_transport_qp *qp)
2171{
2172 int val;
2173
2174 if (!qp)
2175 return;
2176
2177 qp->client_ready = false;
2178
2179 val = ntb_spad_read(qp->ndev, QP_LINKS);
2180
2181 ntb_peer_spad_write(qp->ndev, PIDX, QP_LINKS, val & ~BIT(qp->qp_num));
2182
2183 if (qp->link_is_up)
2184 ntb_send_link_down(qp);
2185 else
2186 cancel_delayed_work_sync(&qp->link_work);
2187}
2188EXPORT_SYMBOL_GPL(ntb_transport_link_down);
2189
2190/**
2191 * ntb_transport_link_query - Query transport link state
2192 * @qp: NTB transport layer queue to be queried
2193 *
2194 * Query connectivity to the remote system of the NTB transport queue
2195 *
2196 * RETURNS: true for link up or false for link down
2197 */
2198bool ntb_transport_link_query(struct ntb_transport_qp *qp)
2199{
2200 if (!qp)
2201 return false;
2202
2203 return qp->link_is_up;
2204}
2205EXPORT_SYMBOL_GPL(ntb_transport_link_query);
2206
2207/**
2208 * ntb_transport_qp_num - Query the qp number
2209 * @qp: NTB transport layer queue to be queried
2210 *
2211 * Query qp number of the NTB transport queue
2212 *
2213 * RETURNS: a zero based number specifying the qp number
2214 */
2215unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp)
2216{
2217 if (!qp)
2218 return 0;
2219
2220 return qp->qp_num;
2221}
2222EXPORT_SYMBOL_GPL(ntb_transport_qp_num);
2223
2224/**
2225 * ntb_transport_max_size - Query the max payload size of a qp
2226 * @qp: NTB transport layer queue to be queried
2227 *
2228 * Query the maximum payload size permissible on the given qp
2229 *
2230 * RETURNS: the max payload size of a qp
2231 */
2232unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp)
2233{
2234 unsigned int max_size;
2235 unsigned int copy_align;
2236 struct dma_chan *rx_chan, *tx_chan;
2237
2238 if (!qp)
2239 return 0;
2240
2241 rx_chan = qp->rx_dma_chan;
2242 tx_chan = qp->tx_dma_chan;
2243
2244 copy_align = max(rx_chan ? rx_chan->device->copy_align : 0,
2245 tx_chan ? tx_chan->device->copy_align : 0);
2246
2247 /* If DMA engine usage is possible, try to find the max size for that */
2248 max_size = qp->tx_max_frame - sizeof(struct ntb_payload_header);
2249 max_size = round_down(max_size, 1 << copy_align);
2250
2251 return max_size;
2252}
2253EXPORT_SYMBOL_GPL(ntb_transport_max_size);
2254
2255unsigned int ntb_transport_tx_free_entry(struct ntb_transport_qp *qp)
2256{
2257 unsigned int head = qp->tx_index;
2258 unsigned int tail = qp->remote_rx_info->entry;
2259
2260 return tail > head ? tail - head : qp->tx_max_entry + tail - head;
2261}
2262EXPORT_SYMBOL_GPL(ntb_transport_tx_free_entry);
2263
2264static void ntb_transport_doorbell_callback(void *data, int vector)
2265{
2266 struct ntb_transport_ctx *nt = data;
2267 struct ntb_transport_qp *qp;
2268 u64 db_bits;
2269 unsigned int qp_num;
2270
2271 db_bits = (nt->qp_bitmap & ~nt->qp_bitmap_free &
2272 ntb_db_vector_mask(nt->ndev, vector));
2273
2274 while (db_bits) {
2275 qp_num = __ffs(db_bits);
2276 qp = &nt->qp_vec[qp_num];
2277
2278 if (qp->active)
2279 tasklet_schedule(&qp->rxc_db_work);
2280
2281 db_bits &= ~BIT_ULL(qp_num);
2282 }
2283}
2284
2285static const struct ntb_ctx_ops ntb_transport_ops = {
2286 .link_event = ntb_transport_event_callback,
2287 .db_event = ntb_transport_doorbell_callback,
2288};
2289
2290static struct ntb_client ntb_transport_client = {
2291 .ops = {
2292 .probe = ntb_transport_probe,
2293 .remove = ntb_transport_free,
2294 },
2295};
2296
2297static int __init ntb_transport_init(void)
2298{
2299 int rc;
2300
2301 pr_info("%s, version %s\n", NTB_TRANSPORT_DESC, NTB_TRANSPORT_VER);
2302
2303 if (debugfs_initialized())
2304 nt_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
2305
2306 rc = bus_register(&ntb_transport_bus);
2307 if (rc)
2308 goto err_bus;
2309
2310 rc = ntb_register_client(&ntb_transport_client);
2311 if (rc)
2312 goto err_client;
2313
2314 return 0;
2315
2316err_client:
2317 bus_unregister(&ntb_transport_bus);
2318err_bus:
2319 debugfs_remove_recursive(nt_debugfs_dir);
2320 return rc;
2321}
2322module_init(ntb_transport_init);
2323
2324static void __exit ntb_transport_exit(void)
2325{
2326 ntb_unregister_client(&ntb_transport_client);
2327 bus_unregister(&ntb_transport_bus);
2328 debugfs_remove_recursive(nt_debugfs_dir);
2329}
2330module_exit(ntb_transport_exit);
2331