1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Microsemi Switchtec(tm) PCIe Management Driver
4	* Copyright (c) 2017, Microsemi Corporation
5	*/
6
7	#include <linux/interrupt.h>
8	#include <linux/io-64-nonatomic-lo-hi.h>
9	#include <linux/delay.h>
10	#include <linux/kthread.h>
11	#include <linux/module.h>
12	#include <linux/ntb.h>
13	#include <linux/pci.h>
14	#include <linux/switchtec.h>
15
16	MODULE_DESCRIPTION("Microsemi Switchtec(tm) NTB Driver");
17	MODULE_VERSION("0.1");
18	MODULE_LICENSE("GPL");
19	MODULE_AUTHOR("Microsemi Corporation");
20
21	static ulong max_mw_size = SZ_2M;
22	module_param(max_mw_size, ulong, 0644);
23	MODULE_PARM_DESC(max_mw_size,
24	"Max memory window size reported to the upper layer");
25
26	static bool use_lut_mws;
27	module_param(use_lut_mws, bool, 0644);
28	MODULE_PARM_DESC(use_lut_mws,
29	"Enable the use of the LUT based memory windows");
30
31	#define SWITCHTEC_NTB_MAGIC 0x45CC0001
32	#define MAX_MWS 128
33
34	struct shared_mw {
35	u32 magic;
36	u32 link_sta;
37	u32 partition_id;
38	u64 mw_sizes[MAX_MWS];
39	u32 spad[128];
40	};
41
42	#define MAX_DIRECT_MW ARRAY_SIZE(((struct ntb_ctrl_regs *)(0))->bar_entry)
43	#define LUT_SIZE SZ_64K
44
45	struct switchtec_ntb {
46	struct ntb_dev ntb;
47	struct switchtec_dev *stdev;
48
49	int self_partition;
50	int peer_partition;
51
52	int doorbell_irq;
53	int message_irq;
54
55	struct ntb_info_regs __iomem *mmio_ntb;
56	struct ntb_ctrl_regs __iomem *mmio_ctrl;
57	struct ntb_dbmsg_regs __iomem *mmio_dbmsg;
58	struct ntb_ctrl_regs __iomem *mmio_self_ctrl;
59	struct ntb_ctrl_regs __iomem *mmio_peer_ctrl;
60	struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg;
61	struct ntb_dbmsg_regs __iomem *mmio_peer_dbmsg;
62
63	void __iomem *mmio_xlink_win;
64
65	struct shared_mw *self_shared;
66	struct shared_mw __iomem *peer_shared;
67	dma_addr_t self_shared_dma;
68
69	u64 db_mask;
70	u64 db_valid_mask;
71	int db_shift;
72	int db_peer_shift;
73
74	/* synchronize rmw access of db_mask and hw reg */
75	spinlock_t db_mask_lock;
76
77	int nr_direct_mw;
78	int nr_lut_mw;
79	int nr_rsvd_luts;
80	int direct_mw_to_bar[MAX_DIRECT_MW];
81
82	int peer_nr_direct_mw;
83	int peer_nr_lut_mw;
84	int peer_direct_mw_to_bar[MAX_DIRECT_MW];
85
86	bool link_is_up;
87	enum ntb_speed link_speed;
88	enum ntb_width link_width;
89	struct work_struct check_link_status_work;
90	bool link_force_down;
91	};
92
93	static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb)
94	{
95	return container_of(ntb, struct switchtec_ntb, ntb);
96	}
97
98	static int switchtec_ntb_part_op(struct switchtec_ntb *sndev,
99	struct ntb_ctrl_regs __iomem *ctl,
100	u32 op, int wait_status)
101	{
102	static const char * const op_text[] = {
103	[NTB_CTRL_PART_OP_LOCK] = "lock",
104	[NTB_CTRL_PART_OP_CFG] = "configure",
105	[NTB_CTRL_PART_OP_RESET] = "reset",
106	};
107
108	int i;
109	u32 ps;
110	int status;
111
112	switch (op) {
113	case NTB_CTRL_PART_OP_LOCK:
114	status = NTB_CTRL_PART_STATUS_LOCKING;
115	break;
116	case NTB_CTRL_PART_OP_CFG:
117	status = NTB_CTRL_PART_STATUS_CONFIGURING;
118	break;
119	case NTB_CTRL_PART_OP_RESET:
120	status = NTB_CTRL_PART_STATUS_RESETTING;
121	break;
122	default:
123	return -EINVAL;
124	}
125
126	iowrite32(op, &ctl->partition_op);
127
128	for (i = 0; i < 1000; i++) {
129	if (msleep_interruptible(msecs: 50) != 0) {
130	iowrite32(NTB_CTRL_PART_OP_RESET, &ctl->partition_op);
131	return -EINTR;
132	}
133
134	ps = ioread32(&ctl->partition_status) & 0xFFFF;
135
136	if (ps != status)
137	break;
138	}
139
140	if (ps == wait_status)
141	return 0;
142
143	if (ps == status) {
144	dev_err(&sndev->stdev->dev,
145	"Timed out while performing %s (%d). (%08x)\n",
146	op_text[op], op,
147	ioread32(&ctl->partition_status));
148
149	return -ETIMEDOUT;
150	}
151
152	return -EIO;
153	}
154
155	static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx,
156	u32 val)
157	{
158	if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_peer_dbmsg->omsg))
159	return -EINVAL;
160
161	iowrite32(val, &sndev->mmio_peer_dbmsg->omsg[idx].msg);
162
163	return 0;
164	}
165
166	static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx)
167	{
168	struct switchtec_ntb *sndev = ntb_sndev(ntb);
169	int nr_direct_mw = sndev->peer_nr_direct_mw;
170	int nr_lut_mw = sndev->peer_nr_lut_mw - sndev->nr_rsvd_luts;
171
172	if (pidx != NTB_DEF_PEER_IDX)
173	return -EINVAL;
174
175	if (!use_lut_mws)
176	nr_lut_mw = 0;
177
178	return nr_direct_mw + nr_lut_mw;
179	}
180
181	static int lut_index(struct switchtec_ntb *sndev, int mw_idx)
182	{
183	return mw_idx - sndev->nr_direct_mw + sndev->nr_rsvd_luts;
184	}
185
186	static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx)
187	{
188	return mw_idx - sndev->peer_nr_direct_mw + sndev->nr_rsvd_luts;
189	}
190
191	static int switchtec_ntb_mw_get_align(struct ntb_dev *ntb, int pidx,
192	int widx, resource_size_t *addr_align,
193	resource_size_t *size_align,
194	resource_size_t *size_max)
195	{
196	struct switchtec_ntb *sndev = ntb_sndev(ntb);
197	int lut;
198	resource_size_t size;
199
200	if (pidx != NTB_DEF_PEER_IDX)
201	return -EINVAL;
202
203	lut = widx >= sndev->peer_nr_direct_mw;
204	size = ioread64(addr: &sndev->peer_shared->mw_sizes[widx]);
205
206	if (size == 0)
207	return -EINVAL;
208
209	if (addr_align)
210	*addr_align = lut ? size : SZ_4K;
211
212	if (size_align)
213	*size_align = lut ? size : SZ_4K;
214
215	if (size_max)
216	*size_max = size;
217
218	return 0;
219	}
220
221	static void switchtec_ntb_mw_clr_direct(struct switchtec_ntb *sndev, int idx)
222	{
223	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
224	int bar = sndev->peer_direct_mw_to_bar[idx];
225	u32 ctl_val;
226
227	ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
228	ctl_val &= ~NTB_CTRL_BAR_DIR_WIN_EN;
229	iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
230	iowrite32(0, &ctl->bar_entry[bar].win_size);
231	iowrite32(0, &ctl->bar_ext_entry[bar].win_size);
232	iowrite64(val: sndev->self_partition, addr: &ctl->bar_entry[bar].xlate_addr);
233	}
234
235	static void switchtec_ntb_mw_clr_lut(struct switchtec_ntb *sndev, int idx)
236	{
237	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
238
239	iowrite64(val: 0, addr: &ctl->lut_entry[peer_lut_index(sndev, mw_idx: idx)]);
240	}
241
242	static void switchtec_ntb_mw_set_direct(struct switchtec_ntb *sndev, int idx,
243	dma_addr_t addr, resource_size_t size)
244	{
245	int xlate_pos = ilog2(size);
246	int bar = sndev->peer_direct_mw_to_bar[idx];
247	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
248	u32 ctl_val;
249
250	ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
251	ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
252
253	iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
254	iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000),
255	&ctl->bar_entry[bar].win_size);
256	iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size);
257	iowrite64(val: sndev->self_partition | addr,
258	addr: &ctl->bar_entry[bar].xlate_addr);
259	}
260
261	static void switchtec_ntb_mw_set_lut(struct switchtec_ntb *sndev, int idx,
262	dma_addr_t addr, resource_size_t size)
263	{
264	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
265
266	iowrite64(val: (NTB_CTRL_LUT_EN | (sndev->self_partition << 1) | addr),
267	addr: &ctl->lut_entry[peer_lut_index(sndev, mw_idx: idx)]);
268	}
269
270	static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
271	dma_addr_t addr, resource_size_t size)
272	{
273	struct switchtec_ntb *sndev = ntb_sndev(ntb);
274	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
275	int xlate_pos = ilog2(size);
276	int nr_direct_mw = sndev->peer_nr_direct_mw;
277	int rc;
278
279	if (pidx != NTB_DEF_PEER_IDX)
280	return -EINVAL;
281
282	dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap\n",
283	widx, pidx, &addr, &size);
284
285	if (widx >= switchtec_ntb_mw_count(ntb, pidx))
286	return -EINVAL;
287
288	if (size != 0 && xlate_pos < 12)
289	return -EINVAL;
290
291	if (!IS_ALIGNED(addr, BIT_ULL(xlate_pos))) {
292	/*
293	* In certain circumstances we can get a buffer that is
294	* not aligned to its size. (Most of the time
295	* dma_alloc_coherent ensures this). This can happen when
296	* using large buffers allocated by the CMA
297	* (see CMA_CONFIG_ALIGNMENT)
298	*/
299	dev_err(&sndev->stdev->dev,
300	"ERROR: Memory window address is not aligned to its size!\n");
301	return -EINVAL;
302	}
303
304	rc = switchtec_ntb_part_op(sndev, ctl, op: NTB_CTRL_PART_OP_LOCK,
305	wait_status: NTB_CTRL_PART_STATUS_LOCKED);
306	if (rc)
307	return rc;
308
309	if (size == 0) {
310	if (widx < nr_direct_mw)
311	switchtec_ntb_mw_clr_direct(sndev, idx: widx);
312	else
313	switchtec_ntb_mw_clr_lut(sndev, idx: widx);
314	} else {
315	if (widx < nr_direct_mw)
316	switchtec_ntb_mw_set_direct(sndev, idx: widx, addr, size);
317	else
318	switchtec_ntb_mw_set_lut(sndev, idx: widx, addr, size);
319	}
320
321	rc = switchtec_ntb_part_op(sndev, ctl, op: NTB_CTRL_PART_OP_CFG,
322	wait_status: NTB_CTRL_PART_STATUS_NORMAL);
323
324	if (rc == -EIO) {
325	dev_err(&sndev->stdev->dev,
326	"Hardware reported an error configuring mw %d: %08x\n",
327	widx, ioread32(&ctl->bar_error));
328
329	if (widx < nr_direct_mw)
330	switchtec_ntb_mw_clr_direct(sndev, idx: widx);
331	else
332	switchtec_ntb_mw_clr_lut(sndev, idx: widx);
333
334	switchtec_ntb_part_op(sndev, ctl, op: NTB_CTRL_PART_OP_CFG,
335	wait_status: NTB_CTRL_PART_STATUS_NORMAL);
336	}
337
338	return rc;
339	}
340
341	static int switchtec_ntb_peer_mw_count(struct ntb_dev *ntb)
342	{
343	struct switchtec_ntb *sndev = ntb_sndev(ntb);
344	int nr_lut_mw = sndev->nr_lut_mw - sndev->nr_rsvd_luts;
345
346	return sndev->nr_direct_mw + (use_lut_mws ? nr_lut_mw : 0);
347	}
348
349	static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev,
350	int idx, phys_addr_t *base,
351	resource_size_t *size)
352	{
353	int bar = sndev->direct_mw_to_bar[idx];
354	size_t offset = 0;
355
356	if (bar < 0)
357	return -EINVAL;
358
359	if (idx == 0) {
360	/*
361	* This is the direct BAR shared with the LUTs
362	* which means the actual window will be offset
363	* by the size of all the LUT entries.
364	*/
365
366	offset = LUT_SIZE * sndev->nr_lut_mw;
367	}
368
369	if (base)
370	*base = pci_resource_start(sndev->ntb.pdev, bar) + offset;
371
372	if (size) {
373	*size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
374	if (offset && *size > offset)
375	*size = offset;
376
377	if (*size > max_mw_size)
378	*size = max_mw_size;
379	}
380
381	return 0;
382	}
383
384	static int switchtec_ntb_lut_get_addr(struct switchtec_ntb *sndev,
385	int idx, phys_addr_t *base,
386	resource_size_t *size)
387	{
388	int bar = sndev->direct_mw_to_bar[0];
389	int offset;
390
391	offset = LUT_SIZE * lut_index(sndev, mw_idx: idx);
392
393	if (base)
394	*base = pci_resource_start(sndev->ntb.pdev, bar) + offset;
395
396	if (size)
397	*size = LUT_SIZE;
398
399	return 0;
400	}
401
402	static int switchtec_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
403	phys_addr_t *base,
404	resource_size_t *size)
405	{
406	struct switchtec_ntb *sndev = ntb_sndev(ntb);
407
408	if (idx < sndev->nr_direct_mw)
409	return switchtec_ntb_direct_get_addr(sndev, idx, base, size);
410	else if (idx < switchtec_ntb_peer_mw_count(ntb))
411	return switchtec_ntb_lut_get_addr(sndev, idx, base, size);
412	else
413	return -EINVAL;
414	}
415
416	static void switchtec_ntb_part_link_speed(struct switchtec_ntb *sndev,
417	int partition,
418	enum ntb_speed *speed,
419	enum ntb_width *width)
420	{
421	struct switchtec_dev *stdev = sndev->stdev;
422	struct part_cfg_regs __iomem *part_cfg =
423	&stdev->mmio_part_cfg_all[partition];
424
425	u32 pff = ioread32(&part_cfg->vep_pff_inst_id) & 0xFF;
426	u32 linksta = ioread32(&stdev->mmio_pff_csr[pff].pci_cap_region[13]);
427
428	if (speed)
429	*speed = (linksta >> 16) & 0xF;
430
431	if (width)
432	*width = (linksta >> 20) & 0x3F;
433	}
434
435	static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev)
436	{
437	enum ntb_speed self_speed, peer_speed;
438	enum ntb_width self_width, peer_width;
439
440	if (!sndev->link_is_up) {
441	sndev->link_speed = NTB_SPEED_NONE;
442	sndev->link_width = NTB_WIDTH_NONE;
443	return;
444	}
445
446	switchtec_ntb_part_link_speed(sndev, partition: sndev->self_partition,
447	speed: &self_speed, width: &self_width);
448	switchtec_ntb_part_link_speed(sndev, partition: sndev->peer_partition,
449	speed: &peer_speed, width: &peer_width);
450
451	sndev->link_speed = min(self_speed, peer_speed);
452	sndev->link_width = min(self_width, peer_width);
453	}
454
455	static int crosslink_is_enabled(struct switchtec_ntb *sndev)
456	{
457	struct ntb_info_regs __iomem *inf = sndev->mmio_ntb;
458
459	return ioread8(&inf->ntp_info[sndev->peer_partition].xlink_enabled);
460	}
461
462	static void crosslink_init_dbmsgs(struct switchtec_ntb *sndev)
463	{
464	int i;
465	u32 msg_map = 0;
466
467	if (!crosslink_is_enabled(sndev))
468	return;
469
470	for (i = 0; i < ARRAY_SIZE(sndev->mmio_peer_dbmsg->imsg); i++) {
471	int m = i | sndev->self_partition << 2;
472
473	msg_map |= m << i * 8;
474	}
475
476	iowrite32(msg_map, &sndev->mmio_peer_dbmsg->msg_map);
477	iowrite64(val: sndev->db_valid_mask << sndev->db_peer_shift,
478	addr: &sndev->mmio_peer_dbmsg->odb_mask);
479	}
480
481	enum switchtec_msg {
482	LINK_MESSAGE = 0,
483	MSG_LINK_UP = 1,
484	MSG_LINK_DOWN = 2,
485	MSG_CHECK_LINK = 3,
486	MSG_LINK_FORCE_DOWN = 4,
487	};
488
489	static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev);
490
491	static void switchtec_ntb_link_status_update(struct switchtec_ntb *sndev)
492	{
493	int link_sta;
494	int old = sndev->link_is_up;
495
496	link_sta = sndev->self_shared->link_sta;
497	if (link_sta) {
498	u64 peer = ioread64(addr: &sndev->peer_shared->magic);
499
500	if ((peer & 0xFFFFFFFF) == SWITCHTEC_NTB_MAGIC)
501	link_sta = peer >> 32;
502	else
503	link_sta = 0;
504	}
505
506	sndev->link_is_up = link_sta;
507	switchtec_ntb_set_link_speed(sndev);
508
509	if (link_sta != old) {
510	switchtec_ntb_send_msg(sndev, idx: LINK_MESSAGE, val: MSG_CHECK_LINK);
511	ntb_link_event(ntb: &sndev->ntb);
512	dev_info(&sndev->stdev->dev, "ntb link %s\n",
513	link_sta ? "up" : "down");
514
515	if (link_sta)
516	crosslink_init_dbmsgs(sndev);
517	}
518	}
519
520	static void check_link_status_work(struct work_struct *work)
521	{
522	struct switchtec_ntb *sndev;
523
524	sndev = container_of(work, struct switchtec_ntb,
525	check_link_status_work);
526
527	if (sndev->link_force_down) {
528	sndev->link_force_down = false;
529	switchtec_ntb_reinit_peer(sndev);
530
531	if (sndev->link_is_up) {
532	sndev->link_is_up = 0;
533	ntb_link_event(ntb: &sndev->ntb);
534	dev_info(&sndev->stdev->dev, "ntb link forced down\n");
535	}
536
537	return;
538	}
539
540	switchtec_ntb_link_status_update(sndev);
541	}
542
543	static void switchtec_ntb_check_link(struct switchtec_ntb *sndev,
544	enum switchtec_msg msg)
545	{
546	if (msg == MSG_LINK_FORCE_DOWN)
547	sndev->link_force_down = true;
548
549	schedule_work(work: &sndev->check_link_status_work);
550	}
551
552	static void switchtec_ntb_link_notification(struct switchtec_dev *stdev)
553	{
554	struct switchtec_ntb *sndev = stdev->sndev;
555
556	switchtec_ntb_check_link(sndev, msg: MSG_CHECK_LINK);
557	}
558
559	static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb,
560	enum ntb_speed *speed,
561	enum ntb_width *width)
562	{
563	struct switchtec_ntb *sndev = ntb_sndev(ntb);
564
565	if (speed)
566	*speed = sndev->link_speed;
567	if (width)
568	*width = sndev->link_width;
569
570	return sndev->link_is_up;
571	}
572
573	static int switchtec_ntb_link_enable(struct ntb_dev *ntb,
574	enum ntb_speed max_speed,
575	enum ntb_width max_width)
576	{
577	struct switchtec_ntb *sndev = ntb_sndev(ntb);
578
579	dev_dbg(&sndev->stdev->dev, "enabling link\n");
580
581	sndev->self_shared->link_sta = 1;
582	switchtec_ntb_send_msg(sndev, idx: LINK_MESSAGE, val: MSG_LINK_UP);
583
584	switchtec_ntb_link_status_update(sndev);
585
586	return 0;
587	}
588
589	static int switchtec_ntb_link_disable(struct ntb_dev *ntb)
590	{
591	struct switchtec_ntb *sndev = ntb_sndev(ntb);
592
593	dev_dbg(&sndev->stdev->dev, "disabling link\n");
594
595	sndev->self_shared->link_sta = 0;
596	switchtec_ntb_send_msg(sndev, idx: LINK_MESSAGE, val: MSG_LINK_DOWN);
597
598	switchtec_ntb_link_status_update(sndev);
599
600	return 0;
601	}
602
603	static u64 switchtec_ntb_db_valid_mask(struct ntb_dev *ntb)
604	{
605	struct switchtec_ntb *sndev = ntb_sndev(ntb);
606
607	return sndev->db_valid_mask;
608	}
609
610	static int switchtec_ntb_db_vector_count(struct ntb_dev *ntb)
611	{
612	return 1;
613	}
614
615	static u64 switchtec_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
616	{
617	struct switchtec_ntb *sndev = ntb_sndev(ntb);
618
619	if (db_vector < 0 || db_vector > 1)
620	return 0;
621
622	return sndev->db_valid_mask;
623	}
624
625	static u64 switchtec_ntb_db_read(struct ntb_dev *ntb)
626	{
627	u64 ret;
628	struct switchtec_ntb *sndev = ntb_sndev(ntb);
629
630	ret = ioread64(addr: &sndev->mmio_self_dbmsg->idb) >> sndev->db_shift;
631
632	return ret & sndev->db_valid_mask;
633	}
634
635	static int switchtec_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
636	{
637	struct switchtec_ntb *sndev = ntb_sndev(ntb);
638
639	iowrite64(val: db_bits << sndev->db_shift, addr: &sndev->mmio_self_dbmsg->idb);
640
641	return 0;
642	}
643
644	static int switchtec_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
645	{
646	unsigned long irqflags;
647	struct switchtec_ntb *sndev = ntb_sndev(ntb);
648
649	if (db_bits & ~sndev->db_valid_mask)
650	return -EINVAL;
651
652	spin_lock_irqsave(&sndev->db_mask_lock, irqflags);
653
654	sndev->db_mask |= db_bits << sndev->db_shift;
655	iowrite64(val: ~sndev->db_mask, addr: &sndev->mmio_self_dbmsg->idb_mask);
656
657	spin_unlock_irqrestore(lock: &sndev->db_mask_lock, flags: irqflags);
658
659	return 0;
660	}
661
662	static int switchtec_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
663	{
664	unsigned long irqflags;
665	struct switchtec_ntb *sndev = ntb_sndev(ntb);
666
667	if (db_bits & ~sndev->db_valid_mask)
668	return -EINVAL;
669
670	spin_lock_irqsave(&sndev->db_mask_lock, irqflags);
671
672	sndev->db_mask &= ~(db_bits << sndev->db_shift);
673	iowrite64(val: ~sndev->db_mask, addr: &sndev->mmio_self_dbmsg->idb_mask);
674
675	spin_unlock_irqrestore(lock: &sndev->db_mask_lock, flags: irqflags);
676
677	return 0;
678	}
679
680	static u64 switchtec_ntb_db_read_mask(struct ntb_dev *ntb)
681	{
682	struct switchtec_ntb *sndev = ntb_sndev(ntb);
683
684	return (sndev->db_mask >> sndev->db_shift) & sndev->db_valid_mask;
685	}
686
687	static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb,
688	phys_addr_t *db_addr,
689	resource_size_t *db_size,
690	u64 *db_data,
691	int db_bit)
692	{
693	struct switchtec_ntb *sndev = ntb_sndev(ntb);
694	unsigned long offset;
695
696	if (unlikely(db_bit >= BITS_PER_LONG_LONG))
697	return -EINVAL;
698
699	offset = (unsigned long)sndev->mmio_peer_dbmsg->odb -
700	(unsigned long)sndev->stdev->mmio;
701
702	offset += sndev->db_shift / 8;
703
704	if (db_addr)
705	*db_addr = pci_resource_start(ntb->pdev, 0) + offset;
706	if (db_size)
707	*db_size = sizeof(u32);
708	if (db_data)
709	*db_data = BIT_ULL(db_bit) << sndev->db_peer_shift;
710
711	return 0;
712	}
713
714	static int switchtec_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
715	{
716	struct switchtec_ntb *sndev = ntb_sndev(ntb);
717
718	iowrite64(val: db_bits << sndev->db_peer_shift,
719	addr: &sndev->mmio_peer_dbmsg->odb);
720
721	return 0;
722	}
723
724	static int switchtec_ntb_spad_count(struct ntb_dev *ntb)
725	{
726	struct switchtec_ntb *sndev = ntb_sndev(ntb);
727
728	return ARRAY_SIZE(sndev->self_shared->spad);
729	}
730
731	static u32 switchtec_ntb_spad_read(struct ntb_dev *ntb, int idx)
732	{
733	struct switchtec_ntb *sndev = ntb_sndev(ntb);
734
735	if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))
736	return 0;
737
738	if (!sndev->self_shared)
739	return 0;
740
741	return sndev->self_shared->spad[idx];
742	}
743
744	static int switchtec_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val)
745	{
746	struct switchtec_ntb *sndev = ntb_sndev(ntb);
747
748	if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))
749	return -EINVAL;
750
751	if (!sndev->self_shared)
752	return -EIO;
753
754	sndev->self_shared->spad[idx] = val;
755
756	return 0;
757	}
758
759	static u32 switchtec_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx,
760	int sidx)
761	{
762	struct switchtec_ntb *sndev = ntb_sndev(ntb);
763
764	if (pidx != NTB_DEF_PEER_IDX)
765	return -EINVAL;
766
767	if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))
768	return 0;
769
770	if (!sndev->peer_shared)
771	return 0;
772
773	return ioread32(&sndev->peer_shared->spad[sidx]);
774	}
775
776	static int switchtec_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
777	int sidx, u32 val)
778	{
779	struct switchtec_ntb *sndev = ntb_sndev(ntb);
780
781	if (pidx != NTB_DEF_PEER_IDX)
782	return -EINVAL;
783
784	if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))
785	return -EINVAL;
786
787	if (!sndev->peer_shared)
788	return -EIO;
789
790	iowrite32(val, &sndev->peer_shared->spad[sidx]);
791
792	return 0;
793	}
794
795	static int switchtec_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx,
796	int sidx, phys_addr_t *spad_addr)
797	{
798	struct switchtec_ntb *sndev = ntb_sndev(ntb);
799	unsigned long offset;
800
801	if (pidx != NTB_DEF_PEER_IDX)
802	return -EINVAL;
803
804	offset = (unsigned long)&sndev->peer_shared->spad[sidx] -
805	(unsigned long)sndev->stdev->mmio;
806
807	if (spad_addr)
808	*spad_addr = pci_resource_start(ntb->pdev, 0) + offset;
809
810	return 0;
811	}
812
813	static const struct ntb_dev_ops switchtec_ntb_ops = {
814	.mw_count = switchtec_ntb_mw_count,
815	.mw_get_align = switchtec_ntb_mw_get_align,
816	.mw_set_trans = switchtec_ntb_mw_set_trans,
817	.peer_mw_count = switchtec_ntb_peer_mw_count,
818	.peer_mw_get_addr = switchtec_ntb_peer_mw_get_addr,
819	.link_is_up = switchtec_ntb_link_is_up,
820	.link_enable = switchtec_ntb_link_enable,
821	.link_disable = switchtec_ntb_link_disable,
822	.db_valid_mask = switchtec_ntb_db_valid_mask,
823	.db_vector_count = switchtec_ntb_db_vector_count,
824	.db_vector_mask = switchtec_ntb_db_vector_mask,
825	.db_read = switchtec_ntb_db_read,
826	.db_clear = switchtec_ntb_db_clear,
827	.db_set_mask = switchtec_ntb_db_set_mask,
828	.db_clear_mask = switchtec_ntb_db_clear_mask,
829	.db_read_mask = switchtec_ntb_db_read_mask,
830	.peer_db_addr = switchtec_ntb_peer_db_addr,
831	.peer_db_set = switchtec_ntb_peer_db_set,
832	.spad_count = switchtec_ntb_spad_count,
833	.spad_read = switchtec_ntb_spad_read,
834	.spad_write = switchtec_ntb_spad_write,
835	.peer_spad_read = switchtec_ntb_peer_spad_read,
836	.peer_spad_write = switchtec_ntb_peer_spad_write,
837	.peer_spad_addr = switchtec_ntb_peer_spad_addr,
838	};
839
840	static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)
841	{
842	u64 tpart_vec;
843	int self;
844	u64 part_map;
845
846	sndev->ntb.pdev = sndev->stdev->pdev;
847	sndev->ntb.topo = NTB_TOPO_SWITCH;
848	sndev->ntb.ops = &switchtec_ntb_ops;
849
850	INIT_WORK(&sndev->check_link_status_work, check_link_status_work);
851	sndev->link_force_down = false;
852
853	sndev->self_partition = sndev->stdev->partition;
854
855	sndev->mmio_ntb = sndev->stdev->mmio_ntb;
856
857	self = sndev->self_partition;
858	tpart_vec = ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_high);
859	tpart_vec <<= 32;
860	tpart_vec |= ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_low);
861
862	part_map = ioread64(addr: &sndev->mmio_ntb->ep_map);
863	tpart_vec &= part_map;
864	part_map &= ~(1 << sndev->self_partition);
865
866	if (!tpart_vec) {
867	if (sndev->stdev->partition_count != 2) {
868	dev_err(&sndev->stdev->dev,
869	"ntb target partition not defined\n");
870	return -ENODEV;
871	}
872
873	if (!part_map) {
874	dev_err(&sndev->stdev->dev,
875	"peer partition is not NT partition\n");
876	return -ENODEV;
877	}
878
879	sndev->peer_partition = __ffs64(word: part_map);
880	} else {
881	if (__ffs64(word: tpart_vec) != (fls64(x: tpart_vec) - 1)) {
882	dev_err(&sndev->stdev->dev,
883	"ntb driver only supports 1 pair of 1-1 ntb mapping\n");
884	return -ENODEV;
885	}
886
887	sndev->peer_partition = __ffs64(word: tpart_vec);
888	if (!(part_map & (1ULL << sndev->peer_partition))) {
889	dev_err(&sndev->stdev->dev,
890	"ntb target partition is not NT partition\n");
891	return -ENODEV;
892	}
893	}
894
895	dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d\n",
896	sndev->self_partition, sndev->stdev->partition_count);
897
898	sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb +
899	SWITCHTEC_NTB_REG_CTRL_OFFSET;
900	sndev->mmio_dbmsg = (void * __iomem)sndev->mmio_ntb +
901	SWITCHTEC_NTB_REG_DBMSG_OFFSET;
902
903	sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition];
904	sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition];
905	sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition];
906	sndev->mmio_peer_dbmsg = sndev->mmio_self_dbmsg;
907
908	return 0;
909	}
910
911	static int config_rsvd_lut_win(struct switchtec_ntb *sndev,
912	struct ntb_ctrl_regs __iomem *ctl,
913	int lut_idx, int partition, u64 addr)
914	{
915	int peer_bar = sndev->peer_direct_mw_to_bar[0];
916	u32 ctl_val;
917	int rc;
918
919	rc = switchtec_ntb_part_op(sndev, ctl, op: NTB_CTRL_PART_OP_LOCK,
920	wait_status: NTB_CTRL_PART_STATUS_LOCKED);
921	if (rc)
922	return rc;
923
924	ctl_val = ioread32(&ctl->bar_entry[peer_bar].ctl);
925	ctl_val &= 0xFF;
926	ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN;
927	ctl_val |= ilog2(LUT_SIZE) << 8;
928	ctl_val |= (sndev->nr_lut_mw - 1) << 14;
929	iowrite32(ctl_val, &ctl->bar_entry[peer_bar].ctl);
930
931	iowrite64(val: (NTB_CTRL_LUT_EN | (partition << 1) | addr),
932	addr: &ctl->lut_entry[lut_idx]);
933
934	rc = switchtec_ntb_part_op(sndev, ctl, op: NTB_CTRL_PART_OP_CFG,
935	wait_status: NTB_CTRL_PART_STATUS_NORMAL);
936	if (rc) {
937	u32 bar_error, lut_error;
938
939	bar_error = ioread32(&ctl->bar_error);
940	lut_error = ioread32(&ctl->lut_error);
941	dev_err(&sndev->stdev->dev,
942	"Error setting up reserved lut window: %08x / %08x\n",
943	bar_error, lut_error);
944	return rc;
945	}
946
947	return 0;
948	}
949
950	static int config_req_id_table(struct switchtec_ntb *sndev,
951	struct ntb_ctrl_regs __iomem *mmio_ctrl,
952	int *req_ids, int count)
953	{
954	int i, rc = 0;
955	u32 error;
956	u32 proxy_id;
957
958	if (ioread16(&mmio_ctrl->req_id_table_size) < count) {
959	dev_err(&sndev->stdev->dev,
960	"Not enough requester IDs available.\n");
961	return -EFAULT;
962	}
963
964	rc = switchtec_ntb_part_op(sndev, ctl: mmio_ctrl,
965	op: NTB_CTRL_PART_OP_LOCK,
966	wait_status: NTB_CTRL_PART_STATUS_LOCKED);
967	if (rc)
968	return rc;
969
970	for (i = 0; i < count; i++) {
971	iowrite32(req_ids[i] << 16 | NTB_CTRL_REQ_ID_EN,
972	&mmio_ctrl->req_id_table[i]);
973
974	proxy_id = ioread32(&mmio_ctrl->req_id_table[i]);
975	dev_dbg(&sndev->stdev->dev,
976	"Requester ID %02X:%02X.%X -> BB:%02X.%X\n",
977	req_ids[i] >> 8, (req_ids[i] >> 3) & 0x1F,
978	req_ids[i] & 0x7, (proxy_id >> 4) & 0x1F,
979	(proxy_id >> 1) & 0x7);
980	}
981
982	rc = switchtec_ntb_part_op(sndev, ctl: mmio_ctrl,
983	op: NTB_CTRL_PART_OP_CFG,
984	wait_status: NTB_CTRL_PART_STATUS_NORMAL);
985
986	if (rc == -EIO) {
987	error = ioread32(&mmio_ctrl->req_id_error);
988	dev_err(&sndev->stdev->dev,
989	"Error setting up the requester ID table: %08x\n",
990	error);
991	}
992
993	return 0;
994	}
995
996	static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx,
997	u64 *mw_addrs, int mw_count)
998	{
999	int rc, i;
1000	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_self_ctrl;
1001	u64 addr;
1002	size_t size, offset;
1003	int bar;
1004	int xlate_pos;
1005	u32 ctl_val;
1006
1007	rc = switchtec_ntb_part_op(sndev, ctl, op: NTB_CTRL_PART_OP_LOCK,
1008	wait_status: NTB_CTRL_PART_STATUS_LOCKED);
1009	if (rc)
1010	return rc;
1011
1012	for (i = 0; i < sndev->nr_lut_mw; i++) {
1013	if (i == ntb_lut_idx)
1014	continue;
1015
1016	addr = mw_addrs[0] + LUT_SIZE * i;
1017
1018	iowrite64(val: (NTB_CTRL_LUT_EN | (sndev->peer_partition << 1) |
1019	addr),
1020	addr: &ctl->lut_entry[i]);
1021	}
1022
1023	sndev->nr_direct_mw = min_t(int, sndev->nr_direct_mw, mw_count);
1024
1025	for (i = 0; i < sndev->nr_direct_mw; i++) {
1026	bar = sndev->direct_mw_to_bar[i];
1027	offset = (i == 0) ? LUT_SIZE * sndev->nr_lut_mw : 0;
1028	addr = mw_addrs[i] + offset;
1029	size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
1030	xlate_pos = ilog2(size);
1031
1032	if (offset && size > offset)
1033	size = offset;
1034
1035	ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
1036	ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
1037
1038	iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
1039	iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000),
1040	&ctl->bar_entry[bar].win_size);
1041	iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size);
1042	iowrite64(val: sndev->peer_partition | addr,
1043	addr: &ctl->bar_entry[bar].xlate_addr);
1044	}
1045
1046	rc = switchtec_ntb_part_op(sndev, ctl, op: NTB_CTRL_PART_OP_CFG,
1047	wait_status: NTB_CTRL_PART_STATUS_NORMAL);
1048	if (rc) {
1049	u32 bar_error, lut_error;
1050
1051	bar_error = ioread32(&ctl->bar_error);
1052	lut_error = ioread32(&ctl->lut_error);
1053	dev_err(&sndev->stdev->dev,
1054	"Error setting up cross link windows: %08x / %08x\n",
1055	bar_error, lut_error);
1056	return rc;
1057	}
1058
1059	return 0;
1060	}
1061
1062	static int crosslink_setup_req_ids(struct switchtec_ntb *sndev,
1063	struct ntb_ctrl_regs __iomem *mmio_ctrl)
1064	{
1065	int req_ids[16];
1066	int i;
1067	u32 proxy_id;
1068
1069	for (i = 0; i < ARRAY_SIZE(req_ids); i++) {
1070	proxy_id = ioread32(&sndev->mmio_self_ctrl->req_id_table[i]);
1071
1072	if (!(proxy_id & NTB_CTRL_REQ_ID_EN))
1073	break;
1074
1075	req_ids[i] = ((proxy_id >> 1) & 0xFF);
1076	}
1077
1078	return config_req_id_table(sndev, mmio_ctrl, req_ids, count: i);
1079	}
1080
1081	/*
1082	* In crosslink configuration there is a virtual partition in the
1083	* middle of the two switches. The BARs in this partition have to be
1084	* enumerated and assigned addresses.
1085	*/
1086	static int crosslink_enum_partition(struct switchtec_ntb *sndev,
1087	u64 *bar_addrs)
1088	{
1089	struct part_cfg_regs __iomem *part_cfg =
1090	&sndev->stdev->mmio_part_cfg_all[sndev->peer_partition];
1091	u32 pff = ioread32(&part_cfg->vep_pff_inst_id) & 0xFF;
1092	struct pff_csr_regs __iomem *mmio_pff =
1093	&sndev->stdev->mmio_pff_csr[pff];
1094	const u64 bar_space = 0x1000000000LL;
1095	u64 bar_addr;
1096	int bar_cnt = 0;
1097	int i;
1098
1099	iowrite16(0x6, &mmio_pff->pcicmd);
1100
1101	for (i = 0; i < ARRAY_SIZE(mmio_pff->pci_bar64); i++) {
1102	iowrite64(val: bar_space * i, addr: &mmio_pff->pci_bar64[i]);
1103	bar_addr = ioread64(addr: &mmio_pff->pci_bar64[i]);
1104	bar_addr &= ~0xf;
1105
1106	dev_dbg(&sndev->stdev->dev,
1107	"Crosslink BAR%d addr: %llx\n",
1108	i*2, bar_addr);
1109
1110	if (bar_addr != bar_space * i)
1111	continue;
1112
1113	bar_addrs[bar_cnt++] = bar_addr;
1114	}
1115
1116	return bar_cnt;
1117	}
1118
1119	static int switchtec_ntb_init_crosslink(struct switchtec_ntb *sndev)
1120	{
1121	int rc;
1122	int bar = sndev->direct_mw_to_bar[0];
1123	const int ntb_lut_idx = 1;
1124	u64 bar_addrs[6];
1125	u64 addr;
1126	int offset;
1127	int bar_cnt;
1128
1129	if (!crosslink_is_enabled(sndev))
1130	return 0;
1131
1132	dev_info(&sndev->stdev->dev, "Using crosslink configuration\n");
1133	sndev->ntb.topo = NTB_TOPO_CROSSLINK;
1134
1135	bar_cnt = crosslink_enum_partition(sndev, bar_addrs);
1136	if (bar_cnt < sndev->nr_direct_mw + 1) {
1137	dev_err(&sndev->stdev->dev,
1138	"Error enumerating crosslink partition\n");
1139	return -EINVAL;
1140	}
1141
1142	addr = (bar_addrs[0] + SWITCHTEC_GAS_NTB_OFFSET +
1143	SWITCHTEC_NTB_REG_DBMSG_OFFSET +
1144	sizeof(struct ntb_dbmsg_regs) * sndev->peer_partition);
1145
1146	offset = addr & (LUT_SIZE - 1);
1147	addr -= offset;
1148
1149	rc = config_rsvd_lut_win(sndev, ctl: sndev->mmio_self_ctrl, lut_idx: ntb_lut_idx,
1150	partition: sndev->peer_partition, addr);
1151	if (rc)
1152	return rc;
1153
1154	rc = crosslink_setup_mws(sndev, ntb_lut_idx, mw_addrs: &bar_addrs[1],
1155	mw_count: bar_cnt - 1);
1156	if (rc)
1157	return rc;
1158
1159	rc = crosslink_setup_req_ids(sndev, mmio_ctrl: sndev->mmio_peer_ctrl);
1160	if (rc)
1161	return rc;
1162
1163	sndev->mmio_xlink_win = pci_iomap_range(dev: sndev->stdev->pdev, bar,
1164	LUT_SIZE, LUT_SIZE);
1165	if (!sndev->mmio_xlink_win) {
1166	rc = -ENOMEM;
1167	return rc;
1168	}
1169
1170	sndev->mmio_peer_dbmsg = sndev->mmio_xlink_win + offset;
1171	sndev->nr_rsvd_luts++;
1172
1173	crosslink_init_dbmsgs(sndev);
1174
1175	return 0;
1176	}
1177
1178	static void switchtec_ntb_deinit_crosslink(struct switchtec_ntb *sndev)
1179	{
1180	if (sndev->mmio_xlink_win)
1181	pci_iounmap(dev: sndev->stdev->pdev, sndev->mmio_xlink_win);
1182	}
1183
1184	static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl)
1185	{
1186	int i;
1187	int cnt = 0;
1188
1189	for (i = 0; i < ARRAY_SIZE(ctrl->bar_entry); i++) {
1190	u32 r = ioread32(&ctrl->bar_entry[i].ctl);
1191
1192	if (r & NTB_CTRL_BAR_VALID)
1193	map[cnt++] = i;
1194	}
1195
1196	return cnt;
1197	}
1198
1199	static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev)
1200	{
1201	sndev->nr_direct_mw = map_bars(map: sndev->direct_mw_to_bar,
1202	ctrl: sndev->mmio_self_ctrl);
1203
1204	sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries);
1205	sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw);
1206
1207	dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n",
1208	sndev->nr_direct_mw, sndev->nr_lut_mw);
1209
1210	sndev->peer_nr_direct_mw = map_bars(map: sndev->peer_direct_mw_to_bar,
1211	ctrl: sndev->mmio_peer_ctrl);
1212
1213	sndev->peer_nr_lut_mw =
1214	ioread16(&sndev->mmio_peer_ctrl->lut_table_entries);
1215	sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw);
1216
1217	dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n",
1218	sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw);
1219
1220	}
1221
1222	/*
1223	* There are 64 doorbells in the switch hardware but this is
1224	* shared among all partitions. So we must split them in half
1225	* (32 for each partition). However, the message interrupts are
1226	* also shared with the top 4 doorbells so we just limit this to
1227	* 28 doorbells per partition.
1228	*
1229	* In crosslink mode, each side has it's own dbmsg register so
1230	* they can each use all 60 of the available doorbells.
1231	*/
1232	static void switchtec_ntb_init_db(struct switchtec_ntb *sndev)
1233	{
1234	sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL;
1235
1236	if (sndev->mmio_peer_dbmsg != sndev->mmio_self_dbmsg) {
1237	sndev->db_shift = 0;
1238	sndev->db_peer_shift = 0;
1239	sndev->db_valid_mask = sndev->db_mask;
1240	} else if (sndev->self_partition < sndev->peer_partition) {
1241	sndev->db_shift = 0;
1242	sndev->db_peer_shift = 32;
1243	sndev->db_valid_mask = 0x0FFFFFFF;
1244	} else {
1245	sndev->db_shift = 32;
1246	sndev->db_peer_shift = 0;
1247	sndev->db_valid_mask = 0x0FFFFFFF;
1248	}
1249
1250	iowrite64(val: ~sndev->db_mask, addr: &sndev->mmio_self_dbmsg->idb_mask);
1251	iowrite64(val: sndev->db_valid_mask << sndev->db_peer_shift,
1252	addr: &sndev->mmio_peer_dbmsg->odb_mask);
1253
1254	dev_dbg(&sndev->stdev->dev, "dbs: shift %d/%d, mask %016llx\n",
1255	sndev->db_shift, sndev->db_peer_shift, sndev->db_valid_mask);
1256	}
1257
1258	static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev)
1259	{
1260	int i;
1261	u32 msg_map = 0;
1262
1263	for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {
1264	int m = i | sndev->peer_partition << 2;
1265
1266	msg_map |= m << i * 8;
1267	}
1268
1269	iowrite32(msg_map, &sndev->mmio_self_dbmsg->msg_map);
1270
1271	for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++)
1272	iowrite64(NTB_DBMSG_IMSG_STATUS | NTB_DBMSG_IMSG_MASK,
1273	addr: &sndev->mmio_self_dbmsg->imsg[i]);
1274	}
1275
1276	static int
1277	switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev)
1278	{
1279	int req_ids[2];
1280
1281	/*
1282	* Root Complex Requester ID (which is 0:00.0)
1283	*/
1284	req_ids[0] = 0;
1285
1286	/*
1287	* Host Bridge Requester ID (as read from the mmap address)
1288	*/
1289	req_ids[1] = ioread16(&sndev->mmio_ntb->requester_id);
1290
1291	return config_req_id_table(sndev, mmio_ctrl: sndev->mmio_self_ctrl, req_ids,
1292	ARRAY_SIZE(req_ids));
1293	}
1294
1295	static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev)
1296	{
1297	int i;
1298
1299	memset(sndev->self_shared, 0, LUT_SIZE);
1300	sndev->self_shared->magic = SWITCHTEC_NTB_MAGIC;
1301	sndev->self_shared->partition_id = sndev->stdev->partition;
1302
1303	for (i = 0; i < sndev->nr_direct_mw; i++) {
1304	int bar = sndev->direct_mw_to_bar[i];
1305	resource_size_t sz = pci_resource_len(sndev->stdev->pdev, bar);
1306
1307	if (i == 0)
1308	sz = min_t(resource_size_t, sz,
1309	LUT_SIZE * sndev->nr_lut_mw);
1310
1311	sndev->self_shared->mw_sizes[i] = sz;
1312	}
1313
1314	for (i = 0; i < sndev->nr_lut_mw; i++) {
1315	int idx = sndev->nr_direct_mw + i;
1316
1317	sndev->self_shared->mw_sizes[idx] = LUT_SIZE;
1318	}
1319	}
1320
1321	static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev)
1322	{
1323	int self_bar = sndev->direct_mw_to_bar[0];
1324	int rc;
1325
1326	sndev->nr_rsvd_luts++;
1327	sndev->self_shared = dma_alloc_coherent(dev: &sndev->stdev->pdev->dev,
1328	LUT_SIZE,
1329	dma_handle: &sndev->self_shared_dma,
1330	GFP_KERNEL);
1331	if (!sndev->self_shared) {
1332	dev_err(&sndev->stdev->dev,
1333	"unable to allocate memory for shared mw\n");
1334	return -ENOMEM;
1335	}
1336
1337	switchtec_ntb_init_shared(sndev);
1338
1339	rc = config_rsvd_lut_win(sndev, ctl: sndev->mmio_peer_ctrl, lut_idx: 0,
1340	partition: sndev->self_partition,
1341	addr: sndev->self_shared_dma);
1342	if (rc)
1343	goto unalloc_and_exit;
1344
1345	sndev->peer_shared = pci_iomap(dev: sndev->stdev->pdev, bar: self_bar, LUT_SIZE);
1346	if (!sndev->peer_shared) {
1347	rc = -ENOMEM;
1348	goto unalloc_and_exit;
1349	}
1350
1351	dev_dbg(&sndev->stdev->dev, "Shared MW Ready\n");
1352	return 0;
1353
1354	unalloc_and_exit:
1355	dma_free_coherent(dev: &sndev->stdev->pdev->dev, LUT_SIZE,
1356	cpu_addr: sndev->self_shared, dma_handle: sndev->self_shared_dma);
1357
1358	return rc;
1359	}
1360
1361	static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev)
1362	{
1363	if (sndev->peer_shared)
1364	pci_iounmap(dev: sndev->stdev->pdev, sndev->peer_shared);
1365
1366	if (sndev->self_shared)
1367	dma_free_coherent(dev: &sndev->stdev->pdev->dev, LUT_SIZE,
1368	cpu_addr: sndev->self_shared,
1369	dma_handle: sndev->self_shared_dma);
1370	sndev->nr_rsvd_luts--;
1371	}
1372
1373	static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev)
1374	{
1375	struct switchtec_ntb *sndev = dev;
1376
1377	dev_dbg(&sndev->stdev->dev, "doorbell\n");
1378
1379	ntb_db_event(ntb: &sndev->ntb, vector: 0);
1380
1381	return IRQ_HANDLED;
1382	}
1383
1384	static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev)
1385	{
1386	int i;
1387	struct switchtec_ntb *sndev = dev;
1388
1389	for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {
1390	u64 msg = ioread64(addr: &sndev->mmio_self_dbmsg->imsg[i]);
1391
1392	if (msg & NTB_DBMSG_IMSG_STATUS) {
1393	dev_dbg(&sndev->stdev->dev, "message: %d %08x\n",
1394	i, (u32)msg);
1395	iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status);
1396
1397	if (i == LINK_MESSAGE)
1398	switchtec_ntb_check_link(sndev, msg);
1399	}
1400	}
1401
1402	return IRQ_HANDLED;
1403	}
1404
1405	static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev)
1406	{
1407	int i;
1408	int rc;
1409	int doorbell_irq = 0;
1410	int message_irq = 0;
1411	int event_irq;
1412	int idb_vecs = sizeof(sndev->mmio_self_dbmsg->idb_vec_map);
1413
1414	event_irq = ioread32(&sndev->stdev->mmio_part_cfg->vep_vector_number);
1415
1416	while (doorbell_irq == event_irq)
1417	doorbell_irq++;
1418	while (message_irq == doorbell_irq ||
1419	message_irq == event_irq)
1420	message_irq++;
1421
1422	dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d\n",
1423	event_irq, doorbell_irq, message_irq);
1424
1425	for (i = 0; i < idb_vecs - 4; i++)
1426	iowrite8(doorbell_irq,
1427	&sndev->mmio_self_dbmsg->idb_vec_map[i]);
1428
1429	for (; i < idb_vecs; i++)
1430	iowrite8(message_irq,
1431	&sndev->mmio_self_dbmsg->idb_vec_map[i]);
1432
1433	sndev->doorbell_irq = pci_irq_vector(dev: sndev->stdev->pdev, nr: doorbell_irq);
1434	sndev->message_irq = pci_irq_vector(dev: sndev->stdev->pdev, nr: message_irq);
1435
1436	rc = request_irq(irq: sndev->doorbell_irq,
1437	handler: switchtec_ntb_doorbell_isr, flags: 0,
1438	name: "switchtec_ntb_doorbell", dev: sndev);
1439	if (rc)
1440	return rc;
1441
1442	rc = request_irq(irq: sndev->message_irq,
1443	handler: switchtec_ntb_message_isr, flags: 0,
1444	name: "switchtec_ntb_message", dev: sndev);
1445	if (rc) {
1446	free_irq(sndev->doorbell_irq, sndev);
1447	return rc;
1448	}
1449
1450	return 0;
1451	}
1452
1453	static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev)
1454	{
1455	free_irq(sndev->doorbell_irq, sndev);
1456	free_irq(sndev->message_irq, sndev);
1457	}
1458
1459	static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev)
1460	{
1461	int rc;
1462
1463	if (crosslink_is_enabled(sndev))
1464	return 0;
1465
1466	dev_info(&sndev->stdev->dev, "reinitialize shared memory window\n");
1467	rc = config_rsvd_lut_win(sndev, ctl: sndev->mmio_peer_ctrl, lut_idx: 0,
1468	partition: sndev->self_partition,
1469	addr: sndev->self_shared_dma);
1470	return rc;
1471	}
1472
1473	static int switchtec_ntb_add(struct device *dev)
1474	{
1475	struct switchtec_dev *stdev = to_stdev(dev);
1476	struct switchtec_ntb *sndev;
1477	int rc;
1478
1479	stdev->sndev = NULL;
1480
1481	if (stdev->pdev->class != (PCI_CLASS_BRIDGE_OTHER << 8))
1482	return -ENODEV;
1483
1484	sndev = kzalloc_node(size: sizeof(*sndev), GFP_KERNEL, node: dev_to_node(dev));
1485	if (!sndev)
1486	return -ENOMEM;
1487
1488	sndev->stdev = stdev;
1489	rc = switchtec_ntb_init_sndev(sndev);
1490	if (rc)
1491	goto free_and_exit;
1492
1493	switchtec_ntb_init_mw(sndev);
1494
1495	rc = switchtec_ntb_init_req_id_table(sndev);
1496	if (rc)
1497	goto free_and_exit;
1498
1499	rc = switchtec_ntb_init_crosslink(sndev);
1500	if (rc)
1501	goto free_and_exit;
1502
1503	switchtec_ntb_init_db(sndev);
1504	switchtec_ntb_init_msgs(sndev);
1505
1506	rc = switchtec_ntb_init_shared_mw(sndev);
1507	if (rc)
1508	goto deinit_crosslink;
1509
1510	rc = switchtec_ntb_init_db_msg_irq(sndev);
1511	if (rc)
1512	goto deinit_shared_and_exit;
1513
1514	/*
1515	* If this host crashed, the other host may think the link is
1516	* still up. Tell them to force it down (it will go back up
1517	* once we register the ntb device).
1518	*/
1519	switchtec_ntb_send_msg(sndev, idx: LINK_MESSAGE, val: MSG_LINK_FORCE_DOWN);
1520
1521	rc = ntb_register_device(ntb: &sndev->ntb);
1522	if (rc)
1523	goto deinit_and_exit;
1524
1525	stdev->sndev = sndev;
1526	stdev->link_notifier = switchtec_ntb_link_notification;
1527	dev_info(dev, "NTB device registered\n");
1528
1529	return 0;
1530
1531	deinit_and_exit:
1532	switchtec_ntb_deinit_db_msg_irq(sndev);
1533	deinit_shared_and_exit:
1534	switchtec_ntb_deinit_shared_mw(sndev);
1535	deinit_crosslink:
1536	switchtec_ntb_deinit_crosslink(sndev);
1537	free_and_exit:
1538	kfree(objp: sndev);
1539	dev_err(dev, "failed to register ntb device: %d\n", rc);
1540	return rc;
1541	}
1542
1543	static void switchtec_ntb_remove(struct device *dev)
1544	{
1545	struct switchtec_dev *stdev = to_stdev(dev);
1546	struct switchtec_ntb *sndev = stdev->sndev;
1547
1548	if (!sndev)
1549	return;
1550
1551	stdev->link_notifier = NULL;
1552	stdev->sndev = NULL;
1553	ntb_unregister_device(ntb: &sndev->ntb);
1554	switchtec_ntb_deinit_db_msg_irq(sndev);
1555	switchtec_ntb_deinit_shared_mw(sndev);
1556	switchtec_ntb_deinit_crosslink(sndev);
1557	kfree(objp: sndev);
1558	dev_info(dev, "ntb device unregistered\n");
1559	}
1560
1561	static struct class_interface switchtec_interface = {
1562	.add_dev = switchtec_ntb_add,
1563	.remove_dev = switchtec_ntb_remove,
1564	};
1565
1566	static int __init switchtec_ntb_init(void)
1567	{
1568	switchtec_interface.class = switchtec_class;
1569	return class_interface_register(&switchtec_interface);
1570	}
1571	module_init(switchtec_ntb_init);
1572
1573	static void __exit switchtec_ntb_exit(void)
1574	{
1575	class_interface_unregister(&switchtec_interface);
1576	}
1577	module_exit(switchtec_ntb_exit);
1578