1	// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2	/*
3	* Copyright(c) 2015 - 2018 Intel Corporation.
4	*/
5
6	#include <linux/spinlock.h>
7	#include <linux/seqlock.h>
8	#include <linux/netdevice.h>
9	#include <linux/moduleparam.h>
10	#include <linux/bitops.h>
11	#include <linux/timer.h>
12	#include <linux/vmalloc.h>
13	#include <linux/highmem.h>
14
15	#include "hfi.h"
16	#include "common.h"
17	#include "qp.h"
18	#include "sdma.h"
19	#include "iowait.h"
20	#include "trace.h"
21
22	/* must be a power of 2 >= 64 <= 32768 */
23	#define SDMA_DESCQ_CNT 2048
24	#define SDMA_DESC_INTR 64
25	#define INVALID_TAIL 0xffff
26	#define SDMA_PAD max_t(size_t, MAX_16B_PADDING, sizeof(u32))
27
28	static uint sdma_descq_cnt = SDMA_DESCQ_CNT;
29	module_param(sdma_descq_cnt, uint, S_IRUGO);
30	MODULE_PARM_DESC(sdma_descq_cnt, "Number of SDMA descq entries");
31
32	static uint sdma_idle_cnt = 250;
33	module_param(sdma_idle_cnt, uint, S_IRUGO);
34	MODULE_PARM_DESC(sdma_idle_cnt, "sdma interrupt idle delay (ns,default 250)");
35
36	uint mod_num_sdma;
37	module_param_named(num_sdma, mod_num_sdma, uint, S_IRUGO);
38	MODULE_PARM_DESC(num_sdma, "Set max number SDMA engines to use");
39
40	static uint sdma_desct_intr = SDMA_DESC_INTR;
41	module_param_named(desct_intr, sdma_desct_intr, uint, S_IRUGO | S_IWUSR);
42	MODULE_PARM_DESC(desct_intr, "Number of SDMA descriptor before interrupt");
43
44	#define SDMA_WAIT_BATCH_SIZE 20
45	/* max wait time for a SDMA engine to indicate it has halted */
46	#define SDMA_ERR_HALT_TIMEOUT 10 /* ms */
47	/* all SDMA engine errors that cause a halt */
48
49	#define SD(name) SEND_DMA_##name
50	#define ALL_SDMA_ENG_HALT_ERRS \
51	(SD(ENG_ERR_STATUS_SDMA_WRONG_DW_ERR_SMASK) \
52	| SD(ENG_ERR_STATUS_SDMA_GEN_MISMATCH_ERR_SMASK) \
53	| SD(ENG_ERR_STATUS_SDMA_TOO_LONG_ERR_SMASK) \
54	| SD(ENG_ERR_STATUS_SDMA_TAIL_OUT_OF_BOUNDS_ERR_SMASK) \
55	| SD(ENG_ERR_STATUS_SDMA_FIRST_DESC_ERR_SMASK) \
56	| SD(ENG_ERR_STATUS_SDMA_MEM_READ_ERR_SMASK) \
57	| SD(ENG_ERR_STATUS_SDMA_HALT_ERR_SMASK) \
58	| SD(ENG_ERR_STATUS_SDMA_LENGTH_MISMATCH_ERR_SMASK) \
59	| SD(ENG_ERR_STATUS_SDMA_PACKET_DESC_OVERFLOW_ERR_SMASK) \
60	| SD(ENG_ERR_STATUS_SDMA_HEADER_SELECT_ERR_SMASK) \
61	| SD(ENG_ERR_STATUS_SDMA_HEADER_ADDRESS_ERR_SMASK) \
62	| SD(ENG_ERR_STATUS_SDMA_HEADER_LENGTH_ERR_SMASK) \
63	| SD(ENG_ERR_STATUS_SDMA_TIMEOUT_ERR_SMASK) \
64	| SD(ENG_ERR_STATUS_SDMA_DESC_TABLE_UNC_ERR_SMASK) \
65	| SD(ENG_ERR_STATUS_SDMA_ASSEMBLY_UNC_ERR_SMASK) \
66	| SD(ENG_ERR_STATUS_SDMA_PACKET_TRACKING_UNC_ERR_SMASK) \
67	| SD(ENG_ERR_STATUS_SDMA_HEADER_STORAGE_UNC_ERR_SMASK) \
68	| SD(ENG_ERR_STATUS_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_SMASK))
69
70	/* sdma_sendctrl operations */
71	#define SDMA_SENDCTRL_OP_ENABLE BIT(0)
72	#define SDMA_SENDCTRL_OP_INTENABLE BIT(1)
73	#define SDMA_SENDCTRL_OP_HALT BIT(2)
74	#define SDMA_SENDCTRL_OP_CLEANUP BIT(3)
75
76	/* handle long defines */
77	#define SDMA_EGRESS_PACKET_OCCUPANCY_SMASK \
78	SEND_EGRESS_SEND_DMA_STATUS_SDMA_EGRESS_PACKET_OCCUPANCY_SMASK
79	#define SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT \
80	SEND_EGRESS_SEND_DMA_STATUS_SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT
81
82	static const char * const sdma_state_names[] = {
83	[sdma_state_s00_hw_down] = "s00_HwDown",
84	[sdma_state_s10_hw_start_up_halt_wait] = "s10_HwStartUpHaltWait",
85	[sdma_state_s15_hw_start_up_clean_wait] = "s15_HwStartUpCleanWait",
86	[sdma_state_s20_idle] = "s20_Idle",
87	[sdma_state_s30_sw_clean_up_wait] = "s30_SwCleanUpWait",
88	[sdma_state_s40_hw_clean_up_wait] = "s40_HwCleanUpWait",
89	[sdma_state_s50_hw_halt_wait] = "s50_HwHaltWait",
90	[sdma_state_s60_idle_halt_wait] = "s60_IdleHaltWait",
91	[sdma_state_s80_hw_freeze] = "s80_HwFreeze",
92	[sdma_state_s82_freeze_sw_clean] = "s82_FreezeSwClean",
93	[sdma_state_s99_running] = "s99_Running",
94	};
95
96	#ifdef CONFIG_SDMA_VERBOSITY
97	static const char * const sdma_event_names[] = {
98	[sdma_event_e00_go_hw_down] = "e00_GoHwDown",
99	[sdma_event_e10_go_hw_start] = "e10_GoHwStart",
100	[sdma_event_e15_hw_halt_done] = "e15_HwHaltDone",
101	[sdma_event_e25_hw_clean_up_done] = "e25_HwCleanUpDone",
102	[sdma_event_e30_go_running] = "e30_GoRunning",
103	[sdma_event_e40_sw_cleaned] = "e40_SwCleaned",
104	[sdma_event_e50_hw_cleaned] = "e50_HwCleaned",
105	[sdma_event_e60_hw_halted] = "e60_HwHalted",
106	[sdma_event_e70_go_idle] = "e70_GoIdle",
107	[sdma_event_e80_hw_freeze] = "e80_HwFreeze",
108	[sdma_event_e81_hw_frozen] = "e81_HwFrozen",
109	[sdma_event_e82_hw_unfreeze] = "e82_HwUnfreeze",
110	[sdma_event_e85_link_down] = "e85_LinkDown",
111	[sdma_event_e90_sw_halted] = "e90_SwHalted",
112	};
113	#endif
114
115	static const struct sdma_set_state_action sdma_action_table[] = {
116	[sdma_state_s00_hw_down] = {
117	.go_s99_running_tofalse = 1,
118	.op_enable = 0,
119	.op_intenable = 0,
120	.op_halt = 0,
121	.op_cleanup = 0,
122	},
123	[sdma_state_s10_hw_start_up_halt_wait] = {
124	.op_enable = 0,
125	.op_intenable = 0,
126	.op_halt = 1,
127	.op_cleanup = 0,
128	},
129	[sdma_state_s15_hw_start_up_clean_wait] = {
130	.op_enable = 0,
131	.op_intenable = 1,
132	.op_halt = 0,
133	.op_cleanup = 1,
134	},
135	[sdma_state_s20_idle] = {
136	.op_enable = 0,
137	.op_intenable = 1,
138	.op_halt = 0,
139	.op_cleanup = 0,
140	},
141	[sdma_state_s30_sw_clean_up_wait] = {
142	.op_enable = 0,
143	.op_intenable = 0,
144	.op_halt = 0,
145	.op_cleanup = 0,
146	},
147	[sdma_state_s40_hw_clean_up_wait] = {
148	.op_enable = 0,
149	.op_intenable = 0,
150	.op_halt = 0,
151	.op_cleanup = 1,
152	},
153	[sdma_state_s50_hw_halt_wait] = {
154	.op_enable = 0,
155	.op_intenable = 0,
156	.op_halt = 0,
157	.op_cleanup = 0,
158	},
159	[sdma_state_s60_idle_halt_wait] = {
160	.go_s99_running_tofalse = 1,
161	.op_enable = 0,
162	.op_intenable = 0,
163	.op_halt = 1,
164	.op_cleanup = 0,
165	},
166	[sdma_state_s80_hw_freeze] = {
167	.op_enable = 0,
168	.op_intenable = 0,
169	.op_halt = 0,
170	.op_cleanup = 0,
171	},
172	[sdma_state_s82_freeze_sw_clean] = {
173	.op_enable = 0,
174	.op_intenable = 0,
175	.op_halt = 0,
176	.op_cleanup = 0,
177	},
178	[sdma_state_s99_running] = {
179	.op_enable = 1,
180	.op_intenable = 1,
181	.op_halt = 0,
182	.op_cleanup = 0,
183	.go_s99_running_totrue = 1,
184	},
185	};
186
187	#define SDMA_TAIL_UPDATE_THRESH 0x1F
188
189	/* declare all statics here rather than keep sorting */
190	static void sdma_complete(struct kref *);
191	static void sdma_finalput(struct sdma_state *);
192	static void sdma_get(struct sdma_state *);
193	static void sdma_hw_clean_up_task(struct tasklet_struct *);
194	static void sdma_put(struct sdma_state *);
195	static void sdma_set_state(struct sdma_engine *, enum sdma_states);
196	static void sdma_start_hw_clean_up(struct sdma_engine *);
197	static void sdma_sw_clean_up_task(struct tasklet_struct *);
198	static void sdma_sendctrl(struct sdma_engine *, unsigned);
199	static void init_sdma_regs(struct sdma_engine *, u32, uint);
200	static void sdma_process_event(
201	struct sdma_engine *sde,
202	enum sdma_events event);
203	static void __sdma_process_event(
204	struct sdma_engine *sde,
205	enum sdma_events event);
206	static void dump_sdma_state(struct sdma_engine *sde);
207	static void sdma_make_progress(struct sdma_engine *sde, u64 status);
208	static void sdma_desc_avail(struct sdma_engine *sde, uint avail);
209	static void sdma_flush_descq(struct sdma_engine *sde);
210
211	/**
212	* sdma_state_name() - return state string from enum
213	* @state: state
214	*/
215	static const char *sdma_state_name(enum sdma_states state)
216	{
217	return sdma_state_names[state];
218	}
219
220	static void sdma_get(struct sdma_state *ss)
221	{
222	kref_get(kref: &ss->kref);
223	}
224
225	static void sdma_complete(struct kref *kref)
226	{
227	struct sdma_state *ss =
228	container_of(kref, struct sdma_state, kref);
229
230	complete(&ss->comp);
231	}
232
233	static void sdma_put(struct sdma_state *ss)
234	{
235	kref_put(kref: &ss->kref, release: sdma_complete);
236	}
237
238	static void sdma_finalput(struct sdma_state *ss)
239	{
240	sdma_put(ss);
241	wait_for_completion(&ss->comp);
242	}
243
244	static inline void write_sde_csr(
245	struct sdma_engine *sde,
246	u32 offset0,
247	u64 value)
248	{
249	write_kctxt_csr(dd: sde->dd, ctxt: sde->this_idx, offset0, value);
250	}
251
252	static inline u64 read_sde_csr(
253	struct sdma_engine *sde,
254	u32 offset0)
255	{
256	return read_kctxt_csr(dd: sde->dd, ctxt: sde->this_idx, offset0);
257	}
258
259	/*
260	* sdma_wait_for_packet_egress() - wait for the VL FIFO occupancy for
261	* sdma engine 'sde' to drop to 0.
262	*/
263	static void sdma_wait_for_packet_egress(struct sdma_engine *sde,
264	int pause)
265	{
266	u64 off = 8 * sde->this_idx;
267	struct hfi1_devdata *dd = sde->dd;
268	int lcnt = 0;
269	u64 reg_prev;
270	u64 reg = 0;
271
272	while (1) {
273	reg_prev = reg;
274	reg = read_csr(dd, offset: off + SEND_EGRESS_SEND_DMA_STATUS);
275
276	reg &= SDMA_EGRESS_PACKET_OCCUPANCY_SMASK;
277	reg >>= SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT;
278	if (reg == 0)
279	break;
280	/* counter is reest if accupancy count changes */
281	if (reg != reg_prev)
282	lcnt = 0;
283	if (lcnt++ > 500) {
284	/* timed out - bounce the link */
285	dd_dev_err(dd, "%s: engine %u timeout waiting for packets to egress, remaining count %u, bouncing link\n",
286	__func__, sde->this_idx, (u32)reg);
287	queue_work(wq: dd->pport->link_wq,
288	work: &dd->pport->link_bounce_work);
289	break;
290	}
291	udelay(1);
292	}
293	}
294
295	/*
296	* sdma_wait() - wait for packet egress to complete for all SDMA engines,
297	* and pause for credit return.
298	*/
299	void sdma_wait(struct hfi1_devdata *dd)
300	{
301	int i;
302
303	for (i = 0; i < dd->num_sdma; i++) {
304	struct sdma_engine *sde = &dd->per_sdma[i];
305
306	sdma_wait_for_packet_egress(sde, pause: 0);
307	}
308	}
309
310	static inline void sdma_set_desc_cnt(struct sdma_engine *sde, unsigned cnt)
311	{
312	u64 reg;
313
314	if (!(sde->dd->flags & HFI1_HAS_SDMA_TIMEOUT))
315	return;
316	reg = cnt;
317	reg &= SD(DESC_CNT_CNT_MASK);
318	reg <<= SD(DESC_CNT_CNT_SHIFT);
319	write_sde_csr(sde, SD(DESC_CNT), value: reg);
320	}
321
322	static inline void complete_tx(struct sdma_engine *sde,
323	struct sdma_txreq *tx,
324	int res)
325	{
326	/* protect against complete modifying */
327	struct iowait *wait = tx->wait;
328	callback_t complete = tx->complete;
329
330	#ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
331	trace_hfi1_sdma_out_sn(sde, sn: tx->sn);
332	if (WARN_ON_ONCE(sde->head_sn != tx->sn))
333	dd_dev_err(sde->dd, "expected %llu got %llu\n",
334	sde->head_sn, tx->sn);
335	sde->head_sn++;
336	#endif
337	__sdma_txclean(sde->dd, tx);
338	if (complete)
339	(*complete)(tx, res);
340	if (iowait_sdma_dec(wait))
341	iowait_drain_wakeup(wait);
342	}
343
344	/*
345	* Complete all the sdma requests with a SDMA_TXREQ_S_ABORTED status
346	*
347	* Depending on timing there can be txreqs in two places:
348	* - in the descq ring
349	* - in the flush list
350	*
351	* To avoid ordering issues the descq ring needs to be flushed
352	* first followed by the flush list.
353	*
354	* This routine is called from two places
355	* - From a work queue item
356	* - Directly from the state machine just before setting the
357	* state to running
358	*
359	* Must be called with head_lock held
360	*
361	*/
362	static void sdma_flush(struct sdma_engine *sde)
363	{
364	struct sdma_txreq *txp, *txp_next;
365	LIST_HEAD(flushlist);
366	unsigned long flags;
367	uint seq;
368
369	/* flush from head to tail */
370	sdma_flush_descq(sde);
371	spin_lock_irqsave(&sde->flushlist_lock, flags);
372	/* copy flush list */
373	list_splice_init(list: &sde->flushlist, head: &flushlist);
374	spin_unlock_irqrestore(lock: &sde->flushlist_lock, flags);
375	/* flush from flush list */
376	list_for_each_entry_safe(txp, txp_next, &flushlist, list)
377	complete_tx(sde, tx: txp, SDMA_TXREQ_S_ABORTED);
378	/* wakeup QPs orphaned on the dmawait list */
379	do {
380	struct iowait *w, *nw;
381
382	seq = read_seqbegin(sl: &sde->waitlock);
383	if (!list_empty(head: &sde->dmawait)) {
384	write_seqlock(sl: &sde->waitlock);
385	list_for_each_entry_safe(w, nw, &sde->dmawait, list) {
386	if (w->wakeup) {
387	w->wakeup(w, SDMA_AVAIL_REASON);
388	list_del_init(entry: &w->list);
389	}
390	}
391	write_sequnlock(sl: &sde->waitlock);
392	}
393	} while (read_seqretry(sl: &sde->waitlock, start: seq));
394	}
395
396	/*
397	* Fields a work request for flushing the descq ring
398	* and the flush list
399	*
400	* If the engine has been brought to running during
401	* the scheduling delay, the flush is ignored, assuming
402	* that the process of bringing the engine to running
403	* would have done this flush prior to going to running.
404	*
405	*/
406	static void sdma_field_flush(struct work_struct *work)
407	{
408	unsigned long flags;
409	struct sdma_engine *sde =
410	container_of(work, struct sdma_engine, flush_worker);
411
412	write_seqlock_irqsave(&sde->head_lock, flags);
413	if (!__sdma_running(engine: sde))
414	sdma_flush(sde);
415	write_sequnlock_irqrestore(sl: &sde->head_lock, flags);
416	}
417
418	static void sdma_err_halt_wait(struct work_struct *work)
419	{
420	struct sdma_engine *sde = container_of(work, struct sdma_engine,
421	err_halt_worker);
422	u64 statuscsr;
423	unsigned long timeout;
424
425	timeout = jiffies + msecs_to_jiffies(SDMA_ERR_HALT_TIMEOUT);
426	while (1) {
427	statuscsr = read_sde_csr(sde, SD(STATUS));
428	statuscsr &= SD(STATUS_ENG_HALTED_SMASK);
429	if (statuscsr)
430	break;
431	if (time_after(jiffies, timeout)) {
432	dd_dev_err(sde->dd,
433	"SDMA engine %d - timeout waiting for engine to halt\n",
434	sde->this_idx);
435	/*
436	* Continue anyway. This could happen if there was
437	* an uncorrectable error in the wrong spot.
438	*/
439	break;
440	}
441	usleep_range(min: 80, max: 120);
442	}
443
444	sdma_process_event(sde, event: sdma_event_e15_hw_halt_done);
445	}
446
447	static void sdma_err_progress_check_schedule(struct sdma_engine *sde)
448	{
449	if (!is_bx(dd: sde->dd) && HFI1_CAP_IS_KSET(SDMA_AHG)) {
450	unsigned index;
451	struct hfi1_devdata *dd = sde->dd;
452
453	for (index = 0; index < dd->num_sdma; index++) {
454	struct sdma_engine *curr_sdma = &dd->per_sdma[index];
455
456	if (curr_sdma != sde)
457	curr_sdma->progress_check_head =
458	curr_sdma->descq_head;
459	}
460	dd_dev_err(sde->dd,
461	"SDMA engine %d - check scheduled\n",
462	sde->this_idx);
463	mod_timer(timer: &sde->err_progress_check_timer, expires: jiffies + 10);
464	}
465	}
466
467	static void sdma_err_progress_check(struct timer_list *t)
468	{
469	unsigned index;
470	struct sdma_engine *sde = from_timer(sde, t, err_progress_check_timer);
471
472	dd_dev_err(sde->dd, "SDE progress check event\n");
473	for (index = 0; index < sde->dd->num_sdma; index++) {
474	struct sdma_engine *curr_sde = &sde->dd->per_sdma[index];
475	unsigned long flags;
476
477	/* check progress on each engine except the current one */
478	if (curr_sde == sde)
479	continue;
480	/*
481	* We must lock interrupts when acquiring sde->lock,
482	* to avoid a deadlock if interrupt triggers and spins on
483	* the same lock on same CPU
484	*/
485	spin_lock_irqsave(&curr_sde->tail_lock, flags);
486	write_seqlock(sl: &curr_sde->head_lock);
487
488	/* skip non-running queues */
489	if (curr_sde->state.current_state != sdma_state_s99_running) {
490	write_sequnlock(sl: &curr_sde->head_lock);
491	spin_unlock_irqrestore(lock: &curr_sde->tail_lock, flags);
492	continue;
493	}
494
495	if ((curr_sde->descq_head != curr_sde->descq_tail) &&
496	(curr_sde->descq_head ==
497	curr_sde->progress_check_head))
498	__sdma_process_event(sde: curr_sde,
499	event: sdma_event_e90_sw_halted);
500	write_sequnlock(sl: &curr_sde->head_lock);
501	spin_unlock_irqrestore(lock: &curr_sde->tail_lock, flags);
502	}
503	schedule_work(work: &sde->err_halt_worker);
504	}
505
506	static void sdma_hw_clean_up_task(struct tasklet_struct *t)
507	{
508	struct sdma_engine *sde = from_tasklet(sde, t,
509	sdma_hw_clean_up_task);
510	u64 statuscsr;
511
512	while (1) {
513	#ifdef CONFIG_SDMA_VERBOSITY
514	dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n",
515	sde->this_idx, slashstrip(__FILE__), __LINE__,
516	__func__);
517	#endif
518	statuscsr = read_sde_csr(sde, SD(STATUS));
519	statuscsr &= SD(STATUS_ENG_CLEANED_UP_SMASK);
520	if (statuscsr)
521	break;
522	udelay(10);
523	}
524
525	sdma_process_event(sde, event: sdma_event_e25_hw_clean_up_done);
526	}
527
528	static inline struct sdma_txreq *get_txhead(struct sdma_engine *sde)
529	{
530	return sde->tx_ring[sde->tx_head & sde->sdma_mask];
531	}
532
533	/*
534	* flush ring for recovery
535	*/
536	static void sdma_flush_descq(struct sdma_engine *sde)
537	{
538	u16 head, tail;
539	int progress = 0;
540	struct sdma_txreq *txp = get_txhead(sde);
541
542	/* The reason for some of the complexity of this code is that
543	* not all descriptors have corresponding txps. So, we have to
544	* be able to skip over descs until we wander into the range of
545	* the next txp on the list.
546	*/
547	head = sde->descq_head & sde->sdma_mask;
548	tail = sde->descq_tail & sde->sdma_mask;
549	while (head != tail) {
550	/* advance head, wrap if needed */
551	head = ++sde->descq_head & sde->sdma_mask;
552	/* if now past this txp's descs, do the callback */
553	if (txp && txp->next_descq_idx == head) {
554	/* remove from list */
555	sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL;
556	complete_tx(sde, tx: txp, SDMA_TXREQ_S_ABORTED);
557	trace_hfi1_sdma_progress(sde, hwhead: head, swhead: tail, txp);
558	txp = get_txhead(sde);
559	}
560	progress++;
561	}
562	if (progress)
563	sdma_desc_avail(sde, avail: sdma_descq_freecnt(sde));
564	}
565
566	static void sdma_sw_clean_up_task(struct tasklet_struct *t)
567	{
568	struct sdma_engine *sde = from_tasklet(sde, t, sdma_sw_clean_up_task);
569	unsigned long flags;
570
571	spin_lock_irqsave(&sde->tail_lock, flags);
572	write_seqlock(sl: &sde->head_lock);
573
574	/*
575	* At this point, the following should always be true:
576	* - We are halted, so no more descriptors are getting retired.
577	* - We are not running, so no one is submitting new work.
578	* - Only we can send the e40_sw_cleaned, so we can't start
579	* running again until we say so. So, the active list and
580	* descq are ours to play with.
581	*/
582
583	/*
584	* In the error clean up sequence, software clean must be called
585	* before the hardware clean so we can use the hardware head in
586	* the progress routine. A hardware clean or SPC unfreeze will
587	* reset the hardware head.
588	*
589	* Process all retired requests. The progress routine will use the
590	* latest physical hardware head - we are not running so speed does
591	* not matter.
592	*/
593	sdma_make_progress(sde, status: 0);
594
595	sdma_flush(sde);
596
597	/*
598	* Reset our notion of head and tail.
599	* Note that the HW registers have been reset via an earlier
600	* clean up.
601	*/
602	sde->descq_tail = 0;
603	sde->descq_head = 0;
604	sde->desc_avail = sdma_descq_freecnt(sde);
605	*sde->head_dma = 0;
606
607	__sdma_process_event(sde, event: sdma_event_e40_sw_cleaned);
608
609	write_sequnlock(sl: &sde->head_lock);
610	spin_unlock_irqrestore(lock: &sde->tail_lock, flags);
611	}
612
613	static void sdma_sw_tear_down(struct sdma_engine *sde)
614	{
615	struct sdma_state *ss = &sde->state;
616
617	/* Releasing this reference means the state machine has stopped. */
618	sdma_put(ss);
619
620	/* stop waiting for all unfreeze events to complete */
621	atomic_set(v: &sde->dd->sdma_unfreeze_count, i: -1);
622	wake_up_interruptible(&sde->dd->sdma_unfreeze_wq);
623	}
624
625	static void sdma_start_hw_clean_up(struct sdma_engine *sde)
626	{
627	tasklet_hi_schedule(t: &sde->sdma_hw_clean_up_task);
628	}
629
630	static void sdma_set_state(struct sdma_engine *sde,
631	enum sdma_states next_state)
632	{
633	struct sdma_state *ss = &sde->state;
634	const struct sdma_set_state_action *action = sdma_action_table;
635	unsigned op = 0;
636
637	trace_hfi1_sdma_state(
638	sde,
639	cstate: sdma_state_names[ss->current_state],
640	nstate: sdma_state_names[next_state]);
641
642	/* debugging bookkeeping */
643	ss->previous_state = ss->current_state;
644	ss->previous_op = ss->current_op;
645	ss->current_state = next_state;
646
647	if (ss->previous_state != sdma_state_s99_running &&
648	next_state == sdma_state_s99_running)
649	sdma_flush(sde);
650
651	if (action[next_state].op_enable)
652	op |= SDMA_SENDCTRL_OP_ENABLE;
653
654	if (action[next_state].op_intenable)
655	op |= SDMA_SENDCTRL_OP_INTENABLE;
656
657	if (action[next_state].op_halt)
658	op |= SDMA_SENDCTRL_OP_HALT;
659
660	if (action[next_state].op_cleanup)
661	op |= SDMA_SENDCTRL_OP_CLEANUP;
662
663	if (action[next_state].go_s99_running_tofalse)
664	ss->go_s99_running = 0;
665
666	if (action[next_state].go_s99_running_totrue)
667	ss->go_s99_running = 1;
668
669	ss->current_op = op;
670	sdma_sendctrl(sde, ss->current_op);
671	}
672
673	/**
674	* sdma_get_descq_cnt() - called when device probed
675	*
676	* Return a validated descq count.
677	*
678	* This is currently only used in the verbs initialization to build the tx
679	* list.
680	*
681	* This will probably be deleted in favor of a more scalable approach to
682	* alloc tx's.
683	*
684	*/
685	u16 sdma_get_descq_cnt(void)
686	{
687	u16 count = sdma_descq_cnt;
688
689	if (!count)
690	return SDMA_DESCQ_CNT;
691	/* count must be a power of 2 greater than 64 and less than
692	* 32768. Otherwise return default.
693	*/
694	if (!is_power_of_2(n: count))
695	return SDMA_DESCQ_CNT;
696	if (count < 64 || count > 32768)
697	return SDMA_DESCQ_CNT;
698	return count;
699	}
700
701	/**
702	* sdma_engine_get_vl() - return vl for a given sdma engine
703	* @sde: sdma engine
704	*
705	* This function returns the vl mapped to a given engine, or an error if
706	* the mapping can't be found. The mapping fields are protected by RCU.
707	*/
708	int sdma_engine_get_vl(struct sdma_engine *sde)
709	{
710	struct hfi1_devdata *dd = sde->dd;
711	struct sdma_vl_map *m;
712	u8 vl;
713
714	if (sde->this_idx >= TXE_NUM_SDMA_ENGINES)
715	return -EINVAL;
716
717	rcu_read_lock();
718	m = rcu_dereference(dd->sdma_map);
719	if (unlikely(!m)) {
720	rcu_read_unlock();
721	return -EINVAL;
722	}
723	vl = m->engine_to_vl[sde->this_idx];
724	rcu_read_unlock();
725
726	return vl;
727	}
728
729	/**
730	* sdma_select_engine_vl() - select sdma engine
731	* @dd: devdata
732	* @selector: a spreading factor
733	* @vl: this vl
734	*
735	*
736	* This function returns an engine based on the selector and a vl. The
737	* mapping fields are protected by RCU.
738	*/
739	struct sdma_engine *sdma_select_engine_vl(
740	struct hfi1_devdata *dd,
741	u32 selector,
742	u8 vl)
743	{
744	struct sdma_vl_map *m;
745	struct sdma_map_elem *e;
746	struct sdma_engine *rval;
747
748	/* NOTE This should only happen if SC->VL changed after the initial
749	* checks on the QP/AH
750	* Default will return engine 0 below
751	*/
752	if (vl >= num_vls) {
753	rval = NULL;
754	goto done;
755	}
756
757	rcu_read_lock();
758	m = rcu_dereference(dd->sdma_map);
759	if (unlikely(!m)) {
760	rcu_read_unlock();
761	return &dd->per_sdma[0];
762	}
763	e = m->map[vl & m->mask];
764	rval = e->sde[selector & e->mask];
765	rcu_read_unlock();
766
767	done:
768	rval = !rval ? &dd->per_sdma[0] : rval;
769	trace_hfi1_sdma_engine_select(dd, sel: selector, vl, idx: rval->this_idx);
770	return rval;
771	}
772
773	/**
774	* sdma_select_engine_sc() - select sdma engine
775	* @dd: devdata
776	* @selector: a spreading factor
777	* @sc5: the 5 bit sc
778	*
779	*
780	* This function returns an engine based on the selector and an sc.
781	*/
782	struct sdma_engine *sdma_select_engine_sc(
783	struct hfi1_devdata *dd,
784	u32 selector,
785	u8 sc5)
786	{
787	u8 vl = sc_to_vlt(dd, sc5);
788
789	return sdma_select_engine_vl(dd, selector, vl);
790	}
791
792	struct sdma_rht_map_elem {
793	u32 mask;
794	u8 ctr;
795	struct sdma_engine *sde[];
796	};
797
798	struct sdma_rht_node {
799	unsigned long cpu_id;
800	struct sdma_rht_map_elem *map[HFI1_MAX_VLS_SUPPORTED];
801	struct rhash_head node;
802	};
803
804	#define NR_CPUS_HINT 192
805
806	static const struct rhashtable_params sdma_rht_params = {
807	.nelem_hint = NR_CPUS_HINT,
808	.head_offset = offsetof(struct sdma_rht_node, node),
809	.key_offset = offsetof(struct sdma_rht_node, cpu_id),
810	.key_len = sizeof_field(struct sdma_rht_node, cpu_id),
811	.max_size = NR_CPUS,
812	.min_size = 8,
813	.automatic_shrinking = true,
814	};
815
816	/*
817	* sdma_select_user_engine() - select sdma engine based on user setup
818	* @dd: devdata
819	* @selector: a spreading factor
820	* @vl: this vl
821	*
822	* This function returns an sdma engine for a user sdma request.
823	* User defined sdma engine affinity setting is honored when applicable,
824	* otherwise system default sdma engine mapping is used. To ensure correct
825	* ordering, the mapping from <selector, vl> to sde must remain unchanged.
826	*/
827	struct sdma_engine *sdma_select_user_engine(struct hfi1_devdata *dd,
828	u32 selector, u8 vl)
829	{
830	struct sdma_rht_node *rht_node;
831	struct sdma_engine *sde = NULL;
832	unsigned long cpu_id;
833
834	/*
835	* To ensure that always the same sdma engine(s) will be
836	* selected make sure the process is pinned to this CPU only.
837	*/
838	if (current->nr_cpus_allowed != 1)
839	goto out;
840
841	rcu_read_lock();
842	cpu_id = smp_processor_id();
843	rht_node = rhashtable_lookup(ht: dd->sdma_rht, key: &cpu_id,
844	params: sdma_rht_params);
845
846	if (rht_node && rht_node->map[vl]) {
847	struct sdma_rht_map_elem *map = rht_node->map[vl];
848
849	sde = map->sde[selector & map->mask];
850	}
851	rcu_read_unlock();
852
853	if (sde)
854	return sde;
855
856	out:
857	return sdma_select_engine_vl(dd, selector, vl);
858	}
859
860	static void sdma_populate_sde_map(struct sdma_rht_map_elem *map)
861	{
862	int i;
863
864	for (i = 0; i < roundup_pow_of_two(map->ctr ? : 1) - map->ctr; i++)
865	map->sde[map->ctr + i] = map->sde[i];
866	}
867
868	static void sdma_cleanup_sde_map(struct sdma_rht_map_elem *map,
869	struct sdma_engine *sde)
870	{
871	unsigned int i, pow;
872
873	/* only need to check the first ctr entries for a match */
874	for (i = 0; i < map->ctr; i++) {
875	if (map->sde[i] == sde) {
876	memmove(&map->sde[i], &map->sde[i + 1],
877	(map->ctr - i - 1) * sizeof(map->sde[0]));
878	map->ctr--;
879	pow = roundup_pow_of_two(map->ctr ? : 1);
880	map->mask = pow - 1;
881	sdma_populate_sde_map(map);
882	break;
883	}
884	}
885	}
886
887	/*
888	* Prevents concurrent reads and writes of the sdma engine cpu_mask
889	*/
890	static DEFINE_MUTEX(process_to_sde_mutex);
891
892	ssize_t sdma_set_cpu_to_sde_map(struct sdma_engine *sde, const char *buf,
893	size_t count)
894	{
895	struct hfi1_devdata *dd = sde->dd;
896	cpumask_var_t mask, new_mask;
897	unsigned long cpu;
898	int ret, vl, sz;
899	struct sdma_rht_node *rht_node;
900
901	vl = sdma_engine_get_vl(sde);
902	if (unlikely(vl < 0 || vl >= ARRAY_SIZE(rht_node->map)))
903	return -EINVAL;
904
905	ret = zalloc_cpumask_var(mask: &mask, GFP_KERNEL);
906	if (!ret)
907	return -ENOMEM;
908
909	ret = zalloc_cpumask_var(mask: &new_mask, GFP_KERNEL);
910	if (!ret) {
911	free_cpumask_var(mask);
912	return -ENOMEM;
913	}
914	ret = cpulist_parse(buf, dstp: mask);
915	if (ret)
916	goto out_free;
917
918	if (!cpumask_subset(src1p: mask, cpu_online_mask)) {
919	dd_dev_warn(sde->dd, "Invalid CPU mask\n");
920	ret = -EINVAL;
921	goto out_free;
922	}
923
924	sz = sizeof(struct sdma_rht_map_elem) +
925	(TXE_NUM_SDMA_ENGINES * sizeof(struct sdma_engine *));
926
927	mutex_lock(&process_to_sde_mutex);
928
929	for_each_cpu(cpu, mask) {
930	/* Check if we have this already mapped */
931	if (cpumask_test_cpu(cpu, cpumask: &sde->cpu_mask)) {
932	cpumask_set_cpu(cpu, dstp: new_mask);
933	continue;
934	}
935
936	rht_node = rhashtable_lookup_fast(ht: dd->sdma_rht, key: &cpu,
937	params: sdma_rht_params);
938	if (!rht_node) {
939	rht_node = kzalloc(size: sizeof(*rht_node), GFP_KERNEL);
940	if (!rht_node) {
941	ret = -ENOMEM;
942	goto out;
943	}
944
945	rht_node->map[vl] = kzalloc(size: sz, GFP_KERNEL);
946	if (!rht_node->map[vl]) {
947	kfree(objp: rht_node);
948	ret = -ENOMEM;
949	goto out;
950	}
951	rht_node->cpu_id = cpu;
952	rht_node->map[vl]->mask = 0;
953	rht_node->map[vl]->ctr = 1;
954	rht_node->map[vl]->sde[0] = sde;
955
956	ret = rhashtable_insert_fast(ht: dd->sdma_rht,
957	obj: &rht_node->node,
958	params: sdma_rht_params);
959	if (ret) {
960	kfree(objp: rht_node->map[vl]);
961	kfree(objp: rht_node);
962	dd_dev_err(sde->dd, "Failed to set process to sde affinity for cpu %lu\n",
963	cpu);
964	goto out;
965	}
966
967	} else {
968	int ctr, pow;
969
970	/* Add new user mappings */
971	if (!rht_node->map[vl])
972	rht_node->map[vl] = kzalloc(size: sz, GFP_KERNEL);
973
974	if (!rht_node->map[vl]) {
975	ret = -ENOMEM;
976	goto out;
977	}
978
979	rht_node->map[vl]->ctr++;
980	ctr = rht_node->map[vl]->ctr;
981	rht_node->map[vl]->sde[ctr - 1] = sde;
982	pow = roundup_pow_of_two(ctr);
983	rht_node->map[vl]->mask = pow - 1;
984
985	/* Populate the sde map table */
986	sdma_populate_sde_map(map: rht_node->map[vl]);
987	}
988	cpumask_set_cpu(cpu, dstp: new_mask);
989	}
990
991	/* Clean up old mappings */
992	for_each_cpu(cpu, cpu_online_mask) {
993	struct sdma_rht_node *rht_node;
994
995	/* Don't cleanup sdes that are set in the new mask */
996	if (cpumask_test_cpu(cpu, cpumask: mask))
997	continue;
998
999	rht_node = rhashtable_lookup_fast(ht: dd->sdma_rht, key: &cpu,
1000	params: sdma_rht_params);
1001	if (rht_node) {
1002	bool empty = true;
1003	int i;
1004
1005	/* Remove mappings for old sde */
1006	for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++)
1007	if (rht_node->map[i])
1008	sdma_cleanup_sde_map(map: rht_node->map[i],
1009	sde);
1010
1011	/* Free empty hash table entries */
1012	for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) {
1013	if (!rht_node->map[i])
1014	continue;
1015
1016	if (rht_node->map[i]->ctr) {
1017	empty = false;
1018	break;
1019	}
1020	}
1021
1022	if (empty) {
1023	ret = rhashtable_remove_fast(ht: dd->sdma_rht,
1024	obj: &rht_node->node,
1025	params: sdma_rht_params);
1026	WARN_ON(ret);
1027
1028	for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++)
1029	kfree(objp: rht_node->map[i]);
1030
1031	kfree(objp: rht_node);
1032	}
1033	}
1034	}
1035
1036	cpumask_copy(dstp: &sde->cpu_mask, srcp: new_mask);
1037	out:
1038	mutex_unlock(lock: &process_to_sde_mutex);
1039	out_free:
1040	free_cpumask_var(mask);
1041	free_cpumask_var(mask: new_mask);
1042	return ret ? : strnlen(p: buf, PAGE_SIZE);
1043	}
1044
1045	ssize_t sdma_get_cpu_to_sde_map(struct sdma_engine *sde, char *buf)
1046	{
1047	mutex_lock(&process_to_sde_mutex);
1048	if (cpumask_empty(srcp: &sde->cpu_mask))
1049	snprintf(buf, PAGE_SIZE, fmt: "%s\n", "empty");
1050	else
1051	cpumap_print_to_pagebuf(list: true, buf, mask: &sde->cpu_mask);
1052	mutex_unlock(lock: &process_to_sde_mutex);
1053	return strnlen(p: buf, PAGE_SIZE);
1054	}
1055
1056	static void sdma_rht_free(void *ptr, void *arg)
1057	{
1058	struct sdma_rht_node *rht_node = ptr;
1059	int i;
1060
1061	for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++)
1062	kfree(objp: rht_node->map[i]);
1063
1064	kfree(objp: rht_node);
1065	}
1066
1067	/**
1068	* sdma_seqfile_dump_cpu_list() - debugfs dump the cpu to sdma mappings
1069	* @s: seq file
1070	* @dd: hfi1_devdata
1071	* @cpuid: cpu id
1072	*
1073	* This routine dumps the process to sde mappings per cpu
1074	*/
1075	void sdma_seqfile_dump_cpu_list(struct seq_file *s,
1076	struct hfi1_devdata *dd,
1077	unsigned long cpuid)
1078	{
1079	struct sdma_rht_node *rht_node;
1080	int i, j;
1081
1082	rht_node = rhashtable_lookup_fast(ht: dd->sdma_rht, key: &cpuid,
1083	params: sdma_rht_params);
1084	if (!rht_node)
1085	return;
1086
1087	seq_printf(m: s, fmt: "cpu%3lu: ", cpuid);
1088	for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) {
1089	if (!rht_node->map[i] || !rht_node->map[i]->ctr)
1090	continue;
1091
1092	seq_printf(m: s, fmt: " vl%d: [", i);
1093
1094	for (j = 0; j < rht_node->map[i]->ctr; j++) {
1095	if (!rht_node->map[i]->sde[j])
1096	continue;
1097
1098	if (j > 0)
1099	seq_puts(m: s, s: ",");
1100
1101	seq_printf(m: s, fmt: " sdma%2d",
1102	rht_node->map[i]->sde[j]->this_idx);
1103	}
1104	seq_puts(m: s, s: " ]");
1105	}
1106
1107	seq_puts(m: s, s: "\n");
1108	}
1109
1110	/*
1111	* Free the indicated map struct
1112	*/
1113	static void sdma_map_free(struct sdma_vl_map *m)
1114	{
1115	int i;
1116
1117	for (i = 0; m && i < m->actual_vls; i++)
1118	kfree(objp: m->map[i]);
1119	kfree(objp: m);
1120	}
1121
1122	/*
1123	* Handle RCU callback
1124	*/
1125	static void sdma_map_rcu_callback(struct rcu_head *list)
1126	{
1127	struct sdma_vl_map *m = container_of(list, struct sdma_vl_map, list);
1128
1129	sdma_map_free(m);
1130	}
1131
1132	/**
1133	* sdma_map_init - called when # vls change
1134	* @dd: hfi1_devdata
1135	* @port: port number
1136	* @num_vls: number of vls
1137	* @vl_engines: per vl engine mapping (optional)
1138	*
1139	* This routine changes the mapping based on the number of vls.
1140	*
1141	* vl_engines is used to specify a non-uniform vl/engine loading. NULL
1142	* implies auto computing the loading and giving each VLs a uniform
1143	* distribution of engines per VL.
1144	*
1145	* The auto algorithm computes the sde_per_vl and the number of extra
1146	* engines. Any extra engines are added from the last VL on down.
1147	*
1148	* rcu locking is used here to control access to the mapping fields.
1149	*
1150	* If either the num_vls or num_sdma are non-power of 2, the array sizes
1151	* in the struct sdma_vl_map and the struct sdma_map_elem are rounded
1152	* up to the next highest power of 2 and the first entry is reused
1153	* in a round robin fashion.
1154	*
1155	* If an error occurs the map change is not done and the mapping is
1156	* not changed.
1157	*
1158	*/
1159	int sdma_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls, u8 *vl_engines)
1160	{
1161	int i, j;
1162	int extra, sde_per_vl;
1163	int engine = 0;
1164	u8 lvl_engines[OPA_MAX_VLS];
1165	struct sdma_vl_map *oldmap, *newmap;
1166
1167	if (!(dd->flags & HFI1_HAS_SEND_DMA))
1168	return 0;
1169
1170	if (!vl_engines) {
1171	/* truncate divide */
1172	sde_per_vl = dd->num_sdma / num_vls;
1173	/* extras */
1174	extra = dd->num_sdma % num_vls;
1175	vl_engines = lvl_engines;
1176	/* add extras from last vl down */
1177	for (i = num_vls - 1; i >= 0; i--, extra--)
1178	vl_engines[i] = sde_per_vl + (extra > 0 ? 1 : 0);
1179	}
1180	/* build new map */
1181	newmap = kzalloc(
1182	size: sizeof(struct sdma_vl_map) +
1183	roundup_pow_of_two(num_vls) *
1184	sizeof(struct sdma_map_elem *),
1185	GFP_KERNEL);
1186	if (!newmap)
1187	goto bail;
1188	newmap->actual_vls = num_vls;
1189	newmap->vls = roundup_pow_of_two(num_vls);
1190	newmap->mask = (1 << ilog2(newmap->vls)) - 1;
1191	/* initialize back-map */
1192	for (i = 0; i < TXE_NUM_SDMA_ENGINES; i++)
1193	newmap->engine_to_vl[i] = -1;
1194	for (i = 0; i < newmap->vls; i++) {
1195	/* save for wrap around */
1196	int first_engine = engine;
1197
1198	if (i < newmap->actual_vls) {
1199	int sz = roundup_pow_of_two(vl_engines[i]);
1200
1201	/* only allocate once */
1202	newmap->map[i] = kzalloc(
1203	size: sizeof(struct sdma_map_elem) +
1204	sz * sizeof(struct sdma_engine *),
1205	GFP_KERNEL);
1206	if (!newmap->map[i])
1207	goto bail;
1208	newmap->map[i]->mask = (1 << ilog2(sz)) - 1;
1209	/* assign engines */
1210	for (j = 0; j < sz; j++) {
1211	newmap->map[i]->sde[j] =
1212	&dd->per_sdma[engine];
1213	if (++engine >= first_engine + vl_engines[i])
1214	/* wrap back to first engine */
1215	engine = first_engine;
1216	}
1217	/* assign back-map */
1218	for (j = 0; j < vl_engines[i]; j++)
1219	newmap->engine_to_vl[first_engine + j] = i;
1220	} else {
1221	/* just re-use entry without allocating */
1222	newmap->map[i] = newmap->map[i % num_vls];
1223	}
1224	engine = first_engine + vl_engines[i];
1225	}
1226	/* newmap in hand, save old map */
1227	spin_lock_irq(lock: &dd->sde_map_lock);
1228	oldmap = rcu_dereference_protected(dd->sdma_map,
1229	lockdep_is_held(&dd->sde_map_lock));
1230
1231	/* publish newmap */
1232	rcu_assign_pointer(dd->sdma_map, newmap);
1233
1234	spin_unlock_irq(lock: &dd->sde_map_lock);
1235	/* success, free any old map after grace period */
1236	if (oldmap)
1237	call_rcu(head: &oldmap->list, func: sdma_map_rcu_callback);
1238	return 0;
1239	bail:
1240	/* free any partial allocation */
1241	sdma_map_free(m: newmap);
1242	return -ENOMEM;
1243	}
1244
1245	/**
1246	* sdma_clean - Clean up allocated memory
1247	* @dd: struct hfi1_devdata
1248	* @num_engines: num sdma engines
1249	*
1250	* This routine can be called regardless of the success of
1251	* sdma_init()
1252	*/
1253	void sdma_clean(struct hfi1_devdata *dd, size_t num_engines)
1254	{
1255	size_t i;
1256	struct sdma_engine *sde;
1257
1258	if (dd->sdma_pad_dma) {
1259	dma_free_coherent(dev: &dd->pcidev->dev, SDMA_PAD,
1260	cpu_addr: (void *)dd->sdma_pad_dma,
1261	dma_handle: dd->sdma_pad_phys);
1262	dd->sdma_pad_dma = NULL;
1263	dd->sdma_pad_phys = 0;
1264	}
1265	if (dd->sdma_heads_dma) {
1266	dma_free_coherent(dev: &dd->pcidev->dev, size: dd->sdma_heads_size,
1267	cpu_addr: (void *)dd->sdma_heads_dma,
1268	dma_handle: dd->sdma_heads_phys);
1269	dd->sdma_heads_dma = NULL;
1270	dd->sdma_heads_phys = 0;
1271	}
1272	for (i = 0; dd->per_sdma && i < num_engines; ++i) {
1273	sde = &dd->per_sdma[i];
1274
1275	sde->head_dma = NULL;
1276	sde->head_phys = 0;
1277
1278	if (sde->descq) {
1279	dma_free_coherent(
1280	dev: &dd->pcidev->dev,
1281	size: sde->descq_cnt * sizeof(u64[2]),
1282	cpu_addr: sde->descq,
1283	dma_handle: sde->descq_phys
1284	);
1285	sde->descq = NULL;
1286	sde->descq_phys = 0;
1287	}
1288	kvfree(addr: sde->tx_ring);
1289	sde->tx_ring = NULL;
1290	}
1291	if (rcu_access_pointer(dd->sdma_map)) {
1292	spin_lock_irq(lock: &dd->sde_map_lock);
1293	sdma_map_free(rcu_access_pointer(dd->sdma_map));
1294	RCU_INIT_POINTER(dd->sdma_map, NULL);
1295	spin_unlock_irq(lock: &dd->sde_map_lock);
1296	synchronize_rcu();
1297	}
1298	kfree(objp: dd->per_sdma);
1299	dd->per_sdma = NULL;
1300
1301	if (dd->sdma_rht) {
1302	rhashtable_free_and_destroy(ht: dd->sdma_rht, free_fn: sdma_rht_free, NULL);
1303	kfree(objp: dd->sdma_rht);
1304	dd->sdma_rht = NULL;
1305	}
1306	}
1307
1308	/**
1309	* sdma_init() - called when device probed
1310	* @dd: hfi1_devdata
1311	* @port: port number (currently only zero)
1312	*
1313	* Initializes each sde and its csrs.
1314	* Interrupts are not required to be enabled.
1315	*
1316	* Returns:
1317	* 0 - success, -errno on failure
1318	*/
1319	int sdma_init(struct hfi1_devdata *dd, u8 port)
1320	{
1321	unsigned this_idx;
1322	struct sdma_engine *sde;
1323	struct rhashtable *tmp_sdma_rht;
1324	u16 descq_cnt;
1325	void *curr_head;
1326	struct hfi1_pportdata *ppd = dd->pport + port;
1327	u32 per_sdma_credits;
1328	uint idle_cnt = sdma_idle_cnt;
1329	size_t num_engines = chip_sdma_engines(dd);
1330	int ret = -ENOMEM;
1331
1332	if (!HFI1_CAP_IS_KSET(SDMA)) {
1333	HFI1_CAP_CLEAR(SDMA_AHG);
1334	return 0;
1335	}
1336	if (mod_num_sdma &&
1337	/* can't exceed chip support */
1338	mod_num_sdma <= chip_sdma_engines(dd) &&
1339	/* count must be >= vls */
1340	mod_num_sdma >= num_vls)
1341	num_engines = mod_num_sdma;
1342
1343	dd_dev_info(dd, "SDMA mod_num_sdma: %u\n", mod_num_sdma);
1344	dd_dev_info(dd, "SDMA chip_sdma_engines: %u\n", chip_sdma_engines(dd));
1345	dd_dev_info(dd, "SDMA chip_sdma_mem_size: %u\n",
1346	chip_sdma_mem_size(dd));
1347
1348	per_sdma_credits =
1349	chip_sdma_mem_size(dd) / (num_engines * SDMA_BLOCK_SIZE);
1350
1351	/* set up freeze waitqueue */
1352	init_waitqueue_head(&dd->sdma_unfreeze_wq);
1353	atomic_set(v: &dd->sdma_unfreeze_count, i: 0);
1354
1355	descq_cnt = sdma_get_descq_cnt();
1356	dd_dev_info(dd, "SDMA engines %zu descq_cnt %u\n",
1357	num_engines, descq_cnt);
1358
1359	/* alloc memory for array of send engines */
1360	dd->per_sdma = kcalloc_node(n: num_engines, size: sizeof(*dd->per_sdma),
1361	GFP_KERNEL, node: dd->node);
1362	if (!dd->per_sdma)
1363	return ret;
1364
1365	idle_cnt = ns_to_cclock(dd, ns: idle_cnt);
1366	if (idle_cnt)
1367	dd->default_desc1 =
1368	SDMA_DESC1_HEAD_TO_HOST_FLAG;
1369	else
1370	dd->default_desc1 =
1371	SDMA_DESC1_INT_REQ_FLAG;
1372
1373	if (!sdma_desct_intr)
1374	sdma_desct_intr = SDMA_DESC_INTR;
1375
1376	/* Allocate memory for SendDMA descriptor FIFOs */
1377	for (this_idx = 0; this_idx < num_engines; ++this_idx) {
1378	sde = &dd->per_sdma[this_idx];
1379	sde->dd = dd;
1380	sde->ppd = ppd;
1381	sde->this_idx = this_idx;
1382	sde->descq_cnt = descq_cnt;
1383	sde->desc_avail = sdma_descq_freecnt(sde);
1384	sde->sdma_shift = ilog2(descq_cnt);
1385	sde->sdma_mask = (1 << sde->sdma_shift) - 1;
1386
1387	/* Create a mask specifically for each interrupt source */
1388	sde->int_mask = (u64)1 << (0 * TXE_NUM_SDMA_ENGINES +
1389	this_idx);
1390	sde->progress_mask = (u64)1 << (1 * TXE_NUM_SDMA_ENGINES +
1391	this_idx);
1392	sde->idle_mask = (u64)1 << (2 * TXE_NUM_SDMA_ENGINES +
1393	this_idx);
1394	/* Create a combined mask to cover all 3 interrupt sources */
1395	sde->imask = sde->int_mask | sde->progress_mask |
1396	sde->idle_mask;
1397
1398	spin_lock_init(&sde->tail_lock);
1399	seqlock_init(&sde->head_lock);
1400	spin_lock_init(&sde->senddmactrl_lock);
1401	spin_lock_init(&sde->flushlist_lock);
1402	seqlock_init(&sde->waitlock);
1403	/* insure there is always a zero bit */
1404	sde->ahg_bits = 0xfffffffe00000000ULL;
1405
1406	sdma_set_state(sde, next_state: sdma_state_s00_hw_down);
1407
1408	/* set up reference counting */
1409	kref_init(kref: &sde->state.kref);
1410	init_completion(x: &sde->state.comp);
1411
1412	INIT_LIST_HEAD(list: &sde->flushlist);
1413	INIT_LIST_HEAD(list: &sde->dmawait);
1414
1415	sde->tail_csr =
1416	get_kctxt_csr_addr(dd, ctxt: this_idx, SD(TAIL));
1417
1418	tasklet_setup(t: &sde->sdma_hw_clean_up_task,
1419	callback: sdma_hw_clean_up_task);
1420	tasklet_setup(t: &sde->sdma_sw_clean_up_task,
1421	callback: sdma_sw_clean_up_task);
1422	INIT_WORK(&sde->err_halt_worker, sdma_err_halt_wait);
1423	INIT_WORK(&sde->flush_worker, sdma_field_flush);
1424
1425	sde->progress_check_head = 0;
1426
1427	timer_setup(&sde->err_progress_check_timer,
1428	sdma_err_progress_check, 0);
1429
1430	sde->descq = dma_alloc_coherent(dev: &dd->pcidev->dev,
1431	size: descq_cnt * sizeof(u64[2]),
1432	dma_handle: &sde->descq_phys, GFP_KERNEL);
1433	if (!sde->descq)
1434	goto bail;
1435	sde->tx_ring =
1436	kvzalloc_node(array_size(descq_cnt,
1437	sizeof(struct sdma_txreq *)),
1438	GFP_KERNEL, node: dd->node);
1439	if (!sde->tx_ring)
1440	goto bail;
1441	}
1442
1443	dd->sdma_heads_size = L1_CACHE_BYTES * num_engines;
1444	/* Allocate memory for DMA of head registers to memory */
1445	dd->sdma_heads_dma = dma_alloc_coherent(dev: &dd->pcidev->dev,
1446	size: dd->sdma_heads_size,
1447	dma_handle: &dd->sdma_heads_phys,
1448	GFP_KERNEL);
1449	if (!dd->sdma_heads_dma) {
1450	dd_dev_err(dd, "failed to allocate SendDMA head memory\n");
1451	goto bail;
1452	}
1453
1454	/* Allocate memory for pad */
1455	dd->sdma_pad_dma = dma_alloc_coherent(dev: &dd->pcidev->dev, SDMA_PAD,
1456	dma_handle: &dd->sdma_pad_phys, GFP_KERNEL);
1457	if (!dd->sdma_pad_dma) {
1458	dd_dev_err(dd, "failed to allocate SendDMA pad memory\n");
1459	goto bail;
1460	}
1461
1462	/* assign each engine to different cacheline and init registers */
1463	curr_head = (void *)dd->sdma_heads_dma;
1464	for (this_idx = 0; this_idx < num_engines; ++this_idx) {
1465	unsigned long phys_offset;
1466
1467	sde = &dd->per_sdma[this_idx];
1468
1469	sde->head_dma = curr_head;
1470	curr_head += L1_CACHE_BYTES;
1471	phys_offset = (unsigned long)sde->head_dma -
1472	(unsigned long)dd->sdma_heads_dma;
1473	sde->head_phys = dd->sdma_heads_phys + phys_offset;
1474	init_sdma_regs(sde, per_sdma_credits, idle_cnt);
1475	}
1476	dd->flags |= HFI1_HAS_SEND_DMA;
1477	dd->flags |= idle_cnt ? HFI1_HAS_SDMA_TIMEOUT : 0;
1478	dd->num_sdma = num_engines;
1479	ret = sdma_map_init(dd, port, num_vls: ppd->vls_operational, NULL);
1480	if (ret < 0)
1481	goto bail;
1482
1483	tmp_sdma_rht = kzalloc(size: sizeof(*tmp_sdma_rht), GFP_KERNEL);
1484	if (!tmp_sdma_rht) {
1485	ret = -ENOMEM;
1486	goto bail;
1487	}
1488
1489	ret = rhashtable_init(ht: tmp_sdma_rht, params: &sdma_rht_params);
1490	if (ret < 0) {
1491	kfree(objp: tmp_sdma_rht);
1492	goto bail;
1493	}
1494
1495	dd->sdma_rht = tmp_sdma_rht;
1496
1497	dd_dev_info(dd, "SDMA num_sdma: %u\n", dd->num_sdma);
1498	return 0;
1499
1500	bail:
1501	sdma_clean(dd, num_engines);
1502	return ret;
1503	}
1504
1505	/**
1506	* sdma_all_running() - called when the link goes up
1507	* @dd: hfi1_devdata
1508	*
1509	* This routine moves all engines to the running state.
1510	*/
1511	void sdma_all_running(struct hfi1_devdata *dd)
1512	{
1513	struct sdma_engine *sde;
1514	unsigned int i;
1515
1516	/* move all engines to running */
1517	for (i = 0; i < dd->num_sdma; ++i) {
1518	sde = &dd->per_sdma[i];
1519	sdma_process_event(sde, event: sdma_event_e30_go_running);
1520	}
1521	}
1522
1523	/**
1524	* sdma_all_idle() - called when the link goes down
1525	* @dd: hfi1_devdata
1526	*
1527	* This routine moves all engines to the idle state.
1528	*/
1529	void sdma_all_idle(struct hfi1_devdata *dd)
1530	{
1531	struct sdma_engine *sde;
1532	unsigned int i;
1533
1534	/* idle all engines */
1535	for (i = 0; i < dd->num_sdma; ++i) {
1536	sde = &dd->per_sdma[i];
1537	sdma_process_event(sde, event: sdma_event_e70_go_idle);
1538	}
1539	}
1540
1541	/**
1542	* sdma_start() - called to kick off state processing for all engines
1543	* @dd: hfi1_devdata
1544	*
1545	* This routine is for kicking off the state processing for all required
1546	* sdma engines. Interrupts need to be working at this point.
1547	*
1548	*/
1549	void sdma_start(struct hfi1_devdata *dd)
1550	{
1551	unsigned i;
1552	struct sdma_engine *sde;
1553
1554	/* kick off the engines state processing */
1555	for (i = 0; i < dd->num_sdma; ++i) {
1556	sde = &dd->per_sdma[i];
1557	sdma_process_event(sde, event: sdma_event_e10_go_hw_start);
1558	}
1559	}
1560
1561	/**
1562	* sdma_exit() - used when module is removed
1563	* @dd: hfi1_devdata
1564	*/
1565	void sdma_exit(struct hfi1_devdata *dd)
1566	{
1567	unsigned this_idx;
1568	struct sdma_engine *sde;
1569
1570	for (this_idx = 0; dd->per_sdma && this_idx < dd->num_sdma;
1571	++this_idx) {
1572	sde = &dd->per_sdma[this_idx];
1573	if (!list_empty(head: &sde->dmawait))
1574	dd_dev_err(dd, "sde %u: dmawait list not empty!\n",
1575	sde->this_idx);
1576	sdma_process_event(sde, event: sdma_event_e00_go_hw_down);
1577
1578	del_timer_sync(timer: &sde->err_progress_check_timer);
1579
1580	/*
1581	* This waits for the state machine to exit so it is not
1582	* necessary to kill the sdma_sw_clean_up_task to make sure
1583	* it is not running.
1584	*/
1585	sdma_finalput(ss: &sde->state);
1586	}
1587	}
1588
1589	/*
1590	* unmap the indicated descriptor
1591	*/
1592	static inline void sdma_unmap_desc(
1593	struct hfi1_devdata *dd,
1594	struct sdma_desc *descp)
1595	{
1596	switch (sdma_mapping_type(d: descp)) {
1597	case SDMA_MAP_SINGLE:
1598	dma_unmap_single(&dd->pcidev->dev, sdma_mapping_addr(descp),
1599	sdma_mapping_len(descp), DMA_TO_DEVICE);
1600	break;
1601	case SDMA_MAP_PAGE:
1602	dma_unmap_page(&dd->pcidev->dev, sdma_mapping_addr(descp),
1603	sdma_mapping_len(descp), DMA_TO_DEVICE);
1604	break;
1605	}
1606
1607	if (descp->pinning_ctx && descp->ctx_put)
1608	descp->ctx_put(descp->pinning_ctx);
1609	descp->pinning_ctx = NULL;
1610	}
1611
1612	/*
1613	* return the mode as indicated by the first
1614	* descriptor in the tx.
1615	*/
1616	static inline u8 ahg_mode(struct sdma_txreq *tx)
1617	{
1618	return (tx->descp[0].qw[1] & SDMA_DESC1_HEADER_MODE_SMASK)
1619	>> SDMA_DESC1_HEADER_MODE_SHIFT;
1620	}
1621
1622	/**
1623	* __sdma_txclean() - clean tx of mappings, descp *kmalloc's
1624	* @dd: hfi1_devdata for unmapping
1625	* @tx: tx request to clean
1626	*
1627	* This is used in the progress routine to clean the tx or
1628	* by the ULP to toss an in-process tx build.
1629	*
1630	* The code can be called multiple times without issue.
1631	*
1632	*/
1633	void __sdma_txclean(
1634	struct hfi1_devdata *dd,
1635	struct sdma_txreq *tx)
1636	{
1637	u16 i;
1638
1639	if (tx->num_desc) {
1640	u8 skip = 0, mode = ahg_mode(tx);
1641
1642	/* unmap first */
1643	sdma_unmap_desc(dd, descp: &tx->descp[0]);
1644	/* determine number of AHG descriptors to skip */
1645	if (mode > SDMA_AHG_APPLY_UPDATE1)
1646	skip = mode >> 1;
1647	for (i = 1 + skip; i < tx->num_desc; i++)
1648	sdma_unmap_desc(dd, descp: &tx->descp[i]);
1649	tx->num_desc = 0;
1650	}
1651	kfree(objp: tx->coalesce_buf);
1652	tx->coalesce_buf = NULL;
1653	/* kmalloc'ed descp */
1654	if (unlikely(tx->desc_limit > ARRAY_SIZE(tx->descs))) {
1655	tx->desc_limit = ARRAY_SIZE(tx->descs);
1656	kfree(objp: tx->descp);
1657	}
1658	}
1659
1660	static inline u16 sdma_gethead(struct sdma_engine *sde)
1661	{
1662	struct hfi1_devdata *dd = sde->dd;
1663	int use_dmahead;
1664	u16 hwhead;
1665
1666	#ifdef CONFIG_SDMA_VERBOSITY
1667	dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n",
1668	sde->this_idx, slashstrip(__FILE__), __LINE__, __func__);
1669	#endif
1670
1671	retry:
1672	use_dmahead = HFI1_CAP_IS_KSET(USE_SDMA_HEAD) && __sdma_running(engine: sde) &&
1673	(dd->flags & HFI1_HAS_SDMA_TIMEOUT);
1674	hwhead = use_dmahead ?
1675	(u16)le64_to_cpu(*sde->head_dma) :
1676	(u16)read_sde_csr(sde, SD(HEAD));
1677
1678	if (unlikely(HFI1_CAP_IS_KSET(SDMA_HEAD_CHECK))) {
1679	u16 cnt;
1680	u16 swtail;
1681	u16 swhead;
1682	int sane;
1683
1684	swhead = sde->descq_head & sde->sdma_mask;
1685	/* this code is really bad for cache line trading */
1686	swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask;
1687	cnt = sde->descq_cnt;
1688
1689	if (swhead < swtail)
1690	/* not wrapped */
1691	sane = (hwhead >= swhead) & (hwhead <= swtail);
1692	else if (swhead > swtail)
1693	/* wrapped around */
1694	sane = ((hwhead >= swhead) && (hwhead < cnt)) ||
1695	(hwhead <= swtail);
1696	else
1697	/* empty */
1698	sane = (hwhead == swhead);
1699
1700	if (unlikely(!sane)) {
1701	dd_dev_err(dd, "SDMA(%u) bad head (%s) hwhd=%u swhd=%u swtl=%u cnt=%u\n",
1702	sde->this_idx,
1703	use_dmahead ? "dma" : "kreg",
1704	hwhead, swhead, swtail, cnt);
1705	if (use_dmahead) {
1706	/* try one more time, using csr */
1707	use_dmahead = 0;
1708	goto retry;
1709	}
1710	/* proceed as if no progress */
1711	hwhead = swhead;
1712	}
1713	}
1714	return hwhead;
1715	}
1716
1717	/*
1718	* This is called when there are send DMA descriptors that might be
1719	* available.
1720	*
1721	* This is called with head_lock held.
1722	*/
1723	static void sdma_desc_avail(struct sdma_engine *sde, uint avail)
1724	{
1725	struct iowait *wait, *nw, *twait;
1726	struct iowait *waits[SDMA_WAIT_BATCH_SIZE];
1727	uint i, n = 0, seq, tidx = 0;
1728
1729	#ifdef CONFIG_SDMA_VERBOSITY
1730	dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx,
1731	slashstrip(__FILE__), __LINE__, __func__);
1732	dd_dev_err(sde->dd, "avail: %u\n", avail);
1733	#endif
1734
1735	do {
1736	seq = read_seqbegin(sl: &sde->waitlock);
1737	if (!list_empty(head: &sde->dmawait)) {
1738	/* at least one item */
1739	write_seqlock(sl: &sde->waitlock);
1740	/* Harvest waiters wanting DMA descriptors */
1741	list_for_each_entry_safe(
1742	wait,
1743	nw,
1744	&sde->dmawait,
1745	list) {
1746	u32 num_desc;
1747
1748	if (!wait->wakeup)
1749	continue;
1750	if (n == ARRAY_SIZE(waits))
1751	break;
1752	iowait_init_priority(w: wait);
1753	num_desc = iowait_get_all_desc(w: wait);
1754	if (num_desc > avail)
1755	break;
1756	avail -= num_desc;
1757	/* Find the top-priority wait memeber */
1758	if (n) {
1759	twait = waits[tidx];
1760	tidx =
1761	iowait_priority_update_top(w: wait,
1762	top: twait,
1763	idx: n,
1764	top_idx: tidx);
1765	}
1766	list_del_init(entry: &wait->list);
1767	waits[n++] = wait;
1768	}
1769	write_sequnlock(sl: &sde->waitlock);
1770	break;
1771	}
1772	} while (read_seqretry(sl: &sde->waitlock, start: seq));
1773
1774	/* Schedule the top-priority entry first */
1775	if (n)
1776	waits[tidx]->wakeup(waits[tidx], SDMA_AVAIL_REASON);
1777
1778	for (i = 0; i < n; i++)
1779	if (i != tidx)
1780	waits[i]->wakeup(waits[i], SDMA_AVAIL_REASON);
1781	}
1782
1783	/* head_lock must be held */
1784	static void sdma_make_progress(struct sdma_engine *sde, u64 status)
1785	{
1786	struct sdma_txreq *txp = NULL;
1787	int progress = 0;
1788	u16 hwhead, swhead;
1789	int idle_check_done = 0;
1790
1791	hwhead = sdma_gethead(sde);
1792
1793	/* The reason for some of the complexity of this code is that
1794	* not all descriptors have corresponding txps. So, we have to
1795	* be able to skip over descs until we wander into the range of
1796	* the next txp on the list.
1797	*/
1798
1799	retry:
1800	txp = get_txhead(sde);
1801	swhead = sde->descq_head & sde->sdma_mask;
1802	trace_hfi1_sdma_progress(sde, hwhead, swhead, txp);
1803	while (swhead != hwhead) {
1804	/* advance head, wrap if needed */
1805	swhead = ++sde->descq_head & sde->sdma_mask;
1806
1807	/* if now past this txp's descs, do the callback */
1808	if (txp && txp->next_descq_idx == swhead) {
1809	/* remove from list */
1810	sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL;
1811	complete_tx(sde, tx: txp, SDMA_TXREQ_S_OK);
1812	/* see if there is another txp */
1813	txp = get_txhead(sde);
1814	}
1815	trace_hfi1_sdma_progress(sde, hwhead, swhead, txp);
1816	progress++;
1817	}
1818
1819	/*
1820	* The SDMA idle interrupt is not guaranteed to be ordered with respect
1821	* to updates to the dma_head location in host memory. The head
1822	* value read might not be fully up to date. If there are pending
1823	* descriptors and the SDMA idle interrupt fired then read from the
1824	* CSR SDMA head instead to get the latest value from the hardware.
1825	* The hardware SDMA head should be read at most once in this invocation
1826	* of sdma_make_progress(..) which is ensured by idle_check_done flag
1827	*/
1828	if ((status & sde->idle_mask) && !idle_check_done) {
1829	u16 swtail;
1830
1831	swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask;
1832	if (swtail != hwhead) {
1833	hwhead = (u16)read_sde_csr(sde, SD(HEAD));
1834	idle_check_done = 1;
1835	goto retry;
1836	}
1837	}
1838
1839	sde->last_status = status;
1840	if (progress)
1841	sdma_desc_avail(sde, avail: sdma_descq_freecnt(sde));
1842	}
1843
1844	/*
1845	* sdma_engine_interrupt() - interrupt handler for engine
1846	* @sde: sdma engine
1847	* @status: sdma interrupt reason
1848	*
1849	* Status is a mask of the 3 possible interrupts for this engine. It will
1850	* contain bits _only_ for this SDMA engine. It will contain at least one
1851	* bit, it may contain more.
1852	*/
1853	void sdma_engine_interrupt(struct sdma_engine *sde, u64 status)
1854	{
1855	trace_hfi1_sdma_engine_interrupt(sde, status);
1856	write_seqlock(sl: &sde->head_lock);
1857	sdma_set_desc_cnt(sde, cnt: sdma_desct_intr);
1858	if (status & sde->idle_mask)
1859	sde->idle_int_cnt++;
1860	else if (status & sde->progress_mask)
1861	sde->progress_int_cnt++;
1862	else if (status & sde->int_mask)
1863	sde->sdma_int_cnt++;
1864	sdma_make_progress(sde, status);
1865	write_sequnlock(sl: &sde->head_lock);
1866	}
1867
1868	/**
1869	* sdma_engine_error() - error handler for engine
1870	* @sde: sdma engine
1871	* @status: sdma interrupt reason
1872	*/
1873	void sdma_engine_error(struct sdma_engine *sde, u64 status)
1874	{
1875	unsigned long flags;
1876
1877	#ifdef CONFIG_SDMA_VERBOSITY
1878	dd_dev_err(sde->dd, "CONFIG SDMA(%u) error status 0x%llx state %s\n",
1879	sde->this_idx,
1880	(unsigned long long)status,
1881	sdma_state_names[sde->state.current_state]);
1882	#endif
1883	spin_lock_irqsave(&sde->tail_lock, flags);
1884	write_seqlock(sl: &sde->head_lock);
1885	if (status & ALL_SDMA_ENG_HALT_ERRS)
1886	__sdma_process_event(sde, event: sdma_event_e60_hw_halted);
1887	if (status & ~SD(ENG_ERR_STATUS_SDMA_HALT_ERR_SMASK)) {
1888	dd_dev_err(sde->dd,
1889	"SDMA (%u) engine error: 0x%llx state %s\n",
1890	sde->this_idx,
1891	(unsigned long long)status,
1892	sdma_state_names[sde->state.current_state]);
1893	dump_sdma_state(sde);
1894	}
1895	write_sequnlock(sl: &sde->head_lock);
1896	spin_unlock_irqrestore(lock: &sde->tail_lock, flags);
1897	}
1898
1899	static void sdma_sendctrl(struct sdma_engine *sde, unsigned op)
1900	{
1901	u64 set_senddmactrl = 0;
1902	u64 clr_senddmactrl = 0;
1903	unsigned long flags;
1904
1905	#ifdef CONFIG_SDMA_VERBOSITY
1906	dd_dev_err(sde->dd, "CONFIG SDMA(%u) senddmactrl E=%d I=%d H=%d C=%d\n",
1907	sde->this_idx,
1908	(op & SDMA_SENDCTRL_OP_ENABLE) ? 1 : 0,
1909	(op & SDMA_SENDCTRL_OP_INTENABLE) ? 1 : 0,
1910	(op & SDMA_SENDCTRL_OP_HALT) ? 1 : 0,
1911	(op & SDMA_SENDCTRL_OP_CLEANUP) ? 1 : 0);
1912	#endif
1913
1914	if (op & SDMA_SENDCTRL_OP_ENABLE)
1915	set_senddmactrl |= SD(CTRL_SDMA_ENABLE_SMASK);
1916	else
1917	clr_senddmactrl |= SD(CTRL_SDMA_ENABLE_SMASK);
1918
1919	if (op & SDMA_SENDCTRL_OP_INTENABLE)
1920	set_senddmactrl |= SD(CTRL_SDMA_INT_ENABLE_SMASK);
1921	else
1922	clr_senddmactrl |= SD(CTRL_SDMA_INT_ENABLE_SMASK);
1923
1924	if (op & SDMA_SENDCTRL_OP_HALT)
1925	set_senddmactrl |= SD(CTRL_SDMA_HALT_SMASK);
1926	else
1927	clr_senddmactrl |= SD(CTRL_SDMA_HALT_SMASK);
1928
1929	spin_lock_irqsave(&sde->senddmactrl_lock, flags);
1930
1931	sde->p_senddmactrl |= set_senddmactrl;
1932	sde->p_senddmactrl &= ~clr_senddmactrl;
1933
1934	if (op & SDMA_SENDCTRL_OP_CLEANUP)
1935	write_sde_csr(sde, SD(CTRL),
1936	value: sde->p_senddmactrl |
1937	SD(CTRL_SDMA_CLEANUP_SMASK));
1938	else
1939	write_sde_csr(sde, SD(CTRL), value: sde->p_senddmactrl);
1940
1941	spin_unlock_irqrestore(lock: &sde->senddmactrl_lock, flags);
1942
1943	#ifdef CONFIG_SDMA_VERBOSITY
1944	sdma_dumpstate(sde);
1945	#endif
1946	}
1947
1948	static void sdma_setlengen(struct sdma_engine *sde)
1949	{
1950	#ifdef CONFIG_SDMA_VERBOSITY
1951	dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n",
1952	sde->this_idx, slashstrip(__FILE__), __LINE__, __func__);
1953	#endif
1954
1955	/*
1956	* Set SendDmaLenGen and clear-then-set the MSB of the generation
1957	* count to enable generation checking and load the internal
1958	* generation counter.
1959	*/
1960	write_sde_csr(sde, SD(LEN_GEN),
1961	value: (sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT));
1962	write_sde_csr(sde, SD(LEN_GEN),
1963	value: ((sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT)) |
1964	(4ULL << SD(LEN_GEN_GENERATION_SHIFT)));
1965	}
1966
1967	static inline void sdma_update_tail(struct sdma_engine *sde, u16 tail)
1968	{
1969	/* Commit writes to memory and advance the tail on the chip */
1970	smp_wmb(); /* see get_txhead() */
1971	writeq(val: tail, addr: sde->tail_csr);
1972	}
1973
1974	/*
1975	* This is called when changing to state s10_hw_start_up_halt_wait as
1976	* a result of send buffer errors or send DMA descriptor errors.
1977	*/
1978	static void sdma_hw_start_up(struct sdma_engine *sde)
1979	{
1980	u64 reg;
1981
1982	#ifdef CONFIG_SDMA_VERBOSITY
1983	dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n",
1984	sde->this_idx, slashstrip(__FILE__), __LINE__, __func__);
1985	#endif
1986
1987	sdma_setlengen(sde);
1988	sdma_update_tail(sde, tail: 0); /* Set SendDmaTail */
1989	*sde->head_dma = 0;
1990
1991	reg = SD(ENG_ERR_CLEAR_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_MASK) <<
1992	SD(ENG_ERR_CLEAR_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_SHIFT);
1993	write_sde_csr(sde, SD(ENG_ERR_CLEAR), value: reg);
1994	}
1995
1996	/*
1997	* set_sdma_integrity
1998	*
1999	* Set the SEND_DMA_CHECK_ENABLE register for send DMA engine 'sde'.
2000	*/
2001	static void set_sdma_integrity(struct sdma_engine *sde)
2002	{
2003	struct hfi1_devdata *dd = sde->dd;
2004
2005	write_sde_csr(sde, SD(CHECK_ENABLE),
2006	value: hfi1_pkt_base_sdma_integrity(dd));
2007	}
2008
2009	static void init_sdma_regs(
2010	struct sdma_engine *sde,
2011	u32 credits,
2012	uint idle_cnt)
2013	{
2014	u8 opval, opmask;
2015	#ifdef CONFIG_SDMA_VERBOSITY
2016	struct hfi1_devdata *dd = sde->dd;
2017
2018	dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n",
2019	sde->this_idx, slashstrip(__FILE__), __LINE__, __func__);
2020	#endif
2021
2022	write_sde_csr(sde, SD(BASE_ADDR), value: sde->descq_phys);
2023	sdma_setlengen(sde);
2024	sdma_update_tail(sde, tail: 0); /* Set SendDmaTail */
2025	write_sde_csr(sde, SD(RELOAD_CNT), value: idle_cnt);
2026	write_sde_csr(sde, SD(DESC_CNT), value: 0);
2027	write_sde_csr(sde, SD(HEAD_ADDR), value: sde->head_phys);
2028	write_sde_csr(sde, SD(MEMORY),
2029	value: ((u64)credits << SD(MEMORY_SDMA_MEMORY_CNT_SHIFT)) |
2030	((u64)(credits * sde->this_idx) <<
2031	SD(MEMORY_SDMA_MEMORY_INDEX_SHIFT)));
2032	write_sde_csr(sde, SD(ENG_ERR_MASK), value: ~0ull);
2033	set_sdma_integrity(sde);
2034	opmask = OPCODE_CHECK_MASK_DISABLED;
2035	opval = OPCODE_CHECK_VAL_DISABLED;
2036	write_sde_csr(sde, SD(CHECK_OPCODE),
2037	value: (opmask << SEND_CTXT_CHECK_OPCODE_MASK_SHIFT) |
2038	(opval << SEND_CTXT_CHECK_OPCODE_VALUE_SHIFT));
2039	}
2040
2041	#ifdef CONFIG_SDMA_VERBOSITY
2042
2043	#define sdma_dumpstate_helper0(reg) do { \
2044	csr = read_csr(sde->dd, reg); \
2045	dd_dev_err(sde->dd, "%36s 0x%016llx\n", #reg, csr); \
2046	} while (0)
2047
2048	#define sdma_dumpstate_helper(reg) do { \
2049	csr = read_sde_csr(sde, reg); \
2050	dd_dev_err(sde->dd, "%36s[%02u] 0x%016llx\n", \
2051	#reg, sde->this_idx, csr); \
2052	} while (0)
2053
2054	#define sdma_dumpstate_helper2(reg) do { \
2055	csr = read_csr(sde->dd, reg + (8 * i)); \
2056	dd_dev_err(sde->dd, "%33s_%02u 0x%016llx\n", \
2057	#reg, i, csr); \
2058	} while (0)
2059
2060	void sdma_dumpstate(struct sdma_engine *sde)
2061	{
2062	u64 csr;
2063	unsigned i;
2064
2065	sdma_dumpstate_helper(SD(CTRL));
2066	sdma_dumpstate_helper(SD(STATUS));
2067	sdma_dumpstate_helper0(SD(ERR_STATUS));
2068	sdma_dumpstate_helper0(SD(ERR_MASK));
2069	sdma_dumpstate_helper(SD(ENG_ERR_STATUS));
2070	sdma_dumpstate_helper(SD(ENG_ERR_MASK));
2071
2072	for (i = 0; i < CCE_NUM_INT_CSRS; ++i) {
2073	sdma_dumpstate_helper2(CCE_INT_STATUS);
2074	sdma_dumpstate_helper2(CCE_INT_MASK);
2075	sdma_dumpstate_helper2(CCE_INT_BLOCKED);
2076	}
2077
2078	sdma_dumpstate_helper(SD(TAIL));
2079	sdma_dumpstate_helper(SD(HEAD));
2080	sdma_dumpstate_helper(SD(PRIORITY_THLD));
2081	sdma_dumpstate_helper(SD(IDLE_CNT));
2082	sdma_dumpstate_helper(SD(RELOAD_CNT));
2083	sdma_dumpstate_helper(SD(DESC_CNT));
2084	sdma_dumpstate_helper(SD(DESC_FETCHED_CNT));
2085	sdma_dumpstate_helper(SD(MEMORY));
2086	sdma_dumpstate_helper0(SD(ENGINES));
2087	sdma_dumpstate_helper0(SD(MEM_SIZE));
2088	/* sdma_dumpstate_helper(SEND_EGRESS_SEND_DMA_STATUS); */
2089	sdma_dumpstate_helper(SD(BASE_ADDR));
2090	sdma_dumpstate_helper(SD(LEN_GEN));
2091	sdma_dumpstate_helper(SD(HEAD_ADDR));
2092	sdma_dumpstate_helper(SD(CHECK_ENABLE));
2093	sdma_dumpstate_helper(SD(CHECK_VL));
2094	sdma_dumpstate_helper(SD(CHECK_JOB_KEY));
2095	sdma_dumpstate_helper(SD(CHECK_PARTITION_KEY));
2096	sdma_dumpstate_helper(SD(CHECK_SLID));
2097	sdma_dumpstate_helper(SD(CHECK_OPCODE));
2098	}
2099	#endif
2100
2101	static void dump_sdma_state(struct sdma_engine *sde)
2102	{
2103	struct hw_sdma_desc *descqp;
2104	u64 desc[2];
2105	u64 addr;
2106	u8 gen;
2107	u16 len;
2108	u16 head, tail, cnt;
2109
2110	head = sde->descq_head & sde->sdma_mask;
2111	tail = sde->descq_tail & sde->sdma_mask;
2112	cnt = sdma_descq_freecnt(sde);
2113
2114	dd_dev_err(sde->dd,
2115	"SDMA (%u) descq_head: %u descq_tail: %u freecnt: %u FLE %d\n",
2116	sde->this_idx, head, tail, cnt,
2117	!list_empty(&sde->flushlist));
2118
2119	/* print info for each entry in the descriptor queue */
2120	while (head != tail) {
2121	char flags[6] = { 'x', 'x', 'x', 'x', 0 };
2122
2123	descqp = &sde->descq[head];
2124	desc[0] = le64_to_cpu(descqp->qw[0]);
2125	desc[1] = le64_to_cpu(descqp->qw[1]);
2126	flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-';
2127	flags[1] = (desc[1] & SDMA_DESC1_HEAD_TO_HOST_FLAG) ?
2128	'H' : '-';
2129	flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-';
2130	flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-';
2131	addr = (desc[0] >> SDMA_DESC0_PHY_ADDR_SHIFT)
2132	& SDMA_DESC0_PHY_ADDR_MASK;
2133	gen = (desc[1] >> SDMA_DESC1_GENERATION_SHIFT)
2134	& SDMA_DESC1_GENERATION_MASK;
2135	len = (desc[0] >> SDMA_DESC0_BYTE_COUNT_SHIFT)
2136	& SDMA_DESC0_BYTE_COUNT_MASK;
2137	dd_dev_err(sde->dd,
2138	"SDMA sdmadesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes\n",
2139	head, flags, addr, gen, len);
2140	dd_dev_err(sde->dd,
2141	"\tdesc0:0x%016llx desc1 0x%016llx\n",
2142	desc[0], desc[1]);
2143	if (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG)
2144	dd_dev_err(sde->dd,
2145	"\taidx: %u amode: %u alen: %u\n",
2146	(u8)((desc[1] &
2147	SDMA_DESC1_HEADER_INDEX_SMASK) >>
2148	SDMA_DESC1_HEADER_INDEX_SHIFT),
2149	(u8)((desc[1] &
2150	SDMA_DESC1_HEADER_MODE_SMASK) >>
2151	SDMA_DESC1_HEADER_MODE_SHIFT),
2152	(u8)((desc[1] &
2153	SDMA_DESC1_HEADER_DWS_SMASK) >>
2154	SDMA_DESC1_HEADER_DWS_SHIFT));
2155	head++;
2156	head &= sde->sdma_mask;
2157	}
2158	}
2159
2160	#define SDE_FMT \
2161	"SDE %u CPU %d STE %s C 0x%llx S 0x%016llx E 0x%llx T(HW) 0x%llx T(SW) 0x%x H(HW) 0x%llx H(SW) 0x%x H(D) 0x%llx DM 0x%llx GL 0x%llx R 0x%llx LIS 0x%llx AHGI 0x%llx TXT %u TXH %u DT %u DH %u FLNE %d DQF %u SLC 0x%llx\n"
2162	/**
2163	* sdma_seqfile_dump_sde() - debugfs dump of sde
2164	* @s: seq file
2165	* @sde: send dma engine to dump
2166	*
2167	* This routine dumps the sde to the indicated seq file.
2168	*/
2169	void sdma_seqfile_dump_sde(struct seq_file *s, struct sdma_engine *sde)
2170	{
2171	u16 head, tail;
2172	struct hw_sdma_desc *descqp;
2173	u64 desc[2];
2174	u64 addr;
2175	u8 gen;
2176	u16 len;
2177
2178	head = sde->descq_head & sde->sdma_mask;
2179	tail = READ_ONCE(sde->descq_tail) & sde->sdma_mask;
2180	seq_printf(m: s, SDE_FMT, sde->this_idx,
2181	sde->cpu,
2182	sdma_state_name(state: sde->state.current_state),
2183	(unsigned long long)read_sde_csr(sde, SD(CTRL)),
2184	(unsigned long long)read_sde_csr(sde, SD(STATUS)),
2185	(unsigned long long)read_sde_csr(sde, SD(ENG_ERR_STATUS)),
2186	(unsigned long long)read_sde_csr(sde, SD(TAIL)), tail,
2187	(unsigned long long)read_sde_csr(sde, SD(HEAD)), head,
2188	(unsigned long long)le64_to_cpu(*sde->head_dma),
2189	(unsigned long long)read_sde_csr(sde, SD(MEMORY)),
2190	(unsigned long long)read_sde_csr(sde, SD(LEN_GEN)),
2191	(unsigned long long)read_sde_csr(sde, SD(RELOAD_CNT)),
2192	(unsigned long long)sde->last_status,
2193	(unsigned long long)sde->ahg_bits,
2194	sde->tx_tail,
2195	sde->tx_head,
2196	sde->descq_tail,
2197	sde->descq_head,
2198	!list_empty(head: &sde->flushlist),
2199	sde->descq_full_count,
2200	(unsigned long long)read_sde_csr(sde, SEND_DMA_CHECK_SLID));
2201
2202	/* print info for each entry in the descriptor queue */
2203	while (head != tail) {
2204	char flags[6] = { 'x', 'x', 'x', 'x', 0 };
2205
2206	descqp = &sde->descq[head];
2207	desc[0] = le64_to_cpu(descqp->qw[0]);
2208	desc[1] = le64_to_cpu(descqp->qw[1]);
2209	flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-';
2210	flags[1] = (desc[1] & SDMA_DESC1_HEAD_TO_HOST_FLAG) ?
2211	'H' : '-';
2212	flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-';
2213	flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-';
2214	addr = (desc[0] >> SDMA_DESC0_PHY_ADDR_SHIFT)
2215	& SDMA_DESC0_PHY_ADDR_MASK;
2216	gen = (desc[1] >> SDMA_DESC1_GENERATION_SHIFT)
2217	& SDMA_DESC1_GENERATION_MASK;
2218	len = (desc[0] >> SDMA_DESC0_BYTE_COUNT_SHIFT)
2219	& SDMA_DESC0_BYTE_COUNT_MASK;
2220	seq_printf(m: s,
2221	fmt: "\tdesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes\n",
2222	head, flags, addr, gen, len);
2223	if (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG)
2224	seq_printf(m: s, fmt: "\t\tahgidx: %u ahgmode: %u\n",
2225	(u8)((desc[1] &
2226	SDMA_DESC1_HEADER_INDEX_SMASK) >>
2227	SDMA_DESC1_HEADER_INDEX_SHIFT),
2228	(u8)((desc[1] &
2229	SDMA_DESC1_HEADER_MODE_SMASK) >>
2230	SDMA_DESC1_HEADER_MODE_SHIFT));
2231	head = (head + 1) & sde->sdma_mask;
2232	}
2233	}
2234
2235	/*
2236	* add the generation number into
2237	* the qw1 and return
2238	*/
2239	static inline u64 add_gen(struct sdma_engine *sde, u64 qw1)
2240	{
2241	u8 generation = (sde->descq_tail >> sde->sdma_shift) & 3;
2242
2243	qw1 &= ~SDMA_DESC1_GENERATION_SMASK;
2244	qw1 |= ((u64)generation & SDMA_DESC1_GENERATION_MASK)
2245	<< SDMA_DESC1_GENERATION_SHIFT;
2246	return qw1;
2247	}
2248
2249	/*
2250	* This routine submits the indicated tx
2251	*
2252	* Space has already been guaranteed and
2253	* tail side of ring is locked.
2254	*
2255	* The hardware tail update is done
2256	* in the caller and that is facilitated
2257	* by returning the new tail.
2258	*
2259	* There is special case logic for ahg
2260	* to not add the generation number for
2261	* up to 2 descriptors that follow the
2262	* first descriptor.
2263	*
2264	*/
2265	static inline u16 submit_tx(struct sdma_engine *sde, struct sdma_txreq *tx)
2266	{
2267	int i;
2268	u16 tail;
2269	struct sdma_desc *descp = tx->descp;
2270	u8 skip = 0, mode = ahg_mode(tx);
2271
2272	tail = sde->descq_tail & sde->sdma_mask;
2273	sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]);
2274	sde->descq[tail].qw[1] = cpu_to_le64(add_gen(sde, descp->qw[1]));
2275	trace_hfi1_sdma_descriptor(sde, desc0: descp->qw[0], desc1: descp->qw[1],
2276	e: tail, descp: &sde->descq[tail]);
2277	tail = ++sde->descq_tail & sde->sdma_mask;
2278	descp++;
2279	if (mode > SDMA_AHG_APPLY_UPDATE1)
2280	skip = mode >> 1;
2281	for (i = 1; i < tx->num_desc; i++, descp++) {
2282	u64 qw1;
2283
2284	sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]);
2285	if (skip) {
2286	/* edits don't have generation */
2287	qw1 = descp->qw[1];
2288	skip--;
2289	} else {
2290	/* replace generation with real one for non-edits */
2291	qw1 = add_gen(sde, qw1: descp->qw[1]);
2292	}
2293	sde->descq[tail].qw[1] = cpu_to_le64(qw1);
2294	trace_hfi1_sdma_descriptor(sde, desc0: descp->qw[0], desc1: qw1,
2295	e: tail, descp: &sde->descq[tail]);
2296	tail = ++sde->descq_tail & sde->sdma_mask;
2297	}
2298	tx->next_descq_idx = tail;
2299	#ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
2300	tx->sn = sde->tail_sn++;
2301	trace_hfi1_sdma_in_sn(sde, sn: tx->sn);
2302	WARN_ON_ONCE(sde->tx_ring[sde->tx_tail & sde->sdma_mask]);
2303	#endif
2304	sde->tx_ring[sde->tx_tail++ & sde->sdma_mask] = tx;
2305	sde->desc_avail -= tx->num_desc;
2306	return tail;
2307	}
2308
2309	/*
2310	* Check for progress
2311	*/
2312	static int sdma_check_progress(
2313	struct sdma_engine *sde,
2314	struct iowait_work *wait,
2315	struct sdma_txreq *tx,
2316	bool pkts_sent)
2317	{
2318	int ret;
2319
2320	sde->desc_avail = sdma_descq_freecnt(sde);
2321	if (tx->num_desc <= sde->desc_avail)
2322	return -EAGAIN;
2323	/* pulse the head_lock */
2324	if (wait && iowait_ioww_to_iow(w: wait)->sleep) {
2325	unsigned seq;
2326
2327	seq = raw_seqcount_begin(
2328	(const seqcount_t *)&sde->head_lock.seqcount);
2329	ret = wait->iow->sleep(sde, wait, tx, seq, pkts_sent);
2330	if (ret == -EAGAIN)
2331	sde->desc_avail = sdma_descq_freecnt(sde);
2332	} else {
2333	ret = -EBUSY;
2334	}
2335	return ret;
2336	}
2337
2338	/**
2339	* sdma_send_txreq() - submit a tx req to ring
2340	* @sde: sdma engine to use
2341	* @wait: SE wait structure to use when full (may be NULL)
2342	* @tx: sdma_txreq to submit
2343	* @pkts_sent: has any packet been sent yet?
2344	*
2345	* The call submits the tx into the ring. If a iowait structure is non-NULL
2346	* the packet will be queued to the list in wait.
2347	*
2348	* Return:
2349	* 0 - Success, -EINVAL - sdma_txreq incomplete, -EBUSY - no space in
2350	* ring (wait == NULL)
2351	* -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state
2352	*/
2353	int sdma_send_txreq(struct sdma_engine *sde,
2354	struct iowait_work *wait,
2355	struct sdma_txreq *tx,
2356	bool pkts_sent)
2357	{
2358	int ret = 0;
2359	u16 tail;
2360	unsigned long flags;
2361
2362	/* user should have supplied entire packet */
2363	if (unlikely(tx->tlen))
2364	return -EINVAL;
2365	tx->wait = iowait_ioww_to_iow(w: wait);
2366	spin_lock_irqsave(&sde->tail_lock, flags);
2367	retry:
2368	if (unlikely(!__sdma_running(sde)))
2369	goto unlock_noconn;
2370	if (unlikely(tx->num_desc > sde->desc_avail))
2371	goto nodesc;
2372	tail = submit_tx(sde, tx);
2373	if (wait)
2374	iowait_sdma_inc(wait: iowait_ioww_to_iow(w: wait));
2375	sdma_update_tail(sde, tail);
2376	unlock:
2377	spin_unlock_irqrestore(lock: &sde->tail_lock, flags);
2378	return ret;
2379	unlock_noconn:
2380	if (wait)
2381	iowait_sdma_inc(wait: iowait_ioww_to_iow(w: wait));
2382	tx->next_descq_idx = 0;
2383	#ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
2384	tx->sn = sde->tail_sn++;
2385	trace_hfi1_sdma_in_sn(sde, sn: tx->sn);
2386	#endif
2387	spin_lock(lock: &sde->flushlist_lock);
2388	list_add_tail(new: &tx->list, head: &sde->flushlist);
2389	spin_unlock(lock: &sde->flushlist_lock);
2390	iowait_inc_wait_count(w: wait, n: tx->num_desc);
2391	queue_work_on(cpu: sde->cpu, wq: system_highpri_wq, work: &sde->flush_worker);
2392	ret = -ECOMM;
2393	goto unlock;
2394	nodesc:
2395	ret = sdma_check_progress(sde, wait, tx, pkts_sent);
2396	if (ret == -EAGAIN) {
2397	ret = 0;
2398	goto retry;
2399	}
2400	sde->descq_full_count++;
2401	goto unlock;
2402	}
2403
2404	/**
2405	* sdma_send_txlist() - submit a list of tx req to ring
2406	* @sde: sdma engine to use
2407	* @wait: SE wait structure to use when full (may be NULL)
2408	* @tx_list: list of sdma_txreqs to submit
2409	* @count_out: pointer to a u16 which, after return will contain the total number of
2410	* sdma_txreqs removed from the tx_list. This will include sdma_txreqs
2411	* whose SDMA descriptors are submitted to the ring and the sdma_txreqs
2412	* which are added to SDMA engine flush list if the SDMA engine state is
2413	* not running.
2414	*
2415	* The call submits the list into the ring.
2416	*
2417	* If the iowait structure is non-NULL and not equal to the iowait list
2418	* the unprocessed part of the list will be appended to the list in wait.
2419	*
2420	* In all cases, the tx_list will be updated so the head of the tx_list is
2421	* the list of descriptors that have yet to be transmitted.
2422	*
2423	* The intent of this call is to provide a more efficient
2424	* way of submitting multiple packets to SDMA while holding the tail
2425	* side locking.
2426	*
2427	* Return:
2428	* 0 - Success,
2429	* -EINVAL - sdma_txreq incomplete, -EBUSY - no space in ring (wait == NULL)
2430	* -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state
2431	*/
2432	int sdma_send_txlist(struct sdma_engine *sde, struct iowait_work *wait,
2433	struct list_head *tx_list, u16 *count_out)
2434	{
2435	struct sdma_txreq *tx, *tx_next;
2436	int ret = 0;
2437	unsigned long flags;
2438	u16 tail = INVALID_TAIL;
2439	u32 submit_count = 0, flush_count = 0, total_count;
2440
2441	spin_lock_irqsave(&sde->tail_lock, flags);
2442	retry:
2443	list_for_each_entry_safe(tx, tx_next, tx_list, list) {
2444	tx->wait = iowait_ioww_to_iow(w: wait);
2445	if (unlikely(!__sdma_running(sde)))
2446	goto unlock_noconn;
2447	if (unlikely(tx->num_desc > sde->desc_avail))
2448	goto nodesc;
2449	if (unlikely(tx->tlen)) {
2450	ret = -EINVAL;
2451	goto update_tail;
2452	}
2453	list_del_init(entry: &tx->list);
2454	tail = submit_tx(sde, tx);
2455	submit_count++;
2456	if (tail != INVALID_TAIL &&
2457	(submit_count & SDMA_TAIL_UPDATE_THRESH) == 0) {
2458	sdma_update_tail(sde, tail);
2459	tail = INVALID_TAIL;
2460	}
2461	}
2462	update_tail:
2463	total_count = submit_count + flush_count;
2464	if (wait) {
2465	iowait_sdma_add(wait: iowait_ioww_to_iow(w: wait), count: total_count);
2466	iowait_starve_clear(pkts_sent: submit_count > 0,
2467	w: iowait_ioww_to_iow(w: wait));
2468	}
2469	if (tail != INVALID_TAIL)
2470	sdma_update_tail(sde, tail);
2471	spin_unlock_irqrestore(lock: &sde->tail_lock, flags);
2472	*count_out = total_count;
2473	return ret;
2474	unlock_noconn:
2475	spin_lock(lock: &sde->flushlist_lock);
2476	list_for_each_entry_safe(tx, tx_next, tx_list, list) {
2477	tx->wait = iowait_ioww_to_iow(w: wait);
2478	list_del_init(entry: &tx->list);
2479	tx->next_descq_idx = 0;
2480	#ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER
2481	tx->sn = sde->tail_sn++;
2482	trace_hfi1_sdma_in_sn(sde, sn: tx->sn);
2483	#endif
2484	list_add_tail(new: &tx->list, head: &sde->flushlist);
2485	flush_count++;
2486	iowait_inc_wait_count(w: wait, n: tx->num_desc);
2487	}
2488	spin_unlock(lock: &sde->flushlist_lock);
2489	queue_work_on(cpu: sde->cpu, wq: system_highpri_wq, work: &sde->flush_worker);
2490	ret = -ECOMM;
2491	goto update_tail;
2492	nodesc:
2493	ret = sdma_check_progress(sde, wait, tx, pkts_sent: submit_count > 0);
2494	if (ret == -EAGAIN) {
2495	ret = 0;
2496	goto retry;
2497	}
2498	sde->descq_full_count++;
2499	goto update_tail;
2500	}
2501
2502	static void sdma_process_event(struct sdma_engine *sde, enum sdma_events event)
2503	{
2504	unsigned long flags;
2505
2506	spin_lock_irqsave(&sde->tail_lock, flags);
2507	write_seqlock(sl: &sde->head_lock);
2508
2509	__sdma_process_event(sde, event);
2510
2511	if (sde->state.current_state == sdma_state_s99_running)
2512	sdma_desc_avail(sde, avail: sdma_descq_freecnt(sde));
2513
2514	write_sequnlock(sl: &sde->head_lock);
2515	spin_unlock_irqrestore(lock: &sde->tail_lock, flags);
2516	}
2517
2518	static void __sdma_process_event(struct sdma_engine *sde,
2519	enum sdma_events event)
2520	{
2521	struct sdma_state *ss = &sde->state;
2522	int need_progress = 0;
2523
2524	/* CONFIG SDMA temporary */
2525	#ifdef CONFIG_SDMA_VERBOSITY
2526	dd_dev_err(sde->dd, "CONFIG SDMA(%u) [%s] %s\n", sde->this_idx,
2527	sdma_state_names[ss->current_state],
2528	sdma_event_names[event]);
2529	#endif
2530
2531	switch (ss->current_state) {
2532	case sdma_state_s00_hw_down:
2533	switch (event) {
2534	case sdma_event_e00_go_hw_down:
2535	break;
2536	case sdma_event_e30_go_running:
2537	/*
2538	* If down, but running requested (usually result
2539	* of link up, then we need to start up.
2540	* This can happen when hw down is requested while
2541	* bringing the link up with traffic active on
2542	* 7220, e.g.
2543	*/
2544	ss->go_s99_running = 1;
2545	fallthrough; /* and start dma engine */
2546	case sdma_event_e10_go_hw_start:
2547	/* This reference means the state machine is started */
2548	sdma_get(ss: &sde->state);
2549	sdma_set_state(sde,
2550	next_state: sdma_state_s10_hw_start_up_halt_wait);
2551	break;
2552	case sdma_event_e15_hw_halt_done:
2553	break;
2554	case sdma_event_e25_hw_clean_up_done:
2555	break;
2556	case sdma_event_e40_sw_cleaned:
2557	sdma_sw_tear_down(sde);
2558	break;
2559	case sdma_event_e50_hw_cleaned:
2560	break;
2561	case sdma_event_e60_hw_halted:
2562	break;
2563	case sdma_event_e70_go_idle:
2564	break;
2565	case sdma_event_e80_hw_freeze:
2566	break;
2567	case sdma_event_e81_hw_frozen:
2568	break;
2569	case sdma_event_e82_hw_unfreeze:
2570	break;
2571	case sdma_event_e85_link_down:
2572	break;
2573	case sdma_event_e90_sw_halted:
2574	break;
2575	}
2576	break;
2577
2578	case sdma_state_s10_hw_start_up_halt_wait:
2579	switch (event) {
2580	case sdma_event_e00_go_hw_down:
2581	sdma_set_state(sde, next_state: sdma_state_s00_hw_down);
2582	sdma_sw_tear_down(sde);
2583	break;
2584	case sdma_event_e10_go_hw_start:
2585	break;
2586	case sdma_event_e15_hw_halt_done:
2587	sdma_set_state(sde,
2588	next_state: sdma_state_s15_hw_start_up_clean_wait);
2589	sdma_start_hw_clean_up(sde);
2590	break;
2591	case sdma_event_e25_hw_clean_up_done:
2592	break;
2593	case sdma_event_e30_go_running:
2594	ss->go_s99_running = 1;
2595	break;
2596	case sdma_event_e40_sw_cleaned:
2597	break;
2598	case sdma_event_e50_hw_cleaned:
2599	break;
2600	case sdma_event_e60_hw_halted:
2601	schedule_work(work: &sde->err_halt_worker);
2602	break;
2603	case sdma_event_e70_go_idle:
2604	ss->go_s99_running = 0;
2605	break;
2606	case sdma_event_e80_hw_freeze:
2607	break;
2608	case sdma_event_e81_hw_frozen:
2609	break;
2610	case sdma_event_e82_hw_unfreeze:
2611	break;
2612	case sdma_event_e85_link_down:
2613	break;
2614	case sdma_event_e90_sw_halted:
2615	break;
2616	}
2617	break;
2618
2619	case sdma_state_s15_hw_start_up_clean_wait:
2620	switch (event) {
2621	case sdma_event_e00_go_hw_down:
2622	sdma_set_state(sde, next_state: sdma_state_s00_hw_down);
2623	sdma_sw_tear_down(sde);
2624	break;
2625	case sdma_event_e10_go_hw_start:
2626	break;
2627	case sdma_event_e15_hw_halt_done:
2628	break;
2629	case sdma_event_e25_hw_clean_up_done:
2630	sdma_hw_start_up(sde);
2631	sdma_set_state(sde, next_state: ss->go_s99_running ?
2632	sdma_state_s99_running :
2633	sdma_state_s20_idle);
2634	break;
2635	case sdma_event_e30_go_running:
2636	ss->go_s99_running = 1;
2637	break;
2638	case sdma_event_e40_sw_cleaned:
2639	break;
2640	case sdma_event_e50_hw_cleaned:
2641	break;
2642	case sdma_event_e60_hw_halted:
2643	break;
2644	case sdma_event_e70_go_idle:
2645	ss->go_s99_running = 0;
2646	break;
2647	case sdma_event_e80_hw_freeze:
2648	break;
2649	case sdma_event_e81_hw_frozen:
2650	break;
2651	case sdma_event_e82_hw_unfreeze:
2652	break;
2653	case sdma_event_e85_link_down:
2654	break;
2655	case sdma_event_e90_sw_halted:
2656	break;
2657	}
2658	break;
2659
2660	case sdma_state_s20_idle:
2661	switch (event) {
2662	case sdma_event_e00_go_hw_down:
2663	sdma_set_state(sde, next_state: sdma_state_s00_hw_down);
2664	sdma_sw_tear_down(sde);
2665	break;
2666	case sdma_event_e10_go_hw_start:
2667	break;
2668	case sdma_event_e15_hw_halt_done:
2669	break;
2670	case sdma_event_e25_hw_clean_up_done:
2671	break;
2672	case sdma_event_e30_go_running:
2673	sdma_set_state(sde, next_state: sdma_state_s99_running);
2674	ss->go_s99_running = 1;
2675	break;
2676	case sdma_event_e40_sw_cleaned:
2677	break;
2678	case sdma_event_e50_hw_cleaned:
2679	break;
2680	case sdma_event_e60_hw_halted:
2681	sdma_set_state(sde, next_state: sdma_state_s50_hw_halt_wait);
2682	schedule_work(work: &sde->err_halt_worker);
2683	break;
2684	case sdma_event_e70_go_idle:
2685	break;
2686	case sdma_event_e85_link_down:
2687	case sdma_event_e80_hw_freeze:
2688	sdma_set_state(sde, next_state: sdma_state_s80_hw_freeze);
2689	atomic_dec(v: &sde->dd->sdma_unfreeze_count);
2690	wake_up_interruptible(&sde->dd->sdma_unfreeze_wq);
2691	break;
2692	case sdma_event_e81_hw_frozen:
2693	break;
2694	case sdma_event_e82_hw_unfreeze:
2695	break;
2696	case sdma_event_e90_sw_halted:
2697	break;
2698	}
2699	break;
2700
2701	case sdma_state_s30_sw_clean_up_wait:
2702	switch (event) {
2703	case sdma_event_e00_go_hw_down:
2704	sdma_set_state(sde, next_state: sdma_state_s00_hw_down);
2705	break;
2706	case sdma_event_e10_go_hw_start:
2707	break;
2708	case sdma_event_e15_hw_halt_done:
2709	break;
2710	case sdma_event_e25_hw_clean_up_done:
2711	break;
2712	case sdma_event_e30_go_running:
2713	ss->go_s99_running = 1;
2714	break;
2715	case sdma_event_e40_sw_cleaned:
2716	sdma_set_state(sde, next_state: sdma_state_s40_hw_clean_up_wait);
2717	sdma_start_hw_clean_up(sde);
2718	break;
2719	case sdma_event_e50_hw_cleaned:
2720	break;
2721	case sdma_event_e60_hw_halted:
2722	break;
2723	case sdma_event_e70_go_idle:
2724	ss->go_s99_running = 0;
2725	break;
2726	case sdma_event_e80_hw_freeze:
2727	break;
2728	case sdma_event_e81_hw_frozen:
2729	break;
2730	case sdma_event_e82_hw_unfreeze:
2731	break;
2732	case sdma_event_e85_link_down:
2733	ss->go_s99_running = 0;
2734	break;
2735	case sdma_event_e90_sw_halted:
2736	break;
2737	}
2738	break;
2739
2740	case sdma_state_s40_hw_clean_up_wait:
2741	switch (event) {
2742	case sdma_event_e00_go_hw_down:
2743	sdma_set_state(sde, next_state: sdma_state_s00_hw_down);
2744	tasklet_hi_schedule(t: &sde->sdma_sw_clean_up_task);
2745	break;
2746	case sdma_event_e10_go_hw_start:
2747	break;
2748	case sdma_event_e15_hw_halt_done:
2749	break;
2750	case sdma_event_e25_hw_clean_up_done:
2751	sdma_hw_start_up(sde);
2752	sdma_set_state(sde, next_state: ss->go_s99_running ?
2753	sdma_state_s99_running :
2754	sdma_state_s20_idle);
2755	break;
2756	case sdma_event_e30_go_running:
2757	ss->go_s99_running = 1;
2758	break;
2759	case sdma_event_e40_sw_cleaned:
2760	break;
2761	case sdma_event_e50_hw_cleaned:
2762	break;
2763	case sdma_event_e60_hw_halted:
2764	break;
2765	case sdma_event_e70_go_idle:
2766	ss->go_s99_running = 0;
2767	break;
2768	case sdma_event_e80_hw_freeze:
2769	break;
2770	case sdma_event_e81_hw_frozen:
2771	break;
2772	case sdma_event_e82_hw_unfreeze:
2773	break;
2774	case sdma_event_e85_link_down:
2775	ss->go_s99_running = 0;
2776	break;
2777	case sdma_event_e90_sw_halted:
2778	break;
2779	}
2780	break;
2781
2782	case sdma_state_s50_hw_halt_wait:
2783	switch (event) {
2784	case sdma_event_e00_go_hw_down:
2785	sdma_set_state(sde, next_state: sdma_state_s00_hw_down);
2786	tasklet_hi_schedule(t: &sde->sdma_sw_clean_up_task);
2787	break;
2788	case sdma_event_e10_go_hw_start:
2789	break;
2790	case sdma_event_e15_hw_halt_done:
2791	sdma_set_state(sde, next_state: sdma_state_s30_sw_clean_up_wait);
2792	tasklet_hi_schedule(t: &sde->sdma_sw_clean_up_task);
2793	break;
2794	case sdma_event_e25_hw_clean_up_done:
2795	break;
2796	case sdma_event_e30_go_running:
2797	ss->go_s99_running = 1;
2798	break;
2799	case sdma_event_e40_sw_cleaned:
2800	break;
2801	case sdma_event_e50_hw_cleaned:
2802	break;
2803	case sdma_event_e60_hw_halted:
2804	schedule_work(work: &sde->err_halt_worker);
2805	break;
2806	case sdma_event_e70_go_idle:
2807	ss->go_s99_running = 0;
2808	break;
2809	case sdma_event_e80_hw_freeze:
2810	break;
2811	case sdma_event_e81_hw_frozen:
2812	break;
2813	case sdma_event_e82_hw_unfreeze:
2814	break;
2815	case sdma_event_e85_link_down:
2816	ss->go_s99_running = 0;
2817	break;
2818	case sdma_event_e90_sw_halted:
2819	break;
2820	}
2821	break;
2822
2823	case sdma_state_s60_idle_halt_wait:
2824	switch (event) {
2825	case sdma_event_e00_go_hw_down:
2826	sdma_set_state(sde, next_state: sdma_state_s00_hw_down);
2827	tasklet_hi_schedule(t: &sde->sdma_sw_clean_up_task);
2828	break;
2829	case sdma_event_e10_go_hw_start:
2830	break;
2831	case sdma_event_e15_hw_halt_done:
2832	sdma_set_state(sde, next_state: sdma_state_s30_sw_clean_up_wait);
2833	tasklet_hi_schedule(t: &sde->sdma_sw_clean_up_task);
2834	break;
2835	case sdma_event_e25_hw_clean_up_done:
2836	break;
2837	case sdma_event_e30_go_running:
2838	ss->go_s99_running = 1;
2839	break;
2840	case sdma_event_e40_sw_cleaned:
2841	break;
2842	case sdma_event_e50_hw_cleaned:
2843	break;
2844	case sdma_event_e60_hw_halted:
2845	schedule_work(work: &sde->err_halt_worker);
2846	break;
2847	case sdma_event_e70_go_idle:
2848	ss->go_s99_running = 0;
2849	break;
2850	case sdma_event_e80_hw_freeze:
2851	break;
2852	case sdma_event_e81_hw_frozen:
2853	break;
2854	case sdma_event_e82_hw_unfreeze:
2855	break;
2856	case sdma_event_e85_link_down:
2857	break;
2858	case sdma_event_e90_sw_halted:
2859	break;
2860	}
2861	break;
2862
2863	case sdma_state_s80_hw_freeze:
2864	switch (event) {
2865	case sdma_event_e00_go_hw_down:
2866	sdma_set_state(sde, next_state: sdma_state_s00_hw_down);
2867	tasklet_hi_schedule(t: &sde->sdma_sw_clean_up_task);
2868	break;
2869	case sdma_event_e10_go_hw_start:
2870	break;
2871	case sdma_event_e15_hw_halt_done:
2872	break;
2873	case sdma_event_e25_hw_clean_up_done:
2874	break;
2875	case sdma_event_e30_go_running:
2876	ss->go_s99_running = 1;
2877	break;
2878	case sdma_event_e40_sw_cleaned:
2879	break;
2880	case sdma_event_e50_hw_cleaned:
2881	break;
2882	case sdma_event_e60_hw_halted:
2883	break;
2884	case sdma_event_e70_go_idle:
2885	ss->go_s99_running = 0;
2886	break;
2887	case sdma_event_e80_hw_freeze:
2888	break;
2889	case sdma_event_e81_hw_frozen:
2890	sdma_set_state(sde, next_state: sdma_state_s82_freeze_sw_clean);
2891	tasklet_hi_schedule(t: &sde->sdma_sw_clean_up_task);
2892	break;
2893	case sdma_event_e82_hw_unfreeze:
2894	break;
2895	case sdma_event_e85_link_down:
2896	break;
2897	case sdma_event_e90_sw_halted:
2898	break;
2899	}
2900	break;
2901
2902	case sdma_state_s82_freeze_sw_clean:
2903	switch (event) {
2904	case sdma_event_e00_go_hw_down:
2905	sdma_set_state(sde, next_state: sdma_state_s00_hw_down);
2906	tasklet_hi_schedule(t: &sde->sdma_sw_clean_up_task);
2907	break;
2908	case sdma_event_e10_go_hw_start:
2909	break;
2910	case sdma_event_e15_hw_halt_done:
2911	break;
2912	case sdma_event_e25_hw_clean_up_done:
2913	break;
2914	case sdma_event_e30_go_running:
2915	ss->go_s99_running = 1;
2916	break;
2917	case sdma_event_e40_sw_cleaned:
2918	/* notify caller this engine is done cleaning */
2919	atomic_dec(v: &sde->dd->sdma_unfreeze_count);
2920	wake_up_interruptible(&sde->dd->sdma_unfreeze_wq);
2921	break;
2922	case sdma_event_e50_hw_cleaned:
2923	break;
2924	case sdma_event_e60_hw_halted:
2925	break;
2926	case sdma_event_e70_go_idle:
2927	ss->go_s99_running = 0;
2928	break;
2929	case sdma_event_e80_hw_freeze:
2930	break;
2931	case sdma_event_e81_hw_frozen:
2932	break;
2933	case sdma_event_e82_hw_unfreeze:
2934	sdma_hw_start_up(sde);
2935	sdma_set_state(sde, next_state: ss->go_s99_running ?
2936	sdma_state_s99_running :
2937	sdma_state_s20_idle);
2938	break;
2939	case sdma_event_e85_link_down:
2940	break;
2941	case sdma_event_e90_sw_halted:
2942	break;
2943	}
2944	break;
2945
2946	case sdma_state_s99_running:
2947	switch (event) {
2948	case sdma_event_e00_go_hw_down:
2949	sdma_set_state(sde, next_state: sdma_state_s00_hw_down);
2950	tasklet_hi_schedule(t: &sde->sdma_sw_clean_up_task);
2951	break;
2952	case sdma_event_e10_go_hw_start:
2953	break;
2954	case sdma_event_e15_hw_halt_done:
2955	break;
2956	case sdma_event_e25_hw_clean_up_done:
2957	break;
2958	case sdma_event_e30_go_running:
2959	break;
2960	case sdma_event_e40_sw_cleaned:
2961	break;
2962	case sdma_event_e50_hw_cleaned:
2963	break;
2964	case sdma_event_e60_hw_halted:
2965	need_progress = 1;
2966	sdma_err_progress_check_schedule(sde);
2967	fallthrough;
2968	case sdma_event_e90_sw_halted:
2969	/*
2970	* SW initiated halt does not perform engines
2971	* progress check
2972	*/
2973	sdma_set_state(sde, next_state: sdma_state_s50_hw_halt_wait);
2974	schedule_work(work: &sde->err_halt_worker);
2975	break;
2976	case sdma_event_e70_go_idle:
2977	sdma_set_state(sde, next_state: sdma_state_s60_idle_halt_wait);
2978	break;
2979	case sdma_event_e85_link_down:
2980	ss->go_s99_running = 0;
2981	fallthrough;
2982	case sdma_event_e80_hw_freeze:
2983	sdma_set_state(sde, next_state: sdma_state_s80_hw_freeze);
2984	atomic_dec(v: &sde->dd->sdma_unfreeze_count);
2985	wake_up_interruptible(&sde->dd->sdma_unfreeze_wq);
2986	break;
2987	case sdma_event_e81_hw_frozen:
2988	break;
2989	case sdma_event_e82_hw_unfreeze:
2990	break;
2991	}
2992	break;
2993	}
2994
2995	ss->last_event = event;
2996	if (need_progress)
2997	sdma_make_progress(sde, status: 0);
2998	}
2999
3000	/*
3001	* _extend_sdma_tx_descs() - helper to extend txreq
3002	*
3003	* This is called once the initial nominal allocation
3004	* of descriptors in the sdma_txreq is exhausted.
3005	*
3006	* The code will bump the allocation up to the max
3007	* of MAX_DESC (64) descriptors. There doesn't seem
3008	* much point in an interim step. The last descriptor
3009	* is reserved for coalesce buffer in order to support
3010	* cases where input packet has >MAX_DESC iovecs.
3011	*
3012	*/
3013	static int _extend_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx)
3014	{
3015	int i;
3016	struct sdma_desc *descp;
3017
3018	/* Handle last descriptor */
3019	if (unlikely((tx->num_desc == (MAX_DESC - 1)))) {
3020	/* if tlen is 0, it is for padding, release last descriptor */
3021	if (!tx->tlen) {
3022	tx->desc_limit = MAX_DESC;
3023	} else if (!tx->coalesce_buf) {
3024	/* allocate coalesce buffer with space for padding */
3025	tx->coalesce_buf = kmalloc(size: tx->tlen + sizeof(u32),
3026	GFP_ATOMIC);
3027	if (!tx->coalesce_buf)
3028	goto enomem;
3029	tx->coalesce_idx = 0;
3030	}
3031	return 0;
3032	}
3033
3034	if (unlikely(tx->num_desc == MAX_DESC))
3035	goto enomem;
3036
3037	descp = kmalloc_array(MAX_DESC, size: sizeof(struct sdma_desc), GFP_ATOMIC);
3038	if (!descp)
3039	goto enomem;
3040	tx->descp = descp;
3041
3042	/* reserve last descriptor for coalescing */
3043	tx->desc_limit = MAX_DESC - 1;
3044	/* copy ones already built */
3045	for (i = 0; i < tx->num_desc; i++)
3046	tx->descp[i] = tx->descs[i];
3047	return 0;
3048	enomem:
3049	__sdma_txclean(dd, tx);
3050	return -ENOMEM;
3051	}
3052
3053	/*
3054	* ext_coal_sdma_tx_descs() - extend or coalesce sdma tx descriptors
3055	*
3056	* This is called once the initial nominal allocation of descriptors
3057	* in the sdma_txreq is exhausted.
3058	*
3059	* This function calls _extend_sdma_tx_descs to extend or allocate
3060	* coalesce buffer. If there is a allocated coalesce buffer, it will
3061	* copy the input packet data into the coalesce buffer. It also adds
3062	* coalesce buffer descriptor once when whole packet is received.
3063	*
3064	* Return:
3065	* <0 - error
3066	* 0 - coalescing, don't populate descriptor
3067	* 1 - continue with populating descriptor
3068	*/
3069	int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx,
3070	int type, void *kvaddr, struct page *page,
3071	unsigned long offset, u16 len)
3072	{
3073	int pad_len, rval;
3074	dma_addr_t addr;
3075
3076	rval = _extend_sdma_tx_descs(dd, tx);
3077	if (rval) {
3078	__sdma_txclean(dd, tx);
3079	return rval;
3080	}
3081
3082	/* If coalesce buffer is allocated, copy data into it */
3083	if (tx->coalesce_buf) {
3084	if (type == SDMA_MAP_NONE) {
3085	__sdma_txclean(dd, tx);
3086	return -EINVAL;
3087	}
3088
3089	if (type == SDMA_MAP_PAGE) {
3090	kvaddr = kmap_local_page(page);
3091	kvaddr += offset;
3092	} else if (WARN_ON(!kvaddr)) {
3093	__sdma_txclean(dd, tx);
3094	return -EINVAL;
3095	}
3096
3097	memcpy(tx->coalesce_buf + tx->coalesce_idx, kvaddr, len);
3098	tx->coalesce_idx += len;
3099	if (type == SDMA_MAP_PAGE)
3100	kunmap_local(kvaddr);
3101
3102	/* If there is more data, return */
3103	if (tx->tlen - tx->coalesce_idx)
3104	return 0;
3105
3106	/* Whole packet is received; add any padding */
3107	pad_len = tx->packet_len & (sizeof(u32) - 1);
3108	if (pad_len) {
3109	pad_len = sizeof(u32) - pad_len;
3110	memset(tx->coalesce_buf + tx->coalesce_idx, 0, pad_len);
3111	/* padding is taken care of for coalescing case */
3112	tx->packet_len += pad_len;
3113	tx->tlen += pad_len;
3114	}
3115
3116	/* dma map the coalesce buffer */
3117	addr = dma_map_single(&dd->pcidev->dev,
3118	tx->coalesce_buf,
3119	tx->tlen,
3120	DMA_TO_DEVICE);
3121
3122	if (unlikely(dma_mapping_error(&dd->pcidev->dev, addr))) {
3123	__sdma_txclean(dd, tx);
3124	return -ENOSPC;
3125	}
3126
3127	/* Add descriptor for coalesce buffer */
3128	tx->desc_limit = MAX_DESC;
3129	return _sdma_txadd_daddr(dd, SDMA_MAP_SINGLE, tx,
3130	addr, len: tx->tlen, NULL, NULL, NULL);
3131	}
3132
3133	return 1;
3134	}
3135
3136	/* Update sdes when the lmc changes */
3137	void sdma_update_lmc(struct hfi1_devdata *dd, u64 mask, u32 lid)
3138	{
3139	struct sdma_engine *sde;
3140	int i;
3141	u64 sreg;
3142
3143	sreg = ((mask & SD(CHECK_SLID_MASK_MASK)) <<
3144	SD(CHECK_SLID_MASK_SHIFT)) |
3145	(((lid & mask) & SD(CHECK_SLID_VALUE_MASK)) <<
3146	SD(CHECK_SLID_VALUE_SHIFT));
3147
3148	for (i = 0; i < dd->num_sdma; i++) {
3149	hfi1_cdbg(LINKVERB, "SendDmaEngine[%d].SLID_CHECK = 0x%x",
3150	i, (u32)sreg);
3151	sde = &dd->per_sdma[i];
3152	write_sde_csr(sde, SD(CHECK_SLID), value: sreg);
3153	}
3154	}
3155
3156	/* tx not dword sized - pad */
3157	int _pad_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx)
3158	{
3159	int rval = 0;
3160
3161	if ((unlikely(tx->num_desc == tx->desc_limit))) {
3162	rval = _extend_sdma_tx_descs(dd, tx);
3163	if (rval) {
3164	__sdma_txclean(dd, tx);
3165	return rval;
3166	}
3167	}
3168
3169	/* finish the one just added */
3170	make_tx_sdma_desc(
3171	tx,
3172	SDMA_MAP_NONE,
3173	addr: dd->sdma_pad_phys,
3174	len: sizeof(u32) - (tx->packet_len & (sizeof(u32) - 1)),
3175	NULL, NULL, NULL);
3176	tx->num_desc++;
3177	_sdma_close_tx(dd, tx);
3178	return rval;
3179	}
3180
3181	/*
3182	* Add ahg to the sdma_txreq
3183	*
3184	* The logic will consume up to 3
3185	* descriptors at the beginning of
3186	* sdma_txreq.
3187	*/
3188	void _sdma_txreq_ahgadd(
3189	struct sdma_txreq *tx,
3190	u8 num_ahg,
3191	u8 ahg_entry,
3192	u32 *ahg,
3193	u8 ahg_hlen)
3194	{
3195	u32 i, shift = 0, desc = 0;
3196	u8 mode;
3197
3198	WARN_ON_ONCE(num_ahg > 9 || (ahg_hlen & 3) || ahg_hlen == 4);
3199	/* compute mode */
3200	if (num_ahg == 1)
3201	mode = SDMA_AHG_APPLY_UPDATE1;
3202	else if (num_ahg <= 5)
3203	mode = SDMA_AHG_APPLY_UPDATE2;
3204	else
3205	mode = SDMA_AHG_APPLY_UPDATE3;
3206	tx->num_desc++;
3207	/* initialize to consumed descriptors to zero */
3208	switch (mode) {
3209	case SDMA_AHG_APPLY_UPDATE3:
3210	tx->num_desc++;
3211	tx->descs[2].qw[0] = 0;
3212	tx->descs[2].qw[1] = 0;
3213	fallthrough;
3214	case SDMA_AHG_APPLY_UPDATE2:
3215	tx->num_desc++;
3216	tx->descs[1].qw[0] = 0;
3217	tx->descs[1].qw[1] = 0;
3218	break;
3219	}
3220	ahg_hlen >>= 2;
3221	tx->descs[0].qw[1] |=
3222	(((u64)ahg_entry & SDMA_DESC1_HEADER_INDEX_MASK)
3223	<< SDMA_DESC1_HEADER_INDEX_SHIFT) |
3224	(((u64)ahg_hlen & SDMA_DESC1_HEADER_DWS_MASK)
3225	<< SDMA_DESC1_HEADER_DWS_SHIFT) |
3226	(((u64)mode & SDMA_DESC1_HEADER_MODE_MASK)
3227	<< SDMA_DESC1_HEADER_MODE_SHIFT) |
3228	(((u64)ahg[0] & SDMA_DESC1_HEADER_UPDATE1_MASK)
3229	<< SDMA_DESC1_HEADER_UPDATE1_SHIFT);
3230	for (i = 0; i < (num_ahg - 1); i++) {
3231	if (!shift && !(i & 2))
3232	desc++;
3233	tx->descs[desc].qw[!!(i & 2)] |=
3234	(((u64)ahg[i + 1])
3235	<< shift);
3236	shift = (shift + 32) & 63;
3237	}
3238	}
3239
3240	/**
3241	* sdma_ahg_alloc - allocate an AHG entry
3242	* @sde: engine to allocate from
3243	*
3244	* Return:
3245	* 0-31 when successful, -EOPNOTSUPP if AHG is not enabled,
3246	* -ENOSPC if an entry is not available
3247	*/
3248	int sdma_ahg_alloc(struct sdma_engine *sde)
3249	{
3250	int nr;
3251	int oldbit;
3252
3253	if (!sde) {
3254	trace_hfi1_ahg_allocate(sde, aidx: -EINVAL);
3255	return -EINVAL;
3256	}
3257	while (1) {
3258	nr = ffz(READ_ONCE(sde->ahg_bits));
3259	if (nr > 31) {
3260	trace_hfi1_ahg_allocate(sde, aidx: -ENOSPC);
3261	return -ENOSPC;
3262	}
3263	oldbit = test_and_set_bit(nr, addr: &sde->ahg_bits);
3264	if (!oldbit)
3265	break;
3266	cpu_relax();
3267	}
3268	trace_hfi1_ahg_allocate(sde, aidx: nr);
3269	return nr;
3270	}
3271
3272	/**
3273	* sdma_ahg_free - free an AHG entry
3274	* @sde: engine to return AHG entry
3275	* @ahg_index: index to free
3276	*
3277	* This routine frees the indicate AHG entry.
3278	*/
3279	void sdma_ahg_free(struct sdma_engine *sde, int ahg_index)
3280	{
3281	if (!sde)
3282	return;
3283	trace_hfi1_ahg_deallocate(sde, aidx: ahg_index);
3284	if (ahg_index < 0 || ahg_index > 31)
3285	return;
3286	clear_bit(nr: ahg_index, addr: &sde->ahg_bits);
3287	}
3288
3289	/*
3290	* SPC freeze handling for SDMA engines. Called when the driver knows
3291	* the SPC is going into a freeze but before the freeze is fully
3292	* settled. Generally an error interrupt.
3293	*
3294	* This event will pull the engine out of running so no more entries can be
3295	* added to the engine's queue.
3296	*/
3297	void sdma_freeze_notify(struct hfi1_devdata *dd, int link_down)
3298	{
3299	int i;
3300	enum sdma_events event = link_down ? sdma_event_e85_link_down :
3301	sdma_event_e80_hw_freeze;
3302
3303	/* set up the wait but do not wait here */
3304	atomic_set(v: &dd->sdma_unfreeze_count, i: dd->num_sdma);
3305
3306	/* tell all engines to stop running and wait */
3307	for (i = 0; i < dd->num_sdma; i++)
3308	sdma_process_event(sde: &dd->per_sdma[i], event);
3309
3310	/* sdma_freeze() will wait for all engines to have stopped */
3311	}
3312
3313	/*
3314	* SPC freeze handling for SDMA engines. Called when the driver knows
3315	* the SPC is fully frozen.
3316	*/
3317	void sdma_freeze(struct hfi1_devdata *dd)
3318	{
3319	int i;
3320	int ret;
3321
3322	/*
3323	* Make sure all engines have moved out of the running state before
3324	* continuing.
3325	*/
3326	ret = wait_event_interruptible(dd->sdma_unfreeze_wq,
3327	atomic_read(&dd->sdma_unfreeze_count) <=
3328	0);
3329	/* interrupted or count is negative, then unloading - just exit */
3330	if (ret || atomic_read(v: &dd->sdma_unfreeze_count) < 0)
3331	return;
3332
3333	/* set up the count for the next wait */
3334	atomic_set(v: &dd->sdma_unfreeze_count, i: dd->num_sdma);
3335
3336	/* tell all engines that the SPC is frozen, they can start cleaning */
3337	for (i = 0; i < dd->num_sdma; i++)
3338	sdma_process_event(sde: &dd->per_sdma[i], event: sdma_event_e81_hw_frozen);
3339
3340	/*
3341	* Wait for everyone to finish software clean before exiting. The
3342	* software clean will read engine CSRs, so must be completed before
3343	* the next step, which will clear the engine CSRs.
3344	*/
3345	(void)wait_event_interruptible(dd->sdma_unfreeze_wq,
3346	atomic_read(&dd->sdma_unfreeze_count) <= 0);
3347	/* no need to check results - done no matter what */
3348	}
3349
3350	/*
3351	* SPC freeze handling for the SDMA engines. Called after the SPC is unfrozen.
3352	*
3353	* The SPC freeze acts like a SDMA halt and a hardware clean combined. All
3354	* that is left is a software clean. We could do it after the SPC is fully
3355	* frozen, but then we'd have to add another state to wait for the unfreeze.
3356	* Instead, just defer the software clean until the unfreeze step.
3357	*/
3358	void sdma_unfreeze(struct hfi1_devdata *dd)
3359	{
3360	int i;
3361
3362	/* tell all engines start freeze clean up */
3363	for (i = 0; i < dd->num_sdma; i++)
3364	sdma_process_event(sde: &dd->per_sdma[i],
3365	event: sdma_event_e82_hw_unfreeze);
3366	}
3367
3368	/**
3369	* _sdma_engine_progress_schedule() - schedule progress on engine
3370	* @sde: sdma_engine to schedule progress
3371	*
3372	*/
3373	void _sdma_engine_progress_schedule(
3374	struct sdma_engine *sde)
3375	{
3376	trace_hfi1_sdma_engine_progress(sde, status: sde->progress_mask);
3377	/* assume we have selected a good cpu */
3378	write_csr(dd: sde->dd,
3379	CCE_INT_FORCE + (8 * (IS_SDMA_START / 64)),
3380	value: sde->progress_mask);
3381	}
3382