1 | /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ |
2 | /* |
3 | * Copyright(c) 2023 - Cornelis Networks, Inc. |
4 | * Copyright(c) 2015 - 2018 Intel Corporation. |
5 | */ |
6 | #ifndef _HFI1_USER_SDMA_H |
7 | #define _HFI1_USER_SDMA_H |
8 | |
9 | #include <linux/device.h> |
10 | #include <linux/wait.h> |
11 | |
12 | #include "common.h" |
13 | #include "iowait.h" |
14 | #include "user_exp_rcv.h" |
15 | #include "mmu_rb.h" |
16 | #include "pinning.h" |
17 | #include "sdma.h" |
18 | |
19 | /* The maximum number of Data io vectors per message/request */ |
20 | #define MAX_VECTORS_PER_REQ 8 |
21 | /* |
22 | * Maximum number of packet to send from each message/request |
23 | * before moving to the next one. |
24 | */ |
25 | #define MAX_PKTS_PER_QUEUE 16 |
26 | |
27 | #define num_pages(x) (1 + ((((x) - 1) & PAGE_MASK) >> PAGE_SHIFT)) |
28 | |
29 | #define req_opcode(x) \ |
30 | (((x) >> HFI1_SDMA_REQ_OPCODE_SHIFT) & HFI1_SDMA_REQ_OPCODE_MASK) |
31 | #define req_version(x) \ |
32 | (((x) >> HFI1_SDMA_REQ_VERSION_SHIFT) & HFI1_SDMA_REQ_OPCODE_MASK) |
33 | #define req_iovcnt(x) \ |
34 | (((x) >> HFI1_SDMA_REQ_IOVCNT_SHIFT) & HFI1_SDMA_REQ_IOVCNT_MASK) |
35 | |
36 | /* Number of BTH.PSN bits used for sequence number in expected rcvs */ |
37 | #define BTH_SEQ_MASK 0x7ffull |
38 | |
39 | #define AHG_KDETH_INTR_SHIFT 12 |
40 | #define AHG_KDETH_SH_SHIFT 13 |
41 | #define AHG_KDETH_ARRAY_SIZE 9 |
42 | |
43 | #define PBC2LRH(x) ((((x) & 0xfff) << 2) - 4) |
44 | #define LRH2PBC(x) ((((x) >> 2) + 1) & 0xfff) |
45 | |
46 | /** |
47 | * Build an SDMA AHG header update descriptor and save it to an array. |
48 | * @arr - Array to save the descriptor to. |
49 | * @idx - Index of the array at which the descriptor will be saved. |
50 | * @array_size - Size of the array arr. |
51 | * @dw - Update index into the header in DWs. |
52 | * @bit - Start bit. |
53 | * @width - Field width. |
54 | * @value - 16 bits of immediate data to write into the field. |
55 | * Returns -ERANGE if idx is invalid. If successful, returns the next index |
56 | * (idx + 1) of the array to be used for the next descriptor. |
57 | */ |
58 | static inline int (u32 *arr, int idx, size_t array_size, |
59 | u8 dw, u8 bit, u8 width, u16 value) |
60 | { |
61 | if ((size_t)idx >= array_size) |
62 | return -ERANGE; |
63 | arr[idx++] = sdma_build_ahg_descriptor(data: value, dwindex: dw, startbit: bit, bits: width); |
64 | return idx; |
65 | } |
66 | |
67 | /* Tx request flag bits */ |
68 | #define TXREQ_FLAGS_REQ_ACK BIT(0) /* Set the ACK bit in the header */ |
69 | #define TXREQ_FLAGS_REQ_DISABLE_SH BIT(1) /* Disable header suppression */ |
70 | |
71 | enum pkt_q_sdma_state { |
72 | SDMA_PKT_Q_ACTIVE, |
73 | SDMA_PKT_Q_DEFERRED, |
74 | }; |
75 | |
76 | #define SDMA_IOWAIT_TIMEOUT 1000 /* in milliseconds */ |
77 | |
78 | #define SDMA_DBG(req, fmt, ...) \ |
79 | hfi1_cdbg(SDMA, "[%u:%u:%u:%u] " fmt, (req)->pq->dd->unit, \ |
80 | (req)->pq->ctxt, (req)->pq->subctxt, (req)->info.comp_idx, \ |
81 | ##__VA_ARGS__) |
82 | |
83 | struct hfi1_user_sdma_pkt_q { |
84 | u16 ctxt; |
85 | u16 subctxt; |
86 | u16 n_max_reqs; |
87 | atomic_t n_reqs; |
88 | u16 reqidx; |
89 | struct hfi1_devdata *dd; |
90 | struct kmem_cache *txreq_cache; |
91 | struct user_sdma_request *reqs; |
92 | unsigned long *req_in_use; |
93 | struct iowait busy; |
94 | enum pkt_q_sdma_state state; |
95 | wait_queue_head_t wait; |
96 | unsigned long unpinned; |
97 | struct mmu_rb_handler *handler; |
98 | atomic_t n_locked; |
99 | }; |
100 | |
101 | struct hfi1_user_sdma_comp_q { |
102 | u16 nentries; |
103 | struct hfi1_sdma_comp_entry *comps; |
104 | }; |
105 | |
106 | struct user_sdma_iovec { |
107 | struct list_head list; |
108 | struct iovec iov; |
109 | /* |
110 | * offset into the virtual address space of the vector at |
111 | * which we last left off. |
112 | */ |
113 | u64 offset; |
114 | }; |
115 | |
116 | /* evict operation argument */ |
117 | struct evict_data { |
118 | u32 cleared; /* count evicted so far */ |
119 | u32 target; /* target count to evict */ |
120 | }; |
121 | |
122 | struct user_sdma_request { |
123 | /* This is the original header from user space */ |
124 | struct hfi1_pkt_header hdr; |
125 | |
126 | /* Read mostly fields */ |
127 | struct hfi1_user_sdma_pkt_q *pq ____cacheline_aligned_in_smp; |
128 | struct hfi1_user_sdma_comp_q *cq; |
129 | /* |
130 | * Pointer to the SDMA engine for this request. |
131 | * Since different request could be on different VLs, |
132 | * each request will need it's own engine pointer. |
133 | */ |
134 | struct sdma_engine *sde; |
135 | struct sdma_req_info info; |
136 | /* TID array values copied from the tid_iov vector */ |
137 | u32 *tids; |
138 | /* total length of the data in the request */ |
139 | u32 data_len; |
140 | /* number of elements copied to the tids array */ |
141 | u16 n_tids; |
142 | /* |
143 | * We copy the iovs for this request (based on |
144 | * info.iovcnt). These are only the data vectors |
145 | */ |
146 | u8 data_iovs; |
147 | s8 ahg_idx; |
148 | |
149 | /* Writeable fields shared with interrupt */ |
150 | u16 seqcomp ____cacheline_aligned_in_smp; |
151 | u16 seqsubmitted; |
152 | |
153 | /* Send side fields */ |
154 | struct list_head txps ____cacheline_aligned_in_smp; |
155 | u16 seqnum; |
156 | /* |
157 | * KDETH.OFFSET (TID) field |
158 | * The offset can cover multiple packets, depending on the |
159 | * size of the TID entry. |
160 | */ |
161 | u32 tidoffset; |
162 | /* |
163 | * KDETH.Offset (Eager) field |
164 | * We need to remember the initial value so the headers |
165 | * can be updated properly. |
166 | */ |
167 | u32 koffset; |
168 | u32 sent; |
169 | /* TID index copied from the tid_iov vector */ |
170 | u16 tididx; |
171 | /* progress index moving along the iovs array */ |
172 | u8 iov_idx; |
173 | u8 has_error; |
174 | |
175 | struct user_sdma_iovec iovs[MAX_VECTORS_PER_REQ]; |
176 | } ____cacheline_aligned_in_smp; |
177 | |
178 | /* |
179 | * A single txreq could span up to 3 physical pages when the MTU |
180 | * is sufficiently large (> 4K). Each of the IOV pointers also |
181 | * needs it's own set of flags so the vector has been handled |
182 | * independently of each other. |
183 | */ |
184 | struct user_sdma_txreq { |
185 | /* Packet header for the txreq */ |
186 | struct hfi1_pkt_header hdr; |
187 | struct sdma_txreq txreq; |
188 | struct list_head list; |
189 | struct user_sdma_request *req; |
190 | u16 flags; |
191 | u16 seqnum; |
192 | }; |
193 | |
194 | int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *uctxt, |
195 | struct hfi1_filedata *fd); |
196 | int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd, |
197 | struct hfi1_ctxtdata *uctxt); |
198 | int hfi1_user_sdma_process_request(struct hfi1_filedata *fd, |
199 | struct iovec *iovec, unsigned long dim, |
200 | unsigned long *count); |
201 | #endif /* _HFI1_USER_SDMA_H */ |
202 | |