1 | /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ |
2 | /* |
3 | * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved. |
4 | */ |
5 | |
6 | #ifndef ENA_H |
7 | #define ENA_H |
8 | |
9 | #include <linux/bitops.h> |
10 | #include <linux/dim.h> |
11 | #include <linux/etherdevice.h> |
12 | #include <linux/if_vlan.h> |
13 | #include <linux/inetdevice.h> |
14 | #include <linux/interrupt.h> |
15 | #include <linux/netdevice.h> |
16 | #include <linux/skbuff.h> |
17 | #include <net/xdp.h> |
18 | #include <uapi/linux/bpf.h> |
19 | |
20 | #include "ena_com.h" |
21 | #include "ena_eth_com.h" |
22 | |
23 | #define DRV_MODULE_GEN_MAJOR 2 |
24 | #define DRV_MODULE_GEN_MINOR 1 |
25 | #define DRV_MODULE_GEN_SUBMINOR 0 |
26 | |
27 | #define DRV_MODULE_NAME "ena" |
28 | |
29 | #define DEVICE_NAME "Elastic Network Adapter (ENA)" |
30 | |
31 | /* 1 for AENQ + ADMIN */ |
32 | #define ENA_ADMIN_MSIX_VEC 1 |
33 | #define ENA_MAX_MSIX_VEC(io_queues) (ENA_ADMIN_MSIX_VEC + (io_queues)) |
34 | |
35 | /* The ENA buffer length fields is 16 bit long. So when PAGE_SIZE == 64kB the |
36 | * driver passes 0. |
37 | * Since the max packet size the ENA handles is ~9kB limit the buffer length to |
38 | * 16kB. |
39 | */ |
40 | #if PAGE_SIZE > SZ_16K |
41 | #define ENA_PAGE_SIZE (_AC(SZ_16K, UL)) |
42 | #else |
43 | #define ENA_PAGE_SIZE PAGE_SIZE |
44 | #endif |
45 | |
46 | #define ENA_MIN_MSIX_VEC 2 |
47 | |
48 | #define ENA_REG_BAR 0 |
49 | #define ENA_MEM_BAR 2 |
50 | #define ENA_BAR_MASK (BIT(ENA_REG_BAR) | BIT(ENA_MEM_BAR)) |
51 | |
52 | #define ENA_DEFAULT_RING_SIZE (1024) |
53 | #define ENA_MIN_RING_SIZE (256) |
54 | |
55 | #define ENA_MIN_RX_BUF_SIZE (2048) |
56 | |
57 | #define ENA_MIN_NUM_IO_QUEUES (1) |
58 | |
59 | #define ENA_TX_WAKEUP_THRESH (MAX_SKB_FRAGS + 2) |
60 | #define ENA_DEFAULT_RX_COPYBREAK (256 - NET_IP_ALIGN) |
61 | |
62 | #define ENA_MIN_MTU 128 |
63 | |
64 | #define ENA_NAME_MAX_LEN 20 |
65 | #define ENA_IRQNAME_SIZE 40 |
66 | |
67 | #define ENA_PKT_MAX_BUFS 19 |
68 | |
69 | #define 7 |
70 | #define (1 << ENA_RX_RSS_TABLE_LOG_SIZE) |
71 | |
72 | /* The number of tx packet completions that will be handled each NAPI poll |
73 | * cycle is ring_size / ENA_TX_POLL_BUDGET_DIVIDER. |
74 | */ |
75 | #define ENA_TX_POLL_BUDGET_DIVIDER 4 |
76 | |
77 | /* Refill Rx queue when number of required descriptors is above |
78 | * QUEUE_SIZE / ENA_RX_REFILL_THRESH_DIVIDER or ENA_RX_REFILL_THRESH_PACKET |
79 | */ |
80 | #define ENA_RX_REFILL_THRESH_DIVIDER 8 |
81 | #define ENA_RX_REFILL_THRESH_PACKET 256 |
82 | |
83 | /* Number of queues to check for missing queues per timer service */ |
84 | #define ENA_MONITORED_TX_QUEUES 4 |
85 | /* Max timeout packets before device reset */ |
86 | #define MAX_NUM_OF_TIMEOUTED_PACKETS 128 |
87 | |
88 | #define ENA_TX_RING_IDX_NEXT(idx, ring_size) (((idx) + 1) & ((ring_size) - 1)) |
89 | |
90 | #define ENA_RX_RING_IDX_NEXT(idx, ring_size) (((idx) + 1) & ((ring_size) - 1)) |
91 | #define ENA_RX_RING_IDX_ADD(idx, n, ring_size) \ |
92 | (((idx) + (n)) & ((ring_size) - 1)) |
93 | |
94 | #define ENA_IO_TXQ_IDX(q) (2 * (q)) |
95 | #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) |
96 | #define ENA_IO_TXQ_IDX_TO_COMBINED_IDX(q) ((q) / 2) |
97 | #define ENA_IO_RXQ_IDX_TO_COMBINED_IDX(q) (((q) - 1) / 2) |
98 | |
99 | #define ENA_MGMNT_IRQ_IDX 0 |
100 | #define ENA_IO_IRQ_FIRST_IDX 1 |
101 | #define ENA_IO_IRQ_IDX(q) (ENA_IO_IRQ_FIRST_IDX + (q)) |
102 | |
103 | #define ENA_ADMIN_POLL_DELAY_US 100 |
104 | |
105 | /* ENA device should send keep alive msg every 1 sec. |
106 | * We wait for 6 sec just to be on the safe side. |
107 | */ |
108 | #define ENA_DEVICE_KALIVE_TIMEOUT (6 * HZ) |
109 | #define ENA_MAX_NO_INTERRUPT_ITERATIONS 3 |
110 | |
111 | #define ENA_MMIO_DISABLE_REG_READ BIT(0) |
112 | |
113 | struct ena_irq { |
114 | irq_handler_t handler; |
115 | void *data; |
116 | int cpu; |
117 | u32 vector; |
118 | cpumask_t affinity_hint_mask; |
119 | char name[ENA_IRQNAME_SIZE]; |
120 | }; |
121 | |
122 | struct ena_napi { |
123 | u8 first_interrupt ____cacheline_aligned; |
124 | u8 interrupts_masked; |
125 | struct napi_struct napi; |
126 | struct ena_ring *tx_ring; |
127 | struct ena_ring *rx_ring; |
128 | u32 qid; |
129 | struct dim dim; |
130 | }; |
131 | |
132 | struct ena_tx_buffer { |
133 | union { |
134 | struct sk_buff *skb; |
135 | /* XDP buffer structure which is used for sending packets in |
136 | * the xdp queues |
137 | */ |
138 | struct xdp_frame *xdpf; |
139 | }; |
140 | /* num of ena desc for this specific skb |
141 | * (includes data desc and metadata desc) |
142 | */ |
143 | u32 tx_descs; |
144 | /* num of buffers used by this skb */ |
145 | u32 num_of_bufs; |
146 | |
147 | /* Total size of all buffers in bytes */ |
148 | u32 total_tx_size; |
149 | |
150 | /* Indicate if bufs[0] map the linear data of the skb. */ |
151 | u8 map_linear_data; |
152 | |
153 | /* Used for detect missing tx packets to limit the number of prints */ |
154 | u8 print_once; |
155 | /* Save the last jiffies to detect missing tx packets |
156 | * |
157 | * sets to non zero value on ena_start_xmit and set to zero on |
158 | * napi and timer_Service_routine. |
159 | * |
160 | * while this value is not protected by lock, |
161 | * a given packet is not expected to be handled by ena_start_xmit |
162 | * and by napi/timer_service at the same time. |
163 | */ |
164 | unsigned long last_jiffies; |
165 | struct ena_com_buf bufs[ENA_PKT_MAX_BUFS]; |
166 | } ____cacheline_aligned; |
167 | |
168 | struct ena_rx_buffer { |
169 | struct sk_buff *skb; |
170 | struct page *page; |
171 | dma_addr_t dma_addr; |
172 | u32 page_offset; |
173 | u32 buf_offset; |
174 | struct ena_com_buf ena_buf; |
175 | } ____cacheline_aligned; |
176 | |
177 | struct ena_stats_tx { |
178 | u64 cnt; |
179 | u64 bytes; |
180 | u64 queue_stop; |
181 | u64 prepare_ctx_err; |
182 | u64 queue_wakeup; |
183 | u64 dma_mapping_err; |
184 | u64 linearize; |
185 | u64 linearize_failed; |
186 | u64 napi_comp; |
187 | u64 tx_poll; |
188 | u64 doorbells; |
189 | u64 bad_req_id; |
190 | u64 llq_buffer_copy; |
191 | u64 missed_tx; |
192 | u64 unmask_interrupt; |
193 | u64 last_napi_jiffies; |
194 | }; |
195 | |
196 | struct ena_stats_rx { |
197 | u64 cnt; |
198 | u64 bytes; |
199 | u64 rx_copybreak_pkt; |
200 | u64 csum_good; |
201 | u64 refil_partial; |
202 | u64 csum_bad; |
203 | u64 page_alloc_fail; |
204 | u64 skb_alloc_fail; |
205 | u64 dma_mapping_err; |
206 | u64 bad_desc_num; |
207 | u64 bad_req_id; |
208 | u64 empty_rx_ring; |
209 | u64 csum_unchecked; |
210 | u64 xdp_aborted; |
211 | u64 xdp_drop; |
212 | u64 xdp_pass; |
213 | u64 xdp_tx; |
214 | u64 xdp_invalid; |
215 | u64 xdp_redirect; |
216 | }; |
217 | |
218 | struct ena_ring { |
219 | /* Holds the empty requests for TX/RX |
220 | * out of order completions |
221 | */ |
222 | u16 *free_ids; |
223 | |
224 | union { |
225 | struct ena_tx_buffer *tx_buffer_info; |
226 | struct ena_rx_buffer *rx_buffer_info; |
227 | }; |
228 | |
229 | /* cache ptr to avoid using the adapter */ |
230 | struct device *dev; |
231 | struct pci_dev *pdev; |
232 | struct napi_struct *napi; |
233 | struct net_device *netdev; |
234 | struct ena_com_dev *ena_dev; |
235 | struct ena_adapter *adapter; |
236 | struct ena_com_io_cq *ena_com_io_cq; |
237 | struct ena_com_io_sq *ena_com_io_sq; |
238 | struct bpf_prog *xdp_bpf_prog; |
239 | struct xdp_rxq_info xdp_rxq; |
240 | spinlock_t xdp_tx_lock; /* synchronize XDP TX/Redirect traffic */ |
241 | /* Used for rx queues only to point to the xdp tx ring, to |
242 | * which traffic should be redirected from this rx ring. |
243 | */ |
244 | struct ena_ring *xdp_ring; |
245 | |
246 | u16 next_to_use; |
247 | u16 next_to_clean; |
248 | u16 rx_copybreak; |
249 | u16 rx_headroom; |
250 | u16 qid; |
251 | u16 mtu; |
252 | u16 sgl_size; |
253 | |
254 | /* The maximum header length the device can handle */ |
255 | u8 ; |
256 | |
257 | bool disable_meta_caching; |
258 | u16 no_interrupt_event_cnt; |
259 | |
260 | /* cpu and NUMA for TPH */ |
261 | int cpu; |
262 | int numa_node; |
263 | |
264 | /* number of tx/rx_buffer_info's entries */ |
265 | int ring_size; |
266 | |
267 | enum ena_admin_placement_policy_type tx_mem_queue_type; |
268 | |
269 | struct ena_com_rx_buf_info ena_bufs[ENA_PKT_MAX_BUFS]; |
270 | u32 smoothed_interval; |
271 | u32 per_napi_packets; |
272 | u16 non_empty_napi_events; |
273 | struct u64_stats_sync syncp; |
274 | union { |
275 | struct ena_stats_tx tx_stats; |
276 | struct ena_stats_rx rx_stats; |
277 | }; |
278 | |
279 | u8 *push_buf_intermediate_buf; |
280 | int empty_rx_queue; |
281 | } ____cacheline_aligned; |
282 | |
283 | struct ena_stats_dev { |
284 | u64 tx_timeout; |
285 | u64 suspend; |
286 | u64 resume; |
287 | u64 wd_expired; |
288 | u64 interface_up; |
289 | u64 interface_down; |
290 | u64 admin_q_pause; |
291 | u64 rx_drops; |
292 | u64 tx_drops; |
293 | }; |
294 | |
295 | enum ena_flags_t { |
296 | ENA_FLAG_DEVICE_RUNNING, |
297 | ENA_FLAG_DEV_UP, |
298 | ENA_FLAG_LINK_UP, |
299 | ENA_FLAG_MSIX_ENABLED, |
300 | ENA_FLAG_TRIGGER_RESET, |
301 | ENA_FLAG_ONGOING_RESET |
302 | }; |
303 | |
304 | /* adapter specific private data structure */ |
305 | struct ena_adapter { |
306 | struct ena_com_dev *ena_dev; |
307 | /* OS defined structs */ |
308 | struct net_device *netdev; |
309 | struct pci_dev *pdev; |
310 | |
311 | /* rx packets that shorter that this len will be copied to the skb |
312 | * header |
313 | */ |
314 | u32 rx_copybreak; |
315 | u32 max_mtu; |
316 | |
317 | u32 num_io_queues; |
318 | u32 max_num_io_queues; |
319 | |
320 | int msix_vecs; |
321 | |
322 | u32 missing_tx_completion_threshold; |
323 | |
324 | u32 requested_tx_ring_size; |
325 | u32 requested_rx_ring_size; |
326 | |
327 | u32 max_tx_ring_size; |
328 | u32 max_rx_ring_size; |
329 | |
330 | u32 msg_enable; |
331 | |
332 | /* large_llq_header_enabled is used for two purposes: |
333 | * 1. Indicates that large LLQ has been requested. |
334 | * 2. Indicates whether large LLQ is set or not after device |
335 | * initialization / configuration. |
336 | */ |
337 | bool ; |
338 | bool ; |
339 | |
340 | u16 max_tx_sgl_size; |
341 | u16 max_rx_sgl_size; |
342 | |
343 | u8 mac_addr[ETH_ALEN]; |
344 | |
345 | unsigned long keep_alive_timeout; |
346 | unsigned long missing_tx_completion_to; |
347 | |
348 | char name[ENA_NAME_MAX_LEN]; |
349 | |
350 | unsigned long flags; |
351 | /* TX */ |
352 | struct ena_ring tx_ring[ENA_MAX_NUM_IO_QUEUES] |
353 | ____cacheline_aligned_in_smp; |
354 | |
355 | /* RX */ |
356 | struct ena_ring rx_ring[ENA_MAX_NUM_IO_QUEUES] |
357 | ____cacheline_aligned_in_smp; |
358 | |
359 | struct ena_napi ena_napi[ENA_MAX_NUM_IO_QUEUES]; |
360 | |
361 | struct ena_irq irq_tbl[ENA_MAX_MSIX_VEC(ENA_MAX_NUM_IO_QUEUES)]; |
362 | |
363 | /* timer service */ |
364 | struct work_struct reset_task; |
365 | struct timer_list timer_service; |
366 | |
367 | bool wd_state; |
368 | bool dev_up_before_reset; |
369 | bool disable_meta_caching; |
370 | unsigned long last_keep_alive_jiffies; |
371 | |
372 | struct u64_stats_sync syncp; |
373 | struct ena_stats_dev dev_stats; |
374 | struct ena_admin_eni_stats eni_stats; |
375 | |
376 | /* last queue index that was checked for uncompleted tx packets */ |
377 | u32 last_monitored_tx_qid; |
378 | |
379 | enum ena_regs_reset_reason_types reset_reason; |
380 | |
381 | struct bpf_prog *xdp_bpf_prog; |
382 | u32 xdp_first_ring; |
383 | u32 xdp_num_queues; |
384 | }; |
385 | |
386 | void ena_set_ethtool_ops(struct net_device *netdev); |
387 | |
388 | void ena_dump_stats_to_dmesg(struct ena_adapter *adapter); |
389 | |
390 | void ena_dump_stats_to_buf(struct ena_adapter *adapter, u8 *buf); |
391 | |
392 | int ena_update_hw_stats(struct ena_adapter *adapter); |
393 | |
394 | int ena_update_queue_params(struct ena_adapter *adapter, |
395 | u32 new_tx_size, |
396 | u32 new_rx_size, |
397 | u32 ); |
398 | |
399 | int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count); |
400 | |
401 | int ena_set_rx_copybreak(struct ena_adapter *adapter, u32 rx_copybreak); |
402 | |
403 | int ena_get_sset_count(struct net_device *netdev, int sset); |
404 | |
405 | static inline void ena_reset_device(struct ena_adapter *adapter, |
406 | enum ena_regs_reset_reason_types reset_reason) |
407 | { |
408 | adapter->reset_reason = reset_reason; |
409 | /* Make sure reset reason is set before triggering the reset */ |
410 | smp_mb__before_atomic(); |
411 | set_bit(nr: ENA_FLAG_TRIGGER_RESET, addr: &adapter->flags); |
412 | } |
413 | |
414 | int handle_invalid_req_id(struct ena_ring *ring, u16 req_id, |
415 | struct ena_tx_buffer *tx_info, bool is_xdp); |
416 | |
417 | /* Increase a stat by cnt while holding syncp seqlock on 32bit machines */ |
418 | static inline void ena_increase_stat(u64 *statp, u64 cnt, |
419 | struct u64_stats_sync *syncp) |
420 | { |
421 | u64_stats_update_begin(syncp); |
422 | (*statp) += cnt; |
423 | u64_stats_update_end(syncp); |
424 | } |
425 | |
426 | static inline void ena_ring_tx_doorbell(struct ena_ring *tx_ring) |
427 | { |
428 | ena_com_write_sq_doorbell(io_sq: tx_ring->ena_com_io_sq); |
429 | ena_increase_stat(statp: &tx_ring->tx_stats.doorbells, cnt: 1, syncp: &tx_ring->syncp); |
430 | } |
431 | |
432 | int ena_xmit_common(struct ena_adapter *adapter, |
433 | struct ena_ring *ring, |
434 | struct ena_tx_buffer *tx_info, |
435 | struct ena_com_tx_ctx *ena_tx_ctx, |
436 | u16 next_to_use, |
437 | u32 bytes); |
438 | void ena_unmap_tx_buff(struct ena_ring *tx_ring, |
439 | struct ena_tx_buffer *tx_info); |
440 | void ena_init_io_rings(struct ena_adapter *adapter, |
441 | int first_index, int count); |
442 | int ena_create_io_tx_queues_in_range(struct ena_adapter *adapter, |
443 | int first_index, int count); |
444 | int ena_setup_tx_resources_in_range(struct ena_adapter *adapter, |
445 | int first_index, int count); |
446 | void ena_free_all_io_tx_resources_in_range(struct ena_adapter *adapter, |
447 | int first_index, int count); |
448 | void ena_free_all_io_tx_resources(struct ena_adapter *adapter); |
449 | void ena_down(struct ena_adapter *adapter); |
450 | int ena_up(struct ena_adapter *adapter); |
451 | void ena_unmask_interrupt(struct ena_ring *tx_ring, struct ena_ring *rx_ring); |
452 | void ena_update_ring_numa_node(struct ena_ring *tx_ring, |
453 | struct ena_ring *rx_ring); |
454 | #endif /* !(ENA_H) */ |
455 | |