1 | /********************************************************************** |
2 | * Author: Cavium, Inc. |
3 | * |
4 | * Contact: support@cavium.com |
5 | * Please include "LiquidIO" in the subject. |
6 | * |
7 | * Copyright (c) 2003-2016 Cavium, Inc. |
8 | * |
9 | * This file is free software; you can redistribute it and/or modify |
10 | * it under the terms of the GNU General Public License, Version 2, as |
11 | * published by the Free Software Foundation. |
12 | * |
13 | * This file is distributed in the hope that it will be useful, but |
14 | * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty |
15 | * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or |
16 | * NONINFRINGEMENT. See the GNU General Public License for more details. |
17 | ***********************************************************************/ |
18 | /*! \file octeon_droq.h |
19 | * \brief Implementation of Octeon Output queues. "Output" is with |
20 | * respect to the Octeon device on the NIC. From this driver's point of |
21 | * view they are ingress queues. |
22 | */ |
23 | |
24 | #ifndef __OCTEON_DROQ_H__ |
25 | #define __OCTEON_DROQ_H__ |
26 | |
27 | /* Default number of packets that will be processed in one iteration. */ |
28 | #define MAX_PACKET_BUDGET 0xFFFFFFFF |
29 | |
30 | /** Octeon descriptor format. |
31 | * The descriptor ring is made of descriptors which have 2 64-bit values: |
32 | * -# Physical (bus) address of the data buffer. |
33 | * -# Physical (bus) address of a octeon_droq_info structure. |
34 | * The Octeon device DMA's incoming packets and its information at the address |
35 | * given by these descriptor fields. |
36 | */ |
37 | struct octeon_droq_desc { |
38 | /** The buffer pointer */ |
39 | u64 buffer_ptr; |
40 | |
41 | /** The Info pointer */ |
42 | u64 info_ptr; |
43 | }; |
44 | |
45 | #define OCT_DROQ_DESC_SIZE (sizeof(struct octeon_droq_desc)) |
46 | |
47 | /** Information about packet DMA'ed by Octeon. |
48 | * The format of the information available at Info Pointer after Octeon |
49 | * has posted a packet. Not all descriptors have valid information. Only |
50 | * the Info field of the first descriptor for a packet has information |
51 | * about the packet. |
52 | */ |
53 | struct octeon_droq_info { |
54 | /** The Length of the packet. */ |
55 | u64 length; |
56 | |
57 | /** The Output Receive Header. */ |
58 | union octeon_rh rh; |
59 | }; |
60 | |
61 | #define OCT_DROQ_INFO_SIZE (sizeof(struct octeon_droq_info)) |
62 | |
63 | struct octeon_skb_page_info { |
64 | /* DMA address for the page */ |
65 | dma_addr_t dma; |
66 | |
67 | /* Page for the rx dma **/ |
68 | struct page *page; |
69 | |
70 | /** which offset into page */ |
71 | unsigned int page_offset; |
72 | }; |
73 | |
74 | /** Pointer to data buffer. |
75 | * Driver keeps a pointer to the data buffer that it made available to |
76 | * the Octeon device. Since the descriptor ring keeps physical (bus) |
77 | * addresses, this field is required for the driver to keep track of |
78 | * the virtual address pointers. |
79 | */ |
80 | struct octeon_recv_buffer { |
81 | /** Packet buffer, including metadata. */ |
82 | void *buffer; |
83 | |
84 | /** Data in the packet buffer. */ |
85 | u8 *data; |
86 | |
87 | /** pg_info **/ |
88 | struct octeon_skb_page_info pg_info; |
89 | }; |
90 | |
91 | #define OCT_DROQ_RECVBUF_SIZE (sizeof(struct octeon_recv_buffer)) |
92 | |
93 | /** Output Queue statistics. Each output queue has four stats fields. */ |
94 | struct oct_droq_stats { |
95 | /** Number of packets received in this queue. */ |
96 | u64 pkts_received; |
97 | |
98 | /** Bytes received by this queue. */ |
99 | u64 bytes_received; |
100 | |
101 | /** Packets dropped due to no dispatch function. */ |
102 | u64 dropped_nodispatch; |
103 | |
104 | /** Packets dropped due to no memory available. */ |
105 | u64 dropped_nomem; |
106 | |
107 | /** Packets dropped due to large number of pkts to process. */ |
108 | u64 dropped_toomany; |
109 | |
110 | /** Number of packets sent to stack from this queue. */ |
111 | u64 rx_pkts_received; |
112 | |
113 | /** Number of Bytes sent to stack from this queue. */ |
114 | u64 rx_bytes_received; |
115 | |
116 | /** Num of Packets dropped due to receive path failures. */ |
117 | u64 rx_dropped; |
118 | |
119 | u64 rx_vxlan; |
120 | |
121 | /** Num of failures of recv_buffer_alloc() */ |
122 | u64 rx_alloc_failure; |
123 | |
124 | }; |
125 | |
126 | /* The maximum number of buffers that can be dispatched from the |
127 | * output/dma queue. Set to 64 assuming 1K buffers in DROQ and the fact that |
128 | * max packet size from DROQ is 64K. |
129 | */ |
130 | #define MAX_RECV_BUFS 64 |
131 | |
132 | /** Receive Packet format used when dispatching output queue packets |
133 | * with non-raw opcodes. |
134 | * The received packet will be sent to the upper layers using this |
135 | * structure which is passed as a parameter to the dispatch function |
136 | */ |
137 | struct octeon_recv_pkt { |
138 | /** Number of buffers in this received packet */ |
139 | u16 buffer_count; |
140 | |
141 | /** Id of the device that is sending the packet up */ |
142 | u16 octeon_id; |
143 | |
144 | /** Length of data in the packet buffer */ |
145 | u32 length; |
146 | |
147 | /** The receive header */ |
148 | union octeon_rh rh; |
149 | |
150 | /** Pointer to the OS-specific packet buffer */ |
151 | void *buffer_ptr[MAX_RECV_BUFS]; |
152 | |
153 | /** Size of the buffers pointed to by ptr's in buffer_ptr */ |
154 | u32 buffer_size[MAX_RECV_BUFS]; |
155 | }; |
156 | |
157 | #define OCT_RECV_PKT_SIZE (sizeof(struct octeon_recv_pkt)) |
158 | |
159 | /** The first parameter of a dispatch function. |
160 | * For a raw mode opcode, the driver dispatches with the device |
161 | * pointer in this structure. |
162 | * For non-raw mode opcode, the driver dispatches the recv_pkt |
163 | * created to contain the buffers with data received from Octeon. |
164 | * --------------------- |
165 | * | *recv_pkt ----|--- |
166 | * |-------------------| | |
167 | * | 0 or more bytes | | |
168 | * | reserved by driver| | |
169 | * |-------------------|<-/ |
170 | * | octeon_recv_pkt | |
171 | * | | |
172 | * |___________________| |
173 | */ |
174 | struct octeon_recv_info { |
175 | void *rsvd; |
176 | struct octeon_recv_pkt *recv_pkt; |
177 | }; |
178 | |
179 | #define OCT_RECV_INFO_SIZE (sizeof(struct octeon_recv_info)) |
180 | |
181 | /** Allocate a recv_info structure. The recv_pkt pointer in the recv_info |
182 | * structure is filled in before this call returns. |
183 | * @param extra_bytes - extra bytes to be allocated at the end of the recv info |
184 | * structure. |
185 | * @return - pointer to a newly allocated recv_info structure. |
186 | */ |
187 | static inline struct octeon_recv_info *octeon_alloc_recv_info(int ) |
188 | { |
189 | struct octeon_recv_info *recv_info; |
190 | u8 *buf; |
191 | |
192 | buf = kmalloc(OCT_RECV_PKT_SIZE + OCT_RECV_INFO_SIZE + |
193 | extra_bytes, GFP_ATOMIC); |
194 | if (!buf) |
195 | return NULL; |
196 | |
197 | recv_info = (struct octeon_recv_info *)buf; |
198 | recv_info->recv_pkt = |
199 | (struct octeon_recv_pkt *)(buf + OCT_RECV_INFO_SIZE); |
200 | recv_info->rsvd = NULL; |
201 | if (extra_bytes) |
202 | recv_info->rsvd = buf + OCT_RECV_INFO_SIZE + OCT_RECV_PKT_SIZE; |
203 | |
204 | return recv_info; |
205 | } |
206 | |
207 | /** Free a recv_info structure. |
208 | * @param recv_info - Pointer to receive_info to be freed |
209 | */ |
210 | static inline void octeon_free_recv_info(struct octeon_recv_info *recv_info) |
211 | { |
212 | kfree(objp: recv_info); |
213 | } |
214 | |
215 | typedef int (*octeon_dispatch_fn_t)(struct octeon_recv_info *, void *); |
216 | |
217 | /** Used by NIC module to register packet handler and to get device |
218 | * information for each octeon device. |
219 | */ |
220 | struct octeon_droq_ops { |
221 | /** This registered function will be called by the driver with |
222 | * the octeon id, pointer to buffer from droq and length of |
223 | * data in the buffer. The receive header gives the port |
224 | * number to the caller. Function pointer is set by caller. |
225 | */ |
226 | void (*fptr)(u32, void *, u32, union octeon_rh *, void *, void *); |
227 | void *farg; |
228 | |
229 | /* This function will be called by the driver for all NAPI related |
230 | * events. The first param is the octeon id. The second param is the |
231 | * output queue number. The third is the NAPI event that occurred. |
232 | */ |
233 | void (*napi_fn)(void *); |
234 | |
235 | u32 poll_mode; |
236 | |
237 | /** Flag indicating if the DROQ handler should drop packets that |
238 | * it cannot handle in one iteration. Set by caller. |
239 | */ |
240 | u32 drop_on_max; |
241 | }; |
242 | |
243 | /** The Descriptor Ring Output Queue structure. |
244 | * This structure has all the information required to implement a |
245 | * Octeon DROQ. |
246 | */ |
247 | struct octeon_droq { |
248 | u32 q_no; |
249 | |
250 | u32 pkt_count; |
251 | |
252 | struct octeon_droq_ops ops; |
253 | |
254 | struct octeon_device *oct_dev; |
255 | |
256 | /** The 8B aligned descriptor ring starts at this address. */ |
257 | struct octeon_droq_desc *desc_ring; |
258 | |
259 | /** Index in the ring where the driver should read the next packet */ |
260 | u32 read_idx; |
261 | |
262 | /** Index in the ring where Octeon will write the next packet */ |
263 | u32 write_idx; |
264 | |
265 | /** Index in the ring where the driver will refill the descriptor's |
266 | * buffer |
267 | */ |
268 | u32 refill_idx; |
269 | |
270 | /** Packets pending to be processed */ |
271 | atomic_t pkts_pending; |
272 | |
273 | /** Number of descriptors in this ring. */ |
274 | u32 max_count; |
275 | |
276 | /** The number of descriptors pending refill. */ |
277 | u32 refill_count; |
278 | |
279 | u32 pkts_per_intr; |
280 | u32 refill_threshold; |
281 | |
282 | /** The max number of descriptors in DROQ without a buffer. |
283 | * This field is used to keep track of empty space threshold. If the |
284 | * refill_count reaches this value, the DROQ cannot accept a max-sized |
285 | * (64K) packet. |
286 | */ |
287 | u32 max_empty_descs; |
288 | |
289 | /** The receive buffer list. This list has the virtual addresses of the |
290 | * buffers. |
291 | */ |
292 | struct octeon_recv_buffer *recv_buf_list; |
293 | |
294 | /** The size of each buffer pointed by the buffer pointer. */ |
295 | u32 buffer_size; |
296 | |
297 | /** Pointer to the mapped packet credit register. |
298 | * Host writes number of info/buffer ptrs available to this register |
299 | */ |
300 | void __iomem *pkts_credit_reg; |
301 | |
302 | /** Pointer to the mapped packet sent register. |
303 | * Octeon writes the number of packets DMA'ed to host memory |
304 | * in this register. |
305 | */ |
306 | void __iomem *pkts_sent_reg; |
307 | |
308 | struct list_head dispatch_list; |
309 | |
310 | /** Statistics for this DROQ. */ |
311 | struct oct_droq_stats stats; |
312 | |
313 | /** DMA mapped address of the DROQ descriptor ring. */ |
314 | size_t desc_ring_dma; |
315 | |
316 | /** application context */ |
317 | void *app_ctx; |
318 | |
319 | struct napi_struct napi; |
320 | |
321 | u32 cpu_id; |
322 | |
323 | call_single_data_t csd; |
324 | }; |
325 | |
326 | #define OCT_DROQ_SIZE (sizeof(struct octeon_droq)) |
327 | |
328 | /** |
329 | * Allocates space for the descriptor ring for the droq and sets the |
330 | * base addr, num desc etc in Octeon registers. |
331 | * |
332 | * @param oct_dev - pointer to the octeon device structure |
333 | * @param q_no - droq no. ranges from 0 - 3. |
334 | * @param app_ctx - pointer to application context |
335 | * @return Success: 0 Failure: 1 |
336 | */ |
337 | int octeon_init_droq(struct octeon_device *oct_dev, |
338 | u32 q_no, |
339 | u32 num_descs, |
340 | u32 desc_size, |
341 | void *app_ctx); |
342 | |
343 | /** |
344 | * Frees the space for descriptor ring for the droq. |
345 | * |
346 | * @param oct_dev - pointer to the octeon device structure |
347 | * @param q_no - droq no. ranges from 0 - 3. |
348 | * @return: Success: 0 Failure: 1 |
349 | */ |
350 | int octeon_delete_droq(struct octeon_device *oct_dev, u32 q_no); |
351 | |
352 | /** Register a change in droq operations. The ops field has a pointer to a |
353 | * function which will called by the DROQ handler for all packets arriving |
354 | * on output queues given by q_no irrespective of the type of packet. |
355 | * The ops field also has a flag which if set tells the DROQ handler to |
356 | * drop packets if it receives more than what it can process in one |
357 | * invocation of the handler. |
358 | * @param oct - octeon device |
359 | * @param q_no - octeon output queue number (0 <= q_no <= MAX_OCTEON_DROQ-1 |
360 | * @param ops - the droq_ops settings for this queue |
361 | * @return - 0 on success, -ENODEV or -EINVAL on error. |
362 | */ |
363 | int |
364 | octeon_register_droq_ops(struct octeon_device *oct, |
365 | u32 q_no, |
366 | struct octeon_droq_ops *ops); |
367 | |
368 | /** Resets the function pointer and flag settings made by |
369 | * octeon_register_droq_ops(). After this routine is called, the DROQ handler |
370 | * will lookup dispatch function for each arriving packet on the output queue |
371 | * given by q_no. |
372 | * @param oct - octeon device |
373 | * @param q_no - octeon output queue number (0 <= q_no <= MAX_OCTEON_DROQ-1 |
374 | * @return - 0 on success, -ENODEV or -EINVAL on error. |
375 | */ |
376 | int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no); |
377 | |
378 | /** Register a dispatch function for a opcode/subcode. The driver will call |
379 | * this dispatch function when it receives a packet with the given |
380 | * opcode/subcode in its output queues along with the user specified |
381 | * argument. |
382 | * @param oct - the octeon device to register with. |
383 | * @param opcode - the opcode for which the dispatch will be registered. |
384 | * @param subcode - the subcode for which the dispatch will be registered |
385 | * @param fn - the dispatch function. |
386 | * @param fn_arg - user specified that will be passed along with the |
387 | * dispatch function by the driver. |
388 | * @return Success: 0; Failure: 1 |
389 | */ |
390 | int octeon_register_dispatch_fn(struct octeon_device *oct, |
391 | u16 opcode, |
392 | u16 subcode, |
393 | octeon_dispatch_fn_t fn, void *fn_arg); |
394 | |
395 | void *octeon_get_dispatch_arg(struct octeon_device *oct, |
396 | u16 opcode, u16 subcode); |
397 | |
398 | void octeon_droq_print_stats(void); |
399 | |
400 | u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq); |
401 | |
402 | int octeon_create_droq(struct octeon_device *oct, u32 q_no, |
403 | u32 num_descs, u32 desc_size, void *app_ctx); |
404 | |
405 | int octeon_droq_process_packets(struct octeon_device *oct, |
406 | struct octeon_droq *droq, |
407 | u32 budget); |
408 | |
409 | int octeon_droq_process_poll_pkts(struct octeon_device *oct, |
410 | struct octeon_droq *droq, u32 budget); |
411 | |
412 | int octeon_enable_irq(struct octeon_device *oct, u32 q_no); |
413 | |
414 | int octeon_retry_droq_refill(struct octeon_droq *droq); |
415 | |
416 | #endif /*__OCTEON_DROQ_H__ */ |
417 | |