1 | /* SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) */ |
2 | |
3 | #ifndef _FUN_QEUEUE_H |
4 | #define _FUN_QEUEUE_H |
5 | |
6 | #include <linux/interrupt.h> |
7 | #include <linux/io.h> |
8 | |
9 | struct device; |
10 | struct fun_dev; |
11 | struct fun_queue; |
12 | struct fun_cqe_info; |
13 | struct fun_rsp_common; |
14 | |
15 | typedef void (*cq_callback_t)(struct fun_queue *funq, void *data, void *msg, |
16 | const struct fun_cqe_info *info); |
17 | |
18 | struct fun_rq_info { |
19 | dma_addr_t dma; |
20 | struct page *page; |
21 | }; |
22 | |
23 | /* A queue group consisting of an SQ, a CQ, and an optional RQ. */ |
24 | struct fun_queue { |
25 | struct fun_dev *fdev; |
26 | spinlock_t sq_lock; |
27 | |
28 | dma_addr_t cq_dma_addr; |
29 | dma_addr_t sq_dma_addr; |
30 | dma_addr_t rq_dma_addr; |
31 | |
32 | u32 __iomem *cq_db; |
33 | u32 __iomem *sq_db; |
34 | u32 __iomem *rq_db; |
35 | |
36 | void *cqes; |
37 | void *sq_cmds; |
38 | struct fun_eprq_rqbuf *rqes; |
39 | struct fun_rq_info *rq_info; |
40 | |
41 | u32 cqid; |
42 | u32 sqid; |
43 | u32 rqid; |
44 | |
45 | u32 cq_depth; |
46 | u32 sq_depth; |
47 | u32 rq_depth; |
48 | |
49 | u16 cq_head; |
50 | u16 sq_tail; |
51 | u16 rq_tail; |
52 | |
53 | u8 cqe_size_log2; |
54 | u8 sqe_size_log2; |
55 | |
56 | u16 cqe_info_offset; |
57 | |
58 | u16 rq_buf_idx; |
59 | int rq_buf_offset; |
60 | u16 num_rqe_to_fill; |
61 | |
62 | u8 cq_intcoal_usec; |
63 | u8 cq_intcoal_nentries; |
64 | u8 sq_intcoal_usec; |
65 | u8 sq_intcoal_nentries; |
66 | |
67 | u16 cq_flags; |
68 | u16 sq_flags; |
69 | u16 rq_flags; |
70 | |
71 | /* SQ head writeback */ |
72 | u16 sq_comp; |
73 | |
74 | volatile __be64 *sq_head; |
75 | |
76 | cq_callback_t cq_cb; |
77 | void *cb_data; |
78 | |
79 | irq_handler_t irq_handler; |
80 | void *irq_data; |
81 | s16 cq_vector; |
82 | u8 cq_phase; |
83 | |
84 | /* I/O q index */ |
85 | u16 qid; |
86 | |
87 | char irqname[24]; |
88 | }; |
89 | |
90 | static inline void *fun_sqe_at(const struct fun_queue *funq, unsigned int pos) |
91 | { |
92 | return funq->sq_cmds + (pos << funq->sqe_size_log2); |
93 | } |
94 | |
95 | static inline void funq_sq_post_tail(struct fun_queue *funq, u16 tail) |
96 | { |
97 | if (++tail == funq->sq_depth) |
98 | tail = 0; |
99 | funq->sq_tail = tail; |
100 | writel(val: tail, addr: funq->sq_db); |
101 | } |
102 | |
103 | static inline struct fun_cqe_info *funq_cqe_info(const struct fun_queue *funq, |
104 | void *cqe) |
105 | { |
106 | return cqe + funq->cqe_info_offset; |
107 | } |
108 | |
109 | static inline void funq_rq_post(struct fun_queue *funq) |
110 | { |
111 | writel(val: funq->rq_tail, addr: funq->rq_db); |
112 | } |
113 | |
114 | struct fun_queue_alloc_req { |
115 | u8 cqe_size_log2; |
116 | u8 sqe_size_log2; |
117 | |
118 | u16 cq_flags; |
119 | u16 sq_flags; |
120 | u16 rq_flags; |
121 | |
122 | u32 cq_depth; |
123 | u32 sq_depth; |
124 | u32 rq_depth; |
125 | |
126 | u8 cq_intcoal_usec; |
127 | u8 cq_intcoal_nentries; |
128 | u8 sq_intcoal_usec; |
129 | u8 sq_intcoal_nentries; |
130 | }; |
131 | |
132 | int fun_sq_create(struct fun_dev *fdev, u16 flags, u32 sqid, u32 cqid, |
133 | u8 sqe_size_log2, u32 sq_depth, dma_addr_t dma_addr, |
134 | u8 coal_nentries, u8 coal_usec, u32 irq_num, |
135 | u32 scan_start_id, u32 scan_end_id, |
136 | u32 rq_buf_size_log2, u32 *sqidp, u32 __iomem **dbp); |
137 | int fun_cq_create(struct fun_dev *fdev, u16 flags, u32 cqid, u32 rqid, |
138 | u8 cqe_size_log2, u32 cq_depth, dma_addr_t dma_addr, |
139 | u16 headroom, u16 tailroom, u8 coal_nentries, u8 coal_usec, |
140 | u32 irq_num, u32 scan_start_id, u32 scan_end_id, |
141 | u32 *cqidp, u32 __iomem **dbp); |
142 | void *fun_alloc_ring_mem(struct device *dma_dev, size_t depth, |
143 | size_t hw_desc_sz, size_t sw_desc_size, bool wb, |
144 | int numa_node, dma_addr_t *dma_addr, void **sw_va, |
145 | volatile __be64 **wb_va); |
146 | void fun_free_ring_mem(struct device *dma_dev, size_t depth, size_t hw_desc_sz, |
147 | bool wb, void *hw_va, dma_addr_t dma_addr, void *sw_va); |
148 | |
149 | #define fun_destroy_sq(fdev, sqid) \ |
150 | fun_res_destroy((fdev), FUN_ADMIN_OP_EPSQ, 0, (sqid)) |
151 | #define fun_destroy_cq(fdev, cqid) \ |
152 | fun_res_destroy((fdev), FUN_ADMIN_OP_EPCQ, 0, (cqid)) |
153 | |
154 | struct fun_queue *fun_alloc_queue(struct fun_dev *fdev, int qid, |
155 | const struct fun_queue_alloc_req *req); |
156 | void fun_free_queue(struct fun_queue *funq); |
157 | |
158 | static inline void fun_set_cq_callback(struct fun_queue *funq, cq_callback_t cb, |
159 | void *cb_data) |
160 | { |
161 | funq->cq_cb = cb; |
162 | funq->cb_data = cb_data; |
163 | } |
164 | |
165 | int fun_create_rq(struct fun_queue *funq); |
166 | int fun_create_queue(struct fun_queue *funq); |
167 | |
168 | void fun_free_irq(struct fun_queue *funq); |
169 | int fun_request_irq(struct fun_queue *funq, const char *devname, |
170 | irq_handler_t handler, void *data); |
171 | |
172 | unsigned int __fun_process_cq(struct fun_queue *funq, unsigned int max); |
173 | unsigned int fun_process_cq(struct fun_queue *funq, unsigned int max); |
174 | |
175 | #endif /* _FUN_QEUEUE_H */ |
176 | |