1 | /* |
2 | * libcxgb_ppm.h: Chelsio common library for T3/T4/T5 iSCSI ddp operation |
3 | * |
4 | * Copyright (c) 2016 Chelsio Communications, Inc. All rights reserved. |
5 | * |
6 | * This software is available to you under a choice of one of two |
7 | * licenses. You may choose to be licensed under the terms of the GNU |
8 | * General Public License (GPL) Version 2, available from the file |
9 | * COPYING in the main directory of this source tree, or the |
10 | * OpenIB.org BSD license below: |
11 | * |
12 | * Redistribution and use in source and binary forms, with or |
13 | * without modification, are permitted provided that the following |
14 | * conditions are met: |
15 | * |
16 | * - Redistributions of source code must retain the above |
17 | * copyright notice, this list of conditions and the following |
18 | * disclaimer. |
19 | * |
20 | * - Redistributions in binary form must reproduce the above |
21 | * copyright notice, this list of conditions and the following |
22 | * disclaimer in the documentation and/or other materials |
23 | * provided with the distribution. |
24 | * |
25 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
26 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
27 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
28 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
29 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
30 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
31 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
32 | * SOFTWARE. |
33 | * |
34 | * Written by: Karen Xie (kxie@chelsio.com) |
35 | */ |
36 | |
37 | #ifndef __LIBCXGB_PPM_H__ |
38 | #define __LIBCXGB_PPM_H__ |
39 | |
40 | #include <linux/kernel.h> |
41 | #include <linux/errno.h> |
42 | #include <linux/types.h> |
43 | #include <linux/debugfs.h> |
44 | #include <linux/list.h> |
45 | #include <linux/netdevice.h> |
46 | #include <linux/scatterlist.h> |
47 | #include <linux/skbuff.h> |
48 | #include <linux/vmalloc.h> |
49 | #include <linux/bitmap.h> |
50 | |
51 | struct cxgbi_pagepod_hdr { |
52 | u32 vld_tid; |
53 | u32 pgsz_tag_clr; |
54 | u32 max_offset; |
55 | u32 page_offset; |
56 | u64 rsvd; |
57 | }; |
58 | |
59 | #define PPOD_PAGES_MAX 4 |
60 | struct cxgbi_pagepod { |
61 | struct cxgbi_pagepod_hdr hdr; |
62 | __be64 addr[PPOD_PAGES_MAX + 1]; |
63 | }; |
64 | |
65 | /* ddp tag format |
66 | * for a 32-bit tag: |
67 | * bit # |
68 | * 31 ..... ..... 0 |
69 | * X Y...Y Z...Z, where |
70 | * ^ ^^^^^ ^^^^ |
71 | * | | |____ when ddp bit = 0: color bits |
72 | * | | |
73 | * | |____ when ddp bit = 0: idx into the ddp memory region |
74 | * | |
75 | * |____ ddp bit: 0 - ddp tag, 1 - non-ddp tag |
76 | * |
77 | * [page selector:2] [sw/free bits] [0] [idx] [color:6] |
78 | */ |
79 | |
80 | #define DDP_PGIDX_MAX 4 |
81 | #define DDP_PGSZ_BASE_SHIFT 12 /* base page 4K */ |
82 | |
83 | struct cxgbi_task_tag_info { |
84 | unsigned char flags; |
85 | #define CXGBI_PPOD_INFO_FLAG_VALID 0x1 |
86 | #define CXGBI_PPOD_INFO_FLAG_MAPPED 0x2 |
87 | unsigned char cid; |
88 | unsigned short pg_shift; |
89 | unsigned int npods; |
90 | unsigned int idx; |
91 | unsigned int tag; |
92 | struct cxgbi_pagepod_hdr hdr; |
93 | int nents; |
94 | int nr_pages; |
95 | struct scatterlist *sgl; |
96 | }; |
97 | |
98 | struct cxgbi_tag_format { |
99 | unsigned char pgsz_order[DDP_PGIDX_MAX]; |
100 | unsigned char pgsz_idx_dflt; |
101 | unsigned char free_bits:4; |
102 | unsigned char color_bits:4; |
103 | unsigned char idx_bits; |
104 | unsigned char rsvd_bits; |
105 | unsigned int no_ddp_mask; |
106 | unsigned int idx_mask; |
107 | unsigned int color_mask; |
108 | unsigned int idx_clr_mask; |
109 | unsigned int rsvd_mask; |
110 | }; |
111 | |
112 | struct cxgbi_ppod_data { |
113 | unsigned char pg_idx:2; |
114 | unsigned char color:6; |
115 | unsigned char chan_id; |
116 | unsigned short npods; |
117 | unsigned long caller_data; |
118 | }; |
119 | |
120 | /* per cpu ppm pool */ |
121 | struct cxgbi_ppm_pool { |
122 | unsigned int base; /* base index */ |
123 | unsigned int next; /* next possible free index */ |
124 | spinlock_t lock; /* ppm pool lock */ |
125 | unsigned long bmap[]; |
126 | } ____cacheline_aligned_in_smp; |
127 | |
128 | struct cxgbi_ppm { |
129 | struct kref refcnt; |
130 | struct net_device *ndev; /* net_device, 1st port */ |
131 | struct pci_dev *pdev; |
132 | void *lldev; |
133 | void **ppm_pp; |
134 | struct cxgbi_tag_format tformat; |
135 | unsigned int ppmax; |
136 | unsigned int llimit; |
137 | unsigned int base_idx; |
138 | |
139 | unsigned int pool_rsvd; |
140 | unsigned int pool_index_max; |
141 | struct cxgbi_ppm_pool __percpu *pool; |
142 | /* map lock */ |
143 | spinlock_t map_lock; /* ppm map lock */ |
144 | unsigned int bmap_index_max; |
145 | unsigned int next; |
146 | unsigned int max_index_in_edram; |
147 | unsigned long *ppod_bmap; |
148 | struct cxgbi_ppod_data ppod_data[]; |
149 | }; |
150 | |
151 | #define DDP_THRESHOLD 512 |
152 | |
153 | #define PPOD_PAGES_SHIFT 2 /* 4 pages per pod */ |
154 | |
155 | #define IPPOD_SIZE sizeof(struct cxgbi_pagepod) /* 64 */ |
156 | #define PPOD_SIZE_SHIFT 6 |
157 | |
158 | /* page pods are allocated in groups of this size (must be power of 2) */ |
159 | #define PPOD_CLUSTER_SIZE 16U |
160 | |
161 | #define ULPMEM_DSGL_MAX_NPPODS 16 /* 1024/PPOD_SIZE */ |
162 | #define ULPMEM_IDATA_MAX_NPPODS 3 /* (PPOD_SIZE * 3 + ulptx hdr) < 256B */ |
163 | #define PCIE_MEMWIN_MAX_NPPODS 16 /* 1024/PPOD_SIZE */ |
164 | |
165 | #define PPOD_COLOR_SHIFT 0 |
166 | #define PPOD_COLOR(x) ((x) << PPOD_COLOR_SHIFT) |
167 | |
168 | #define PPOD_IDX_SHIFT 6 |
169 | #define PPOD_IDX_MAX_SIZE 24 |
170 | |
171 | #define PPOD_TID_SHIFT 0 |
172 | #define PPOD_TID(x) ((x) << PPOD_TID_SHIFT) |
173 | |
174 | #define PPOD_TAG_SHIFT 6 |
175 | #define PPOD_TAG(x) ((x) << PPOD_TAG_SHIFT) |
176 | |
177 | #define PPOD_VALID_SHIFT 24 |
178 | #define PPOD_VALID(x) ((x) << PPOD_VALID_SHIFT) |
179 | #define PPOD_VALID_FLAG PPOD_VALID(1U) |
180 | |
181 | #define 31 |
182 | #define (x) ((x) << PPOD_PI_EXTRACT_CTL_SHIFT) |
183 | #define V_PPOD_PI_EXTRACT_CTL(1U) |
184 | |
185 | #define PPOD_PI_TYPE_SHIFT 29 |
186 | #define PPOD_PI_TYPE_MASK 0x3 |
187 | #define PPOD_PI_TYPE(x) ((x) << PPOD_PI_TYPE_SHIFT) |
188 | |
189 | #define PPOD_PI_CHECK_CTL_SHIFT 27 |
190 | #define PPOD_PI_CHECK_CTL_MASK 0x3 |
191 | #define PPOD_PI_CHECK_CTL(x) ((x) << PPOD_PI_CHECK_CTL_SHIFT) |
192 | |
193 | #define PPOD_PI_REPORT_CTL_SHIFT 25 |
194 | #define PPOD_PI_REPORT_CTL_MASK 0x3 |
195 | #define PPOD_PI_REPORT_CTL(x) ((x) << PPOD_PI_REPORT_CTL_SHIFT) |
196 | |
197 | static inline int cxgbi_ppm_is_ddp_tag(struct cxgbi_ppm *ppm, u32 tag) |
198 | { |
199 | return !(tag & ppm->tformat.no_ddp_mask); |
200 | } |
201 | |
202 | static inline int cxgbi_ppm_sw_tag_is_usable(struct cxgbi_ppm *ppm, |
203 | u32 tag) |
204 | { |
205 | /* the sw tag must be using <= 31 bits */ |
206 | return !(tag & 0x80000000U); |
207 | } |
208 | |
209 | static inline int cxgbi_ppm_make_non_ddp_tag(struct cxgbi_ppm *ppm, |
210 | u32 sw_tag, |
211 | u32 *final_tag) |
212 | { |
213 | struct cxgbi_tag_format *tformat = &ppm->tformat; |
214 | |
215 | if (!cxgbi_ppm_sw_tag_is_usable(ppm, tag: sw_tag)) { |
216 | pr_info("sw_tag 0x%x NOT usable.\n" , sw_tag); |
217 | return -EINVAL; |
218 | } |
219 | |
220 | if (!sw_tag) { |
221 | *final_tag = tformat->no_ddp_mask; |
222 | } else { |
223 | unsigned int shift = tformat->idx_bits + tformat->color_bits; |
224 | u32 lower = sw_tag & tformat->idx_clr_mask; |
225 | u32 upper = (sw_tag >> shift) << (shift + 1); |
226 | |
227 | *final_tag = upper | tformat->no_ddp_mask | lower; |
228 | } |
229 | return 0; |
230 | } |
231 | |
232 | static inline u32 cxgbi_ppm_decode_non_ddp_tag(struct cxgbi_ppm *ppm, |
233 | u32 tag) |
234 | { |
235 | struct cxgbi_tag_format *tformat = &ppm->tformat; |
236 | unsigned int shift = tformat->idx_bits + tformat->color_bits; |
237 | u32 lower = tag & tformat->idx_clr_mask; |
238 | u32 upper = (tag >> tformat->rsvd_bits) << shift; |
239 | |
240 | return upper | lower; |
241 | } |
242 | |
243 | static inline u32 cxgbi_ppm_ddp_tag_get_idx(struct cxgbi_ppm *ppm, |
244 | u32 ddp_tag) |
245 | { |
246 | u32 hw_idx = (ddp_tag >> PPOD_IDX_SHIFT) & |
247 | ppm->tformat.idx_mask; |
248 | |
249 | return hw_idx - ppm->base_idx; |
250 | } |
251 | |
252 | static inline u32 cxgbi_ppm_make_ddp_tag(unsigned int hw_idx, |
253 | unsigned char color) |
254 | { |
255 | return (hw_idx << PPOD_IDX_SHIFT) | ((u32)color); |
256 | } |
257 | |
258 | static inline unsigned long |
259 | cxgbi_ppm_get_tag_caller_data(struct cxgbi_ppm *ppm, |
260 | u32 ddp_tag) |
261 | { |
262 | u32 idx = cxgbi_ppm_ddp_tag_get_idx(ppm, ddp_tag); |
263 | |
264 | return ppm->ppod_data[idx].caller_data; |
265 | } |
266 | |
267 | /* sw bits are the free bits */ |
268 | static inline int cxgbi_ppm_ddp_tag_update_sw_bits(struct cxgbi_ppm *ppm, |
269 | u32 val, u32 orig_tag, |
270 | u32 *final_tag) |
271 | { |
272 | struct cxgbi_tag_format *tformat = &ppm->tformat; |
273 | u32 v = val >> tformat->free_bits; |
274 | |
275 | if (v) { |
276 | pr_info("sw_bits 0x%x too large, avail bits %u.\n" , |
277 | val, tformat->free_bits); |
278 | return -EINVAL; |
279 | } |
280 | if (!cxgbi_ppm_is_ddp_tag(ppm, tag: orig_tag)) |
281 | return -EINVAL; |
282 | |
283 | *final_tag = (val << tformat->rsvd_bits) | |
284 | (orig_tag & ppm->tformat.rsvd_mask); |
285 | return 0; |
286 | } |
287 | |
288 | static inline void cxgbi_ppm_ppod_clear(struct cxgbi_pagepod *ppod) |
289 | { |
290 | ppod->hdr.vld_tid = 0U; |
291 | } |
292 | |
293 | static inline void cxgbi_tagmask_check(unsigned int tagmask, |
294 | struct cxgbi_tag_format *tformat) |
295 | { |
296 | unsigned int bits = fls(x: tagmask); |
297 | |
298 | /* reserve top most 2 bits for page selector */ |
299 | tformat->free_bits = 32 - 2 - bits; |
300 | tformat->rsvd_bits = bits; |
301 | tformat->color_bits = PPOD_IDX_SHIFT; |
302 | tformat->idx_bits = bits - 1 - PPOD_IDX_SHIFT; |
303 | tformat->no_ddp_mask = 1 << (bits - 1); |
304 | tformat->idx_mask = (1 << tformat->idx_bits) - 1; |
305 | tformat->color_mask = (1 << PPOD_IDX_SHIFT) - 1; |
306 | tformat->idx_clr_mask = (1 << (bits - 1)) - 1; |
307 | tformat->rsvd_mask = (1 << bits) - 1; |
308 | |
309 | pr_info("ippm: tagmask 0x%x, rsvd %u=%u+%u+1, mask 0x%x,0x%x, " |
310 | "pg %u,%u,%u,%u.\n" , |
311 | tagmask, tformat->rsvd_bits, tformat->idx_bits, |
312 | tformat->color_bits, tformat->no_ddp_mask, tformat->rsvd_mask, |
313 | tformat->pgsz_order[0], tformat->pgsz_order[1], |
314 | tformat->pgsz_order[2], tformat->pgsz_order[3]); |
315 | } |
316 | |
317 | int cxgbi_ppm_find_page_index(struct cxgbi_ppm *ppm, unsigned long pgsz); |
318 | void cxgbi_ppm_make_ppod_hdr(struct cxgbi_ppm *ppm, u32 tag, |
319 | unsigned int tid, unsigned int offset, |
320 | unsigned int length, |
321 | struct cxgbi_pagepod_hdr *hdr); |
322 | void cxgbi_ppm_ppod_release(struct cxgbi_ppm *, u32 idx); |
323 | int cxgbi_ppm_ppods_reserve(struct cxgbi_ppm *, unsigned short nr_pages, |
324 | u32 per_tag_pg_idx, u32 *ppod_idx, u32 *ddp_tag, |
325 | unsigned long caller_data); |
326 | int cxgbi_ppm_init(void **ppm_pp, struct net_device *, struct pci_dev *, |
327 | void *lldev, struct cxgbi_tag_format *, |
328 | unsigned int iscsi_size, unsigned int llimit, |
329 | unsigned int start, unsigned int reserve_factor, |
330 | unsigned int edram_start, unsigned int edram_size); |
331 | int cxgbi_ppm_release(struct cxgbi_ppm *ppm); |
332 | void cxgbi_tagmask_check(unsigned int tagmask, struct cxgbi_tag_format *); |
333 | unsigned int cxgbi_tagmask_set(unsigned int ppmax); |
334 | |
335 | #endif /*__LIBCXGB_PPM_H__*/ |
336 | |