1 | // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) |
2 | /* Copyright (C) 2015-2018 Netronome Systems, Inc. */ |
3 | |
4 | /* |
5 | * nfp6000_pcie.c |
6 | * Authors: Jakub Kicinski <jakub.kicinski@netronome.com> |
7 | * Jason McMullan <jason.mcmullan@netronome.com> |
8 | * Rolf Neugebauer <rolf.neugebauer@netronome.com> |
9 | * |
10 | * Multiplexes the NFP BARs between NFP internal resources and |
11 | * implements the PCIe specific interface for generic CPP bus access. |
12 | * |
13 | * The BARs are managed with refcounts and are allocated/acquired |
14 | * using target, token and offset/size matching. The generic CPP bus |
15 | * abstraction builds upon this BAR interface. |
16 | */ |
17 | |
18 | #include <asm/unaligned.h> |
19 | #include <linux/kernel.h> |
20 | #include <linux/module.h> |
21 | #include <linux/kref.h> |
22 | #include <linux/io.h> |
23 | #include <linux/delay.h> |
24 | #include <linux/interrupt.h> |
25 | #include <linux/sort.h> |
26 | #include <linux/sched.h> |
27 | #include <linux/types.h> |
28 | #include <linux/pci.h> |
29 | |
30 | #include "nfp_cpp.h" |
31 | #include "nfp_dev.h" |
32 | |
33 | #include "nfp6000/nfp6000.h" |
34 | |
35 | #include "nfp6000_pcie.h" |
36 | |
37 | #define NFP_PCIE_BAR(_pf) (0x30000 + ((_pf) & 7) * 0xc0) |
38 | #define NFP_PCIE_BAR_EXPLICIT_BAR0(_x, _y) \ |
39 | (0x00000080 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3))) |
40 | #define NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(_x) (((_x) & 0x3) << 30) |
41 | #define NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType_of(_x) (((_x) >> 30) & 0x3) |
42 | #define NFP_PCIE_BAR_EXPLICIT_BAR0_Token(_x) (((_x) & 0x3) << 28) |
43 | #define NFP_PCIE_BAR_EXPLICIT_BAR0_Token_of(_x) (((_x) >> 28) & 0x3) |
44 | #define NFP_PCIE_BAR_EXPLICIT_BAR0_Address(_x) (((_x) & 0xffffff) << 0) |
45 | #define NFP_PCIE_BAR_EXPLICIT_BAR0_Address_of(_x) (((_x) >> 0) & 0xffffff) |
46 | #define NFP_PCIE_BAR_EXPLICIT_BAR1(_x, _y) \ |
47 | (0x00000084 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3))) |
48 | #define NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(_x) (((_x) & 0x7f) << 24) |
49 | #define NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef_of(_x) (((_x) >> 24) & 0x7f) |
50 | #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(_x) (((_x) & 0x3ff) << 14) |
51 | #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster_of(_x) (((_x) >> 14) & 0x3ff) |
52 | #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(_x) (((_x) & 0x3fff) << 0) |
53 | #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef_of(_x) (((_x) >> 0) & 0x3fff) |
54 | #define NFP_PCIE_BAR_EXPLICIT_BAR2(_x, _y) \ |
55 | (0x00000088 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3))) |
56 | #define NFP_PCIE_BAR_EXPLICIT_BAR2_Target(_x) (((_x) & 0xf) << 28) |
57 | #define NFP_PCIE_BAR_EXPLICIT_BAR2_Target_of(_x) (((_x) >> 28) & 0xf) |
58 | #define NFP_PCIE_BAR_EXPLICIT_BAR2_Action(_x) (((_x) & 0x1f) << 23) |
59 | #define NFP_PCIE_BAR_EXPLICIT_BAR2_Action_of(_x) (((_x) >> 23) & 0x1f) |
60 | #define NFP_PCIE_BAR_EXPLICIT_BAR2_Length(_x) (((_x) & 0x1f) << 18) |
61 | #define NFP_PCIE_BAR_EXPLICIT_BAR2_Length_of(_x) (((_x) >> 18) & 0x1f) |
62 | #define NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(_x) (((_x) & 0xff) << 10) |
63 | #define NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask_of(_x) (((_x) >> 10) & 0xff) |
64 | #define NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(_x) (((_x) & 0x3ff) << 0) |
65 | #define NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster_of(_x) (((_x) >> 0) & 0x3ff) |
66 | |
67 | #define NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(_x) (((_x) & 0x1f) << 16) |
68 | #define NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(_x) (((_x) >> 16) & 0x1f) |
69 | #define NFP_PCIE_BAR_PCIE2CPP_BaseAddress(_x) (((_x) & 0xffff) << 0) |
70 | #define NFP_PCIE_BAR_PCIE2CPP_BaseAddress_of(_x) (((_x) >> 0) & 0xffff) |
71 | #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect(_x) (((_x) & 0x3) << 27) |
72 | #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(_x) (((_x) >> 27) & 0x3) |
73 | #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT 0 |
74 | #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT 1 |
75 | #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE 3 |
76 | #define NFP_PCIE_BAR_PCIE2CPP_MapType(_x) (((_x) & 0x7) << 29) |
77 | #define NFP_PCIE_BAR_PCIE2CPP_MapType_of(_x) (((_x) >> 29) & 0x7) |
78 | #define NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED 0 |
79 | #define NFP_PCIE_BAR_PCIE2CPP_MapType_BULK 1 |
80 | #define NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET 2 |
81 | #define NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL 3 |
82 | #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0 4 |
83 | #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1 5 |
84 | #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2 6 |
85 | #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3 7 |
86 | #define NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(_x) (((_x) & 0xf) << 23) |
87 | #define NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(_x) (((_x) >> 23) & 0xf) |
88 | #define NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(_x) (((_x) & 0x3) << 21) |
89 | #define NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(_x) (((_x) >> 21) & 0x3) |
90 | #define NFP_PCIE_EM 0x020000 |
91 | #define NFP_PCIE_SRAM 0x000000 |
92 | |
93 | /* Minimal size of the PCIe cfg memory we depend on being mapped, |
94 | * queue controller and DMA controller don't have to be covered. |
95 | */ |
96 | #define NFP_PCI_MIN_MAP_SIZE 0x080000 |
97 | |
98 | #define NFP_PCIE_P2C_FIXED_SIZE(bar) (1 << (bar)->bitsize) |
99 | #define NFP_PCIE_P2C_BULK_SIZE(bar) (1 << (bar)->bitsize) |
100 | #define NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(bar, x) ((x) << ((bar)->bitsize - 2)) |
101 | #define NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(bar, x) ((x) << ((bar)->bitsize - 4)) |
102 | #define NFP_PCIE_P2C_GENERAL_SIZE(bar) (1 << ((bar)->bitsize - 4)) |
103 | |
104 | #define NFP_PCIE_P2C_EXPBAR_OFFSET(bar_index) ((bar_index) * 4) |
105 | |
106 | /* The number of explicit BARs to reserve. |
107 | * Minimum is 0, maximum is 4 on the NFP6000. |
108 | * The NFP3800 can have only one per PF. |
109 | */ |
110 | #define NFP_PCIE_EXPLICIT_BARS 2 |
111 | |
112 | struct nfp6000_pcie; |
113 | struct nfp6000_area_priv; |
114 | |
115 | /** |
116 | * struct nfp_bar - describes BAR configuration and usage |
117 | * @nfp: backlink to owner |
118 | * @barcfg: cached contents of BAR config CSR |
119 | * @base: the BAR's base CPP offset |
120 | * @mask: mask for the BAR aperture (read only) |
121 | * @bitsize: bitsize of BAR aperture (read only) |
122 | * @index: index of the BAR |
123 | * @refcnt: number of current users |
124 | * @iomem: mapped IO memory |
125 | * @resource: iomem resource window |
126 | */ |
127 | struct nfp_bar { |
128 | struct nfp6000_pcie *nfp; |
129 | u32 barcfg; |
130 | u64 base; /* CPP address base */ |
131 | u64 mask; /* Bit mask of the bar */ |
132 | u32 bitsize; /* Bit size of the bar */ |
133 | int index; |
134 | atomic_t refcnt; |
135 | |
136 | void __iomem *iomem; |
137 | struct resource *resource; |
138 | }; |
139 | |
140 | #define NFP_PCI_BAR_MAX (PCI_64BIT_BAR_COUNT * 8) |
141 | |
142 | struct nfp6000_pcie { |
143 | struct pci_dev *pdev; |
144 | struct device *dev; |
145 | const struct nfp_dev_info *dev_info; |
146 | |
147 | /* PCI BAR management */ |
148 | spinlock_t bar_lock; /* Protect the PCI2CPP BAR cache */ |
149 | int bars; |
150 | struct nfp_bar bar[NFP_PCI_BAR_MAX]; |
151 | wait_queue_head_t bar_waiters; |
152 | |
153 | /* Reserved BAR access */ |
154 | struct { |
155 | void __iomem *csr; |
156 | void __iomem *em; |
157 | void __iomem *expl[4]; |
158 | } iomem; |
159 | |
160 | /* Explicit IO access */ |
161 | struct { |
162 | struct mutex mutex; /* Lock access to this explicit group */ |
163 | u8 master_id; |
164 | u8 signal_ref; |
165 | void __iomem *data; |
166 | struct { |
167 | void __iomem *addr; |
168 | int bitsize; |
169 | int free[4]; |
170 | } group[4]; |
171 | } expl; |
172 | }; |
173 | |
174 | static u32 nfp_bar_maptype(struct nfp_bar *bar) |
175 | { |
176 | return NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg); |
177 | } |
178 | |
179 | static resource_size_t nfp_bar_resource_len(struct nfp_bar *bar) |
180 | { |
181 | return pci_resource_len(bar->nfp->pdev, (bar->index / 8) * 2) / 8; |
182 | } |
183 | |
184 | static resource_size_t nfp_bar_resource_start(struct nfp_bar *bar) |
185 | { |
186 | return pci_resource_start(bar->nfp->pdev, (bar->index / 8) * 2) |
187 | + nfp_bar_resource_len(bar) * (bar->index & 7); |
188 | } |
189 | |
190 | #define TARGET_WIDTH_32 4 |
191 | #define TARGET_WIDTH_64 8 |
192 | |
193 | static int |
194 | compute_bar(const struct nfp6000_pcie *nfp, const struct nfp_bar *bar, |
195 | u32 *bar_config, u64 *bar_base, |
196 | int tgt, int act, int tok, u64 offset, size_t size, int width) |
197 | { |
198 | int bitsize; |
199 | u32 newcfg; |
200 | |
201 | if (tgt >= NFP_CPP_NUM_TARGETS) |
202 | return -EINVAL; |
203 | |
204 | switch (width) { |
205 | case 8: |
206 | newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect( |
207 | NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT); |
208 | break; |
209 | case 4: |
210 | newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect( |
211 | NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT); |
212 | break; |
213 | case 0: |
214 | newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect( |
215 | NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE); |
216 | break; |
217 | default: |
218 | return -EINVAL; |
219 | } |
220 | |
221 | if (act != NFP_CPP_ACTION_RW && act != 0) { |
222 | /* Fixed CPP mapping with specific action */ |
223 | u64 mask = ~(NFP_PCIE_P2C_FIXED_SIZE(bar) - 1); |
224 | |
225 | newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType( |
226 | NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED); |
227 | newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt); |
228 | newcfg |= NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(act); |
229 | newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok); |
230 | |
231 | if ((offset & mask) != ((offset + size - 1) & mask)) |
232 | return -EINVAL; |
233 | offset &= mask; |
234 | |
235 | bitsize = 40 - 16; |
236 | } else { |
237 | u64 mask = ~(NFP_PCIE_P2C_BULK_SIZE(bar) - 1); |
238 | |
239 | /* Bulk mapping */ |
240 | newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType( |
241 | NFP_PCIE_BAR_PCIE2CPP_MapType_BULK); |
242 | newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt); |
243 | newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok); |
244 | |
245 | if ((offset & mask) != ((offset + size - 1) & mask)) |
246 | return -EINVAL; |
247 | |
248 | offset &= mask; |
249 | |
250 | bitsize = 40 - 21; |
251 | } |
252 | |
253 | if (bar->bitsize < bitsize) |
254 | return -EINVAL; |
255 | |
256 | newcfg |= offset >> bitsize; |
257 | |
258 | if (bar_base) |
259 | *bar_base = offset; |
260 | |
261 | if (bar_config) |
262 | *bar_config = newcfg; |
263 | |
264 | return 0; |
265 | } |
266 | |
267 | static int |
268 | nfp6000_bar_write(struct nfp6000_pcie *nfp, struct nfp_bar *bar, u32 newcfg) |
269 | { |
270 | unsigned int xbar; |
271 | |
272 | xbar = NFP_PCIE_P2C_EXPBAR_OFFSET(bar->index); |
273 | |
274 | if (nfp->iomem.csr) { |
275 | writel(val: newcfg, addr: nfp->iomem.csr + xbar); |
276 | /* Readback to ensure BAR is flushed */ |
277 | readl(addr: nfp->iomem.csr + xbar); |
278 | } else { |
279 | xbar += nfp->dev_info->pcie_cfg_expbar_offset; |
280 | pci_write_config_dword(dev: nfp->pdev, where: xbar, val: newcfg); |
281 | } |
282 | |
283 | bar->barcfg = newcfg; |
284 | |
285 | return 0; |
286 | } |
287 | |
288 | static int |
289 | reconfigure_bar(struct nfp6000_pcie *nfp, struct nfp_bar *bar, |
290 | int tgt, int act, int tok, u64 offset, size_t size, int width) |
291 | { |
292 | u64 newbase; |
293 | u32 newcfg; |
294 | int err; |
295 | |
296 | err = compute_bar(nfp, bar, bar_config: &newcfg, bar_base: &newbase, |
297 | tgt, act, tok, offset, size, width); |
298 | if (err) |
299 | return err; |
300 | |
301 | bar->base = newbase; |
302 | |
303 | return nfp6000_bar_write(nfp, bar, newcfg); |
304 | } |
305 | |
306 | /* Check if BAR can be used with the given parameters. */ |
307 | static int matching_bar(struct nfp_bar *bar, u32 tgt, u32 act, u32 tok, |
308 | u64 offset, size_t size, int width) |
309 | { |
310 | int bartgt, baract, bartok; |
311 | int barwidth; |
312 | u32 maptype; |
313 | |
314 | maptype = NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg); |
315 | bartgt = NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(bar->barcfg); |
316 | bartok = NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(bar->barcfg); |
317 | baract = NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(bar->barcfg); |
318 | |
319 | barwidth = NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(bar->barcfg); |
320 | switch (barwidth) { |
321 | case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT: |
322 | barwidth = 4; |
323 | break; |
324 | case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT: |
325 | barwidth = 8; |
326 | break; |
327 | case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE: |
328 | barwidth = 0; |
329 | break; |
330 | default: |
331 | barwidth = -1; |
332 | break; |
333 | } |
334 | |
335 | switch (maptype) { |
336 | case NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET: |
337 | bartok = -1; |
338 | fallthrough; |
339 | case NFP_PCIE_BAR_PCIE2CPP_MapType_BULK: |
340 | baract = NFP_CPP_ACTION_RW; |
341 | if (act == 0) |
342 | act = NFP_CPP_ACTION_RW; |
343 | fallthrough; |
344 | case NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED: |
345 | break; |
346 | default: |
347 | /* We don't match explicit bars through the area interface */ |
348 | return 0; |
349 | } |
350 | |
351 | /* Make sure to match up the width */ |
352 | if (barwidth != width) |
353 | return 0; |
354 | |
355 | if ((bartgt < 0 || bartgt == tgt) && |
356 | (bartok < 0 || bartok == tok) && |
357 | (baract == act) && |
358 | bar->base <= offset && |
359 | (bar->base + (1 << bar->bitsize)) >= (offset + size)) |
360 | return 1; |
361 | |
362 | /* No match */ |
363 | return 0; |
364 | } |
365 | |
366 | static int |
367 | find_matching_bar(struct nfp6000_pcie *nfp, |
368 | u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width) |
369 | { |
370 | int n; |
371 | |
372 | for (n = 0; n < nfp->bars; n++) { |
373 | struct nfp_bar *bar = &nfp->bar[n]; |
374 | |
375 | if (matching_bar(bar, tgt, act, tok, offset, size, width)) |
376 | return n; |
377 | } |
378 | |
379 | return -1; |
380 | } |
381 | |
382 | /* Return EAGAIN if no resource is available */ |
383 | static int |
384 | find_unused_bar_noblock(const struct nfp6000_pcie *nfp, |
385 | int tgt, int act, int tok, |
386 | u64 offset, size_t size, int width) |
387 | { |
388 | int n, busy = 0; |
389 | |
390 | for (n = 0; n < nfp->bars; n++) { |
391 | const struct nfp_bar *bar = &nfp->bar[n]; |
392 | int err; |
393 | |
394 | if (!bar->bitsize) |
395 | continue; |
396 | |
397 | /* Just check to see if we can make it fit... */ |
398 | err = compute_bar(nfp, bar, NULL, NULL, |
399 | tgt, act, tok, offset, size, width); |
400 | if (err) |
401 | continue; |
402 | |
403 | if (!atomic_read(v: &bar->refcnt)) |
404 | return n; |
405 | |
406 | busy++; |
407 | } |
408 | |
409 | if (WARN(!busy, "No suitable BAR found for request tgt:0x%x act:0x%x tok:0x%x off:0x%llx size:%zd width:%d\n" , |
410 | tgt, act, tok, offset, size, width)) |
411 | return -EINVAL; |
412 | |
413 | return -EAGAIN; |
414 | } |
415 | |
416 | static int |
417 | find_unused_bar_and_lock(struct nfp6000_pcie *nfp, |
418 | int tgt, int act, int tok, |
419 | u64 offset, size_t size, int width) |
420 | { |
421 | unsigned long flags; |
422 | int n; |
423 | |
424 | spin_lock_irqsave(&nfp->bar_lock, flags); |
425 | |
426 | n = find_unused_bar_noblock(nfp, tgt, act, tok, offset, size, width); |
427 | if (n < 0) |
428 | spin_unlock_irqrestore(lock: &nfp->bar_lock, flags); |
429 | else |
430 | __release(&nfp->bar_lock); |
431 | |
432 | return n; |
433 | } |
434 | |
435 | static void nfp_bar_get(struct nfp6000_pcie *nfp, struct nfp_bar *bar) |
436 | { |
437 | atomic_inc(v: &bar->refcnt); |
438 | } |
439 | |
440 | static void nfp_bar_put(struct nfp6000_pcie *nfp, struct nfp_bar *bar) |
441 | { |
442 | if (atomic_dec_and_test(v: &bar->refcnt)) |
443 | wake_up_interruptible(&nfp->bar_waiters); |
444 | } |
445 | |
446 | static int |
447 | nfp_wait_for_bar(struct nfp6000_pcie *nfp, int *barnum, |
448 | u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width) |
449 | { |
450 | return wait_event_interruptible(nfp->bar_waiters, |
451 | (*barnum = find_unused_bar_and_lock(nfp, tgt, act, tok, |
452 | offset, size, width)) |
453 | != -EAGAIN); |
454 | } |
455 | |
456 | static int |
457 | nfp_alloc_bar(struct nfp6000_pcie *nfp, |
458 | u32 tgt, u32 act, u32 tok, |
459 | u64 offset, size_t size, int width, int nonblocking) |
460 | { |
461 | unsigned long irqflags; |
462 | int barnum, retval; |
463 | |
464 | if (size > (1 << 24)) |
465 | return -EINVAL; |
466 | |
467 | spin_lock_irqsave(&nfp->bar_lock, irqflags); |
468 | barnum = find_matching_bar(nfp, tgt, act, tok, offset, size, width); |
469 | if (barnum >= 0) { |
470 | /* Found a perfect match. */ |
471 | nfp_bar_get(nfp, bar: &nfp->bar[barnum]); |
472 | spin_unlock_irqrestore(lock: &nfp->bar_lock, flags: irqflags); |
473 | return barnum; |
474 | } |
475 | |
476 | barnum = find_unused_bar_noblock(nfp, tgt, act, tok, |
477 | offset, size, width); |
478 | if (barnum < 0) { |
479 | if (nonblocking) |
480 | goto err_nobar; |
481 | |
482 | /* Wait until a BAR becomes available. The |
483 | * find_unused_bar function will reclaim the bar_lock |
484 | * if a free BAR is found. |
485 | */ |
486 | spin_unlock_irqrestore(lock: &nfp->bar_lock, flags: irqflags); |
487 | retval = nfp_wait_for_bar(nfp, barnum: &barnum, tgt, act, tok, |
488 | offset, size, width); |
489 | if (retval) |
490 | return retval; |
491 | __acquire(&nfp->bar_lock); |
492 | } |
493 | |
494 | nfp_bar_get(nfp, bar: &nfp->bar[barnum]); |
495 | retval = reconfigure_bar(nfp, bar: &nfp->bar[barnum], |
496 | tgt, act, tok, offset, size, width); |
497 | if (retval < 0) { |
498 | nfp_bar_put(nfp, bar: &nfp->bar[barnum]); |
499 | barnum = retval; |
500 | } |
501 | |
502 | err_nobar: |
503 | spin_unlock_irqrestore(lock: &nfp->bar_lock, flags: irqflags); |
504 | return barnum; |
505 | } |
506 | |
507 | static void disable_bars(struct nfp6000_pcie *nfp); |
508 | |
509 | static int bar_cmp(const void *aptr, const void *bptr) |
510 | { |
511 | const struct nfp_bar *a = aptr, *b = bptr; |
512 | |
513 | if (a->bitsize == b->bitsize) |
514 | return a->index - b->index; |
515 | else |
516 | return a->bitsize - b->bitsize; |
517 | } |
518 | |
519 | /* Map all PCI bars and fetch the actual BAR configurations from the |
520 | * board. We assume that the BAR with the PCIe config block is |
521 | * already mapped. |
522 | * |
523 | * BAR0.0: Reserved for General Mapping (for MSI-X access to PCIe SRAM) |
524 | * BAR0.1: Reserved for XPB access (for MSI-X access to PCIe PBA) |
525 | * BAR0.2: -- |
526 | * BAR0.3: -- |
527 | * BAR0.4: Reserved for Explicit 0.0-0.3 access |
528 | * BAR0.5: Reserved for Explicit 1.0-1.3 access |
529 | * BAR0.6: Reserved for Explicit 2.0-2.3 access |
530 | * BAR0.7: Reserved for Explicit 3.0-3.3 access |
531 | * |
532 | * BAR1.0-BAR1.7: -- |
533 | * BAR2.0-BAR2.7: -- |
534 | */ |
535 | static int enable_bars(struct nfp6000_pcie *nfp, u16 interface) |
536 | { |
537 | const u32 barcfg_msix_general = |
538 | NFP_PCIE_BAR_PCIE2CPP_MapType( |
539 | NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) | |
540 | NFP_PCIE_BAR_PCIE2CPP_LengthSelect( |
541 | NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT); |
542 | const u32 barcfg_msix_xpb = |
543 | NFP_PCIE_BAR_PCIE2CPP_MapType( |
544 | NFP_PCIE_BAR_PCIE2CPP_MapType_BULK) | |
545 | NFP_PCIE_BAR_PCIE2CPP_LengthSelect( |
546 | NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT) | |
547 | NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress( |
548 | NFP_CPP_TARGET_ISLAND_XPB); |
549 | const u32 barcfg_explicit[4] = { |
550 | NFP_PCIE_BAR_PCIE2CPP_MapType( |
551 | NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0), |
552 | NFP_PCIE_BAR_PCIE2CPP_MapType( |
553 | NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1), |
554 | NFP_PCIE_BAR_PCIE2CPP_MapType( |
555 | NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2), |
556 | NFP_PCIE_BAR_PCIE2CPP_MapType( |
557 | NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3), |
558 | }; |
559 | char status_msg[196] = {}; |
560 | int i, err, bars_free; |
561 | struct nfp_bar *bar; |
562 | int expl_groups; |
563 | char *msg, *end; |
564 | |
565 | msg = status_msg + |
566 | snprintf(buf: status_msg, size: sizeof(status_msg) - 1, fmt: "RESERVED BARs: " ); |
567 | end = status_msg + sizeof(status_msg) - 1; |
568 | |
569 | bar = &nfp->bar[0]; |
570 | for (i = 0; i < ARRAY_SIZE(nfp->bar); i++, bar++) { |
571 | struct resource *res; |
572 | |
573 | res = &nfp->pdev->resource[(i >> 3) * 2]; |
574 | |
575 | /* Skip over BARs that are not IORESOURCE_MEM */ |
576 | if (!(resource_type(res) & IORESOURCE_MEM)) { |
577 | bar--; |
578 | continue; |
579 | } |
580 | |
581 | bar->resource = res; |
582 | bar->barcfg = 0; |
583 | |
584 | bar->nfp = nfp; |
585 | bar->index = i; |
586 | bar->mask = nfp_bar_resource_len(bar) - 1; |
587 | bar->bitsize = fls(x: bar->mask); |
588 | bar->base = 0; |
589 | bar->iomem = NULL; |
590 | } |
591 | |
592 | nfp->bars = bar - &nfp->bar[0]; |
593 | if (nfp->bars < 8) { |
594 | dev_err(nfp->dev, "No usable BARs found!\n" ); |
595 | return -EINVAL; |
596 | } |
597 | |
598 | bars_free = nfp->bars; |
599 | |
600 | /* Convert unit ID (0..3) to signal master/data master ID (0x40..0x70) |
601 | */ |
602 | mutex_init(&nfp->expl.mutex); |
603 | |
604 | nfp->expl.master_id = ((NFP_CPP_INTERFACE_UNIT_of(interface) & 3) + 4) |
605 | << 4; |
606 | nfp->expl.signal_ref = 0x10; |
607 | |
608 | /* Configure, and lock, BAR0.0 for General Target use (MSI-X SRAM) */ |
609 | bar = &nfp->bar[0]; |
610 | if (nfp_bar_resource_len(bar) >= NFP_PCI_MIN_MAP_SIZE) |
611 | bar->iomem = ioremap(offset: nfp_bar_resource_start(bar), |
612 | size: nfp_bar_resource_len(bar)); |
613 | if (bar->iomem) { |
614 | int pf; |
615 | |
616 | msg += scnprintf(buf: msg, size: end - msg, fmt: "0.0: General/MSI-X SRAM, " ); |
617 | atomic_inc(v: &bar->refcnt); |
618 | bars_free--; |
619 | |
620 | nfp6000_bar_write(nfp, bar, newcfg: barcfg_msix_general); |
621 | |
622 | nfp->expl.data = bar->iomem + NFP_PCIE_SRAM + |
623 | nfp->dev_info->pcie_expl_offset; |
624 | |
625 | switch (nfp->pdev->device) { |
626 | case PCI_DEVICE_ID_NFP3800: |
627 | pf = nfp->pdev->devfn & 7; |
628 | nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(pf); |
629 | break; |
630 | case PCI_DEVICE_ID_NFP4000: |
631 | case PCI_DEVICE_ID_NFP5000: |
632 | case PCI_DEVICE_ID_NFP6000: |
633 | nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(0); |
634 | break; |
635 | default: |
636 | dev_err(nfp->dev, "Unsupported device ID: %04hx!\n" , |
637 | nfp->pdev->device); |
638 | err = -EINVAL; |
639 | goto err_unmap_bar0; |
640 | } |
641 | nfp->iomem.em = bar->iomem + NFP_PCIE_EM; |
642 | } |
643 | |
644 | switch (nfp->pdev->device) { |
645 | case PCI_DEVICE_ID_NFP3800: |
646 | expl_groups = 1; |
647 | break; |
648 | case PCI_DEVICE_ID_NFP4000: |
649 | case PCI_DEVICE_ID_NFP5000: |
650 | case PCI_DEVICE_ID_NFP6000: |
651 | expl_groups = 4; |
652 | break; |
653 | default: |
654 | dev_err(nfp->dev, "Unsupported device ID: %04hx!\n" , |
655 | nfp->pdev->device); |
656 | err = -EINVAL; |
657 | goto err_unmap_bar0; |
658 | } |
659 | |
660 | /* Configure, and lock, BAR0.1 for PCIe XPB (MSI-X PBA) */ |
661 | bar = &nfp->bar[1]; |
662 | msg += scnprintf(buf: msg, size: end - msg, fmt: "0.1: PCIe XPB/MSI-X PBA, " ); |
663 | atomic_inc(v: &bar->refcnt); |
664 | bars_free--; |
665 | |
666 | nfp6000_bar_write(nfp, bar, newcfg: barcfg_msix_xpb); |
667 | |
668 | /* Use BAR0.4..BAR0.7 for EXPL IO */ |
669 | for (i = 0; i < 4; i++) { |
670 | int j; |
671 | |
672 | if (i >= NFP_PCIE_EXPLICIT_BARS || i >= expl_groups) { |
673 | nfp->expl.group[i].bitsize = 0; |
674 | continue; |
675 | } |
676 | |
677 | bar = &nfp->bar[4 + i]; |
678 | bar->iomem = ioremap(offset: nfp_bar_resource_start(bar), |
679 | size: nfp_bar_resource_len(bar)); |
680 | if (bar->iomem) { |
681 | msg += scnprintf(buf: msg, size: end - msg, |
682 | fmt: "0.%d: Explicit%d, " , 4 + i, i); |
683 | atomic_inc(v: &bar->refcnt); |
684 | bars_free--; |
685 | |
686 | nfp->expl.group[i].bitsize = bar->bitsize; |
687 | nfp->expl.group[i].addr = bar->iomem; |
688 | nfp6000_bar_write(nfp, bar, newcfg: barcfg_explicit[i]); |
689 | |
690 | for (j = 0; j < 4; j++) |
691 | nfp->expl.group[i].free[j] = true; |
692 | } |
693 | nfp->iomem.expl[i] = bar->iomem; |
694 | } |
695 | |
696 | /* Sort bars by bit size - use the smallest possible first. */ |
697 | sort(base: &nfp->bar[0], num: nfp->bars, size: sizeof(nfp->bar[0]), |
698 | cmp_func: bar_cmp, NULL); |
699 | |
700 | dev_info(nfp->dev, "%sfree: %d/%d\n" , status_msg, bars_free, nfp->bars); |
701 | |
702 | return 0; |
703 | |
704 | err_unmap_bar0: |
705 | if (nfp->bar[0].iomem) |
706 | iounmap(addr: nfp->bar[0].iomem); |
707 | return err; |
708 | } |
709 | |
710 | static void disable_bars(struct nfp6000_pcie *nfp) |
711 | { |
712 | struct nfp_bar *bar = &nfp->bar[0]; |
713 | int n; |
714 | |
715 | for (n = 0; n < nfp->bars; n++, bar++) { |
716 | if (bar->iomem) { |
717 | iounmap(addr: bar->iomem); |
718 | bar->iomem = NULL; |
719 | } |
720 | } |
721 | } |
722 | |
723 | /* |
724 | * Generic CPP bus access interface. |
725 | */ |
726 | |
727 | struct nfp6000_area_priv { |
728 | atomic_t refcnt; |
729 | |
730 | struct nfp_bar *bar; |
731 | u32 bar_offset; |
732 | |
733 | u32 target; |
734 | u32 action; |
735 | u32 token; |
736 | u64 offset; |
737 | struct { |
738 | int read; |
739 | int write; |
740 | int bar; |
741 | } width; |
742 | size_t size; |
743 | |
744 | void __iomem *iomem; |
745 | phys_addr_t phys; |
746 | struct resource resource; |
747 | }; |
748 | |
749 | static int nfp6000_area_init(struct nfp_cpp_area *area, u32 dest, |
750 | unsigned long long address, unsigned long size) |
751 | { |
752 | struct nfp6000_area_priv *priv = nfp_cpp_area_priv(cpp_area: area); |
753 | u32 target = NFP_CPP_ID_TARGET_of(id: dest); |
754 | u32 action = NFP_CPP_ID_ACTION_of(id: dest); |
755 | u32 token = NFP_CPP_ID_TOKEN_of(id: dest); |
756 | int pp; |
757 | |
758 | pp = nfp_target_pushpull(NFP_CPP_ID(target, action, token), address); |
759 | if (pp < 0) |
760 | return pp; |
761 | |
762 | priv->width.read = PUSH_WIDTH(pp); |
763 | priv->width.write = PULL_WIDTH(pp); |
764 | if (priv->width.read > 0 && |
765 | priv->width.write > 0 && |
766 | priv->width.read != priv->width.write) { |
767 | return -EINVAL; |
768 | } |
769 | |
770 | if (priv->width.read > 0) |
771 | priv->width.bar = priv->width.read; |
772 | else |
773 | priv->width.bar = priv->width.write; |
774 | |
775 | atomic_set(v: &priv->refcnt, i: 0); |
776 | priv->bar = NULL; |
777 | |
778 | priv->target = target; |
779 | priv->action = action; |
780 | priv->token = token; |
781 | priv->offset = address; |
782 | priv->size = size; |
783 | memset(&priv->resource, 0, sizeof(priv->resource)); |
784 | |
785 | return 0; |
786 | } |
787 | |
788 | static void nfp6000_area_cleanup(struct nfp_cpp_area *area) |
789 | { |
790 | } |
791 | |
792 | static void priv_area_get(struct nfp_cpp_area *area) |
793 | { |
794 | struct nfp6000_area_priv *priv = nfp_cpp_area_priv(cpp_area: area); |
795 | |
796 | atomic_inc(v: &priv->refcnt); |
797 | } |
798 | |
799 | static int priv_area_put(struct nfp_cpp_area *area) |
800 | { |
801 | struct nfp6000_area_priv *priv = nfp_cpp_area_priv(cpp_area: area); |
802 | |
803 | if (WARN_ON(!atomic_read(&priv->refcnt))) |
804 | return 0; |
805 | |
806 | return atomic_dec_and_test(v: &priv->refcnt); |
807 | } |
808 | |
809 | static int nfp6000_area_acquire(struct nfp_cpp_area *area) |
810 | { |
811 | struct nfp6000_pcie *nfp = nfp_cpp_priv(priv: nfp_cpp_area_cpp(cpp_area: area)); |
812 | struct nfp6000_area_priv *priv = nfp_cpp_area_priv(cpp_area: area); |
813 | int barnum, err; |
814 | |
815 | if (priv->bar) { |
816 | /* Already allocated. */ |
817 | priv_area_get(area); |
818 | return 0; |
819 | } |
820 | |
821 | barnum = nfp_alloc_bar(nfp, tgt: priv->target, act: priv->action, tok: priv->token, |
822 | offset: priv->offset, size: priv->size, width: priv->width.bar, nonblocking: 1); |
823 | |
824 | if (barnum < 0) { |
825 | err = barnum; |
826 | goto err_alloc_bar; |
827 | } |
828 | priv->bar = &nfp->bar[barnum]; |
829 | |
830 | /* Calculate offset into BAR. */ |
831 | if (nfp_bar_maptype(bar: priv->bar) == |
832 | NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) { |
833 | priv->bar_offset = priv->offset & |
834 | (NFP_PCIE_P2C_GENERAL_SIZE(priv->bar) - 1); |
835 | priv->bar_offset += NFP_PCIE_P2C_GENERAL_TARGET_OFFSET( |
836 | priv->bar, priv->target); |
837 | priv->bar_offset += NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET( |
838 | priv->bar, priv->token); |
839 | } else { |
840 | priv->bar_offset = priv->offset & priv->bar->mask; |
841 | } |
842 | |
843 | /* We don't actually try to acquire the resource area using |
844 | * request_resource. This would prevent sharing the mapped |
845 | * BAR between multiple CPP areas and prevent us from |
846 | * effectively utilizing the limited amount of BAR resources. |
847 | */ |
848 | priv->phys = nfp_bar_resource_start(bar: priv->bar) + priv->bar_offset; |
849 | priv->resource.name = nfp_cpp_area_name(cpp_area: area); |
850 | priv->resource.start = priv->phys; |
851 | priv->resource.end = priv->resource.start + priv->size - 1; |
852 | priv->resource.flags = IORESOURCE_MEM; |
853 | |
854 | /* If the bar is already mapped in, use its mapping */ |
855 | if (priv->bar->iomem) |
856 | priv->iomem = priv->bar->iomem + priv->bar_offset; |
857 | else |
858 | /* Must have been too big. Sub-allocate. */ |
859 | priv->iomem = ioremap(offset: priv->phys, size: priv->size); |
860 | |
861 | if (IS_ERR_OR_NULL(ptr: priv->iomem)) { |
862 | dev_err(nfp->dev, "Can't ioremap() a %d byte region of BAR %d\n" , |
863 | (int)priv->size, priv->bar->index); |
864 | err = !priv->iomem ? -ENOMEM : PTR_ERR(ptr: priv->iomem); |
865 | priv->iomem = NULL; |
866 | goto err_iomem_remap; |
867 | } |
868 | |
869 | priv_area_get(area); |
870 | return 0; |
871 | |
872 | err_iomem_remap: |
873 | nfp_bar_put(nfp, bar: priv->bar); |
874 | priv->bar = NULL; |
875 | err_alloc_bar: |
876 | return err; |
877 | } |
878 | |
879 | static void nfp6000_area_release(struct nfp_cpp_area *area) |
880 | { |
881 | struct nfp6000_pcie *nfp = nfp_cpp_priv(priv: nfp_cpp_area_cpp(cpp_area: area)); |
882 | struct nfp6000_area_priv *priv = nfp_cpp_area_priv(cpp_area: area); |
883 | |
884 | if (!priv_area_put(area)) |
885 | return; |
886 | |
887 | if (!priv->bar->iomem) |
888 | iounmap(addr: priv->iomem); |
889 | |
890 | nfp_bar_put(nfp, bar: priv->bar); |
891 | |
892 | priv->bar = NULL; |
893 | priv->iomem = NULL; |
894 | } |
895 | |
896 | static phys_addr_t nfp6000_area_phys(struct nfp_cpp_area *area) |
897 | { |
898 | struct nfp6000_area_priv *priv = nfp_cpp_area_priv(cpp_area: area); |
899 | |
900 | return priv->phys; |
901 | } |
902 | |
903 | static void __iomem *nfp6000_area_iomem(struct nfp_cpp_area *area) |
904 | { |
905 | struct nfp6000_area_priv *priv = nfp_cpp_area_priv(cpp_area: area); |
906 | |
907 | return priv->iomem; |
908 | } |
909 | |
910 | static struct resource *nfp6000_area_resource(struct nfp_cpp_area *area) |
911 | { |
912 | /* Use the BAR resource as the resource for the CPP area. |
913 | * This enables us to share the BAR among multiple CPP areas |
914 | * without resource conflicts. |
915 | */ |
916 | struct nfp6000_area_priv *priv = nfp_cpp_area_priv(cpp_area: area); |
917 | |
918 | return priv->bar->resource; |
919 | } |
920 | |
921 | static int nfp6000_area_read(struct nfp_cpp_area *area, void *kernel_vaddr, |
922 | unsigned long offset, unsigned int length) |
923 | { |
924 | u64 __maybe_unused *wrptr64 = kernel_vaddr; |
925 | const u64 __iomem __maybe_unused *rdptr64; |
926 | struct nfp6000_area_priv *priv; |
927 | u32 *wrptr32 = kernel_vaddr; |
928 | const u32 __iomem *rdptr32; |
929 | int n, width; |
930 | |
931 | priv = nfp_cpp_area_priv(cpp_area: area); |
932 | rdptr64 = priv->iomem + offset; |
933 | rdptr32 = priv->iomem + offset; |
934 | |
935 | if (offset + length > priv->size) |
936 | return -EFAULT; |
937 | |
938 | width = priv->width.read; |
939 | if (width <= 0) |
940 | return -EINVAL; |
941 | |
942 | /* MU reads via a PCIe2CPP BAR support 32bit (and other) lengths */ |
943 | if (priv->target == (NFP_CPP_TARGET_MU & NFP_CPP_TARGET_ID_MASK) && |
944 | priv->action == NFP_CPP_ACTION_RW && |
945 | (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4)) |
946 | width = TARGET_WIDTH_32; |
947 | |
948 | /* Unaligned? Translate to an explicit access */ |
949 | if ((priv->offset + offset) & (width - 1)) |
950 | return nfp_cpp_explicit_read(cpp: nfp_cpp_area_cpp(cpp_area: area), |
951 | NFP_CPP_ID(priv->target, |
952 | priv->action, |
953 | priv->token), |
954 | addr: priv->offset + offset, |
955 | buff: kernel_vaddr, len: length, width_read: width); |
956 | |
957 | if (WARN_ON(!priv->bar)) |
958 | return -EFAULT; |
959 | |
960 | switch (width) { |
961 | case TARGET_WIDTH_32: |
962 | if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0) |
963 | return -EINVAL; |
964 | |
965 | for (n = 0; n < length; n += sizeof(u32)) |
966 | *wrptr32++ = __raw_readl(addr: rdptr32++); |
967 | return n; |
968 | #ifdef __raw_readq |
969 | case TARGET_WIDTH_64: |
970 | if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0) |
971 | return -EINVAL; |
972 | |
973 | for (n = 0; n < length; n += sizeof(u64)) |
974 | *wrptr64++ = __raw_readq(addr: rdptr64++); |
975 | return n; |
976 | #endif |
977 | default: |
978 | return -EINVAL; |
979 | } |
980 | } |
981 | |
982 | static int |
983 | nfp6000_area_write(struct nfp_cpp_area *area, |
984 | const void *kernel_vaddr, |
985 | unsigned long offset, unsigned int length) |
986 | { |
987 | const u64 __maybe_unused *rdptr64 = kernel_vaddr; |
988 | u64 __iomem __maybe_unused *wrptr64; |
989 | const u32 *rdptr32 = kernel_vaddr; |
990 | struct nfp6000_area_priv *priv; |
991 | u32 __iomem *wrptr32; |
992 | int n, width; |
993 | |
994 | priv = nfp_cpp_area_priv(cpp_area: area); |
995 | wrptr64 = priv->iomem + offset; |
996 | wrptr32 = priv->iomem + offset; |
997 | |
998 | if (offset + length > priv->size) |
999 | return -EFAULT; |
1000 | |
1001 | width = priv->width.write; |
1002 | if (width <= 0) |
1003 | return -EINVAL; |
1004 | |
1005 | /* MU writes via a PCIe2CPP BAR support 32bit (and other) lengths */ |
1006 | if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) && |
1007 | priv->action == NFP_CPP_ACTION_RW && |
1008 | (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4)) |
1009 | width = TARGET_WIDTH_32; |
1010 | |
1011 | /* Unaligned? Translate to an explicit access */ |
1012 | if ((priv->offset + offset) & (width - 1)) |
1013 | return nfp_cpp_explicit_write(cpp: nfp_cpp_area_cpp(cpp_area: area), |
1014 | NFP_CPP_ID(priv->target, |
1015 | priv->action, |
1016 | priv->token), |
1017 | addr: priv->offset + offset, |
1018 | buff: kernel_vaddr, len: length, width_write: width); |
1019 | |
1020 | if (WARN_ON(!priv->bar)) |
1021 | return -EFAULT; |
1022 | |
1023 | switch (width) { |
1024 | case TARGET_WIDTH_32: |
1025 | if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0) |
1026 | return -EINVAL; |
1027 | |
1028 | for (n = 0; n < length; n += sizeof(u32)) { |
1029 | __raw_writel(val: *rdptr32++, addr: wrptr32++); |
1030 | wmb(); |
1031 | } |
1032 | return n; |
1033 | #ifdef __raw_writeq |
1034 | case TARGET_WIDTH_64: |
1035 | if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0) |
1036 | return -EINVAL; |
1037 | |
1038 | for (n = 0; n < length; n += sizeof(u64)) { |
1039 | __raw_writeq(val: *rdptr64++, addr: wrptr64++); |
1040 | wmb(); |
1041 | } |
1042 | return n; |
1043 | #endif |
1044 | default: |
1045 | return -EINVAL; |
1046 | } |
1047 | } |
1048 | |
1049 | struct nfp6000_explicit_priv { |
1050 | struct nfp6000_pcie *nfp; |
1051 | struct { |
1052 | int group; |
1053 | int area; |
1054 | } bar; |
1055 | int bitsize; |
1056 | void __iomem *data; |
1057 | void __iomem *addr; |
1058 | }; |
1059 | |
1060 | static int nfp6000_explicit_acquire(struct nfp_cpp_explicit *expl) |
1061 | { |
1062 | struct nfp6000_pcie *nfp = nfp_cpp_priv(priv: nfp_cpp_explicit_cpp(expl)); |
1063 | struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(cpp_explicit: expl); |
1064 | int i, j; |
1065 | |
1066 | mutex_lock(&nfp->expl.mutex); |
1067 | for (i = 0; i < ARRAY_SIZE(nfp->expl.group); i++) { |
1068 | if (!nfp->expl.group[i].bitsize) |
1069 | continue; |
1070 | |
1071 | for (j = 0; j < ARRAY_SIZE(nfp->expl.group[i].free); j++) { |
1072 | u16 data_offset; |
1073 | |
1074 | if (!nfp->expl.group[i].free[j]) |
1075 | continue; |
1076 | |
1077 | priv->nfp = nfp; |
1078 | priv->bar.group = i; |
1079 | priv->bar.area = j; |
1080 | priv->bitsize = nfp->expl.group[i].bitsize - 2; |
1081 | |
1082 | data_offset = (priv->bar.group << 9) + |
1083 | (priv->bar.area << 7); |
1084 | priv->data = nfp->expl.data + data_offset; |
1085 | priv->addr = nfp->expl.group[i].addr + |
1086 | (priv->bar.area << priv->bitsize); |
1087 | nfp->expl.group[i].free[j] = false; |
1088 | |
1089 | mutex_unlock(lock: &nfp->expl.mutex); |
1090 | return 0; |
1091 | } |
1092 | } |
1093 | mutex_unlock(lock: &nfp->expl.mutex); |
1094 | |
1095 | return -EAGAIN; |
1096 | } |
1097 | |
1098 | static void nfp6000_explicit_release(struct nfp_cpp_explicit *expl) |
1099 | { |
1100 | struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(cpp_explicit: expl); |
1101 | struct nfp6000_pcie *nfp = priv->nfp; |
1102 | |
1103 | mutex_lock(&nfp->expl.mutex); |
1104 | nfp->expl.group[priv->bar.group].free[priv->bar.area] = true; |
1105 | mutex_unlock(lock: &nfp->expl.mutex); |
1106 | } |
1107 | |
1108 | static int nfp6000_explicit_put(struct nfp_cpp_explicit *expl, |
1109 | const void *buff, size_t len) |
1110 | { |
1111 | struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(cpp_explicit: expl); |
1112 | const u32 *src = buff; |
1113 | size_t i; |
1114 | |
1115 | for (i = 0; i < len; i += sizeof(u32)) |
1116 | writel(val: *(src++), addr: priv->data + i); |
1117 | |
1118 | return i; |
1119 | } |
1120 | |
1121 | static int |
1122 | nfp6000_explicit_do(struct nfp_cpp_explicit *expl, |
1123 | const struct nfp_cpp_explicit_command *cmd, u64 address) |
1124 | { |
1125 | struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(cpp_explicit: expl); |
1126 | u8 signal_master, signal_ref, data_master; |
1127 | struct nfp6000_pcie *nfp = priv->nfp; |
1128 | int sigmask = 0; |
1129 | u16 data_ref; |
1130 | u32 csr[3]; |
1131 | |
1132 | if (cmd->siga_mode) |
1133 | sigmask |= 1 << cmd->siga; |
1134 | if (cmd->sigb_mode) |
1135 | sigmask |= 1 << cmd->sigb; |
1136 | |
1137 | signal_master = cmd->signal_master; |
1138 | if (!signal_master) |
1139 | signal_master = nfp->expl.master_id; |
1140 | |
1141 | signal_ref = cmd->signal_ref; |
1142 | if (signal_master == nfp->expl.master_id) |
1143 | signal_ref = nfp->expl.signal_ref + |
1144 | ((priv->bar.group * 4 + priv->bar.area) << 1); |
1145 | |
1146 | data_master = cmd->data_master; |
1147 | if (!data_master) |
1148 | data_master = nfp->expl.master_id; |
1149 | |
1150 | data_ref = cmd->data_ref; |
1151 | if (data_master == nfp->expl.master_id) |
1152 | data_ref = 0x1000 + |
1153 | (priv->bar.group << 9) + (priv->bar.area << 7); |
1154 | |
1155 | csr[0] = NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(sigmask) | |
1156 | NFP_PCIE_BAR_EXPLICIT_BAR0_Token( |
1157 | NFP_CPP_ID_TOKEN_of(cmd->cpp_id)) | |
1158 | NFP_PCIE_BAR_EXPLICIT_BAR0_Address(address >> 16); |
1159 | |
1160 | csr[1] = NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(signal_ref) | |
1161 | NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(data_master) | |
1162 | NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(data_ref); |
1163 | |
1164 | csr[2] = NFP_PCIE_BAR_EXPLICIT_BAR2_Target( |
1165 | NFP_CPP_ID_TARGET_of(cmd->cpp_id)) | |
1166 | NFP_PCIE_BAR_EXPLICIT_BAR2_Action( |
1167 | NFP_CPP_ID_ACTION_of(cmd->cpp_id)) | |
1168 | NFP_PCIE_BAR_EXPLICIT_BAR2_Length(cmd->len) | |
1169 | NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(cmd->byte_mask) | |
1170 | NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(signal_master); |
1171 | |
1172 | if (nfp->iomem.csr) { |
1173 | writel(val: csr[0], addr: nfp->iomem.csr + |
1174 | NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group, |
1175 | priv->bar.area)); |
1176 | writel(val: csr[1], addr: nfp->iomem.csr + |
1177 | NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group, |
1178 | priv->bar.area)); |
1179 | writel(val: csr[2], addr: nfp->iomem.csr + |
1180 | NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group, |
1181 | priv->bar.area)); |
1182 | /* Readback to ensure BAR is flushed */ |
1183 | readl(addr: nfp->iomem.csr + |
1184 | NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group, |
1185 | priv->bar.area)); |
1186 | readl(addr: nfp->iomem.csr + |
1187 | NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group, |
1188 | priv->bar.area)); |
1189 | readl(addr: nfp->iomem.csr + |
1190 | NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group, |
1191 | priv->bar.area)); |
1192 | } else { |
1193 | pci_write_config_dword(dev: nfp->pdev, where: 0x400 + |
1194 | NFP_PCIE_BAR_EXPLICIT_BAR0( |
1195 | priv->bar.group, priv->bar.area), |
1196 | val: csr[0]); |
1197 | |
1198 | pci_write_config_dword(dev: nfp->pdev, where: 0x400 + |
1199 | NFP_PCIE_BAR_EXPLICIT_BAR1( |
1200 | priv->bar.group, priv->bar.area), |
1201 | val: csr[1]); |
1202 | |
1203 | pci_write_config_dword(dev: nfp->pdev, where: 0x400 + |
1204 | NFP_PCIE_BAR_EXPLICIT_BAR2( |
1205 | priv->bar.group, priv->bar.area), |
1206 | val: csr[2]); |
1207 | } |
1208 | |
1209 | /* Issue the 'kickoff' transaction */ |
1210 | readb(addr: priv->addr + (address & ((1 << priv->bitsize) - 1))); |
1211 | |
1212 | return sigmask; |
1213 | } |
1214 | |
1215 | static int nfp6000_explicit_get(struct nfp_cpp_explicit *expl, |
1216 | void *buff, size_t len) |
1217 | { |
1218 | struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(cpp_explicit: expl); |
1219 | u32 *dst = buff; |
1220 | size_t i; |
1221 | |
1222 | for (i = 0; i < len; i += sizeof(u32)) |
1223 | *(dst++) = readl(addr: priv->data + i); |
1224 | |
1225 | return i; |
1226 | } |
1227 | |
1228 | static int nfp6000_init(struct nfp_cpp *cpp) |
1229 | { |
1230 | nfp_cpp_area_cache_add(cpp, SZ_64K); |
1231 | nfp_cpp_area_cache_add(cpp, SZ_64K); |
1232 | nfp_cpp_area_cache_add(cpp, SZ_256K); |
1233 | |
1234 | return 0; |
1235 | } |
1236 | |
1237 | static void nfp6000_free(struct nfp_cpp *cpp) |
1238 | { |
1239 | struct nfp6000_pcie *nfp = nfp_cpp_priv(priv: cpp); |
1240 | |
1241 | disable_bars(nfp); |
1242 | kfree(objp: nfp); |
1243 | } |
1244 | |
1245 | static int nfp6000_read_serial(struct device *dev, u8 *serial) |
1246 | { |
1247 | struct pci_dev *pdev = to_pci_dev(dev); |
1248 | u64 dsn; |
1249 | |
1250 | dsn = pci_get_dsn(dev: pdev); |
1251 | if (!dsn) { |
1252 | dev_err(dev, "can't find PCIe Serial Number Capability\n" ); |
1253 | return -EINVAL; |
1254 | } |
1255 | |
1256 | put_unaligned_be32(val: (u32)(dsn >> 32), p: serial); |
1257 | put_unaligned_be16(val: (u16)(dsn >> 16), p: serial + 4); |
1258 | |
1259 | return 0; |
1260 | } |
1261 | |
1262 | static int nfp6000_get_interface(struct device *dev) |
1263 | { |
1264 | struct pci_dev *pdev = to_pci_dev(dev); |
1265 | u64 dsn; |
1266 | |
1267 | dsn = pci_get_dsn(dev: pdev); |
1268 | if (!dsn) { |
1269 | dev_err(dev, "can't find PCIe Serial Number Capability\n" ); |
1270 | return -EINVAL; |
1271 | } |
1272 | |
1273 | return dsn & 0xffff; |
1274 | } |
1275 | |
1276 | static const struct nfp_cpp_operations nfp6000_pcie_ops = { |
1277 | .owner = THIS_MODULE, |
1278 | |
1279 | .init = nfp6000_init, |
1280 | .free = nfp6000_free, |
1281 | |
1282 | .read_serial = nfp6000_read_serial, |
1283 | .get_interface = nfp6000_get_interface, |
1284 | |
1285 | .area_priv_size = sizeof(struct nfp6000_area_priv), |
1286 | .area_init = nfp6000_area_init, |
1287 | .area_cleanup = nfp6000_area_cleanup, |
1288 | .area_acquire = nfp6000_area_acquire, |
1289 | .area_release = nfp6000_area_release, |
1290 | .area_phys = nfp6000_area_phys, |
1291 | .area_iomem = nfp6000_area_iomem, |
1292 | .area_resource = nfp6000_area_resource, |
1293 | .area_read = nfp6000_area_read, |
1294 | .area_write = nfp6000_area_write, |
1295 | |
1296 | .explicit_priv_size = sizeof(struct nfp6000_explicit_priv), |
1297 | .explicit_acquire = nfp6000_explicit_acquire, |
1298 | .explicit_release = nfp6000_explicit_release, |
1299 | .explicit_put = nfp6000_explicit_put, |
1300 | .explicit_do = nfp6000_explicit_do, |
1301 | .explicit_get = nfp6000_explicit_get, |
1302 | }; |
1303 | |
1304 | /** |
1305 | * nfp_cpp_from_nfp6000_pcie() - Build a NFP CPP bus from a NFP6000 PCI device |
1306 | * @pdev: NFP6000 PCI device |
1307 | * @dev_info: NFP ASIC params |
1308 | * |
1309 | * Return: NFP CPP handle |
1310 | */ |
1311 | struct nfp_cpp * |
1312 | nfp_cpp_from_nfp6000_pcie(struct pci_dev *pdev, const struct nfp_dev_info *dev_info) |
1313 | { |
1314 | struct nfp6000_pcie *nfp; |
1315 | u16 interface; |
1316 | int err; |
1317 | |
1318 | /* Finished with card initialization. */ |
1319 | dev_info(&pdev->dev, "Network Flow Processor %s PCIe Card Probe\n" , |
1320 | dev_info->chip_names); |
1321 | pcie_print_link_status(dev: pdev); |
1322 | |
1323 | nfp = kzalloc(size: sizeof(*nfp), GFP_KERNEL); |
1324 | if (!nfp) { |
1325 | err = -ENOMEM; |
1326 | goto err_ret; |
1327 | } |
1328 | |
1329 | nfp->dev = &pdev->dev; |
1330 | nfp->pdev = pdev; |
1331 | nfp->dev_info = dev_info; |
1332 | init_waitqueue_head(&nfp->bar_waiters); |
1333 | spin_lock_init(&nfp->bar_lock); |
1334 | |
1335 | interface = nfp6000_get_interface(dev: &pdev->dev); |
1336 | |
1337 | if (NFP_CPP_INTERFACE_TYPE_of(interface) != |
1338 | NFP_CPP_INTERFACE_TYPE_PCI) { |
1339 | dev_err(&pdev->dev, |
1340 | "Interface type %d is not the expected %d\n" , |
1341 | NFP_CPP_INTERFACE_TYPE_of(interface), |
1342 | NFP_CPP_INTERFACE_TYPE_PCI); |
1343 | err = -ENODEV; |
1344 | goto err_free_nfp; |
1345 | } |
1346 | |
1347 | if (NFP_CPP_INTERFACE_CHANNEL_of(interface) != |
1348 | NFP_CPP_INTERFACE_CHANNEL_PEROPENER) { |
1349 | dev_err(&pdev->dev, "Interface channel %d is not the expected %d\n" , |
1350 | NFP_CPP_INTERFACE_CHANNEL_of(interface), |
1351 | NFP_CPP_INTERFACE_CHANNEL_PEROPENER); |
1352 | err = -ENODEV; |
1353 | goto err_free_nfp; |
1354 | } |
1355 | |
1356 | err = enable_bars(nfp, interface); |
1357 | if (err) |
1358 | goto err_free_nfp; |
1359 | |
1360 | /* Probe for all the common NFP devices */ |
1361 | return nfp_cpp_from_operations(ops: &nfp6000_pcie_ops, parent: &pdev->dev, priv: nfp); |
1362 | |
1363 | err_free_nfp: |
1364 | kfree(objp: nfp); |
1365 | err_ret: |
1366 | dev_err(&pdev->dev, "NFP6000 PCI setup failed\n" ); |
1367 | return ERR_PTR(error: err); |
1368 | } |
1369 | |