1/* Copyright 2008 - 2016 Freescale Semiconductor, Inc.
2 *
3 * Redistribution and use in source and binary forms, with or without
4 * modification, are permitted provided that the following conditions are met:
5 * * Redistributions of source code must retain the above copyright
6 * notice, this list of conditions and the following disclaimer.
7 * * Redistributions in binary form must reproduce the above copyright
8 * notice, this list of conditions and the following disclaimer in the
9 * documentation and/or other materials provided with the distribution.
10 * * Neither the name of Freescale Semiconductor nor the
11 * names of its contributors may be used to endorse or promote products
12 * derived from this software without specific prior written permission.
13 *
14 * ALTERNATIVELY, this software may be distributed under the terms of the
15 * GNU General Public License ("GPL") as published by the Free Software
16 * Foundation, either version 2 of that License or (at your option) any
17 * later version.
18 *
19 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
20 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
23 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
26 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31#include "bman_priv.h"
32
33#define IRQNAME "BMan portal %d"
34#define MAX_IRQNAME 16 /* big enough for "BMan portal %d" */
35
36/* Portal register assists */
37
38#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
39/* Cache-inhibited register offsets */
40#define BM_REG_RCR_PI_CINH 0x3000
41#define BM_REG_RCR_CI_CINH 0x3100
42#define BM_REG_RCR_ITR 0x3200
43#define BM_REG_CFG 0x3300
44#define BM_REG_SCN(n) (0x3400 + ((n) << 6))
45#define BM_REG_ISR 0x3e00
46#define BM_REG_IER 0x3e40
47#define BM_REG_ISDR 0x3e80
48#define BM_REG_IIR 0x3ec0
49
50/* Cache-enabled register offsets */
51#define BM_CL_CR 0x0000
52#define BM_CL_RR0 0x0100
53#define BM_CL_RR1 0x0140
54#define BM_CL_RCR 0x1000
55#define BM_CL_RCR_PI_CENA 0x3000
56#define BM_CL_RCR_CI_CENA 0x3100
57
58#else
59/* Cache-inhibited register offsets */
60#define BM_REG_RCR_PI_CINH 0x0000
61#define BM_REG_RCR_CI_CINH 0x0004
62#define BM_REG_RCR_ITR 0x0008
63#define BM_REG_CFG 0x0100
64#define BM_REG_SCN(n) (0x0200 + ((n) << 2))
65#define BM_REG_ISR 0x0e00
66#define BM_REG_IER 0x0e04
67#define BM_REG_ISDR 0x0e08
68#define BM_REG_IIR 0x0e0c
69
70/* Cache-enabled register offsets */
71#define BM_CL_CR 0x0000
72#define BM_CL_RR0 0x0100
73#define BM_CL_RR1 0x0140
74#define BM_CL_RCR 0x1000
75#define BM_CL_RCR_PI_CENA 0x3000
76#define BM_CL_RCR_CI_CENA 0x3100
77#endif
78
79/*
80 * Portal modes.
81 * Enum types;
82 * pmode == production mode
83 * cmode == consumption mode,
84 * Enum values use 3 letter codes. First letter matches the portal mode,
85 * remaining two letters indicate;
86 * ci == cache-inhibited portal register
87 * ce == cache-enabled portal register
88 * vb == in-band valid-bit (cache-enabled)
89 */
90enum bm_rcr_pmode { /* matches BCSP_CFG::RPM */
91 bm_rcr_pci = 0, /* PI index, cache-inhibited */
92 bm_rcr_pce = 1, /* PI index, cache-enabled */
93 bm_rcr_pvb = 2 /* valid-bit */
94};
95enum bm_rcr_cmode { /* s/w-only */
96 bm_rcr_cci, /* CI index, cache-inhibited */
97 bm_rcr_cce /* CI index, cache-enabled */
98};
99
100
101/* --- Portal structures --- */
102
103#define BM_RCR_SIZE 8
104
105/* Release Command */
106struct bm_rcr_entry {
107 union {
108 struct {
109 u8 _ncw_verb; /* writes to this are non-coherent */
110 u8 bpid; /* used with BM_RCR_VERB_CMD_BPID_SINGLE */
111 u8 __reserved1[62];
112 };
113 struct bm_buffer bufs[8];
114 };
115};
116#define BM_RCR_VERB_VBIT 0x80
117#define BM_RCR_VERB_CMD_MASK 0x70 /* one of two values; */
118#define BM_RCR_VERB_CMD_BPID_SINGLE 0x20
119#define BM_RCR_VERB_CMD_BPID_MULTI 0x30
120#define BM_RCR_VERB_BUFCOUNT_MASK 0x0f /* values 1..8 */
121
122struct bm_rcr {
123 struct bm_rcr_entry *ring, *cursor;
124 u8 ci, available, ithresh, vbit;
125#ifdef CONFIG_FSL_DPAA_CHECKING
126 u32 busy;
127 enum bm_rcr_pmode pmode;
128 enum bm_rcr_cmode cmode;
129#endif
130};
131
132/* MC (Management Command) command */
133struct bm_mc_command {
134 u8 _ncw_verb; /* writes to this are non-coherent */
135 u8 bpid; /* used by acquire command */
136 u8 __reserved[62];
137};
138#define BM_MCC_VERB_VBIT 0x80
139#define BM_MCC_VERB_CMD_MASK 0x70 /* where the verb contains; */
140#define BM_MCC_VERB_CMD_ACQUIRE 0x10
141#define BM_MCC_VERB_CMD_QUERY 0x40
142#define BM_MCC_VERB_ACQUIRE_BUFCOUNT 0x0f /* values 1..8 go here */
143
144/* MC result, Acquire and Query Response */
145union bm_mc_result {
146 struct {
147 u8 verb;
148 u8 bpid;
149 u8 __reserved[62];
150 };
151 struct bm_buffer bufs[8];
152};
153#define BM_MCR_VERB_VBIT 0x80
154#define BM_MCR_VERB_CMD_MASK BM_MCC_VERB_CMD_MASK
155#define BM_MCR_VERB_CMD_ACQUIRE BM_MCC_VERB_CMD_ACQUIRE
156#define BM_MCR_VERB_CMD_QUERY BM_MCC_VERB_CMD_QUERY
157#define BM_MCR_VERB_CMD_ERR_INVALID 0x60
158#define BM_MCR_VERB_CMD_ERR_ECC 0x70
159#define BM_MCR_VERB_ACQUIRE_BUFCOUNT BM_MCC_VERB_ACQUIRE_BUFCOUNT /* 0..8 */
160#define BM_MCR_TIMEOUT 10000 /* us */
161
162struct bm_mc {
163 struct bm_mc_command *cr;
164 union bm_mc_result *rr;
165 u8 rridx, vbit;
166#ifdef CONFIG_FSL_DPAA_CHECKING
167 enum {
168 /* Can only be _mc_start()ed */
169 mc_idle,
170 /* Can only be _mc_commit()ed or _mc_abort()ed */
171 mc_user,
172 /* Can only be _mc_retry()ed */
173 mc_hw
174 } state;
175#endif
176};
177
178struct bm_addr {
179 void *ce; /* cache-enabled */
180 __be32 *ce_be; /* Same as above but for direct access */
181 void __iomem *ci; /* cache-inhibited */
182};
183
184struct bm_portal {
185 struct bm_addr addr;
186 struct bm_rcr rcr;
187 struct bm_mc mc;
188} ____cacheline_aligned;
189
190/* Cache-inhibited register access. */
191static inline u32 bm_in(struct bm_portal *p, u32 offset)
192{
193 return ioread32be(p->addr.ci + offset);
194}
195
196static inline void bm_out(struct bm_portal *p, u32 offset, u32 val)
197{
198 iowrite32be(val, p->addr.ci + offset);
199}
200
201/* Cache Enabled Portal Access */
202static inline void bm_cl_invalidate(struct bm_portal *p, u32 offset)
203{
204 dpaa_invalidate(p->addr.ce + offset);
205}
206
207static inline void bm_cl_touch_ro(struct bm_portal *p, u32 offset)
208{
209 dpaa_touch_ro(p: p->addr.ce + offset);
210}
211
212static inline u32 bm_ce_in(struct bm_portal *p, u32 offset)
213{
214 return be32_to_cpu(*(p->addr.ce_be + (offset/4)));
215}
216
217struct bman_portal {
218 struct bm_portal p;
219 /* interrupt sources processed by portal_isr(), configurable */
220 unsigned long irq_sources;
221 /* probing time config params for cpu-affine portals */
222 const struct bm_portal_config *config;
223 char irqname[MAX_IRQNAME];
224};
225
226static cpumask_t affine_mask;
227static DEFINE_SPINLOCK(affine_mask_lock);
228static DEFINE_PER_CPU(struct bman_portal, bman_affine_portal);
229
230static inline struct bman_portal *get_affine_portal(void)
231{
232 return &get_cpu_var(bman_affine_portal);
233}
234
235static inline void put_affine_portal(void)
236{
237 put_cpu_var(bman_affine_portal);
238}
239
240/*
241 * This object type refers to a pool, it isn't *the* pool. There may be
242 * more than one such object per BMan buffer pool, eg. if different users of the
243 * pool are operating via different portals.
244 */
245struct bman_pool {
246 /* index of the buffer pool to encapsulate (0-63) */
247 u32 bpid;
248 /* Used for hash-table admin when using depletion notifications. */
249 struct bman_portal *portal;
250 struct bman_pool *next;
251};
252
253static u32 poll_portal_slow(struct bman_portal *p, u32 is);
254
255static irqreturn_t portal_isr(int irq, void *ptr)
256{
257 struct bman_portal *p = ptr;
258 struct bm_portal *portal = &p->p;
259 u32 clear = p->irq_sources;
260 u32 is = bm_in(p: portal, BM_REG_ISR) & p->irq_sources;
261
262 if (unlikely(!is))
263 return IRQ_NONE;
264
265 clear |= poll_portal_slow(p, is);
266 bm_out(p: portal, BM_REG_ISR, val: clear);
267 return IRQ_HANDLED;
268}
269
270/* --- RCR API --- */
271
272#define RCR_SHIFT ilog2(sizeof(struct bm_rcr_entry))
273#define RCR_CARRY (uintptr_t)(BM_RCR_SIZE << RCR_SHIFT)
274
275/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
276static struct bm_rcr_entry *rcr_carryclear(struct bm_rcr_entry *p)
277{
278 uintptr_t addr = (uintptr_t)p;
279
280 addr &= ~RCR_CARRY;
281
282 return (struct bm_rcr_entry *)addr;
283}
284
285#ifdef CONFIG_FSL_DPAA_CHECKING
286/* Bit-wise logic to convert a ring pointer to a ring index */
287static int rcr_ptr2idx(struct bm_rcr_entry *e)
288{
289 return ((uintptr_t)e >> RCR_SHIFT) & (BM_RCR_SIZE - 1);
290}
291#endif
292
293/* Increment the 'cursor' ring pointer, taking 'vbit' into account */
294static inline void rcr_inc(struct bm_rcr *rcr)
295{
296 /* increment to the next RCR pointer and handle overflow and 'vbit' */
297 struct bm_rcr_entry *partial = rcr->cursor + 1;
298
299 rcr->cursor = rcr_carryclear(p: partial);
300 if (partial != rcr->cursor)
301 rcr->vbit ^= BM_RCR_VERB_VBIT;
302}
303
304static int bm_rcr_get_avail(struct bm_portal *portal)
305{
306 struct bm_rcr *rcr = &portal->rcr;
307
308 return rcr->available;
309}
310
311static int bm_rcr_get_fill(struct bm_portal *portal)
312{
313 struct bm_rcr *rcr = &portal->rcr;
314
315 return BM_RCR_SIZE - 1 - rcr->available;
316}
317
318static void bm_rcr_set_ithresh(struct bm_portal *portal, u8 ithresh)
319{
320 struct bm_rcr *rcr = &portal->rcr;
321
322 rcr->ithresh = ithresh;
323 bm_out(p: portal, BM_REG_RCR_ITR, val: ithresh);
324}
325
326static void bm_rcr_cce_prefetch(struct bm_portal *portal)
327{
328 __maybe_unused struct bm_rcr *rcr = &portal->rcr;
329
330 DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
331 bm_cl_touch_ro(p: portal, BM_CL_RCR_CI_CENA);
332}
333
334static u8 bm_rcr_cce_update(struct bm_portal *portal)
335{
336 struct bm_rcr *rcr = &portal->rcr;
337 u8 diff, old_ci = rcr->ci;
338
339 DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
340 rcr->ci = bm_ce_in(p: portal, BM_CL_RCR_CI_CENA) & (BM_RCR_SIZE - 1);
341 bm_cl_invalidate(p: portal, BM_CL_RCR_CI_CENA);
342 diff = dpaa_cyc_diff(BM_RCR_SIZE, first: old_ci, last: rcr->ci);
343 rcr->available += diff;
344 return diff;
345}
346
347static inline struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal)
348{
349 struct bm_rcr *rcr = &portal->rcr;
350
351 DPAA_ASSERT(!rcr->busy);
352 if (!rcr->available)
353 return NULL;
354#ifdef CONFIG_FSL_DPAA_CHECKING
355 rcr->busy = 1;
356#endif
357 dpaa_zero(rcr->cursor);
358 return rcr->cursor;
359}
360
361static inline void bm_rcr_pvb_commit(struct bm_portal *portal, u8 myverb)
362{
363 struct bm_rcr *rcr = &portal->rcr;
364 struct bm_rcr_entry *rcursor;
365
366 DPAA_ASSERT(rcr->busy);
367 DPAA_ASSERT(rcr->pmode == bm_rcr_pvb);
368 DPAA_ASSERT(rcr->available >= 1);
369 dma_wmb();
370 rcursor = rcr->cursor;
371 rcursor->_ncw_verb = myverb | rcr->vbit;
372 dpaa_flush(p: rcursor);
373 rcr_inc(rcr);
374 rcr->available--;
375#ifdef CONFIG_FSL_DPAA_CHECKING
376 rcr->busy = 0;
377#endif
378}
379
380static int bm_rcr_init(struct bm_portal *portal, enum bm_rcr_pmode pmode,
381 enum bm_rcr_cmode cmode)
382{
383 struct bm_rcr *rcr = &portal->rcr;
384 u32 cfg;
385 u8 pi;
386
387 rcr->ring = portal->addr.ce + BM_CL_RCR;
388 rcr->ci = bm_in(p: portal, BM_REG_RCR_CI_CINH) & (BM_RCR_SIZE - 1);
389 pi = bm_in(p: portal, BM_REG_RCR_PI_CINH) & (BM_RCR_SIZE - 1);
390 rcr->cursor = rcr->ring + pi;
391 rcr->vbit = (bm_in(p: portal, BM_REG_RCR_PI_CINH) & BM_RCR_SIZE) ?
392 BM_RCR_VERB_VBIT : 0;
393 rcr->available = BM_RCR_SIZE - 1
394 - dpaa_cyc_diff(BM_RCR_SIZE, first: rcr->ci, last: pi);
395 rcr->ithresh = bm_in(p: portal, BM_REG_RCR_ITR);
396#ifdef CONFIG_FSL_DPAA_CHECKING
397 rcr->busy = 0;
398 rcr->pmode = pmode;
399 rcr->cmode = cmode;
400#endif
401 cfg = (bm_in(p: portal, BM_REG_CFG) & 0xffffffe0)
402 | (pmode & 0x3); /* BCSP_CFG::RPM */
403 bm_out(p: portal, BM_REG_CFG, val: cfg);
404 return 0;
405}
406
407static void bm_rcr_finish(struct bm_portal *portal)
408{
409#ifdef CONFIG_FSL_DPAA_CHECKING
410 struct bm_rcr *rcr = &portal->rcr;
411 int i;
412
413 DPAA_ASSERT(!rcr->busy);
414
415 i = bm_in(portal, BM_REG_RCR_PI_CINH) & (BM_RCR_SIZE - 1);
416 if (i != rcr_ptr2idx(rcr->cursor))
417 pr_crit("losing uncommitted RCR entries\n");
418
419 i = bm_in(portal, BM_REG_RCR_CI_CINH) & (BM_RCR_SIZE - 1);
420 if (i != rcr->ci)
421 pr_crit("missing existing RCR completions\n");
422 if (rcr->ci != rcr_ptr2idx(rcr->cursor))
423 pr_crit("RCR destroyed unquiesced\n");
424#endif
425}
426
427/* --- Management command API --- */
428static int bm_mc_init(struct bm_portal *portal)
429{
430 struct bm_mc *mc = &portal->mc;
431
432 mc->cr = portal->addr.ce + BM_CL_CR;
433 mc->rr = portal->addr.ce + BM_CL_RR0;
434 mc->rridx = (mc->cr->_ncw_verb & BM_MCC_VERB_VBIT) ?
435 0 : 1;
436 mc->vbit = mc->rridx ? BM_MCC_VERB_VBIT : 0;
437#ifdef CONFIG_FSL_DPAA_CHECKING
438 mc->state = mc_idle;
439#endif
440 return 0;
441}
442
443static void bm_mc_finish(struct bm_portal *portal)
444{
445#ifdef CONFIG_FSL_DPAA_CHECKING
446 struct bm_mc *mc = &portal->mc;
447
448 DPAA_ASSERT(mc->state == mc_idle);
449 if (mc->state != mc_idle)
450 pr_crit("Losing incomplete MC command\n");
451#endif
452}
453
454static inline struct bm_mc_command *bm_mc_start(struct bm_portal *portal)
455{
456 struct bm_mc *mc = &portal->mc;
457
458 DPAA_ASSERT(mc->state == mc_idle);
459#ifdef CONFIG_FSL_DPAA_CHECKING
460 mc->state = mc_user;
461#endif
462 dpaa_zero(mc->cr);
463 return mc->cr;
464}
465
466static inline void bm_mc_commit(struct bm_portal *portal, u8 myverb)
467{
468 struct bm_mc *mc = &portal->mc;
469 union bm_mc_result *rr = mc->rr + mc->rridx;
470
471 DPAA_ASSERT(mc->state == mc_user);
472 dma_wmb();
473 mc->cr->_ncw_verb = myverb | mc->vbit;
474 dpaa_flush(p: mc->cr);
475 dpaa_invalidate_touch_ro(p: rr);
476#ifdef CONFIG_FSL_DPAA_CHECKING
477 mc->state = mc_hw;
478#endif
479}
480
481static inline union bm_mc_result *bm_mc_result(struct bm_portal *portal)
482{
483 struct bm_mc *mc = &portal->mc;
484 union bm_mc_result *rr = mc->rr + mc->rridx;
485
486 DPAA_ASSERT(mc->state == mc_hw);
487 /*
488 * The inactive response register's verb byte always returns zero until
489 * its command is submitted and completed. This includes the valid-bit,
490 * in case you were wondering...
491 */
492 if (!rr->verb) {
493 dpaa_invalidate_touch_ro(p: rr);
494 return NULL;
495 }
496 mc->rridx ^= 1;
497 mc->vbit ^= BM_MCC_VERB_VBIT;
498#ifdef CONFIG_FSL_DPAA_CHECKING
499 mc->state = mc_idle;
500#endif
501 return rr;
502}
503
504static inline int bm_mc_result_timeout(struct bm_portal *portal,
505 union bm_mc_result **mcr)
506{
507 int timeout = BM_MCR_TIMEOUT;
508
509 do {
510 *mcr = bm_mc_result(portal);
511 if (*mcr)
512 break;
513 udelay(1);
514 } while (--timeout);
515
516 return timeout;
517}
518
519/* Disable all BSCN interrupts for the portal */
520static void bm_isr_bscn_disable(struct bm_portal *portal)
521{
522 bm_out(p: portal, BM_REG_SCN(0), val: 0);
523 bm_out(p: portal, BM_REG_SCN(1), val: 0);
524}
525
526static int bman_create_portal(struct bman_portal *portal,
527 const struct bm_portal_config *c)
528{
529 struct bm_portal *p;
530 int ret;
531
532 p = &portal->p;
533 /*
534 * prep the low-level portal struct with the mapped addresses from the
535 * config, everything that follows depends on it and "config" is more
536 * for (de)reference...
537 */
538 p->addr.ce = c->addr_virt_ce;
539 p->addr.ce_be = c->addr_virt_ce;
540 p->addr.ci = c->addr_virt_ci;
541 if (bm_rcr_init(portal: p, pmode: bm_rcr_pvb, cmode: bm_rcr_cce)) {
542 dev_err(c->dev, "RCR initialisation failed\n");
543 goto fail_rcr;
544 }
545 if (bm_mc_init(portal: p)) {
546 dev_err(c->dev, "MC initialisation failed\n");
547 goto fail_mc;
548 }
549 /*
550 * Default to all BPIDs disabled, we enable as required at
551 * run-time.
552 */
553 bm_isr_bscn_disable(portal: p);
554
555 /* Write-to-clear any stale interrupt status bits */
556 bm_out(p, BM_REG_ISDR, val: 0xffffffff);
557 portal->irq_sources = 0;
558 bm_out(p, BM_REG_IER, val: 0);
559 bm_out(p, BM_REG_ISR, val: 0xffffffff);
560 snprintf(buf: portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);
561 if (request_irq(irq: c->irq, handler: portal_isr, flags: 0, name: portal->irqname, dev: portal)) {
562 dev_err(c->dev, "request_irq() failed\n");
563 goto fail_irq;
564 }
565
566 if (dpaa_set_portal_irq_affinity(dev: c->dev, irq: c->irq, cpu: c->cpu))
567 goto fail_affinity;
568
569 /* Need RCR to be empty before continuing */
570 ret = bm_rcr_get_fill(portal: p);
571 if (ret) {
572 dev_err(c->dev, "RCR unclean\n");
573 goto fail_rcr_empty;
574 }
575 /* Success */
576 portal->config = c;
577
578 bm_out(p, BM_REG_ISDR, val: 0);
579 bm_out(p, BM_REG_IIR, val: 0);
580
581 return 0;
582
583fail_rcr_empty:
584fail_affinity:
585 free_irq(c->irq, portal);
586fail_irq:
587 bm_mc_finish(portal: p);
588fail_mc:
589 bm_rcr_finish(portal: p);
590fail_rcr:
591 return -EIO;
592}
593
594struct bman_portal *bman_create_affine_portal(const struct bm_portal_config *c)
595{
596 struct bman_portal *portal;
597 int err;
598
599 portal = &per_cpu(bman_affine_portal, c->cpu);
600 err = bman_create_portal(portal, c);
601 if (err)
602 return NULL;
603
604 spin_lock(lock: &affine_mask_lock);
605 cpumask_set_cpu(cpu: c->cpu, dstp: &affine_mask);
606 spin_unlock(lock: &affine_mask_lock);
607
608 return portal;
609}
610
611static u32 poll_portal_slow(struct bman_portal *p, u32 is)
612{
613 u32 ret = is;
614
615 if (is & BM_PIRQ_RCRI) {
616 bm_rcr_cce_update(portal: &p->p);
617 bm_rcr_set_ithresh(portal: &p->p, ithresh: 0);
618 bm_out(p: &p->p, BM_REG_ISR, BM_PIRQ_RCRI);
619 is &= ~BM_PIRQ_RCRI;
620 }
621
622 /* There should be no status register bits left undefined */
623 DPAA_ASSERT(!is);
624 return ret;
625}
626
627int bman_p_irqsource_add(struct bman_portal *p, u32 bits)
628{
629 unsigned long irqflags;
630
631 local_irq_save(irqflags);
632 p->irq_sources |= bits & BM_PIRQ_VISIBLE;
633 bm_out(p: &p->p, BM_REG_IER, val: p->irq_sources);
634 local_irq_restore(irqflags);
635 return 0;
636}
637
638int bm_shutdown_pool(u32 bpid)
639{
640 int err = 0;
641 struct bm_mc_command *bm_cmd;
642 union bm_mc_result *bm_res;
643
644
645 struct bman_portal *p = get_affine_portal();
646 while (1) {
647 /* Acquire buffers until empty */
648 bm_cmd = bm_mc_start(portal: &p->p);
649 bm_cmd->bpid = bpid;
650 bm_mc_commit(portal: &p->p, BM_MCC_VERB_CMD_ACQUIRE | 1);
651 if (!bm_mc_result_timeout(portal: &p->p, mcr: &bm_res)) {
652 pr_crit("BMan Acquire Command timedout\n");
653 err = -ETIMEDOUT;
654 goto done;
655 }
656 if (!(bm_res->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT)) {
657 /* Pool is empty */
658 goto done;
659 }
660 }
661done:
662 put_affine_portal();
663 return err;
664}
665
666struct gen_pool *bm_bpalloc;
667
668static int bm_alloc_bpid_range(u32 *result, u32 count)
669{
670 unsigned long addr;
671
672 addr = gen_pool_alloc(pool: bm_bpalloc, size: count);
673 if (!addr)
674 return -ENOMEM;
675
676 *result = addr & ~DPAA_GENALLOC_OFF;
677
678 return 0;
679}
680
681static int bm_release_bpid(u32 bpid)
682{
683 int ret;
684
685 ret = bm_shutdown_pool(bpid);
686 if (ret) {
687 pr_debug("BPID %d leaked\n", bpid);
688 return ret;
689 }
690
691 gen_pool_free(pool: bm_bpalloc, addr: bpid | DPAA_GENALLOC_OFF, size: 1);
692 return 0;
693}
694
695struct bman_pool *bman_new_pool(void)
696{
697 struct bman_pool *pool = NULL;
698 u32 bpid;
699
700 if (bm_alloc_bpid_range(result: &bpid, count: 1))
701 return NULL;
702
703 pool = kmalloc(size: sizeof(*pool), GFP_KERNEL);
704 if (!pool)
705 goto err;
706
707 pool->bpid = bpid;
708
709 return pool;
710err:
711 bm_release_bpid(bpid);
712 return NULL;
713}
714EXPORT_SYMBOL(bman_new_pool);
715
716void bman_free_pool(struct bman_pool *pool)
717{
718 bm_release_bpid(bpid: pool->bpid);
719
720 kfree(objp: pool);
721}
722EXPORT_SYMBOL(bman_free_pool);
723
724int bman_get_bpid(const struct bman_pool *pool)
725{
726 return pool->bpid;
727}
728EXPORT_SYMBOL(bman_get_bpid);
729
730static void update_rcr_ci(struct bman_portal *p, int avail)
731{
732 if (avail)
733 bm_rcr_cce_prefetch(portal: &p->p);
734 else
735 bm_rcr_cce_update(portal: &p->p);
736}
737
738int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num)
739{
740 struct bman_portal *p;
741 struct bm_rcr_entry *r;
742 unsigned long irqflags;
743 int avail, timeout = 1000; /* 1ms */
744 int i = num - 1;
745
746 DPAA_ASSERT(num > 0 && num <= 8);
747
748 do {
749 p = get_affine_portal();
750 local_irq_save(irqflags);
751 avail = bm_rcr_get_avail(portal: &p->p);
752 if (avail < 2)
753 update_rcr_ci(p, avail);
754 r = bm_rcr_start(portal: &p->p);
755 local_irq_restore(irqflags);
756 put_affine_portal();
757 if (likely(r))
758 break;
759
760 udelay(1);
761 } while (--timeout);
762
763 if (unlikely(!timeout))
764 return -ETIMEDOUT;
765
766 p = get_affine_portal();
767 local_irq_save(irqflags);
768 /*
769 * we can copy all but the first entry, as this can trigger badness
770 * with the valid-bit
771 */
772 bm_buffer_set64(buf: r->bufs, addr: bm_buffer_get64(buf: bufs));
773 bm_buffer_set_bpid(buf: r->bufs, bpid: pool->bpid);
774 if (i)
775 memcpy(&r->bufs[1], &bufs[1], i * sizeof(bufs[0]));
776
777 bm_rcr_pvb_commit(portal: &p->p, BM_RCR_VERB_CMD_BPID_SINGLE |
778 (num & BM_RCR_VERB_BUFCOUNT_MASK));
779
780 local_irq_restore(irqflags);
781 put_affine_portal();
782 return 0;
783}
784EXPORT_SYMBOL(bman_release);
785
786int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num)
787{
788 struct bman_portal *p = get_affine_portal();
789 struct bm_mc_command *mcc;
790 union bm_mc_result *mcr;
791 int ret;
792
793 DPAA_ASSERT(num > 0 && num <= 8);
794
795 mcc = bm_mc_start(portal: &p->p);
796 mcc->bpid = pool->bpid;
797 bm_mc_commit(portal: &p->p, BM_MCC_VERB_CMD_ACQUIRE |
798 (num & BM_MCC_VERB_ACQUIRE_BUFCOUNT));
799 if (!bm_mc_result_timeout(portal: &p->p, mcr: &mcr)) {
800 put_affine_portal();
801 pr_crit("BMan Acquire Timeout\n");
802 return -ETIMEDOUT;
803 }
804 ret = mcr->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT;
805 if (bufs)
806 memcpy(&bufs[0], &mcr->bufs[0], num * sizeof(bufs[0]));
807
808 put_affine_portal();
809 if (ret != num)
810 ret = -ENOMEM;
811 return ret;
812}
813EXPORT_SYMBOL(bman_acquire);
814
815const struct bm_portal_config *
816bman_get_bm_portal_config(const struct bman_portal *portal)
817{
818 return portal->config;
819}
820

source code of linux/drivers/soc/fsl/qbman/bman.c