aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/soc/fsl/qbman/bman.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/soc/fsl/qbman/bman.c')
-rw-r--r--drivers/soc/fsl/qbman/bman.c797
1 files changed, 797 insertions, 0 deletions
diff --git a/drivers/soc/fsl/qbman/bman.c b/drivers/soc/fsl/qbman/bman.c
new file mode 100644
index 000000000000..ffa48fdbb1a9
--- /dev/null
+++ b/drivers/soc/fsl/qbman/bman.c
@@ -0,0 +1,797 @@
1/* Copyright 2008 - 2016 Freescale Semiconductor, Inc.
2 *
3 * Redistribution and use in source and binary forms, with or without
4 * modification, are permitted provided that the following conditions are met:
5 * * Redistributions of source code must retain the above copyright
6 * notice, this list of conditions and the following disclaimer.
7 * * Redistributions in binary form must reproduce the above copyright
8 * notice, this list of conditions and the following disclaimer in the
9 * documentation and/or other materials provided with the distribution.
10 * * Neither the name of Freescale Semiconductor nor the
11 * names of its contributors may be used to endorse or promote products
12 * derived from this software without specific prior written permission.
13 *
14 * ALTERNATIVELY, this software may be distributed under the terms of the
15 * GNU General Public License ("GPL") as published by the Free Software
16 * Foundation, either version 2 of that License or (at your option) any
17 * later version.
18 *
19 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
20 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
23 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
26 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31#include "bman_priv.h"
32
33#define IRQNAME "BMan portal %d"
34#define MAX_IRQNAME 16 /* big enough for "BMan portal %d" */
35
36/* Portal register assists */
37
38/* Cache-inhibited register offsets */
39#define BM_REG_RCR_PI_CINH 0x0000
40#define BM_REG_RCR_CI_CINH 0x0004
41#define BM_REG_RCR_ITR 0x0008
42#define BM_REG_CFG 0x0100
43#define BM_REG_SCN(n) (0x0200 + ((n) << 2))
44#define BM_REG_ISR 0x0e00
45#define BM_REG_IER 0x0e04
46#define BM_REG_ISDR 0x0e08
47#define BM_REG_IIR 0x0e0c
48
49/* Cache-enabled register offsets */
50#define BM_CL_CR 0x0000
51#define BM_CL_RR0 0x0100
52#define BM_CL_RR1 0x0140
53#define BM_CL_RCR 0x1000
54#define BM_CL_RCR_PI_CENA 0x3000
55#define BM_CL_RCR_CI_CENA 0x3100
56
57/*
58 * Portal modes.
59 * Enum types;
60 * pmode == production mode
61 * cmode == consumption mode,
62 * Enum values use 3 letter codes. First letter matches the portal mode,
63 * remaining two letters indicate;
64 * ci == cache-inhibited portal register
65 * ce == cache-enabled portal register
66 * vb == in-band valid-bit (cache-enabled)
67 */
68enum bm_rcr_pmode { /* matches BCSP_CFG::RPM */
69 bm_rcr_pci = 0, /* PI index, cache-inhibited */
70 bm_rcr_pce = 1, /* PI index, cache-enabled */
71 bm_rcr_pvb = 2 /* valid-bit */
72};
73enum bm_rcr_cmode { /* s/w-only */
74 bm_rcr_cci, /* CI index, cache-inhibited */
75 bm_rcr_cce /* CI index, cache-enabled */
76};
77
78
79/* --- Portal structures --- */
80
81#define BM_RCR_SIZE 8
82
83/* Release Command */
84struct bm_rcr_entry {
85 union {
86 struct {
87 u8 _ncw_verb; /* writes to this are non-coherent */
88 u8 bpid; /* used with BM_RCR_VERB_CMD_BPID_SINGLE */
89 u8 __reserved1[62];
90 };
91 struct bm_buffer bufs[8];
92 };
93};
94#define BM_RCR_VERB_VBIT 0x80
95#define BM_RCR_VERB_CMD_MASK 0x70 /* one of two values; */
96#define BM_RCR_VERB_CMD_BPID_SINGLE 0x20
97#define BM_RCR_VERB_CMD_BPID_MULTI 0x30
98#define BM_RCR_VERB_BUFCOUNT_MASK 0x0f /* values 1..8 */
99
100struct bm_rcr {
101 struct bm_rcr_entry *ring, *cursor;
102 u8 ci, available, ithresh, vbit;
103#ifdef CONFIG_FSL_DPAA_CHECKING
104 u32 busy;
105 enum bm_rcr_pmode pmode;
106 enum bm_rcr_cmode cmode;
107#endif
108};
109
110/* MC (Management Command) command */
111struct bm_mc_command {
112 u8 _ncw_verb; /* writes to this are non-coherent */
113 u8 bpid; /* used by acquire command */
114 u8 __reserved[62];
115};
116#define BM_MCC_VERB_VBIT 0x80
117#define BM_MCC_VERB_CMD_MASK 0x70 /* where the verb contains; */
118#define BM_MCC_VERB_CMD_ACQUIRE 0x10
119#define BM_MCC_VERB_CMD_QUERY 0x40
120#define BM_MCC_VERB_ACQUIRE_BUFCOUNT 0x0f /* values 1..8 go here */
121
122/* MC result, Acquire and Query Response */
123union bm_mc_result {
124 struct {
125 u8 verb;
126 u8 bpid;
127 u8 __reserved[62];
128 };
129 struct bm_buffer bufs[8];
130};
131#define BM_MCR_VERB_VBIT 0x80
132#define BM_MCR_VERB_CMD_MASK BM_MCC_VERB_CMD_MASK
133#define BM_MCR_VERB_CMD_ACQUIRE BM_MCC_VERB_CMD_ACQUIRE
134#define BM_MCR_VERB_CMD_QUERY BM_MCC_VERB_CMD_QUERY
135#define BM_MCR_VERB_CMD_ERR_INVALID 0x60
136#define BM_MCR_VERB_CMD_ERR_ECC 0x70
137#define BM_MCR_VERB_ACQUIRE_BUFCOUNT BM_MCC_VERB_ACQUIRE_BUFCOUNT /* 0..8 */
138#define BM_MCR_TIMEOUT 10000 /* us */
139
140struct bm_mc {
141 struct bm_mc_command *cr;
142 union bm_mc_result *rr;
143 u8 rridx, vbit;
144#ifdef CONFIG_FSL_DPAA_CHECKING
145 enum {
146 /* Can only be _mc_start()ed */
147 mc_idle,
148 /* Can only be _mc_commit()ed or _mc_abort()ed */
149 mc_user,
150 /* Can only be _mc_retry()ed */
151 mc_hw
152 } state;
153#endif
154};
155
156struct bm_addr {
157 void __iomem *ce; /* cache-enabled */
158 void __iomem *ci; /* cache-inhibited */
159};
160
161struct bm_portal {
162 struct bm_addr addr;
163 struct bm_rcr rcr;
164 struct bm_mc mc;
165} ____cacheline_aligned;
166
167/* Cache-inhibited register access. */
168static inline u32 bm_in(struct bm_portal *p, u32 offset)
169{
170 return __raw_readl(p->addr.ci + offset);
171}
172
173static inline void bm_out(struct bm_portal *p, u32 offset, u32 val)
174{
175 __raw_writel(val, p->addr.ci + offset);
176}
177
178/* Cache Enabled Portal Access */
179static inline void bm_cl_invalidate(struct bm_portal *p, u32 offset)
180{
181 dpaa_invalidate(p->addr.ce + offset);
182}
183
184static inline void bm_cl_touch_ro(struct bm_portal *p, u32 offset)
185{
186 dpaa_touch_ro(p->addr.ce + offset);
187}
188
189static inline u32 bm_ce_in(struct bm_portal *p, u32 offset)
190{
191 return __raw_readl(p->addr.ce + offset);
192}
193
194struct bman_portal {
195 struct bm_portal p;
196 /* interrupt sources processed by portal_isr(), configurable */
197 unsigned long irq_sources;
198 /* probing time config params for cpu-affine portals */
199 const struct bm_portal_config *config;
200 char irqname[MAX_IRQNAME];
201};
202
203static cpumask_t affine_mask;
204static DEFINE_SPINLOCK(affine_mask_lock);
205static DEFINE_PER_CPU(struct bman_portal, bman_affine_portal);
206
207static inline struct bman_portal *get_affine_portal(void)
208{
209 return &get_cpu_var(bman_affine_portal);
210}
211
212static inline void put_affine_portal(void)
213{
214 put_cpu_var(bman_affine_portal);
215}
216
217/*
218 * This object type refers to a pool, it isn't *the* pool. There may be
219 * more than one such object per BMan buffer pool, eg. if different users of the
220 * pool are operating via different portals.
221 */
222struct bman_pool {
223 /* index of the buffer pool to encapsulate (0-63) */
224 u32 bpid;
225 /* Used for hash-table admin when using depletion notifications. */
226 struct bman_portal *portal;
227 struct bman_pool *next;
228};
229
230static u32 poll_portal_slow(struct bman_portal *p, u32 is);
231
232static irqreturn_t portal_isr(int irq, void *ptr)
233{
234 struct bman_portal *p = ptr;
235 struct bm_portal *portal = &p->p;
236 u32 clear = p->irq_sources;
237 u32 is = bm_in(portal, BM_REG_ISR) & p->irq_sources;
238
239 if (unlikely(!is))
240 return IRQ_NONE;
241
242 clear |= poll_portal_slow(p, is);
243 bm_out(portal, BM_REG_ISR, clear);
244 return IRQ_HANDLED;
245}
246
247/* --- RCR API --- */
248
249#define RCR_SHIFT ilog2(sizeof(struct bm_rcr_entry))
250#define RCR_CARRY (uintptr_t)(BM_RCR_SIZE << RCR_SHIFT)
251
252/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
253static struct bm_rcr_entry *rcr_carryclear(struct bm_rcr_entry *p)
254{
255 uintptr_t addr = (uintptr_t)p;
256
257 addr &= ~RCR_CARRY;
258
259 return (struct bm_rcr_entry *)addr;
260}
261
262#ifdef CONFIG_FSL_DPAA_CHECKING
263/* Bit-wise logic to convert a ring pointer to a ring index */
264static int rcr_ptr2idx(struct bm_rcr_entry *e)
265{
266 return ((uintptr_t)e >> RCR_SHIFT) & (BM_RCR_SIZE - 1);
267}
268#endif
269
270/* Increment the 'cursor' ring pointer, taking 'vbit' into account */
271static inline void rcr_inc(struct bm_rcr *rcr)
272{
273 /* increment to the next RCR pointer and handle overflow and 'vbit' */
274 struct bm_rcr_entry *partial = rcr->cursor + 1;
275
276 rcr->cursor = rcr_carryclear(partial);
277 if (partial != rcr->cursor)
278 rcr->vbit ^= BM_RCR_VERB_VBIT;
279}
280
281static int bm_rcr_get_avail(struct bm_portal *portal)
282{
283 struct bm_rcr *rcr = &portal->rcr;
284
285 return rcr->available;
286}
287
288static int bm_rcr_get_fill(struct bm_portal *portal)
289{
290 struct bm_rcr *rcr = &portal->rcr;
291
292 return BM_RCR_SIZE - 1 - rcr->available;
293}
294
295static void bm_rcr_set_ithresh(struct bm_portal *portal, u8 ithresh)
296{
297 struct bm_rcr *rcr = &portal->rcr;
298
299 rcr->ithresh = ithresh;
300 bm_out(portal, BM_REG_RCR_ITR, ithresh);
301}
302
303static void bm_rcr_cce_prefetch(struct bm_portal *portal)
304{
305 __maybe_unused struct bm_rcr *rcr = &portal->rcr;
306
307 DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
308 bm_cl_touch_ro(portal, BM_CL_RCR_CI_CENA);
309}
310
311static u8 bm_rcr_cce_update(struct bm_portal *portal)
312{
313 struct bm_rcr *rcr = &portal->rcr;
314 u8 diff, old_ci = rcr->ci;
315
316 DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
317 rcr->ci = bm_ce_in(portal, BM_CL_RCR_CI_CENA) & (BM_RCR_SIZE - 1);
318 bm_cl_invalidate(portal, BM_CL_RCR_CI_CENA);
319 diff = dpaa_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
320 rcr->available += diff;
321 return diff;
322}
323
324static inline struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal)
325{
326 struct bm_rcr *rcr = &portal->rcr;
327
328 DPAA_ASSERT(!rcr->busy);
329 if (!rcr->available)
330 return NULL;
331#ifdef CONFIG_FSL_DPAA_CHECKING
332 rcr->busy = 1;
333#endif
334 dpaa_zero(rcr->cursor);
335 return rcr->cursor;
336}
337
338static inline void bm_rcr_pvb_commit(struct bm_portal *portal, u8 myverb)
339{
340 struct bm_rcr *rcr = &portal->rcr;
341 struct bm_rcr_entry *rcursor;
342
343 DPAA_ASSERT(rcr->busy);
344 DPAA_ASSERT(rcr->pmode == bm_rcr_pvb);
345 DPAA_ASSERT(rcr->available >= 1);
346 dma_wmb();
347 rcursor = rcr->cursor;
348 rcursor->_ncw_verb = myverb | rcr->vbit;
349 dpaa_flush(rcursor);
350 rcr_inc(rcr);
351 rcr->available--;
352#ifdef CONFIG_FSL_DPAA_CHECKING
353 rcr->busy = 0;
354#endif
355}
356
357static int bm_rcr_init(struct bm_portal *portal, enum bm_rcr_pmode pmode,
358 enum bm_rcr_cmode cmode)
359{
360 struct bm_rcr *rcr = &portal->rcr;
361 u32 cfg;
362 u8 pi;
363
364 rcr->ring = portal->addr.ce + BM_CL_RCR;
365 rcr->ci = bm_in(portal, BM_REG_RCR_CI_CINH) & (BM_RCR_SIZE - 1);
366 pi = bm_in(portal, BM_REG_RCR_PI_CINH) & (BM_RCR_SIZE - 1);
367 rcr->cursor = rcr->ring + pi;
368 rcr->vbit = (bm_in(portal, BM_REG_RCR_PI_CINH) & BM_RCR_SIZE) ?
369 BM_RCR_VERB_VBIT : 0;
370 rcr->available = BM_RCR_SIZE - 1
371 - dpaa_cyc_diff(BM_RCR_SIZE, rcr->ci, pi);
372 rcr->ithresh = bm_in(portal, BM_REG_RCR_ITR);
373#ifdef CONFIG_FSL_DPAA_CHECKING
374 rcr->busy = 0;
375 rcr->pmode = pmode;
376 rcr->cmode = cmode;
377#endif
378 cfg = (bm_in(portal, BM_REG_CFG) & 0xffffffe0)
379 | (pmode & 0x3); /* BCSP_CFG::RPM */
380 bm_out(portal, BM_REG_CFG, cfg);
381 return 0;
382}
383
384static void bm_rcr_finish(struct bm_portal *portal)
385{
386#ifdef CONFIG_FSL_DPAA_CHECKING
387 struct bm_rcr *rcr = &portal->rcr;
388 int i;
389
390 DPAA_ASSERT(!rcr->busy);
391
392 i = bm_in(portal, BM_REG_RCR_PI_CINH) & (BM_RCR_SIZE - 1);
393 if (i != rcr_ptr2idx(rcr->cursor))
394 pr_crit("losing uncommited RCR entries\n");
395
396 i = bm_in(portal, BM_REG_RCR_CI_CINH) & (BM_RCR_SIZE - 1);
397 if (i != rcr->ci)
398 pr_crit("missing existing RCR completions\n");
399 if (rcr->ci != rcr_ptr2idx(rcr->cursor))
400 pr_crit("RCR destroyed unquiesced\n");
401#endif
402}
403
404/* --- Management command API --- */
405static int bm_mc_init(struct bm_portal *portal)
406{
407 struct bm_mc *mc = &portal->mc;
408
409 mc->cr = portal->addr.ce + BM_CL_CR;
410 mc->rr = portal->addr.ce + BM_CL_RR0;
411 mc->rridx = (__raw_readb(&mc->cr->_ncw_verb) & BM_MCC_VERB_VBIT) ?
412 0 : 1;
413 mc->vbit = mc->rridx ? BM_MCC_VERB_VBIT : 0;
414#ifdef CONFIG_FSL_DPAA_CHECKING
415 mc->state = mc_idle;
416#endif
417 return 0;
418}
419
420static void bm_mc_finish(struct bm_portal *portal)
421{
422#ifdef CONFIG_FSL_DPAA_CHECKING
423 struct bm_mc *mc = &portal->mc;
424
425 DPAA_ASSERT(mc->state == mc_idle);
426 if (mc->state != mc_idle)
427 pr_crit("Losing incomplete MC command\n");
428#endif
429}
430
431static inline struct bm_mc_command *bm_mc_start(struct bm_portal *portal)
432{
433 struct bm_mc *mc = &portal->mc;
434
435 DPAA_ASSERT(mc->state == mc_idle);
436#ifdef CONFIG_FSL_DPAA_CHECKING
437 mc->state = mc_user;
438#endif
439 dpaa_zero(mc->cr);
440 return mc->cr;
441}
442
443static inline void bm_mc_commit(struct bm_portal *portal, u8 myverb)
444{
445 struct bm_mc *mc = &portal->mc;
446 union bm_mc_result *rr = mc->rr + mc->rridx;
447
448 DPAA_ASSERT(mc->state == mc_user);
449 dma_wmb();
450 mc->cr->_ncw_verb = myverb | mc->vbit;
451 dpaa_flush(mc->cr);
452 dpaa_invalidate_touch_ro(rr);
453#ifdef CONFIG_FSL_DPAA_CHECKING
454 mc->state = mc_hw;
455#endif
456}
457
458static inline union bm_mc_result *bm_mc_result(struct bm_portal *portal)
459{
460 struct bm_mc *mc = &portal->mc;
461 union bm_mc_result *rr = mc->rr + mc->rridx;
462
463 DPAA_ASSERT(mc->state == mc_hw);
464 /*
465 * The inactive response register's verb byte always returns zero until
466 * its command is submitted and completed. This includes the valid-bit,
467 * in case you were wondering...
468 */
469 if (!__raw_readb(&rr->verb)) {
470 dpaa_invalidate_touch_ro(rr);
471 return NULL;
472 }
473 mc->rridx ^= 1;
474 mc->vbit ^= BM_MCC_VERB_VBIT;
475#ifdef CONFIG_FSL_DPAA_CHECKING
476 mc->state = mc_idle;
477#endif
478 return rr;
479}
480
481static inline int bm_mc_result_timeout(struct bm_portal *portal,
482 union bm_mc_result **mcr)
483{
484 int timeout = BM_MCR_TIMEOUT;
485
486 do {
487 *mcr = bm_mc_result(portal);
488 if (*mcr)
489 break;
490 udelay(1);
491 } while (--timeout);
492
493 return timeout;
494}
495
496/* Disable all BSCN interrupts for the portal */
497static void bm_isr_bscn_disable(struct bm_portal *portal)
498{
499 bm_out(portal, BM_REG_SCN(0), 0);
500 bm_out(portal, BM_REG_SCN(1), 0);
501}
502
503static int bman_create_portal(struct bman_portal *portal,
504 const struct bm_portal_config *c)
505{
506 struct bm_portal *p;
507 int ret;
508
509 p = &portal->p;
510 /*
511 * prep the low-level portal struct with the mapped addresses from the
512 * config, everything that follows depends on it and "config" is more
513 * for (de)reference...
514 */
515 p->addr.ce = c->addr_virt[DPAA_PORTAL_CE];
516 p->addr.ci = c->addr_virt[DPAA_PORTAL_CI];
517 if (bm_rcr_init(p, bm_rcr_pvb, bm_rcr_cce)) {
518 dev_err(c->dev, "RCR initialisation failed\n");
519 goto fail_rcr;
520 }
521 if (bm_mc_init(p)) {
522 dev_err(c->dev, "MC initialisation failed\n");
523 goto fail_mc;
524 }
525 /*
526 * Default to all BPIDs disabled, we enable as required at
527 * run-time.
528 */
529 bm_isr_bscn_disable(p);
530
531 /* Write-to-clear any stale interrupt status bits */
532 bm_out(p, BM_REG_ISDR, 0xffffffff);
533 portal->irq_sources = 0;
534 bm_out(p, BM_REG_IER, 0);
535 bm_out(p, BM_REG_ISR, 0xffffffff);
536 snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);
537 if (request_irq(c->irq, portal_isr, 0, portal->irqname, portal)) {
538 dev_err(c->dev, "request_irq() failed\n");
539 goto fail_irq;
540 }
541 if (c->cpu != -1 && irq_can_set_affinity(c->irq) &&
542 irq_set_affinity(c->irq, cpumask_of(c->cpu))) {
543 dev_err(c->dev, "irq_set_affinity() failed\n");
544 goto fail_affinity;
545 }
546
547 /* Need RCR to be empty before continuing */
548 ret = bm_rcr_get_fill(p);
549 if (ret) {
550 dev_err(c->dev, "RCR unclean\n");
551 goto fail_rcr_empty;
552 }
553 /* Success */
554 portal->config = c;
555
556 bm_out(p, BM_REG_ISDR, 0);
557 bm_out(p, BM_REG_IIR, 0);
558
559 return 0;
560
561fail_rcr_empty:
562fail_affinity:
563 free_irq(c->irq, portal);
564fail_irq:
565 bm_mc_finish(p);
566fail_mc:
567 bm_rcr_finish(p);
568fail_rcr:
569 return -EIO;
570}
571
572struct bman_portal *bman_create_affine_portal(const struct bm_portal_config *c)
573{
574 struct bman_portal *portal;
575 int err;
576
577 portal = &per_cpu(bman_affine_portal, c->cpu);
578 err = bman_create_portal(portal, c);
579 if (err)
580 return NULL;
581
582 spin_lock(&affine_mask_lock);
583 cpumask_set_cpu(c->cpu, &affine_mask);
584 spin_unlock(&affine_mask_lock);
585
586 return portal;
587}
588
589static u32 poll_portal_slow(struct bman_portal *p, u32 is)
590{
591 u32 ret = is;
592
593 if (is & BM_PIRQ_RCRI) {
594 bm_rcr_cce_update(&p->p);
595 bm_rcr_set_ithresh(&p->p, 0);
596 bm_out(&p->p, BM_REG_ISR, BM_PIRQ_RCRI);
597 is &= ~BM_PIRQ_RCRI;
598 }
599
600 /* There should be no status register bits left undefined */
601 DPAA_ASSERT(!is);
602 return ret;
603}
604
605int bman_p_irqsource_add(struct bman_portal *p, u32 bits)
606{
607 unsigned long irqflags;
608
609 local_irq_save(irqflags);
610 set_bits(bits & BM_PIRQ_VISIBLE, &p->irq_sources);
611 bm_out(&p->p, BM_REG_IER, p->irq_sources);
612 local_irq_restore(irqflags);
613 return 0;
614}
615
616static int bm_shutdown_pool(u32 bpid)
617{
618 struct bm_mc_command *bm_cmd;
619 union bm_mc_result *bm_res;
620
621 while (1) {
622 struct bman_portal *p = get_affine_portal();
623 /* Acquire buffers until empty */
624 bm_cmd = bm_mc_start(&p->p);
625 bm_cmd->bpid = bpid;
626 bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE | 1);
627 if (!bm_mc_result_timeout(&p->p, &bm_res)) {
628 put_affine_portal();
629 pr_crit("BMan Acquire Command timedout\n");
630 return -ETIMEDOUT;
631 }
632 if (!(bm_res->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT)) {
633 put_affine_portal();
634 /* Pool is empty */
635 return 0;
636 }
637 put_affine_portal();
638 }
639
640 return 0;
641}
642
643struct gen_pool *bm_bpalloc;
644
645static int bm_alloc_bpid_range(u32 *result, u32 count)
646{
647 unsigned long addr;
648
649 addr = gen_pool_alloc(bm_bpalloc, count);
650 if (!addr)
651 return -ENOMEM;
652
653 *result = addr & ~DPAA_GENALLOC_OFF;
654
655 return 0;
656}
657
658static int bm_release_bpid(u32 bpid)
659{
660 int ret;
661
662 ret = bm_shutdown_pool(bpid);
663 if (ret) {
664 pr_debug("BPID %d leaked\n", bpid);
665 return ret;
666 }
667
668 gen_pool_free(bm_bpalloc, bpid | DPAA_GENALLOC_OFF, 1);
669 return 0;
670}
671
672struct bman_pool *bman_new_pool(void)
673{
674 struct bman_pool *pool = NULL;
675 u32 bpid;
676
677 if (bm_alloc_bpid_range(&bpid, 1))
678 return NULL;
679
680 pool = kmalloc(sizeof(*pool), GFP_KERNEL);
681 if (!pool)
682 goto err;
683
684 pool->bpid = bpid;
685
686 return pool;
687err:
688 bm_release_bpid(bpid);
689 kfree(pool);
690 return NULL;
691}
692EXPORT_SYMBOL(bman_new_pool);
693
694void bman_free_pool(struct bman_pool *pool)
695{
696 bm_release_bpid(pool->bpid);
697
698 kfree(pool);
699}
700EXPORT_SYMBOL(bman_free_pool);
701
702int bman_get_bpid(const struct bman_pool *pool)
703{
704 return pool->bpid;
705}
706EXPORT_SYMBOL(bman_get_bpid);
707
708static void update_rcr_ci(struct bman_portal *p, int avail)
709{
710 if (avail)
711 bm_rcr_cce_prefetch(&p->p);
712 else
713 bm_rcr_cce_update(&p->p);
714}
715
716int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num)
717{
718 struct bman_portal *p;
719 struct bm_rcr_entry *r;
720 unsigned long irqflags;
721 int avail, timeout = 1000; /* 1ms */
722 int i = num - 1;
723
724 DPAA_ASSERT(num > 0 && num <= 8);
725
726 do {
727 p = get_affine_portal();
728 local_irq_save(irqflags);
729 avail = bm_rcr_get_avail(&p->p);
730 if (avail < 2)
731 update_rcr_ci(p, avail);
732 r = bm_rcr_start(&p->p);
733 local_irq_restore(irqflags);
734 put_affine_portal();
735 if (likely(r))
736 break;
737
738 udelay(1);
739 } while (--timeout);
740
741 if (unlikely(!timeout))
742 return -ETIMEDOUT;
743
744 p = get_affine_portal();
745 local_irq_save(irqflags);
746 /*
747 * we can copy all but the first entry, as this can trigger badness
748 * with the valid-bit
749 */
750 bm_buffer_set64(r->bufs, bm_buffer_get64(bufs));
751 bm_buffer_set_bpid(r->bufs, pool->bpid);
752 if (i)
753 memcpy(&r->bufs[1], &bufs[1], i * sizeof(bufs[0]));
754
755 bm_rcr_pvb_commit(&p->p, BM_RCR_VERB_CMD_BPID_SINGLE |
756 (num & BM_RCR_VERB_BUFCOUNT_MASK));
757
758 local_irq_restore(irqflags);
759 put_affine_portal();
760 return 0;
761}
762EXPORT_SYMBOL(bman_release);
763
764int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num)
765{
766 struct bman_portal *p = get_affine_portal();
767 struct bm_mc_command *mcc;
768 union bm_mc_result *mcr;
769 int ret;
770
771 DPAA_ASSERT(num > 0 && num <= 8);
772
773 mcc = bm_mc_start(&p->p);
774 mcc->bpid = pool->bpid;
775 bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE |
776 (num & BM_MCC_VERB_ACQUIRE_BUFCOUNT));
777 if (!bm_mc_result_timeout(&p->p, &mcr)) {
778 put_affine_portal();
779 pr_crit("BMan Acquire Timeout\n");
780 return -ETIMEDOUT;
781 }
782 ret = mcr->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT;
783 if (bufs)
784 memcpy(&bufs[0], &mcr->bufs[0], num * sizeof(bufs[0]));
785
786 put_affine_portal();
787 if (ret != num)
788 ret = -ENOMEM;
789 return ret;
790}
791EXPORT_SYMBOL(bman_acquire);
792
793const struct bm_portal_config *
794bman_get_bm_portal_config(const struct bman_portal *portal)
795{
796 return portal->config;
797}
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-27 01:47:50 -0500