1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright 2008-2010 Cisco Systems, Inc. All rights reserved. |
4 | * Copyright 2007 Nuova Systems, Inc. All rights reserved. |
5 | */ |
6 | |
7 | #include <linux/kernel.h> |
8 | #include <linux/errno.h> |
9 | #include <linux/types.h> |
10 | #include <linux/pci.h> |
11 | #include <linux/delay.h> |
12 | #include <linux/slab.h> |
13 | |
14 | #include "vnic_dev.h" |
15 | #include "vnic_rq.h" |
16 | #include "enic.h" |
17 | |
18 | static int vnic_rq_alloc_bufs(struct vnic_rq *rq) |
19 | { |
20 | struct vnic_rq_buf *buf; |
21 | unsigned int i, j, count = rq->ring.desc_count; |
22 | unsigned int blks = VNIC_RQ_BUF_BLKS_NEEDED(count); |
23 | |
24 | for (i = 0; i < blks; i++) { |
25 | rq->bufs[i] = kzalloc(VNIC_RQ_BUF_BLK_SZ(count), GFP_KERNEL); |
26 | if (!rq->bufs[i]) |
27 | return -ENOMEM; |
28 | } |
29 | |
30 | for (i = 0; i < blks; i++) { |
31 | buf = rq->bufs[i]; |
32 | for (j = 0; j < VNIC_RQ_BUF_BLK_ENTRIES(count); j++) { |
33 | buf->index = i * VNIC_RQ_BUF_BLK_ENTRIES(count) + j; |
34 | buf->desc = (u8 *)rq->ring.descs + |
35 | rq->ring.desc_size * buf->index; |
36 | if (buf->index + 1 == count) { |
37 | buf->next = rq->bufs[0]; |
38 | break; |
39 | } else if (j + 1 == VNIC_RQ_BUF_BLK_ENTRIES(count)) { |
40 | buf->next = rq->bufs[i + 1]; |
41 | } else { |
42 | buf->next = buf + 1; |
43 | buf++; |
44 | } |
45 | } |
46 | } |
47 | |
48 | rq->to_use = rq->to_clean = rq->bufs[0]; |
49 | |
50 | return 0; |
51 | } |
52 | |
53 | void vnic_rq_free(struct vnic_rq *rq) |
54 | { |
55 | struct vnic_dev *vdev; |
56 | unsigned int i; |
57 | |
58 | vdev = rq->vdev; |
59 | |
60 | vnic_dev_free_desc_ring(vdev, ring: &rq->ring); |
61 | |
62 | for (i = 0; i < VNIC_RQ_BUF_BLKS_MAX; i++) { |
63 | if (rq->bufs[i]) { |
64 | kfree(objp: rq->bufs[i]); |
65 | rq->bufs[i] = NULL; |
66 | } |
67 | } |
68 | |
69 | rq->ctrl = NULL; |
70 | } |
71 | |
72 | int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, |
73 | unsigned int desc_count, unsigned int desc_size) |
74 | { |
75 | int err; |
76 | |
77 | rq->index = index; |
78 | rq->vdev = vdev; |
79 | |
80 | rq->ctrl = vnic_dev_get_res(vdev, type: RES_TYPE_RQ, index); |
81 | if (!rq->ctrl) { |
82 | vdev_err(vdev, "Failed to hook RQ[%d] resource\n" , index); |
83 | return -EINVAL; |
84 | } |
85 | |
86 | vnic_rq_disable(rq); |
87 | |
88 | err = vnic_dev_alloc_desc_ring(vdev, ring: &rq->ring, desc_count, desc_size); |
89 | if (err) |
90 | return err; |
91 | |
92 | err = vnic_rq_alloc_bufs(rq); |
93 | if (err) { |
94 | vnic_rq_free(rq); |
95 | return err; |
96 | } |
97 | |
98 | return 0; |
99 | } |
100 | |
101 | static void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, |
102 | unsigned int fetch_index, unsigned int posted_index, |
103 | unsigned int error_interrupt_enable, |
104 | unsigned int error_interrupt_offset) |
105 | { |
106 | u64 paddr; |
107 | unsigned int count = rq->ring.desc_count; |
108 | |
109 | paddr = (u64)rq->ring.base_addr | VNIC_PADDR_TARGET; |
110 | writeq(val: paddr, addr: &rq->ctrl->ring_base); |
111 | iowrite32(count, &rq->ctrl->ring_size); |
112 | iowrite32(cq_index, &rq->ctrl->cq_index); |
113 | iowrite32(error_interrupt_enable, &rq->ctrl->error_interrupt_enable); |
114 | iowrite32(error_interrupt_offset, &rq->ctrl->error_interrupt_offset); |
115 | iowrite32(0, &rq->ctrl->dropped_packet_count); |
116 | iowrite32(0, &rq->ctrl->error_status); |
117 | iowrite32(fetch_index, &rq->ctrl->fetch_index); |
118 | iowrite32(posted_index, &rq->ctrl->posted_index); |
119 | |
120 | rq->to_use = rq->to_clean = |
121 | &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)] |
122 | [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)]; |
123 | } |
124 | |
125 | void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, |
126 | unsigned int error_interrupt_enable, |
127 | unsigned int error_interrupt_offset) |
128 | { |
129 | vnic_rq_init_start(rq, cq_index, fetch_index: 0, posted_index: 0, error_interrupt_enable, |
130 | error_interrupt_offset); |
131 | } |
132 | |
133 | unsigned int vnic_rq_error_status(struct vnic_rq *rq) |
134 | { |
135 | return ioread32(&rq->ctrl->error_status); |
136 | } |
137 | |
138 | void vnic_rq_enable(struct vnic_rq *rq) |
139 | { |
140 | iowrite32(1, &rq->ctrl->enable); |
141 | } |
142 | |
143 | int vnic_rq_disable(struct vnic_rq *rq) |
144 | { |
145 | unsigned int wait; |
146 | struct vnic_dev *vdev = rq->vdev; |
147 | int i; |
148 | |
149 | /* Due to a race condition with clearing RQ "mini-cache" in hw, we need |
150 | * to disable the RQ twice to guarantee that stale descriptors are not |
151 | * used when this RQ is re-enabled. |
152 | */ |
153 | for (i = 0; i < 2; i++) { |
154 | iowrite32(0, &rq->ctrl->enable); |
155 | |
156 | /* Wait for HW to ACK disable request */ |
157 | for (wait = 20000; wait > 0; wait--) |
158 | if (!ioread32(&rq->ctrl->running)) |
159 | break; |
160 | if (!wait) { |
161 | vdev_neterr(vdev, "Failed to disable RQ[%d]\n" , |
162 | rq->index); |
163 | |
164 | return -ETIMEDOUT; |
165 | } |
166 | } |
167 | |
168 | return 0; |
169 | } |
170 | |
171 | void vnic_rq_clean(struct vnic_rq *rq, |
172 | void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf)) |
173 | { |
174 | struct vnic_rq_buf *buf; |
175 | u32 fetch_index; |
176 | unsigned int count = rq->ring.desc_count; |
177 | int i; |
178 | |
179 | buf = rq->to_clean; |
180 | |
181 | for (i = 0; i < rq->ring.desc_count; i++) { |
182 | (*buf_clean)(rq, buf); |
183 | buf = buf->next; |
184 | } |
185 | rq->ring.desc_avail = rq->ring.desc_count - 1; |
186 | |
187 | /* Use current fetch_index as the ring starting point */ |
188 | fetch_index = ioread32(&rq->ctrl->fetch_index); |
189 | |
190 | if (fetch_index == 0xFFFFFFFF) { /* check for hardware gone */ |
191 | /* Hardware surprise removal: reset fetch_index */ |
192 | fetch_index = 0; |
193 | } |
194 | rq->to_use = rq->to_clean = |
195 | &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)] |
196 | [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)]; |
197 | iowrite32(fetch_index, &rq->ctrl->posted_index); |
198 | |
199 | /* Anytime we write fetch_index, we need to re-write 0 to rq->enable |
200 | * to re-sync internal VIC state. |
201 | */ |
202 | iowrite32(0, &rq->ctrl->enable); |
203 | |
204 | vnic_dev_clear_desc_ring(ring: &rq->ring); |
205 | } |
206 | |