1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 2017 Marvell |
4 | * |
5 | * Antoine Tenart <antoine.tenart@free-electrons.com> |
6 | */ |
7 | |
8 | #include <linux/dma-mapping.h> |
9 | #include <linux/spinlock.h> |
10 | |
11 | #include "safexcel.h" |
12 | |
13 | int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv, |
14 | struct safexcel_desc_ring *cdr, |
15 | struct safexcel_desc_ring *rdr) |
16 | { |
17 | int i; |
18 | struct safexcel_command_desc *cdesc; |
19 | dma_addr_t atok; |
20 | |
21 | /* Actual command descriptor ring */ |
22 | cdr->offset = priv->config.cd_offset; |
23 | cdr->base = dmam_alloc_coherent(dev: priv->dev, |
24 | size: cdr->offset * EIP197_DEFAULT_RING_SIZE, |
25 | dma_handle: &cdr->base_dma, GFP_KERNEL); |
26 | if (!cdr->base) |
27 | return -ENOMEM; |
28 | cdr->write = cdr->base; |
29 | cdr->base_end = cdr->base + cdr->offset * (EIP197_DEFAULT_RING_SIZE - 1); |
30 | cdr->read = cdr->base; |
31 | |
32 | /* Command descriptor shadow ring for storing additional token data */ |
33 | cdr->shoffset = priv->config.cdsh_offset; |
34 | cdr->shbase = dmam_alloc_coherent(dev: priv->dev, |
35 | size: cdr->shoffset * |
36 | EIP197_DEFAULT_RING_SIZE, |
37 | dma_handle: &cdr->shbase_dma, GFP_KERNEL); |
38 | if (!cdr->shbase) |
39 | return -ENOMEM; |
40 | cdr->shwrite = cdr->shbase; |
41 | cdr->shbase_end = cdr->shbase + cdr->shoffset * |
42 | (EIP197_DEFAULT_RING_SIZE - 1); |
43 | |
44 | /* |
45 | * Populate command descriptors with physical pointers to shadow descs. |
46 | * Note that we only need to do this once if we don't overwrite them. |
47 | */ |
48 | cdesc = cdr->base; |
49 | atok = cdr->shbase_dma; |
50 | for (i = 0; i < EIP197_DEFAULT_RING_SIZE; i++) { |
51 | cdesc->atok_lo = lower_32_bits(atok); |
52 | cdesc->atok_hi = upper_32_bits(atok); |
53 | cdesc = (void *)cdesc + cdr->offset; |
54 | atok += cdr->shoffset; |
55 | } |
56 | |
57 | rdr->offset = priv->config.rd_offset; |
58 | /* Use shoffset for result token offset here */ |
59 | rdr->shoffset = priv->config.res_offset; |
60 | rdr->base = dmam_alloc_coherent(dev: priv->dev, |
61 | size: rdr->offset * EIP197_DEFAULT_RING_SIZE, |
62 | dma_handle: &rdr->base_dma, GFP_KERNEL); |
63 | if (!rdr->base) |
64 | return -ENOMEM; |
65 | rdr->write = rdr->base; |
66 | rdr->base_end = rdr->base + rdr->offset * (EIP197_DEFAULT_RING_SIZE - 1); |
67 | rdr->read = rdr->base; |
68 | |
69 | return 0; |
70 | } |
71 | |
72 | inline int safexcel_select_ring(struct safexcel_crypto_priv *priv) |
73 | { |
74 | return (atomic_inc_return(v: &priv->ring_used) % priv->config.rings); |
75 | } |
76 | |
77 | static void *safexcel_ring_next_cwptr(struct safexcel_crypto_priv *priv, |
78 | struct safexcel_desc_ring *ring, |
79 | bool first, |
80 | struct safexcel_token **atoken) |
81 | { |
82 | void *ptr = ring->write; |
83 | |
84 | if (first) |
85 | *atoken = ring->shwrite; |
86 | |
87 | if ((ring->write == ring->read - ring->offset) || |
88 | (ring->read == ring->base && ring->write == ring->base_end)) |
89 | return ERR_PTR(error: -ENOMEM); |
90 | |
91 | if (ring->write == ring->base_end) { |
92 | ring->write = ring->base; |
93 | ring->shwrite = ring->shbase; |
94 | } else { |
95 | ring->write += ring->offset; |
96 | ring->shwrite += ring->shoffset; |
97 | } |
98 | |
99 | return ptr; |
100 | } |
101 | |
102 | static void *safexcel_ring_next_rwptr(struct safexcel_crypto_priv *priv, |
103 | struct safexcel_desc_ring *ring, |
104 | struct result_data_desc **rtoken) |
105 | { |
106 | void *ptr = ring->write; |
107 | |
108 | /* Result token at relative offset shoffset */ |
109 | *rtoken = ring->write + ring->shoffset; |
110 | |
111 | if ((ring->write == ring->read - ring->offset) || |
112 | (ring->read == ring->base && ring->write == ring->base_end)) |
113 | return ERR_PTR(error: -ENOMEM); |
114 | |
115 | if (ring->write == ring->base_end) |
116 | ring->write = ring->base; |
117 | else |
118 | ring->write += ring->offset; |
119 | |
120 | return ptr; |
121 | } |
122 | |
123 | void *safexcel_ring_next_rptr(struct safexcel_crypto_priv *priv, |
124 | struct safexcel_desc_ring *ring) |
125 | { |
126 | void *ptr = ring->read; |
127 | |
128 | if (ring->write == ring->read) |
129 | return ERR_PTR(error: -ENOENT); |
130 | |
131 | if (ring->read == ring->base_end) |
132 | ring->read = ring->base; |
133 | else |
134 | ring->read += ring->offset; |
135 | |
136 | return ptr; |
137 | } |
138 | |
139 | inline void *safexcel_ring_curr_rptr(struct safexcel_crypto_priv *priv, |
140 | int ring) |
141 | { |
142 | struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr; |
143 | |
144 | return rdr->read; |
145 | } |
146 | |
147 | inline int safexcel_ring_first_rdr_index(struct safexcel_crypto_priv *priv, |
148 | int ring) |
149 | { |
150 | struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr; |
151 | |
152 | return (rdr->read - rdr->base) / rdr->offset; |
153 | } |
154 | |
155 | inline int safexcel_ring_rdr_rdesc_index(struct safexcel_crypto_priv *priv, |
156 | int ring, |
157 | struct safexcel_result_desc *rdesc) |
158 | { |
159 | struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr; |
160 | |
161 | return ((void *)rdesc - rdr->base) / rdr->offset; |
162 | } |
163 | |
164 | void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv, |
165 | struct safexcel_desc_ring *ring) |
166 | { |
167 | if (ring->write == ring->read) |
168 | return; |
169 | |
170 | if (ring->write == ring->base) { |
171 | ring->write = ring->base_end; |
172 | ring->shwrite = ring->shbase_end; |
173 | } else { |
174 | ring->write -= ring->offset; |
175 | ring->shwrite -= ring->shoffset; |
176 | } |
177 | } |
178 | |
179 | struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv, |
180 | int ring_id, |
181 | bool first, bool last, |
182 | dma_addr_t data, u32 data_len, |
183 | u32 full_data_len, |
184 | dma_addr_t context, |
185 | struct safexcel_token **atoken) |
186 | { |
187 | struct safexcel_command_desc *cdesc; |
188 | |
189 | cdesc = safexcel_ring_next_cwptr(priv, ring: &priv->ring[ring_id].cdr, |
190 | first, atoken); |
191 | if (IS_ERR(ptr: cdesc)) |
192 | return cdesc; |
193 | |
194 | cdesc->particle_size = data_len; |
195 | cdesc->rsvd0 = 0; |
196 | cdesc->last_seg = last; |
197 | cdesc->first_seg = first; |
198 | cdesc->additional_cdata_size = 0; |
199 | cdesc->rsvd1 = 0; |
200 | cdesc->data_lo = lower_32_bits(data); |
201 | cdesc->data_hi = upper_32_bits(data); |
202 | |
203 | if (first) { |
204 | /* |
205 | * Note that the length here MUST be >0 or else the EIP(1)97 |
206 | * may hang. Newer EIP197 firmware actually incorporates this |
207 | * fix already, but that doesn't help the EIP97 and we may |
208 | * also be running older firmware. |
209 | */ |
210 | cdesc->control_data.packet_length = full_data_len ?: 1; |
211 | cdesc->control_data.options = EIP197_OPTION_MAGIC_VALUE | |
212 | EIP197_OPTION_64BIT_CTX | |
213 | EIP197_OPTION_CTX_CTRL_IN_CMD | |
214 | EIP197_OPTION_RC_AUTO; |
215 | cdesc->control_data.type = EIP197_TYPE_BCLA; |
216 | cdesc->control_data.context_lo = lower_32_bits(context) | |
217 | EIP197_CONTEXT_SMALL; |
218 | cdesc->control_data.context_hi = upper_32_bits(context); |
219 | } |
220 | |
221 | return cdesc; |
222 | } |
223 | |
224 | struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *priv, |
225 | int ring_id, |
226 | bool first, bool last, |
227 | dma_addr_t data, u32 len) |
228 | { |
229 | struct safexcel_result_desc *rdesc; |
230 | struct result_data_desc *rtoken; |
231 | |
232 | rdesc = safexcel_ring_next_rwptr(priv, ring: &priv->ring[ring_id].rdr, |
233 | rtoken: &rtoken); |
234 | if (IS_ERR(ptr: rdesc)) |
235 | return rdesc; |
236 | |
237 | rdesc->particle_size = len; |
238 | rdesc->rsvd0 = 0; |
239 | rdesc->descriptor_overflow = 1; /* assume error */ |
240 | rdesc->buffer_overflow = 1; /* assume error */ |
241 | rdesc->last_seg = last; |
242 | rdesc->first_seg = first; |
243 | rdesc->result_size = EIP197_RD64_RESULT_SIZE; |
244 | rdesc->rsvd1 = 0; |
245 | rdesc->data_lo = lower_32_bits(data); |
246 | rdesc->data_hi = upper_32_bits(data); |
247 | |
248 | /* Clear length in result token */ |
249 | rtoken->packet_length = 0; |
250 | /* Assume errors - HW will clear if not the case */ |
251 | rtoken->error_code = 0x7fff; |
252 | |
253 | return rdesc; |
254 | } |
255 | |