1 | /* |
2 | * x86_64/AVX/AES-NI assembler implementation of Camellia |
3 | * |
4 | * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> |
5 | * |
6 | * This program is free software; you can redistribute it and/or modify |
7 | * it under the terms of the GNU General Public License as published by |
8 | * the Free Software Foundation; either version 2 of the License, or |
9 | * (at your option) any later version. |
10 | * |
11 | */ |
12 | |
13 | /* |
14 | * Version licensed under 2-clause BSD License is available at: |
15 | * http://koti.mbnet.fi/axh/crypto/camellia-BSD-1.2.0-aesni1.tar.xz |
16 | */ |
17 | |
18 | #include <linux/linkage.h> |
19 | #include <asm/frame.h> |
20 | |
21 | #define CAMELLIA_TABLE_BYTE_LEN 272 |
22 | |
23 | /* struct camellia_ctx: */ |
24 | #define key_table 0 |
25 | #define key_length CAMELLIA_TABLE_BYTE_LEN |
26 | |
27 | /* register macros */ |
28 | #define CTX %rdi |
29 | |
30 | /********************************************************************** |
31 | 16-way camellia |
32 | **********************************************************************/ |
33 | #define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \ |
34 | vpand x, mask4bit, tmp0; \ |
35 | vpandn x, mask4bit, x; \ |
36 | vpsrld $4, x, x; \ |
37 | \ |
38 | vpshufb tmp0, lo_t, tmp0; \ |
39 | vpshufb x, hi_t, x; \ |
40 | vpxor tmp0, x, x; |
41 | |
42 | /* |
43 | * IN: |
44 | * x0..x7: byte-sliced AB state |
45 | * mem_cd: register pointer storing CD state |
46 | * key: index for key material |
47 | * OUT: |
48 | * x0..x7: new byte-sliced CD state |
49 | */ |
50 | #define roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, t0, t1, t2, t3, t4, t5, t6, \ |
51 | t7, mem_cd, key) \ |
52 | /* \ |
53 | * S-function with AES subbytes \ |
54 | */ \ |
55 | vmovdqa .Linv_shift_row(%rip), t4; \ |
56 | vbroadcastss .L0f0f0f0f(%rip), t7; \ |
57 | vmovdqa .Lpre_tf_lo_s1(%rip), t0; \ |
58 | vmovdqa .Lpre_tf_hi_s1(%rip), t1; \ |
59 | \ |
60 | /* AES inverse shift rows */ \ |
61 | vpshufb t4, x0, x0; \ |
62 | vpshufb t4, x7, x7; \ |
63 | vpshufb t4, x1, x1; \ |
64 | vpshufb t4, x4, x4; \ |
65 | vpshufb t4, x2, x2; \ |
66 | vpshufb t4, x5, x5; \ |
67 | vpshufb t4, x3, x3; \ |
68 | vpshufb t4, x6, x6; \ |
69 | \ |
70 | /* prefilter sboxes 1, 2 and 3 */ \ |
71 | vmovdqa .Lpre_tf_lo_s4(%rip), t2; \ |
72 | vmovdqa .Lpre_tf_hi_s4(%rip), t3; \ |
73 | filter_8bit(x0, t0, t1, t7, t6); \ |
74 | filter_8bit(x7, t0, t1, t7, t6); \ |
75 | filter_8bit(x1, t0, t1, t7, t6); \ |
76 | filter_8bit(x4, t0, t1, t7, t6); \ |
77 | filter_8bit(x2, t0, t1, t7, t6); \ |
78 | filter_8bit(x5, t0, t1, t7, t6); \ |
79 | \ |
80 | /* prefilter sbox 4 */ \ |
81 | vpxor t4, t4, t4; \ |
82 | filter_8bit(x3, t2, t3, t7, t6); \ |
83 | filter_8bit(x6, t2, t3, t7, t6); \ |
84 | \ |
85 | /* AES subbytes + AES shift rows */ \ |
86 | vmovdqa .Lpost_tf_lo_s1(%rip), t0; \ |
87 | vmovdqa .Lpost_tf_hi_s1(%rip), t1; \ |
88 | vaesenclast t4, x0, x0; \ |
89 | vaesenclast t4, x7, x7; \ |
90 | vaesenclast t4, x1, x1; \ |
91 | vaesenclast t4, x4, x4; \ |
92 | vaesenclast t4, x2, x2; \ |
93 | vaesenclast t4, x5, x5; \ |
94 | vaesenclast t4, x3, x3; \ |
95 | vaesenclast t4, x6, x6; \ |
96 | \ |
97 | /* postfilter sboxes 1 and 4 */ \ |
98 | vmovdqa .Lpost_tf_lo_s3(%rip), t2; \ |
99 | vmovdqa .Lpost_tf_hi_s3(%rip), t3; \ |
100 | filter_8bit(x0, t0, t1, t7, t6); \ |
101 | filter_8bit(x7, t0, t1, t7, t6); \ |
102 | filter_8bit(x3, t0, t1, t7, t6); \ |
103 | filter_8bit(x6, t0, t1, t7, t6); \ |
104 | \ |
105 | /* postfilter sbox 3 */ \ |
106 | vmovdqa .Lpost_tf_lo_s2(%rip), t4; \ |
107 | vmovdqa .Lpost_tf_hi_s2(%rip), t5; \ |
108 | filter_8bit(x2, t2, t3, t7, t6); \ |
109 | filter_8bit(x5, t2, t3, t7, t6); \ |
110 | \ |
111 | vpxor t6, t6, t6; \ |
112 | vmovq key, t0; \ |
113 | \ |
114 | /* postfilter sbox 2 */ \ |
115 | filter_8bit(x1, t4, t5, t7, t2); \ |
116 | filter_8bit(x4, t4, t5, t7, t2); \ |
117 | \ |
118 | vpsrldq $5, t0, t5; \ |
119 | vpsrldq $1, t0, t1; \ |
120 | vpsrldq $2, t0, t2; \ |
121 | vpsrldq $3, t0, t3; \ |
122 | vpsrldq $4, t0, t4; \ |
123 | vpshufb t6, t0, t0; \ |
124 | vpshufb t6, t1, t1; \ |
125 | vpshufb t6, t2, t2; \ |
126 | vpshufb t6, t3, t3; \ |
127 | vpshufb t6, t4, t4; \ |
128 | vpsrldq $2, t5, t7; \ |
129 | vpshufb t6, t7, t7; \ |
130 | \ |
131 | /* \ |
132 | * P-function \ |
133 | */ \ |
134 | vpxor x5, x0, x0; \ |
135 | vpxor x6, x1, x1; \ |
136 | vpxor x7, x2, x2; \ |
137 | vpxor x4, x3, x3; \ |
138 | \ |
139 | vpxor x2, x4, x4; \ |
140 | vpxor x3, x5, x5; \ |
141 | vpxor x0, x6, x6; \ |
142 | vpxor x1, x7, x7; \ |
143 | \ |
144 | vpxor x7, x0, x0; \ |
145 | vpxor x4, x1, x1; \ |
146 | vpxor x5, x2, x2; \ |
147 | vpxor x6, x3, x3; \ |
148 | \ |
149 | vpxor x3, x4, x4; \ |
150 | vpxor x0, x5, x5; \ |
151 | vpxor x1, x6, x6; \ |
152 | vpxor x2, x7, x7; /* note: high and low parts swapped */ \ |
153 | \ |
154 | /* \ |
155 | * Add key material and result to CD (x becomes new CD) \ |
156 | */ \ |
157 | \ |
158 | vpxor t3, x4, x4; \ |
159 | vpxor 0 * 16(mem_cd), x4, x4; \ |
160 | \ |
161 | vpxor t2, x5, x5; \ |
162 | vpxor 1 * 16(mem_cd), x5, x5; \ |
163 | \ |
164 | vpsrldq $1, t5, t3; \ |
165 | vpshufb t6, t5, t5; \ |
166 | vpshufb t6, t3, t6; \ |
167 | \ |
168 | vpxor t1, x6, x6; \ |
169 | vpxor 2 * 16(mem_cd), x6, x6; \ |
170 | \ |
171 | vpxor t0, x7, x7; \ |
172 | vpxor 3 * 16(mem_cd), x7, x7; \ |
173 | \ |
174 | vpxor t7, x0, x0; \ |
175 | vpxor 4 * 16(mem_cd), x0, x0; \ |
176 | \ |
177 | vpxor t6, x1, x1; \ |
178 | vpxor 5 * 16(mem_cd), x1, x1; \ |
179 | \ |
180 | vpxor t5, x2, x2; \ |
181 | vpxor 6 * 16(mem_cd), x2, x2; \ |
182 | \ |
183 | vpxor t4, x3, x3; \ |
184 | vpxor 7 * 16(mem_cd), x3, x3; |
185 | |
186 | /* |
187 | * Size optimization... with inlined roundsm16, binary would be over 5 times |
188 | * larger and would only be 0.5% faster (on sandy-bridge). |
189 | */ |
190 | .align 8 |
191 | SYM_FUNC_START_LOCAL(roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd) |
192 | roundsm16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
193 | %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15, |
194 | %rcx, (%r9)); |
195 | RET; |
196 | SYM_FUNC_END(roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd) |
197 | |
198 | .align 8 |
199 | SYM_FUNC_START_LOCAL(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) |
200 | roundsm16(%xmm4, %xmm5, %xmm6, %xmm7, %xmm0, %xmm1, %xmm2, %xmm3, |
201 | %xmm12, %xmm13, %xmm14, %xmm15, %xmm8, %xmm9, %xmm10, %xmm11, |
202 | %rax, (%r9)); |
203 | RET; |
204 | SYM_FUNC_END(roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) |
205 | |
206 | /* |
207 | * IN/OUT: |
208 | * x0..x7: byte-sliced AB state preloaded |
209 | * mem_ab: byte-sliced AB state in memory |
210 | * mem_cb: byte-sliced CD state in memory |
211 | */ |
212 | #define two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ |
213 | y6, y7, mem_ab, mem_cd, i, dir, store_ab) \ |
214 | leaq (key_table + (i) * 8)(CTX), %r9; \ |
215 | call roundsm16_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd; \ |
216 | \ |
217 | vmovdqu x4, 0 * 16(mem_cd); \ |
218 | vmovdqu x5, 1 * 16(mem_cd); \ |
219 | vmovdqu x6, 2 * 16(mem_cd); \ |
220 | vmovdqu x7, 3 * 16(mem_cd); \ |
221 | vmovdqu x0, 4 * 16(mem_cd); \ |
222 | vmovdqu x1, 5 * 16(mem_cd); \ |
223 | vmovdqu x2, 6 * 16(mem_cd); \ |
224 | vmovdqu x3, 7 * 16(mem_cd); \ |
225 | \ |
226 | leaq (key_table + ((i) + (dir)) * 8)(CTX), %r9; \ |
227 | call roundsm16_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab; \ |
228 | \ |
229 | store_ab(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab); |
230 | |
231 | #define dummy_store(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab) /* do nothing */ |
232 | |
233 | #define store_ab_state(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab) \ |
234 | /* Store new AB state */ \ |
235 | vmovdqu x0, 0 * 16(mem_ab); \ |
236 | vmovdqu x1, 1 * 16(mem_ab); \ |
237 | vmovdqu x2, 2 * 16(mem_ab); \ |
238 | vmovdqu x3, 3 * 16(mem_ab); \ |
239 | vmovdqu x4, 4 * 16(mem_ab); \ |
240 | vmovdqu x5, 5 * 16(mem_ab); \ |
241 | vmovdqu x6, 6 * 16(mem_ab); \ |
242 | vmovdqu x7, 7 * 16(mem_ab); |
243 | |
244 | #define enc_rounds16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ |
245 | y6, y7, mem_ab, mem_cd, i) \ |
246 | two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ |
247 | y6, y7, mem_ab, mem_cd, (i) + 2, 1, store_ab_state); \ |
248 | two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ |
249 | y6, y7, mem_ab, mem_cd, (i) + 4, 1, store_ab_state); \ |
250 | two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ |
251 | y6, y7, mem_ab, mem_cd, (i) + 6, 1, dummy_store); |
252 | |
253 | #define dec_rounds16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ |
254 | y6, y7, mem_ab, mem_cd, i) \ |
255 | two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ |
256 | y6, y7, mem_ab, mem_cd, (i) + 7, -1, store_ab_state); \ |
257 | two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ |
258 | y6, y7, mem_ab, mem_cd, (i) + 5, -1, store_ab_state); \ |
259 | two_roundsm16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ |
260 | y6, y7, mem_ab, mem_cd, (i) + 3, -1, dummy_store); |
261 | |
262 | /* |
263 | * IN: |
264 | * v0..3: byte-sliced 32-bit integers |
265 | * OUT: |
266 | * v0..3: (IN <<< 1) |
267 | */ |
268 | #define rol32_1_16(v0, v1, v2, v3, t0, t1, t2, zero) \ |
269 | vpcmpgtb v0, zero, t0; \ |
270 | vpaddb v0, v0, v0; \ |
271 | vpabsb t0, t0; \ |
272 | \ |
273 | vpcmpgtb v1, zero, t1; \ |
274 | vpaddb v1, v1, v1; \ |
275 | vpabsb t1, t1; \ |
276 | \ |
277 | vpcmpgtb v2, zero, t2; \ |
278 | vpaddb v2, v2, v2; \ |
279 | vpabsb t2, t2; \ |
280 | \ |
281 | vpor t0, v1, v1; \ |
282 | \ |
283 | vpcmpgtb v3, zero, t0; \ |
284 | vpaddb v3, v3, v3; \ |
285 | vpabsb t0, t0; \ |
286 | \ |
287 | vpor t1, v2, v2; \ |
288 | vpor t2, v3, v3; \ |
289 | vpor t0, v0, v0; |
290 | |
291 | /* |
292 | * IN: |
293 | * r: byte-sliced AB state in memory |
294 | * l: byte-sliced CD state in memory |
295 | * OUT: |
296 | * x0..x7: new byte-sliced CD state |
297 | */ |
298 | #define fls16(l, l0, l1, l2, l3, l4, l5, l6, l7, r, t0, t1, t2, t3, tt0, \ |
299 | tt1, tt2, tt3, kll, klr, krl, krr) \ |
300 | /* \ |
301 | * t0 = kll; \ |
302 | * t0 &= ll; \ |
303 | * lr ^= rol32(t0, 1); \ |
304 | */ \ |
305 | vpxor tt0, tt0, tt0; \ |
306 | vmovd kll, t0; \ |
307 | vpshufb tt0, t0, t3; \ |
308 | vpsrldq $1, t0, t0; \ |
309 | vpshufb tt0, t0, t2; \ |
310 | vpsrldq $1, t0, t0; \ |
311 | vpshufb tt0, t0, t1; \ |
312 | vpsrldq $1, t0, t0; \ |
313 | vpshufb tt0, t0, t0; \ |
314 | \ |
315 | vpand l0, t0, t0; \ |
316 | vpand l1, t1, t1; \ |
317 | vpand l2, t2, t2; \ |
318 | vpand l3, t3, t3; \ |
319 | \ |
320 | rol32_1_16(t3, t2, t1, t0, tt1, tt2, tt3, tt0); \ |
321 | \ |
322 | vpxor l4, t0, l4; \ |
323 | vmovdqu l4, 4 * 16(l); \ |
324 | vpxor l5, t1, l5; \ |
325 | vmovdqu l5, 5 * 16(l); \ |
326 | vpxor l6, t2, l6; \ |
327 | vmovdqu l6, 6 * 16(l); \ |
328 | vpxor l7, t3, l7; \ |
329 | vmovdqu l7, 7 * 16(l); \ |
330 | \ |
331 | /* \ |
332 | * t2 = krr; \ |
333 | * t2 |= rr; \ |
334 | * rl ^= t2; \ |
335 | */ \ |
336 | \ |
337 | vmovd krr, t0; \ |
338 | vpshufb tt0, t0, t3; \ |
339 | vpsrldq $1, t0, t0; \ |
340 | vpshufb tt0, t0, t2; \ |
341 | vpsrldq $1, t0, t0; \ |
342 | vpshufb tt0, t0, t1; \ |
343 | vpsrldq $1, t0, t0; \ |
344 | vpshufb tt0, t0, t0; \ |
345 | \ |
346 | vpor 4 * 16(r), t0, t0; \ |
347 | vpor 5 * 16(r), t1, t1; \ |
348 | vpor 6 * 16(r), t2, t2; \ |
349 | vpor 7 * 16(r), t3, t3; \ |
350 | \ |
351 | vpxor 0 * 16(r), t0, t0; \ |
352 | vpxor 1 * 16(r), t1, t1; \ |
353 | vpxor 2 * 16(r), t2, t2; \ |
354 | vpxor 3 * 16(r), t3, t3; \ |
355 | vmovdqu t0, 0 * 16(r); \ |
356 | vmovdqu t1, 1 * 16(r); \ |
357 | vmovdqu t2, 2 * 16(r); \ |
358 | vmovdqu t3, 3 * 16(r); \ |
359 | \ |
360 | /* \ |
361 | * t2 = krl; \ |
362 | * t2 &= rl; \ |
363 | * rr ^= rol32(t2, 1); \ |
364 | */ \ |
365 | vmovd krl, t0; \ |
366 | vpshufb tt0, t0, t3; \ |
367 | vpsrldq $1, t0, t0; \ |
368 | vpshufb tt0, t0, t2; \ |
369 | vpsrldq $1, t0, t0; \ |
370 | vpshufb tt0, t0, t1; \ |
371 | vpsrldq $1, t0, t0; \ |
372 | vpshufb tt0, t0, t0; \ |
373 | \ |
374 | vpand 0 * 16(r), t0, t0; \ |
375 | vpand 1 * 16(r), t1, t1; \ |
376 | vpand 2 * 16(r), t2, t2; \ |
377 | vpand 3 * 16(r), t3, t3; \ |
378 | \ |
379 | rol32_1_16(t3, t2, t1, t0, tt1, tt2, tt3, tt0); \ |
380 | \ |
381 | vpxor 4 * 16(r), t0, t0; \ |
382 | vpxor 5 * 16(r), t1, t1; \ |
383 | vpxor 6 * 16(r), t2, t2; \ |
384 | vpxor 7 * 16(r), t3, t3; \ |
385 | vmovdqu t0, 4 * 16(r); \ |
386 | vmovdqu t1, 5 * 16(r); \ |
387 | vmovdqu t2, 6 * 16(r); \ |
388 | vmovdqu t3, 7 * 16(r); \ |
389 | \ |
390 | /* \ |
391 | * t0 = klr; \ |
392 | * t0 |= lr; \ |
393 | * ll ^= t0; \ |
394 | */ \ |
395 | \ |
396 | vmovd klr, t0; \ |
397 | vpshufb tt0, t0, t3; \ |
398 | vpsrldq $1, t0, t0; \ |
399 | vpshufb tt0, t0, t2; \ |
400 | vpsrldq $1, t0, t0; \ |
401 | vpshufb tt0, t0, t1; \ |
402 | vpsrldq $1, t0, t0; \ |
403 | vpshufb tt0, t0, t0; \ |
404 | \ |
405 | vpor l4, t0, t0; \ |
406 | vpor l5, t1, t1; \ |
407 | vpor l6, t2, t2; \ |
408 | vpor l7, t3, t3; \ |
409 | \ |
410 | vpxor l0, t0, l0; \ |
411 | vmovdqu l0, 0 * 16(l); \ |
412 | vpxor l1, t1, l1; \ |
413 | vmovdqu l1, 1 * 16(l); \ |
414 | vpxor l2, t2, l2; \ |
415 | vmovdqu l2, 2 * 16(l); \ |
416 | vpxor l3, t3, l3; \ |
417 | vmovdqu l3, 3 * 16(l); |
418 | |
419 | #define transpose_4x4(x0, x1, x2, x3, t1, t2) \ |
420 | vpunpckhdq x1, x0, t2; \ |
421 | vpunpckldq x1, x0, x0; \ |
422 | \ |
423 | vpunpckldq x3, x2, t1; \ |
424 | vpunpckhdq x3, x2, x2; \ |
425 | \ |
426 | vpunpckhqdq t1, x0, x1; \ |
427 | vpunpcklqdq t1, x0, x0; \ |
428 | \ |
429 | vpunpckhqdq x2, t2, x3; \ |
430 | vpunpcklqdq x2, t2, x2; |
431 | |
432 | #define byteslice_16x16b(a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, a3, \ |
433 | b3, c3, d3, st0, st1) \ |
434 | vmovdqu d2, st0; \ |
435 | vmovdqu d3, st1; \ |
436 | transpose_4x4(a0, a1, a2, a3, d2, d3); \ |
437 | transpose_4x4(b0, b1, b2, b3, d2, d3); \ |
438 | vmovdqu st0, d2; \ |
439 | vmovdqu st1, d3; \ |
440 | \ |
441 | vmovdqu a0, st0; \ |
442 | vmovdqu a1, st1; \ |
443 | transpose_4x4(c0, c1, c2, c3, a0, a1); \ |
444 | transpose_4x4(d0, d1, d2, d3, a0, a1); \ |
445 | \ |
446 | vmovdqu .Lshufb_16x16b(%rip), a0; \ |
447 | vmovdqu st1, a1; \ |
448 | vpshufb a0, a2, a2; \ |
449 | vpshufb a0, a3, a3; \ |
450 | vpshufb a0, b0, b0; \ |
451 | vpshufb a0, b1, b1; \ |
452 | vpshufb a0, b2, b2; \ |
453 | vpshufb a0, b3, b3; \ |
454 | vpshufb a0, a1, a1; \ |
455 | vpshufb a0, c0, c0; \ |
456 | vpshufb a0, c1, c1; \ |
457 | vpshufb a0, c2, c2; \ |
458 | vpshufb a0, c3, c3; \ |
459 | vpshufb a0, d0, d0; \ |
460 | vpshufb a0, d1, d1; \ |
461 | vpshufb a0, d2, d2; \ |
462 | vpshufb a0, d3, d3; \ |
463 | vmovdqu d3, st1; \ |
464 | vmovdqu st0, d3; \ |
465 | vpshufb a0, d3, a0; \ |
466 | vmovdqu d2, st0; \ |
467 | \ |
468 | transpose_4x4(a0, b0, c0, d0, d2, d3); \ |
469 | transpose_4x4(a1, b1, c1, d1, d2, d3); \ |
470 | vmovdqu st0, d2; \ |
471 | vmovdqu st1, d3; \ |
472 | \ |
473 | vmovdqu b0, st0; \ |
474 | vmovdqu b1, st1; \ |
475 | transpose_4x4(a2, b2, c2, d2, b0, b1); \ |
476 | transpose_4x4(a3, b3, c3, d3, b0, b1); \ |
477 | vmovdqu st0, b0; \ |
478 | vmovdqu st1, b1; \ |
479 | /* does not adjust output bytes inside vectors */ |
480 | |
481 | /* load blocks to registers and apply pre-whitening */ |
482 | #define inpack16_pre(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ |
483 | y6, y7, rio, key) \ |
484 | vmovq key, x0; \ |
485 | vpshufb .Lpack_bswap(%rip), x0, x0; \ |
486 | \ |
487 | vpxor 0 * 16(rio), x0, y7; \ |
488 | vpxor 1 * 16(rio), x0, y6; \ |
489 | vpxor 2 * 16(rio), x0, y5; \ |
490 | vpxor 3 * 16(rio), x0, y4; \ |
491 | vpxor 4 * 16(rio), x0, y3; \ |
492 | vpxor 5 * 16(rio), x0, y2; \ |
493 | vpxor 6 * 16(rio), x0, y1; \ |
494 | vpxor 7 * 16(rio), x0, y0; \ |
495 | vpxor 8 * 16(rio), x0, x7; \ |
496 | vpxor 9 * 16(rio), x0, x6; \ |
497 | vpxor 10 * 16(rio), x0, x5; \ |
498 | vpxor 11 * 16(rio), x0, x4; \ |
499 | vpxor 12 * 16(rio), x0, x3; \ |
500 | vpxor 13 * 16(rio), x0, x2; \ |
501 | vpxor 14 * 16(rio), x0, x1; \ |
502 | vpxor 15 * 16(rio), x0, x0; |
503 | |
504 | /* byteslice pre-whitened blocks and store to temporary memory */ |
505 | #define inpack16_post(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ |
506 | y6, y7, mem_ab, mem_cd) \ |
507 | byteslice_16x16b(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, \ |
508 | y5, y6, y7, (mem_ab), (mem_cd)); \ |
509 | \ |
510 | vmovdqu x0, 0 * 16(mem_ab); \ |
511 | vmovdqu x1, 1 * 16(mem_ab); \ |
512 | vmovdqu x2, 2 * 16(mem_ab); \ |
513 | vmovdqu x3, 3 * 16(mem_ab); \ |
514 | vmovdqu x4, 4 * 16(mem_ab); \ |
515 | vmovdqu x5, 5 * 16(mem_ab); \ |
516 | vmovdqu x6, 6 * 16(mem_ab); \ |
517 | vmovdqu x7, 7 * 16(mem_ab); \ |
518 | vmovdqu y0, 0 * 16(mem_cd); \ |
519 | vmovdqu y1, 1 * 16(mem_cd); \ |
520 | vmovdqu y2, 2 * 16(mem_cd); \ |
521 | vmovdqu y3, 3 * 16(mem_cd); \ |
522 | vmovdqu y4, 4 * 16(mem_cd); \ |
523 | vmovdqu y5, 5 * 16(mem_cd); \ |
524 | vmovdqu y6, 6 * 16(mem_cd); \ |
525 | vmovdqu y7, 7 * 16(mem_cd); |
526 | |
527 | /* de-byteslice, apply post-whitening and store blocks */ |
528 | #define outunpack16(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, \ |
529 | y5, y6, y7, key, stack_tmp0, stack_tmp1) \ |
530 | byteslice_16x16b(y0, y4, x0, x4, y1, y5, x1, x5, y2, y6, x2, x6, y3, \ |
531 | y7, x3, x7, stack_tmp0, stack_tmp1); \ |
532 | \ |
533 | vmovdqu x0, stack_tmp0; \ |
534 | \ |
535 | vmovq key, x0; \ |
536 | vpshufb .Lpack_bswap(%rip), x0, x0; \ |
537 | \ |
538 | vpxor x0, y7, y7; \ |
539 | vpxor x0, y6, y6; \ |
540 | vpxor x0, y5, y5; \ |
541 | vpxor x0, y4, y4; \ |
542 | vpxor x0, y3, y3; \ |
543 | vpxor x0, y2, y2; \ |
544 | vpxor x0, y1, y1; \ |
545 | vpxor x0, y0, y0; \ |
546 | vpxor x0, x7, x7; \ |
547 | vpxor x0, x6, x6; \ |
548 | vpxor x0, x5, x5; \ |
549 | vpxor x0, x4, x4; \ |
550 | vpxor x0, x3, x3; \ |
551 | vpxor x0, x2, x2; \ |
552 | vpxor x0, x1, x1; \ |
553 | vpxor stack_tmp0, x0, x0; |
554 | |
555 | #define write_output(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ |
556 | y6, y7, rio) \ |
557 | vmovdqu x0, 0 * 16(rio); \ |
558 | vmovdqu x1, 1 * 16(rio); \ |
559 | vmovdqu x2, 2 * 16(rio); \ |
560 | vmovdqu x3, 3 * 16(rio); \ |
561 | vmovdqu x4, 4 * 16(rio); \ |
562 | vmovdqu x5, 5 * 16(rio); \ |
563 | vmovdqu x6, 6 * 16(rio); \ |
564 | vmovdqu x7, 7 * 16(rio); \ |
565 | vmovdqu y0, 8 * 16(rio); \ |
566 | vmovdqu y1, 9 * 16(rio); \ |
567 | vmovdqu y2, 10 * 16(rio); \ |
568 | vmovdqu y3, 11 * 16(rio); \ |
569 | vmovdqu y4, 12 * 16(rio); \ |
570 | vmovdqu y5, 13 * 16(rio); \ |
571 | vmovdqu y6, 14 * 16(rio); \ |
572 | vmovdqu y7, 15 * 16(rio); |
573 | |
574 | |
575 | /* NB: section is mergeable, all elements must be aligned 16-byte blocks */ |
576 | .section .rodata.cst16, "aM" , @progbits, 16 |
577 | .align 16 |
578 | |
579 | #define SHUFB_BYTES(idx) \ |
580 | 0 + (idx), 4 + (idx), 8 + (idx), 12 + (idx) |
581 | |
582 | .Lshufb_16x16b: |
583 | .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3); |
584 | |
585 | .Lpack_bswap: |
586 | .long 0x00010203 |
587 | .long 0x04050607 |
588 | .long 0x80808080 |
589 | .long 0x80808080 |
590 | |
591 | /* |
592 | * pre-SubByte transform |
593 | * |
594 | * pre-lookup for sbox1, sbox2, sbox3: |
595 | * swap_bitendianness( |
596 | * isom_map_camellia_to_aes( |
597 | * camellia_f( |
598 | * swap_bitendianess(in) |
599 | * ) |
600 | * ) |
601 | * ) |
602 | * |
603 | * (note: '⊕ 0xc5' inside camellia_f()) |
604 | */ |
605 | .Lpre_tf_lo_s1: |
606 | .byte 0x45, 0xe8, 0x40, 0xed, 0x2e, 0x83, 0x2b, 0x86 |
607 | .byte 0x4b, 0xe6, 0x4e, 0xe3, 0x20, 0x8d, 0x25, 0x88 |
608 | .Lpre_tf_hi_s1: |
609 | .byte 0x00, 0x51, 0xf1, 0xa0, 0x8a, 0xdb, 0x7b, 0x2a |
610 | .byte 0x09, 0x58, 0xf8, 0xa9, 0x83, 0xd2, 0x72, 0x23 |
611 | |
612 | /* |
613 | * pre-SubByte transform |
614 | * |
615 | * pre-lookup for sbox4: |
616 | * swap_bitendianness( |
617 | * isom_map_camellia_to_aes( |
618 | * camellia_f( |
619 | * swap_bitendianess(in <<< 1) |
620 | * ) |
621 | * ) |
622 | * ) |
623 | * |
624 | * (note: '⊕ 0xc5' inside camellia_f()) |
625 | */ |
626 | .Lpre_tf_lo_s4: |
627 | .byte 0x45, 0x40, 0x2e, 0x2b, 0x4b, 0x4e, 0x20, 0x25 |
628 | .byte 0x14, 0x11, 0x7f, 0x7a, 0x1a, 0x1f, 0x71, 0x74 |
629 | .Lpre_tf_hi_s4: |
630 | .byte 0x00, 0xf1, 0x8a, 0x7b, 0x09, 0xf8, 0x83, 0x72 |
631 | .byte 0xad, 0x5c, 0x27, 0xd6, 0xa4, 0x55, 0x2e, 0xdf |
632 | |
633 | /* |
634 | * post-SubByte transform |
635 | * |
636 | * post-lookup for sbox1, sbox4: |
637 | * swap_bitendianness( |
638 | * camellia_h( |
639 | * isom_map_aes_to_camellia( |
640 | * swap_bitendianness( |
641 | * aes_inverse_affine_transform(in) |
642 | * ) |
643 | * ) |
644 | * ) |
645 | * ) |
646 | * |
647 | * (note: '⊕ 0x6e' inside camellia_h()) |
648 | */ |
649 | .Lpost_tf_lo_s1: |
650 | .byte 0x3c, 0xcc, 0xcf, 0x3f, 0x32, 0xc2, 0xc1, 0x31 |
651 | .byte 0xdc, 0x2c, 0x2f, 0xdf, 0xd2, 0x22, 0x21, 0xd1 |
652 | .Lpost_tf_hi_s1: |
653 | .byte 0x00, 0xf9, 0x86, 0x7f, 0xd7, 0x2e, 0x51, 0xa8 |
654 | .byte 0xa4, 0x5d, 0x22, 0xdb, 0x73, 0x8a, 0xf5, 0x0c |
655 | |
656 | /* |
657 | * post-SubByte transform |
658 | * |
659 | * post-lookup for sbox2: |
660 | * swap_bitendianness( |
661 | * camellia_h( |
662 | * isom_map_aes_to_camellia( |
663 | * swap_bitendianness( |
664 | * aes_inverse_affine_transform(in) |
665 | * ) |
666 | * ) |
667 | * ) |
668 | * ) <<< 1 |
669 | * |
670 | * (note: '⊕ 0x6e' inside camellia_h()) |
671 | */ |
672 | .Lpost_tf_lo_s2: |
673 | .byte 0x78, 0x99, 0x9f, 0x7e, 0x64, 0x85, 0x83, 0x62 |
674 | .byte 0xb9, 0x58, 0x5e, 0xbf, 0xa5, 0x44, 0x42, 0xa3 |
675 | .Lpost_tf_hi_s2: |
676 | .byte 0x00, 0xf3, 0x0d, 0xfe, 0xaf, 0x5c, 0xa2, 0x51 |
677 | .byte 0x49, 0xba, 0x44, 0xb7, 0xe6, 0x15, 0xeb, 0x18 |
678 | |
679 | /* |
680 | * post-SubByte transform |
681 | * |
682 | * post-lookup for sbox3: |
683 | * swap_bitendianness( |
684 | * camellia_h( |
685 | * isom_map_aes_to_camellia( |
686 | * swap_bitendianness( |
687 | * aes_inverse_affine_transform(in) |
688 | * ) |
689 | * ) |
690 | * ) |
691 | * ) >>> 1 |
692 | * |
693 | * (note: '⊕ 0x6e' inside camellia_h()) |
694 | */ |
695 | .Lpost_tf_lo_s3: |
696 | .byte 0x1e, 0x66, 0xe7, 0x9f, 0x19, 0x61, 0xe0, 0x98 |
697 | .byte 0x6e, 0x16, 0x97, 0xef, 0x69, 0x11, 0x90, 0xe8 |
698 | .Lpost_tf_hi_s3: |
699 | .byte 0x00, 0xfc, 0x43, 0xbf, 0xeb, 0x17, 0xa8, 0x54 |
700 | .byte 0x52, 0xae, 0x11, 0xed, 0xb9, 0x45, 0xfa, 0x06 |
701 | |
702 | /* For isolating SubBytes from AESENCLAST, inverse shift row */ |
703 | .Linv_shift_row: |
704 | .byte 0x00, 0x0d, 0x0a, 0x07, 0x04, 0x01, 0x0e, 0x0b |
705 | .byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03 |
706 | |
707 | /* 4-bit mask */ |
708 | .section .rodata.cst4.L0f0f0f0f, "aM" , @progbits, 4 |
709 | .align 4 |
710 | .L0f0f0f0f: |
711 | .long 0x0f0f0f0f |
712 | |
713 | .text |
714 | |
715 | SYM_FUNC_START_LOCAL(__camellia_enc_blk16) |
716 | /* input: |
717 | * %rdi: ctx, CTX |
718 | * %rax: temporary storage, 256 bytes |
719 | * %xmm0..%xmm15: 16 plaintext blocks |
720 | * output: |
721 | * %xmm0..%xmm15: 16 encrypted blocks, order swapped: |
722 | * 7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8 |
723 | */ |
724 | FRAME_BEGIN |
725 | |
726 | leaq 8 * 16(%rax), %rcx; |
727 | |
728 | inpack16_post(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
729 | %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
730 | %xmm15, %rax, %rcx); |
731 | |
732 | enc_rounds16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
733 | %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
734 | %xmm15, %rax, %rcx, 0); |
735 | |
736 | fls16(%rax, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
737 | %rcx, %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
738 | %xmm15, |
739 | ((key_table + (8) * 8) + 0)(CTX), |
740 | ((key_table + (8) * 8) + 4)(CTX), |
741 | ((key_table + (8) * 8) + 8)(CTX), |
742 | ((key_table + (8) * 8) + 12)(CTX)); |
743 | |
744 | enc_rounds16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
745 | %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
746 | %xmm15, %rax, %rcx, 8); |
747 | |
748 | fls16(%rax, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
749 | %rcx, %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
750 | %xmm15, |
751 | ((key_table + (16) * 8) + 0)(CTX), |
752 | ((key_table + (16) * 8) + 4)(CTX), |
753 | ((key_table + (16) * 8) + 8)(CTX), |
754 | ((key_table + (16) * 8) + 12)(CTX)); |
755 | |
756 | enc_rounds16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
757 | %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
758 | %xmm15, %rax, %rcx, 16); |
759 | |
760 | movl $24, %r8d; |
761 | cmpl $16, key_length(CTX); |
762 | jne .Lenc_max32; |
763 | |
764 | .Lenc_done: |
765 | /* load CD for output */ |
766 | vmovdqu 0 * 16(%rcx), %xmm8; |
767 | vmovdqu 1 * 16(%rcx), %xmm9; |
768 | vmovdqu 2 * 16(%rcx), %xmm10; |
769 | vmovdqu 3 * 16(%rcx), %xmm11; |
770 | vmovdqu 4 * 16(%rcx), %xmm12; |
771 | vmovdqu 5 * 16(%rcx), %xmm13; |
772 | vmovdqu 6 * 16(%rcx), %xmm14; |
773 | vmovdqu 7 * 16(%rcx), %xmm15; |
774 | |
775 | outunpack16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
776 | %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
777 | %xmm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 16(%rax)); |
778 | |
779 | FRAME_END |
780 | RET; |
781 | |
782 | .align 8 |
783 | .Lenc_max32: |
784 | movl $32, %r8d; |
785 | |
786 | fls16(%rax, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
787 | %rcx, %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
788 | %xmm15, |
789 | ((key_table + (24) * 8) + 0)(CTX), |
790 | ((key_table + (24) * 8) + 4)(CTX), |
791 | ((key_table + (24) * 8) + 8)(CTX), |
792 | ((key_table + (24) * 8) + 12)(CTX)); |
793 | |
794 | enc_rounds16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
795 | %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
796 | %xmm15, %rax, %rcx, 24); |
797 | |
798 | jmp .Lenc_done; |
799 | SYM_FUNC_END(__camellia_enc_blk16) |
800 | |
801 | SYM_FUNC_START_LOCAL(__camellia_dec_blk16) |
802 | /* input: |
803 | * %rdi: ctx, CTX |
804 | * %rax: temporary storage, 256 bytes |
805 | * %r8d: 24 for 16 byte key, 32 for larger |
806 | * %xmm0..%xmm15: 16 encrypted blocks |
807 | * output: |
808 | * %xmm0..%xmm15: 16 plaintext blocks, order swapped: |
809 | * 7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8 |
810 | */ |
811 | FRAME_BEGIN |
812 | |
813 | leaq 8 * 16(%rax), %rcx; |
814 | |
815 | inpack16_post(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
816 | %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
817 | %xmm15, %rax, %rcx); |
818 | |
819 | cmpl $32, %r8d; |
820 | je .Ldec_max32; |
821 | |
822 | .Ldec_max24: |
823 | dec_rounds16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
824 | %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
825 | %xmm15, %rax, %rcx, 16); |
826 | |
827 | fls16(%rax, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
828 | %rcx, %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
829 | %xmm15, |
830 | ((key_table + (16) * 8) + 8)(CTX), |
831 | ((key_table + (16) * 8) + 12)(CTX), |
832 | ((key_table + (16) * 8) + 0)(CTX), |
833 | ((key_table + (16) * 8) + 4)(CTX)); |
834 | |
835 | dec_rounds16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
836 | %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
837 | %xmm15, %rax, %rcx, 8); |
838 | |
839 | fls16(%rax, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
840 | %rcx, %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
841 | %xmm15, |
842 | ((key_table + (8) * 8) + 8)(CTX), |
843 | ((key_table + (8) * 8) + 12)(CTX), |
844 | ((key_table + (8) * 8) + 0)(CTX), |
845 | ((key_table + (8) * 8) + 4)(CTX)); |
846 | |
847 | dec_rounds16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
848 | %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
849 | %xmm15, %rax, %rcx, 0); |
850 | |
851 | /* load CD for output */ |
852 | vmovdqu 0 * 16(%rcx), %xmm8; |
853 | vmovdqu 1 * 16(%rcx), %xmm9; |
854 | vmovdqu 2 * 16(%rcx), %xmm10; |
855 | vmovdqu 3 * 16(%rcx), %xmm11; |
856 | vmovdqu 4 * 16(%rcx), %xmm12; |
857 | vmovdqu 5 * 16(%rcx), %xmm13; |
858 | vmovdqu 6 * 16(%rcx), %xmm14; |
859 | vmovdqu 7 * 16(%rcx), %xmm15; |
860 | |
861 | outunpack16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
862 | %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
863 | %xmm15, (key_table)(CTX), (%rax), 1 * 16(%rax)); |
864 | |
865 | FRAME_END |
866 | RET; |
867 | |
868 | .align 8 |
869 | .Ldec_max32: |
870 | dec_rounds16(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
871 | %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
872 | %xmm15, %rax, %rcx, 24); |
873 | |
874 | fls16(%rax, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
875 | %rcx, %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
876 | %xmm15, |
877 | ((key_table + (24) * 8) + 8)(CTX), |
878 | ((key_table + (24) * 8) + 12)(CTX), |
879 | ((key_table + (24) * 8) + 0)(CTX), |
880 | ((key_table + (24) * 8) + 4)(CTX)); |
881 | |
882 | jmp .Ldec_max24; |
883 | SYM_FUNC_END(__camellia_dec_blk16) |
884 | |
885 | SYM_FUNC_START(camellia_ecb_enc_16way) |
886 | /* input: |
887 | * %rdi: ctx, CTX |
888 | * %rsi: dst (16 blocks) |
889 | * %rdx: src (16 blocks) |
890 | */ |
891 | FRAME_BEGIN |
892 | |
893 | inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
894 | %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
895 | %xmm15, %rdx, (key_table)(CTX)); |
896 | |
897 | /* now dst can be used as temporary buffer (even in src == dst case) */ |
898 | movq %rsi, %rax; |
899 | |
900 | call __camellia_enc_blk16; |
901 | |
902 | write_output(%xmm7, %xmm6, %xmm5, %xmm4, %xmm3, %xmm2, %xmm1, %xmm0, |
903 | %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9, |
904 | %xmm8, %rsi); |
905 | |
906 | FRAME_END |
907 | RET; |
908 | SYM_FUNC_END(camellia_ecb_enc_16way) |
909 | |
910 | SYM_FUNC_START(camellia_ecb_dec_16way) |
911 | /* input: |
912 | * %rdi: ctx, CTX |
913 | * %rsi: dst (16 blocks) |
914 | * %rdx: src (16 blocks) |
915 | */ |
916 | FRAME_BEGIN |
917 | |
918 | cmpl $16, key_length(CTX); |
919 | movl $32, %r8d; |
920 | movl $24, %eax; |
921 | cmovel %eax, %r8d; /* max */ |
922 | |
923 | inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
924 | %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
925 | %xmm15, %rdx, (key_table)(CTX, %r8, 8)); |
926 | |
927 | /* now dst can be used as temporary buffer (even in src == dst case) */ |
928 | movq %rsi, %rax; |
929 | |
930 | call __camellia_dec_blk16; |
931 | |
932 | write_output(%xmm7, %xmm6, %xmm5, %xmm4, %xmm3, %xmm2, %xmm1, %xmm0, |
933 | %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9, |
934 | %xmm8, %rsi); |
935 | |
936 | FRAME_END |
937 | RET; |
938 | SYM_FUNC_END(camellia_ecb_dec_16way) |
939 | |
940 | SYM_FUNC_START(camellia_cbc_dec_16way) |
941 | /* input: |
942 | * %rdi: ctx, CTX |
943 | * %rsi: dst (16 blocks) |
944 | * %rdx: src (16 blocks) |
945 | */ |
946 | FRAME_BEGIN |
947 | |
948 | cmpl $16, key_length(CTX); |
949 | movl $32, %r8d; |
950 | movl $24, %eax; |
951 | cmovel %eax, %r8d; /* max */ |
952 | |
953 | inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, |
954 | %xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, |
955 | %xmm15, %rdx, (key_table)(CTX, %r8, 8)); |
956 | |
957 | /* |
958 | * dst might still be in-use (in case dst == src), so use stack for |
959 | * temporary storage. |
960 | */ |
961 | subq $(16 * 16), %rsp; |
962 | movq %rsp, %rax; |
963 | |
964 | call __camellia_dec_blk16; |
965 | |
966 | addq $(16 * 16), %rsp; |
967 | |
968 | vpxor (0 * 16)(%rdx), %xmm6, %xmm6; |
969 | vpxor (1 * 16)(%rdx), %xmm5, %xmm5; |
970 | vpxor (2 * 16)(%rdx), %xmm4, %xmm4; |
971 | vpxor (3 * 16)(%rdx), %xmm3, %xmm3; |
972 | vpxor (4 * 16)(%rdx), %xmm2, %xmm2; |
973 | vpxor (5 * 16)(%rdx), %xmm1, %xmm1; |
974 | vpxor (6 * 16)(%rdx), %xmm0, %xmm0; |
975 | vpxor (7 * 16)(%rdx), %xmm15, %xmm15; |
976 | vpxor (8 * 16)(%rdx), %xmm14, %xmm14; |
977 | vpxor (9 * 16)(%rdx), %xmm13, %xmm13; |
978 | vpxor (10 * 16)(%rdx), %xmm12, %xmm12; |
979 | vpxor (11 * 16)(%rdx), %xmm11, %xmm11; |
980 | vpxor (12 * 16)(%rdx), %xmm10, %xmm10; |
981 | vpxor (13 * 16)(%rdx), %xmm9, %xmm9; |
982 | vpxor (14 * 16)(%rdx), %xmm8, %xmm8; |
983 | write_output(%xmm7, %xmm6, %xmm5, %xmm4, %xmm3, %xmm2, %xmm1, %xmm0, |
984 | %xmm15, %xmm14, %xmm13, %xmm12, %xmm11, %xmm10, %xmm9, |
985 | %xmm8, %rsi); |
986 | |
987 | FRAME_END |
988 | RET; |
989 | SYM_FUNC_END(camellia_cbc_dec_16way) |
990 | |