| 1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
|---|
| 2 | # |
|---|
| 3 | # Accelerated AES-GCM stitched implementation for ppc64le. |
|---|
| 4 | # |
|---|
| 5 | # Copyright 2022- IBM Inc. All rights reserved |
|---|
| 6 | # |
|---|
| 7 | #=================================================================================== |
|---|
| 8 | # Written by Danny Tsen <dtsen@linux.ibm.com> |
|---|
| 9 | # |
|---|
| 10 | # GHASH is based on the Karatsuba multiplication method. |
|---|
| 11 | # |
|---|
| 12 | # Xi xor X1 |
|---|
| 13 | # |
|---|
| 14 | # X1 * H^4 + X2 * H^3 + x3 * H^2 + X4 * H = |
|---|
| 15 | # (X1.h * H4.h + xX.l * H4.l + X1 * H4) + |
|---|
| 16 | # (X2.h * H3.h + X2.l * H3.l + X2 * H3) + |
|---|
| 17 | # (X3.h * H2.h + X3.l * H2.l + X3 * H2) + |
|---|
| 18 | # (X4.h * H.h + X4.l * H.l + X4 * H) |
|---|
| 19 | # |
|---|
| 20 | # Xi = v0 |
|---|
| 21 | # H Poly = v2 |
|---|
| 22 | # Hash keys = v3 - v14 |
|---|
| 23 | # ( H.l, H, H.h) |
|---|
| 24 | # ( H^2.l, H^2, H^2.h) |
|---|
| 25 | # ( H^3.l, H^3, H^3.h) |
|---|
| 26 | # ( H^4.l, H^4, H^4.h) |
|---|
| 27 | # |
|---|
| 28 | # v30 is IV |
|---|
| 29 | # v31 - counter 1 |
|---|
| 30 | # |
|---|
| 31 | # AES used, |
|---|
| 32 | # vs0 - vs14 for round keys |
|---|
| 33 | # v15, v16, v17, v18, v19, v20, v21, v22 for 8 blocks (encrypted) |
|---|
| 34 | # |
|---|
| 35 | # This implementation uses stitched AES-GCM approach to improve overall performance. |
|---|
| 36 | # AES is implemented with 8x blocks and GHASH is using 2 4x blocks. |
|---|
| 37 | # |
|---|
| 38 | # =================================================================================== |
|---|
| 39 | # |
|---|
| 40 | |
|---|
| 41 | #include <asm/ppc_asm.h> |
|---|
| 42 | #include <linux/linkage.h> |
|---|
| 43 | |
|---|
| 44 | .machine "any" |
|---|
| 45 | .text |
|---|
| 46 | |
|---|
| 47 | # 4x loops |
|---|
| 48 | # v15 - v18 - input states |
|---|
| 49 | # vs1 - vs9 - round keys |
|---|
| 50 | # |
|---|
| 51 | .macro Loop_aes_middle4x |
|---|
| 52 | xxlor 19+32, 1, 1 |
|---|
| 53 | xxlor 20+32, 2, 2 |
|---|
| 54 | xxlor 21+32, 3, 3 |
|---|
| 55 | xxlor 22+32, 4, 4 |
|---|
| 56 | |
|---|
| 57 | vcipher 15, 15, 19 |
|---|
| 58 | vcipher 16, 16, 19 |
|---|
| 59 | vcipher 17, 17, 19 |
|---|
| 60 | vcipher 18, 18, 19 |
|---|
| 61 | |
|---|
| 62 | vcipher 15, 15, 20 |
|---|
| 63 | vcipher 16, 16, 20 |
|---|
| 64 | vcipher 17, 17, 20 |
|---|
| 65 | vcipher 18, 18, 20 |
|---|
| 66 | |
|---|
| 67 | vcipher 15, 15, 21 |
|---|
| 68 | vcipher 16, 16, 21 |
|---|
| 69 | vcipher 17, 17, 21 |
|---|
| 70 | vcipher 18, 18, 21 |
|---|
| 71 | |
|---|
| 72 | vcipher 15, 15, 22 |
|---|
| 73 | vcipher 16, 16, 22 |
|---|
| 74 | vcipher 17, 17, 22 |
|---|
| 75 | vcipher 18, 18, 22 |
|---|
| 76 | |
|---|
| 77 | xxlor 19+32, 5, 5 |
|---|
| 78 | xxlor 20+32, 6, 6 |
|---|
| 79 | xxlor 21+32, 7, 7 |
|---|
| 80 | xxlor 22+32, 8, 8 |
|---|
| 81 | |
|---|
| 82 | vcipher 15, 15, 19 |
|---|
| 83 | vcipher 16, 16, 19 |
|---|
| 84 | vcipher 17, 17, 19 |
|---|
| 85 | vcipher 18, 18, 19 |
|---|
| 86 | |
|---|
| 87 | vcipher 15, 15, 20 |
|---|
| 88 | vcipher 16, 16, 20 |
|---|
| 89 | vcipher 17, 17, 20 |
|---|
| 90 | vcipher 18, 18, 20 |
|---|
| 91 | |
|---|
| 92 | vcipher 15, 15, 21 |
|---|
| 93 | vcipher 16, 16, 21 |
|---|
| 94 | vcipher 17, 17, 21 |
|---|
| 95 | vcipher 18, 18, 21 |
|---|
| 96 | |
|---|
| 97 | vcipher 15, 15, 22 |
|---|
| 98 | vcipher 16, 16, 22 |
|---|
| 99 | vcipher 17, 17, 22 |
|---|
| 100 | vcipher 18, 18, 22 |
|---|
| 101 | |
|---|
| 102 | xxlor 23+32, 9, 9 |
|---|
| 103 | vcipher 15, 15, 23 |
|---|
| 104 | vcipher 16, 16, 23 |
|---|
| 105 | vcipher 17, 17, 23 |
|---|
| 106 | vcipher 18, 18, 23 |
|---|
| 107 | .endm |
|---|
| 108 | |
|---|
| 109 | # 8x loops |
|---|
| 110 | # v15 - v22 - input states |
|---|
| 111 | # vs1 - vs9 - round keys |
|---|
| 112 | # |
|---|
| 113 | .macro Loop_aes_middle8x |
|---|
| 114 | xxlor 23+32, 1, 1 |
|---|
| 115 | xxlor 24+32, 2, 2 |
|---|
| 116 | xxlor 25+32, 3, 3 |
|---|
| 117 | xxlor 26+32, 4, 4 |
|---|
| 118 | |
|---|
| 119 | vcipher 15, 15, 23 |
|---|
| 120 | vcipher 16, 16, 23 |
|---|
| 121 | vcipher 17, 17, 23 |
|---|
| 122 | vcipher 18, 18, 23 |
|---|
| 123 | vcipher 19, 19, 23 |
|---|
| 124 | vcipher 20, 20, 23 |
|---|
| 125 | vcipher 21, 21, 23 |
|---|
| 126 | vcipher 22, 22, 23 |
|---|
| 127 | |
|---|
| 128 | vcipher 15, 15, 24 |
|---|
| 129 | vcipher 16, 16, 24 |
|---|
| 130 | vcipher 17, 17, 24 |
|---|
| 131 | vcipher 18, 18, 24 |
|---|
| 132 | vcipher 19, 19, 24 |
|---|
| 133 | vcipher 20, 20, 24 |
|---|
| 134 | vcipher 21, 21, 24 |
|---|
| 135 | vcipher 22, 22, 24 |
|---|
| 136 | |
|---|
| 137 | vcipher 15, 15, 25 |
|---|
| 138 | vcipher 16, 16, 25 |
|---|
| 139 | vcipher 17, 17, 25 |
|---|
| 140 | vcipher 18, 18, 25 |
|---|
| 141 | vcipher 19, 19, 25 |
|---|
| 142 | vcipher 20, 20, 25 |
|---|
| 143 | vcipher 21, 21, 25 |
|---|
| 144 | vcipher 22, 22, 25 |
|---|
| 145 | |
|---|
| 146 | vcipher 15, 15, 26 |
|---|
| 147 | vcipher 16, 16, 26 |
|---|
| 148 | vcipher 17, 17, 26 |
|---|
| 149 | vcipher 18, 18, 26 |
|---|
| 150 | vcipher 19, 19, 26 |
|---|
| 151 | vcipher 20, 20, 26 |
|---|
| 152 | vcipher 21, 21, 26 |
|---|
| 153 | vcipher 22, 22, 26 |
|---|
| 154 | |
|---|
| 155 | xxlor 23+32, 5, 5 |
|---|
| 156 | xxlor 24+32, 6, 6 |
|---|
| 157 | xxlor 25+32, 7, 7 |
|---|
| 158 | xxlor 26+32, 8, 8 |
|---|
| 159 | |
|---|
| 160 | vcipher 15, 15, 23 |
|---|
| 161 | vcipher 16, 16, 23 |
|---|
| 162 | vcipher 17, 17, 23 |
|---|
| 163 | vcipher 18, 18, 23 |
|---|
| 164 | vcipher 19, 19, 23 |
|---|
| 165 | vcipher 20, 20, 23 |
|---|
| 166 | vcipher 21, 21, 23 |
|---|
| 167 | vcipher 22, 22, 23 |
|---|
| 168 | |
|---|
| 169 | vcipher 15, 15, 24 |
|---|
| 170 | vcipher 16, 16, 24 |
|---|
| 171 | vcipher 17, 17, 24 |
|---|
| 172 | vcipher 18, 18, 24 |
|---|
| 173 | vcipher 19, 19, 24 |
|---|
| 174 | vcipher 20, 20, 24 |
|---|
| 175 | vcipher 21, 21, 24 |
|---|
| 176 | vcipher 22, 22, 24 |
|---|
| 177 | |
|---|
| 178 | vcipher 15, 15, 25 |
|---|
| 179 | vcipher 16, 16, 25 |
|---|
| 180 | vcipher 17, 17, 25 |
|---|
| 181 | vcipher 18, 18, 25 |
|---|
| 182 | vcipher 19, 19, 25 |
|---|
| 183 | vcipher 20, 20, 25 |
|---|
| 184 | vcipher 21, 21, 25 |
|---|
| 185 | vcipher 22, 22, 25 |
|---|
| 186 | |
|---|
| 187 | vcipher 15, 15, 26 |
|---|
| 188 | vcipher 16, 16, 26 |
|---|
| 189 | vcipher 17, 17, 26 |
|---|
| 190 | vcipher 18, 18, 26 |
|---|
| 191 | vcipher 19, 19, 26 |
|---|
| 192 | vcipher 20, 20, 26 |
|---|
| 193 | vcipher 21, 21, 26 |
|---|
| 194 | vcipher 22, 22, 26 |
|---|
| 195 | |
|---|
| 196 | xxlor 23+32, 9, 9 |
|---|
| 197 | vcipher 15, 15, 23 |
|---|
| 198 | vcipher 16, 16, 23 |
|---|
| 199 | vcipher 17, 17, 23 |
|---|
| 200 | vcipher 18, 18, 23 |
|---|
| 201 | vcipher 19, 19, 23 |
|---|
| 202 | vcipher 20, 20, 23 |
|---|
| 203 | vcipher 21, 21, 23 |
|---|
| 204 | vcipher 22, 22, 23 |
|---|
| 205 | .endm |
|---|
| 206 | |
|---|
| 207 | .macro Loop_aes_middle_1x |
|---|
| 208 | xxlor 19+32, 1, 1 |
|---|
| 209 | xxlor 20+32, 2, 2 |
|---|
| 210 | xxlor 21+32, 3, 3 |
|---|
| 211 | xxlor 22+32, 4, 4 |
|---|
| 212 | |
|---|
| 213 | vcipher 15, 15, 19 |
|---|
| 214 | vcipher 15, 15, 20 |
|---|
| 215 | vcipher 15, 15, 21 |
|---|
| 216 | vcipher 15, 15, 22 |
|---|
| 217 | |
|---|
| 218 | xxlor 19+32, 5, 5 |
|---|
| 219 | xxlor 20+32, 6, 6 |
|---|
| 220 | xxlor 21+32, 7, 7 |
|---|
| 221 | xxlor 22+32, 8, 8 |
|---|
| 222 | |
|---|
| 223 | vcipher 15, 15, 19 |
|---|
| 224 | vcipher 15, 15, 20 |
|---|
| 225 | vcipher 15, 15, 21 |
|---|
| 226 | vcipher 15, 15, 22 |
|---|
| 227 | |
|---|
| 228 | xxlor 19+32, 9, 9 |
|---|
| 229 | vcipher 15, 15, 19 |
|---|
| 230 | .endm |
|---|
| 231 | |
|---|
| 232 | # |
|---|
| 233 | # Compute 4x hash values based on Karatsuba method. |
|---|
| 234 | # |
|---|
| 235 | .macro ppc_aes_gcm_ghash |
|---|
| 236 | vxor 15, 15, 0 |
|---|
| 237 | |
|---|
| 238 | vpmsumd 23, 12, 15 # H4.L * X.L |
|---|
| 239 | vpmsumd 24, 9, 16 |
|---|
| 240 | vpmsumd 25, 6, 17 |
|---|
| 241 | vpmsumd 26, 3, 18 |
|---|
| 242 | |
|---|
| 243 | vxor 23, 23, 24 |
|---|
| 244 | vxor 23, 23, 25 |
|---|
| 245 | vxor 23, 23, 26 # L |
|---|
| 246 | |
|---|
| 247 | vpmsumd 24, 13, 15 # H4.L * X.H + H4.H * X.L |
|---|
| 248 | vpmsumd 25, 10, 16 # H3.L * X1.H + H3.H * X1.L |
|---|
| 249 | vpmsumd 26, 7, 17 |
|---|
| 250 | vpmsumd 27, 4, 18 |
|---|
| 251 | |
|---|
| 252 | vxor 24, 24, 25 |
|---|
| 253 | vxor 24, 24, 26 |
|---|
| 254 | vxor 24, 24, 27 # M |
|---|
| 255 | |
|---|
| 256 | # sum hash and reduction with H Poly |
|---|
| 257 | vpmsumd 28, 23, 2 # reduction |
|---|
| 258 | |
|---|
| 259 | vxor 29, 29, 29 |
|---|
| 260 | vsldoi 26, 24, 29, 8 # mL |
|---|
| 261 | vsldoi 29, 29, 24, 8 # mH |
|---|
| 262 | vxor 23, 23, 26 # mL + L |
|---|
| 263 | |
|---|
| 264 | vsldoi 23, 23, 23, 8 # swap |
|---|
| 265 | vxor 23, 23, 28 |
|---|
| 266 | |
|---|
| 267 | vpmsumd 24, 14, 15 # H4.H * X.H |
|---|
| 268 | vpmsumd 25, 11, 16 |
|---|
| 269 | vpmsumd 26, 8, 17 |
|---|
| 270 | vpmsumd 27, 5, 18 |
|---|
| 271 | |
|---|
| 272 | vxor 24, 24, 25 |
|---|
| 273 | vxor 24, 24, 26 |
|---|
| 274 | vxor 24, 24, 27 |
|---|
| 275 | |
|---|
| 276 | vxor 24, 24, 29 |
|---|
| 277 | |
|---|
| 278 | # sum hash and reduction with H Poly |
|---|
| 279 | vsldoi 27, 23, 23, 8 # swap |
|---|
| 280 | vpmsumd 23, 23, 2 |
|---|
| 281 | vxor 27, 27, 24 |
|---|
| 282 | vxor 23, 23, 27 |
|---|
| 283 | |
|---|
| 284 | xxlor 32, 23+32, 23+32 # update hash |
|---|
| 285 | |
|---|
| 286 | .endm |
|---|
| 287 | |
|---|
| 288 | # |
|---|
| 289 | # Combine two 4x ghash |
|---|
| 290 | # v15 - v22 - input blocks |
|---|
| 291 | # |
|---|
| 292 | .macro ppc_aes_gcm_ghash2_4x |
|---|
| 293 | # first 4x hash |
|---|
| 294 | vxor 15, 15, 0 # Xi + X |
|---|
| 295 | |
|---|
| 296 | vpmsumd 23, 12, 15 # H4.L * X.L |
|---|
| 297 | vpmsumd 24, 9, 16 |
|---|
| 298 | vpmsumd 25, 6, 17 |
|---|
| 299 | vpmsumd 26, 3, 18 |
|---|
| 300 | |
|---|
| 301 | vxor 23, 23, 24 |
|---|
| 302 | vxor 23, 23, 25 |
|---|
| 303 | vxor 23, 23, 26 # L |
|---|
| 304 | |
|---|
| 305 | vpmsumd 24, 13, 15 # H4.L * X.H + H4.H * X.L |
|---|
| 306 | vpmsumd 25, 10, 16 # H3.L * X1.H + H3.H * X1.L |
|---|
| 307 | vpmsumd 26, 7, 17 |
|---|
| 308 | vpmsumd 27, 4, 18 |
|---|
| 309 | |
|---|
| 310 | vxor 24, 24, 25 |
|---|
| 311 | vxor 24, 24, 26 |
|---|
| 312 | |
|---|
| 313 | # sum hash and reduction with H Poly |
|---|
| 314 | vpmsumd 28, 23, 2 # reduction |
|---|
| 315 | |
|---|
| 316 | vxor 29, 29, 29 |
|---|
| 317 | |
|---|
| 318 | vxor 24, 24, 27 # M |
|---|
| 319 | vsldoi 26, 24, 29, 8 # mL |
|---|
| 320 | vsldoi 29, 29, 24, 8 # mH |
|---|
| 321 | vxor 23, 23, 26 # mL + L |
|---|
| 322 | |
|---|
| 323 | vsldoi 23, 23, 23, 8 # swap |
|---|
| 324 | vxor 23, 23, 28 |
|---|
| 325 | |
|---|
| 326 | vpmsumd 24, 14, 15 # H4.H * X.H |
|---|
| 327 | vpmsumd 25, 11, 16 |
|---|
| 328 | vpmsumd 26, 8, 17 |
|---|
| 329 | vpmsumd 27, 5, 18 |
|---|
| 330 | |
|---|
| 331 | vxor 24, 24, 25 |
|---|
| 332 | vxor 24, 24, 26 |
|---|
| 333 | vxor 24, 24, 27 # H |
|---|
| 334 | |
|---|
| 335 | vxor 24, 24, 29 # H + mH |
|---|
| 336 | |
|---|
| 337 | # sum hash and reduction with H Poly |
|---|
| 338 | vsldoi 27, 23, 23, 8 # swap |
|---|
| 339 | vpmsumd 23, 23, 2 |
|---|
| 340 | vxor 27, 27, 24 |
|---|
| 341 | vxor 27, 23, 27 # 1st Xi |
|---|
| 342 | |
|---|
| 343 | # 2nd 4x hash |
|---|
| 344 | vpmsumd 24, 9, 20 |
|---|
| 345 | vpmsumd 25, 6, 21 |
|---|
| 346 | vpmsumd 26, 3, 22 |
|---|
| 347 | vxor 19, 19, 27 # Xi + X |
|---|
| 348 | vpmsumd 23, 12, 19 # H4.L * X.L |
|---|
| 349 | |
|---|
| 350 | vxor 23, 23, 24 |
|---|
| 351 | vxor 23, 23, 25 |
|---|
| 352 | vxor 23, 23, 26 # L |
|---|
| 353 | |
|---|
| 354 | vpmsumd 24, 13, 19 # H4.L * X.H + H4.H * X.L |
|---|
| 355 | vpmsumd 25, 10, 20 # H3.L * X1.H + H3.H * X1.L |
|---|
| 356 | vpmsumd 26, 7, 21 |
|---|
| 357 | vpmsumd 27, 4, 22 |
|---|
| 358 | |
|---|
| 359 | vxor 24, 24, 25 |
|---|
| 360 | vxor 24, 24, 26 |
|---|
| 361 | |
|---|
| 362 | # sum hash and reduction with H Poly |
|---|
| 363 | vpmsumd 28, 23, 2 # reduction |
|---|
| 364 | |
|---|
| 365 | vxor 29, 29, 29 |
|---|
| 366 | |
|---|
| 367 | vxor 24, 24, 27 # M |
|---|
| 368 | vsldoi 26, 24, 29, 8 # mL |
|---|
| 369 | vsldoi 29, 29, 24, 8 # mH |
|---|
| 370 | vxor 23, 23, 26 # mL + L |
|---|
| 371 | |
|---|
| 372 | vsldoi 23, 23, 23, 8 # swap |
|---|
| 373 | vxor 23, 23, 28 |
|---|
| 374 | |
|---|
| 375 | vpmsumd 24, 14, 19 # H4.H * X.H |
|---|
| 376 | vpmsumd 25, 11, 20 |
|---|
| 377 | vpmsumd 26, 8, 21 |
|---|
| 378 | vpmsumd 27, 5, 22 |
|---|
| 379 | |
|---|
| 380 | vxor 24, 24, 25 |
|---|
| 381 | vxor 24, 24, 26 |
|---|
| 382 | vxor 24, 24, 27 # H |
|---|
| 383 | |
|---|
| 384 | vxor 24, 24, 29 # H + mH |
|---|
| 385 | |
|---|
| 386 | # sum hash and reduction with H Poly |
|---|
| 387 | vsldoi 27, 23, 23, 8 # swap |
|---|
| 388 | vpmsumd 23, 23, 2 |
|---|
| 389 | vxor 27, 27, 24 |
|---|
| 390 | vxor 23, 23, 27 |
|---|
| 391 | |
|---|
| 392 | xxlor 32, 23+32, 23+32 # update hash |
|---|
| 393 | |
|---|
| 394 | .endm |
|---|
| 395 | |
|---|
| 396 | # |
|---|
| 397 | # Compute update single hash |
|---|
| 398 | # |
|---|
| 399 | .macro ppc_update_hash_1x |
|---|
| 400 | vxor 28, 28, 0 |
|---|
| 401 | |
|---|
| 402 | vxor 19, 19, 19 |
|---|
| 403 | |
|---|
| 404 | vpmsumd 22, 3, 28 # L |
|---|
| 405 | vpmsumd 23, 4, 28 # M |
|---|
| 406 | vpmsumd 24, 5, 28 # H |
|---|
| 407 | |
|---|
| 408 | vpmsumd 27, 22, 2 # reduction |
|---|
| 409 | |
|---|
| 410 | vsldoi 25, 23, 19, 8 # mL |
|---|
| 411 | vsldoi 26, 19, 23, 8 # mH |
|---|
| 412 | vxor 22, 22, 25 # LL + LL |
|---|
| 413 | vxor 24, 24, 26 # HH + HH |
|---|
| 414 | |
|---|
| 415 | vsldoi 22, 22, 22, 8 # swap |
|---|
| 416 | vxor 22, 22, 27 |
|---|
| 417 | |
|---|
| 418 | vsldoi 20, 22, 22, 8 # swap |
|---|
| 419 | vpmsumd 22, 22, 2 # reduction |
|---|
| 420 | vxor 20, 20, 24 |
|---|
| 421 | vxor 22, 22, 20 |
|---|
| 422 | |
|---|
| 423 | vmr 0, 22 # update hash |
|---|
| 424 | |
|---|
| 425 | .endm |
|---|
| 426 | |
|---|
| 427 | .macro SAVE_REGS |
|---|
| 428 | stdu 1,-640(1) |
|---|
| 429 | mflr 0 |
|---|
| 430 | |
|---|
| 431 | std 14,112(1) |
|---|
| 432 | std 15,120(1) |
|---|
| 433 | std 16,128(1) |
|---|
| 434 | std 17,136(1) |
|---|
| 435 | std 18,144(1) |
|---|
| 436 | std 19,152(1) |
|---|
| 437 | std 20,160(1) |
|---|
| 438 | std 21,168(1) |
|---|
| 439 | li 9, 256 |
|---|
| 440 | stvx 20, 9, 1 |
|---|
| 441 | addi 9, 9, 16 |
|---|
| 442 | stvx 21, 9, 1 |
|---|
| 443 | addi 9, 9, 16 |
|---|
| 444 | stvx 22, 9, 1 |
|---|
| 445 | addi 9, 9, 16 |
|---|
| 446 | stvx 23, 9, 1 |
|---|
| 447 | addi 9, 9, 16 |
|---|
| 448 | stvx 24, 9, 1 |
|---|
| 449 | addi 9, 9, 16 |
|---|
| 450 | stvx 25, 9, 1 |
|---|
| 451 | addi 9, 9, 16 |
|---|
| 452 | stvx 26, 9, 1 |
|---|
| 453 | addi 9, 9, 16 |
|---|
| 454 | stvx 27, 9, 1 |
|---|
| 455 | addi 9, 9, 16 |
|---|
| 456 | stvx 28, 9, 1 |
|---|
| 457 | addi 9, 9, 16 |
|---|
| 458 | stvx 29, 9, 1 |
|---|
| 459 | addi 9, 9, 16 |
|---|
| 460 | stvx 30, 9, 1 |
|---|
| 461 | addi 9, 9, 16 |
|---|
| 462 | stvx 31, 9, 1 |
|---|
| 463 | stxv 14, 464(1) |
|---|
| 464 | stxv 15, 480(1) |
|---|
| 465 | stxv 16, 496(1) |
|---|
| 466 | stxv 17, 512(1) |
|---|
| 467 | stxv 18, 528(1) |
|---|
| 468 | stxv 19, 544(1) |
|---|
| 469 | stxv 20, 560(1) |
|---|
| 470 | stxv 21, 576(1) |
|---|
| 471 | stxv 22, 592(1) |
|---|
| 472 | std 0, 656(1) |
|---|
| 473 | .endm |
|---|
| 474 | |
|---|
| 475 | .macro RESTORE_REGS |
|---|
| 476 | lxv 14, 464(1) |
|---|
| 477 | lxv 15, 480(1) |
|---|
| 478 | lxv 16, 496(1) |
|---|
| 479 | lxv 17, 512(1) |
|---|
| 480 | lxv 18, 528(1) |
|---|
| 481 | lxv 19, 544(1) |
|---|
| 482 | lxv 20, 560(1) |
|---|
| 483 | lxv 21, 576(1) |
|---|
| 484 | lxv 22, 592(1) |
|---|
| 485 | li 9, 256 |
|---|
| 486 | lvx 20, 9, 1 |
|---|
| 487 | addi 9, 9, 16 |
|---|
| 488 | lvx 21, 9, 1 |
|---|
| 489 | addi 9, 9, 16 |
|---|
| 490 | lvx 22, 9, 1 |
|---|
| 491 | addi 9, 9, 16 |
|---|
| 492 | lvx 23, 9, 1 |
|---|
| 493 | addi 9, 9, 16 |
|---|
| 494 | lvx 24, 9, 1 |
|---|
| 495 | addi 9, 9, 16 |
|---|
| 496 | lvx 25, 9, 1 |
|---|
| 497 | addi 9, 9, 16 |
|---|
| 498 | lvx 26, 9, 1 |
|---|
| 499 | addi 9, 9, 16 |
|---|
| 500 | lvx 27, 9, 1 |
|---|
| 501 | addi 9, 9, 16 |
|---|
| 502 | lvx 28, 9, 1 |
|---|
| 503 | addi 9, 9, 16 |
|---|
| 504 | lvx 29, 9, 1 |
|---|
| 505 | addi 9, 9, 16 |
|---|
| 506 | lvx 30, 9, 1 |
|---|
| 507 | addi 9, 9, 16 |
|---|
| 508 | lvx 31, 9, 1 |
|---|
| 509 | |
|---|
| 510 | ld 0, 656(1) |
|---|
| 511 | ld 14,112(1) |
|---|
| 512 | ld 15,120(1) |
|---|
| 513 | ld 16,128(1) |
|---|
| 514 | ld 17,136(1) |
|---|
| 515 | ld 18,144(1) |
|---|
| 516 | ld 19,152(1) |
|---|
| 517 | ld 20,160(1) |
|---|
| 518 | ld 21,168(1) |
|---|
| 519 | |
|---|
| 520 | mtlr 0 |
|---|
| 521 | addi 1, 1, 640 |
|---|
| 522 | .endm |
|---|
| 523 | |
|---|
| 524 | .macro LOAD_HASH_TABLE |
|---|
| 525 | # Load Xi |
|---|
| 526 | lxvb16x 32, 0, 8 # load Xi |
|---|
| 527 | |
|---|
| 528 | # load Hash - h^4, h^3, h^2, h |
|---|
| 529 | li 10, 32 |
|---|
| 530 | lxvd2x 2+32, 10, 8 # H Poli |
|---|
| 531 | li 10, 48 |
|---|
| 532 | lxvd2x 3+32, 10, 8 # Hl |
|---|
| 533 | li 10, 64 |
|---|
| 534 | lxvd2x 4+32, 10, 8 # H |
|---|
| 535 | li 10, 80 |
|---|
| 536 | lxvd2x 5+32, 10, 8 # Hh |
|---|
| 537 | |
|---|
| 538 | li 10, 96 |
|---|
| 539 | lxvd2x 6+32, 10, 8 # H^2l |
|---|
| 540 | li 10, 112 |
|---|
| 541 | lxvd2x 7+32, 10, 8 # H^2 |
|---|
| 542 | li 10, 128 |
|---|
| 543 | lxvd2x 8+32, 10, 8 # H^2h |
|---|
| 544 | |
|---|
| 545 | li 10, 144 |
|---|
| 546 | lxvd2x 9+32, 10, 8 # H^3l |
|---|
| 547 | li 10, 160 |
|---|
| 548 | lxvd2x 10+32, 10, 8 # H^3 |
|---|
| 549 | li 10, 176 |
|---|
| 550 | lxvd2x 11+32, 10, 8 # H^3h |
|---|
| 551 | |
|---|
| 552 | li 10, 192 |
|---|
| 553 | lxvd2x 12+32, 10, 8 # H^4l |
|---|
| 554 | li 10, 208 |
|---|
| 555 | lxvd2x 13+32, 10, 8 # H^4 |
|---|
| 556 | li 10, 224 |
|---|
| 557 | lxvd2x 14+32, 10, 8 # H^4h |
|---|
| 558 | .endm |
|---|
| 559 | |
|---|
| 560 | # |
|---|
| 561 | # aes_p10_gcm_encrypt (const void *inp, void *out, size_t len, |
|---|
| 562 | # const char *rk, unsigned char iv[16], void *Xip); |
|---|
| 563 | # |
|---|
| 564 | # r3 - inp |
|---|
| 565 | # r4 - out |
|---|
| 566 | # r5 - len |
|---|
| 567 | # r6 - AES round keys |
|---|
| 568 | # r7 - iv and other data |
|---|
| 569 | # r8 - Xi, HPoli, hash keys |
|---|
| 570 | # |
|---|
| 571 | # rounds is at offset 240 in rk |
|---|
| 572 | # Xi is at 0 in gcm_table (Xip). |
|---|
| 573 | # |
|---|
| 574 | _GLOBAL(aes_p10_gcm_encrypt) |
|---|
| 575 | .align 5 |
|---|
| 576 | |
|---|
| 577 | SAVE_REGS |
|---|
| 578 | |
|---|
| 579 | LOAD_HASH_TABLE |
|---|
| 580 | |
|---|
| 581 | # initialize ICB: GHASH( IV ), IV - r7 |
|---|
| 582 | lxvb16x 30+32, 0, 7 # load IV - v30 |
|---|
| 583 | |
|---|
| 584 | mr 12, 5 # length |
|---|
| 585 | li 11, 0 # block index |
|---|
| 586 | |
|---|
| 587 | # counter 1 |
|---|
| 588 | vxor 31, 31, 31 |
|---|
| 589 | vspltisb 22, 1 |
|---|
| 590 | vsldoi 31, 31, 22,1 # counter 1 |
|---|
| 591 | |
|---|
| 592 | # load round key to VSR |
|---|
| 593 | lxv 0, 0(6) |
|---|
| 594 | lxv 1, 0x10(6) |
|---|
| 595 | lxv 2, 0x20(6) |
|---|
| 596 | lxv 3, 0x30(6) |
|---|
| 597 | lxv 4, 0x40(6) |
|---|
| 598 | lxv 5, 0x50(6) |
|---|
| 599 | lxv 6, 0x60(6) |
|---|
| 600 | lxv 7, 0x70(6) |
|---|
| 601 | lxv 8, 0x80(6) |
|---|
| 602 | lxv 9, 0x90(6) |
|---|
| 603 | lxv 10, 0xa0(6) |
|---|
| 604 | |
|---|
| 605 | # load rounds - 10 (128), 12 (192), 14 (256) |
|---|
| 606 | lwz 9,240(6) |
|---|
| 607 | |
|---|
| 608 | # |
|---|
| 609 | # vxor state, state, w # addroundkey |
|---|
| 610 | xxlor 32+29, 0, 0 |
|---|
| 611 | vxor 15, 30, 29 # IV + round key - add round key 0 |
|---|
| 612 | |
|---|
| 613 | cmpdi 9, 10 |
|---|
| 614 | beq Loop_aes_gcm_8x |
|---|
| 615 | |
|---|
| 616 | # load 2 more round keys (v11, v12) |
|---|
| 617 | lxv 11, 0xb0(6) |
|---|
| 618 | lxv 12, 0xc0(6) |
|---|
| 619 | |
|---|
| 620 | cmpdi 9, 12 |
|---|
| 621 | beq Loop_aes_gcm_8x |
|---|
| 622 | |
|---|
| 623 | # load 2 more round keys (v11, v12, v13, v14) |
|---|
| 624 | lxv 13, 0xd0(6) |
|---|
| 625 | lxv 14, 0xe0(6) |
|---|
| 626 | cmpdi 9, 14 |
|---|
| 627 | beq Loop_aes_gcm_8x |
|---|
| 628 | |
|---|
| 629 | b aes_gcm_out |
|---|
| 630 | |
|---|
| 631 | .align 5 |
|---|
| 632 | Loop_aes_gcm_8x: |
|---|
| 633 | mr 14, 3 |
|---|
| 634 | mr 9, 4 |
|---|
| 635 | |
|---|
| 636 | # |
|---|
| 637 | # check partial block |
|---|
| 638 | # |
|---|
| 639 | Continue_partial_check: |
|---|
| 640 | ld 15, 56(7) |
|---|
| 641 | cmpdi 15, 0 |
|---|
| 642 | beq Continue |
|---|
| 643 | bgt Final_block |
|---|
| 644 | cmpdi 15, 16 |
|---|
| 645 | blt Final_block |
|---|
| 646 | |
|---|
| 647 | Continue: |
|---|
| 648 | # n blcoks |
|---|
| 649 | li 10, 128 |
|---|
| 650 | divdu 10, 12, 10 # n 128 bytes-blocks |
|---|
| 651 | cmpdi 10, 0 |
|---|
| 652 | beq Loop_last_block |
|---|
| 653 | |
|---|
| 654 | vaddudm 30, 30, 31 # IV + counter |
|---|
| 655 | vxor 16, 30, 29 |
|---|
| 656 | vaddudm 30, 30, 31 |
|---|
| 657 | vxor 17, 30, 29 |
|---|
| 658 | vaddudm 30, 30, 31 |
|---|
| 659 | vxor 18, 30, 29 |
|---|
| 660 | vaddudm 30, 30, 31 |
|---|
| 661 | vxor 19, 30, 29 |
|---|
| 662 | vaddudm 30, 30, 31 |
|---|
| 663 | vxor 20, 30, 29 |
|---|
| 664 | vaddudm 30, 30, 31 |
|---|
| 665 | vxor 21, 30, 29 |
|---|
| 666 | vaddudm 30, 30, 31 |
|---|
| 667 | vxor 22, 30, 29 |
|---|
| 668 | |
|---|
| 669 | mtctr 10 |
|---|
| 670 | |
|---|
| 671 | li 15, 16 |
|---|
| 672 | li 16, 32 |
|---|
| 673 | li 17, 48 |
|---|
| 674 | li 18, 64 |
|---|
| 675 | li 19, 80 |
|---|
| 676 | li 20, 96 |
|---|
| 677 | li 21, 112 |
|---|
| 678 | |
|---|
| 679 | lwz 10, 240(6) |
|---|
| 680 | |
|---|
| 681 | Loop_8x_block: |
|---|
| 682 | |
|---|
| 683 | lxvb16x 15, 0, 14 # load block |
|---|
| 684 | lxvb16x 16, 15, 14 # load block |
|---|
| 685 | lxvb16x 17, 16, 14 # load block |
|---|
| 686 | lxvb16x 18, 17, 14 # load block |
|---|
| 687 | lxvb16x 19, 18, 14 # load block |
|---|
| 688 | lxvb16x 20, 19, 14 # load block |
|---|
| 689 | lxvb16x 21, 20, 14 # load block |
|---|
| 690 | lxvb16x 22, 21, 14 # load block |
|---|
| 691 | addi 14, 14, 128 |
|---|
| 692 | |
|---|
| 693 | Loop_aes_middle8x |
|---|
| 694 | |
|---|
| 695 | xxlor 23+32, 10, 10 |
|---|
| 696 | |
|---|
| 697 | cmpdi 10, 10 |
|---|
| 698 | beq Do_next_ghash |
|---|
| 699 | |
|---|
| 700 | # 192 bits |
|---|
| 701 | xxlor 24+32, 11, 11 |
|---|
| 702 | |
|---|
| 703 | vcipher 15, 15, 23 |
|---|
| 704 | vcipher 16, 16, 23 |
|---|
| 705 | vcipher 17, 17, 23 |
|---|
| 706 | vcipher 18, 18, 23 |
|---|
| 707 | vcipher 19, 19, 23 |
|---|
| 708 | vcipher 20, 20, 23 |
|---|
| 709 | vcipher 21, 21, 23 |
|---|
| 710 | vcipher 22, 22, 23 |
|---|
| 711 | |
|---|
| 712 | vcipher 15, 15, 24 |
|---|
| 713 | vcipher 16, 16, 24 |
|---|
| 714 | vcipher 17, 17, 24 |
|---|
| 715 | vcipher 18, 18, 24 |
|---|
| 716 | vcipher 19, 19, 24 |
|---|
| 717 | vcipher 20, 20, 24 |
|---|
| 718 | vcipher 21, 21, 24 |
|---|
| 719 | vcipher 22, 22, 24 |
|---|
| 720 | |
|---|
| 721 | xxlor 23+32, 12, 12 |
|---|
| 722 | |
|---|
| 723 | cmpdi 10, 12 |
|---|
| 724 | beq Do_next_ghash |
|---|
| 725 | |
|---|
| 726 | # 256 bits |
|---|
| 727 | xxlor 24+32, 13, 13 |
|---|
| 728 | |
|---|
| 729 | vcipher 15, 15, 23 |
|---|
| 730 | vcipher 16, 16, 23 |
|---|
| 731 | vcipher 17, 17, 23 |
|---|
| 732 | vcipher 18, 18, 23 |
|---|
| 733 | vcipher 19, 19, 23 |
|---|
| 734 | vcipher 20, 20, 23 |
|---|
| 735 | vcipher 21, 21, 23 |
|---|
| 736 | vcipher 22, 22, 23 |
|---|
| 737 | |
|---|
| 738 | vcipher 15, 15, 24 |
|---|
| 739 | vcipher 16, 16, 24 |
|---|
| 740 | vcipher 17, 17, 24 |
|---|
| 741 | vcipher 18, 18, 24 |
|---|
| 742 | vcipher 19, 19, 24 |
|---|
| 743 | vcipher 20, 20, 24 |
|---|
| 744 | vcipher 21, 21, 24 |
|---|
| 745 | vcipher 22, 22, 24 |
|---|
| 746 | |
|---|
| 747 | xxlor 23+32, 14, 14 |
|---|
| 748 | |
|---|
| 749 | cmpdi 10, 14 |
|---|
| 750 | beq Do_next_ghash |
|---|
| 751 | b aes_gcm_out |
|---|
| 752 | |
|---|
| 753 | Do_next_ghash: |
|---|
| 754 | |
|---|
| 755 | # |
|---|
| 756 | # last round |
|---|
| 757 | vcipherlast 15, 15, 23 |
|---|
| 758 | vcipherlast 16, 16, 23 |
|---|
| 759 | |
|---|
| 760 | xxlxor 47, 47, 15 |
|---|
| 761 | stxvb16x 47, 0, 9 # store output |
|---|
| 762 | xxlxor 48, 48, 16 |
|---|
| 763 | stxvb16x 48, 15, 9 # store output |
|---|
| 764 | |
|---|
| 765 | vcipherlast 17, 17, 23 |
|---|
| 766 | vcipherlast 18, 18, 23 |
|---|
| 767 | |
|---|
| 768 | xxlxor 49, 49, 17 |
|---|
| 769 | stxvb16x 49, 16, 9 # store output |
|---|
| 770 | xxlxor 50, 50, 18 |
|---|
| 771 | stxvb16x 50, 17, 9 # store output |
|---|
| 772 | |
|---|
| 773 | vcipherlast 19, 19, 23 |
|---|
| 774 | vcipherlast 20, 20, 23 |
|---|
| 775 | |
|---|
| 776 | xxlxor 51, 51, 19 |
|---|
| 777 | stxvb16x 51, 18, 9 # store output |
|---|
| 778 | xxlxor 52, 52, 20 |
|---|
| 779 | stxvb16x 52, 19, 9 # store output |
|---|
| 780 | |
|---|
| 781 | vcipherlast 21, 21, 23 |
|---|
| 782 | vcipherlast 22, 22, 23 |
|---|
| 783 | |
|---|
| 784 | xxlxor 53, 53, 21 |
|---|
| 785 | stxvb16x 53, 20, 9 # store output |
|---|
| 786 | xxlxor 54, 54, 22 |
|---|
| 787 | stxvb16x 54, 21, 9 # store output |
|---|
| 788 | |
|---|
| 789 | addi 9, 9, 128 |
|---|
| 790 | |
|---|
| 791 | # ghash here |
|---|
| 792 | ppc_aes_gcm_ghash2_4x |
|---|
| 793 | |
|---|
| 794 | xxlor 27+32, 0, 0 |
|---|
| 795 | vaddudm 30, 30, 31 # IV + counter |
|---|
| 796 | vmr 29, 30 |
|---|
| 797 | vxor 15, 30, 27 # add round key |
|---|
| 798 | vaddudm 30, 30, 31 |
|---|
| 799 | vxor 16, 30, 27 |
|---|
| 800 | vaddudm 30, 30, 31 |
|---|
| 801 | vxor 17, 30, 27 |
|---|
| 802 | vaddudm 30, 30, 31 |
|---|
| 803 | vxor 18, 30, 27 |
|---|
| 804 | vaddudm 30, 30, 31 |
|---|
| 805 | vxor 19, 30, 27 |
|---|
| 806 | vaddudm 30, 30, 31 |
|---|
| 807 | vxor 20, 30, 27 |
|---|
| 808 | vaddudm 30, 30, 31 |
|---|
| 809 | vxor 21, 30, 27 |
|---|
| 810 | vaddudm 30, 30, 31 |
|---|
| 811 | vxor 22, 30, 27 |
|---|
| 812 | |
|---|
| 813 | addi 12, 12, -128 |
|---|
| 814 | addi 11, 11, 128 |
|---|
| 815 | |
|---|
| 816 | bdnz Loop_8x_block |
|---|
| 817 | |
|---|
| 818 | vmr 30, 29 |
|---|
| 819 | stxvb16x 30+32, 0, 7 # update IV |
|---|
| 820 | |
|---|
| 821 | Loop_last_block: |
|---|
| 822 | cmpdi 12, 0 |
|---|
| 823 | beq aes_gcm_out |
|---|
| 824 | |
|---|
| 825 | # loop last few blocks |
|---|
| 826 | li 10, 16 |
|---|
| 827 | divdu 10, 12, 10 |
|---|
| 828 | |
|---|
| 829 | mtctr 10 |
|---|
| 830 | |
|---|
| 831 | lwz 10, 240(6) |
|---|
| 832 | |
|---|
| 833 | cmpdi 12, 16 |
|---|
| 834 | blt Final_block |
|---|
| 835 | |
|---|
| 836 | Next_rem_block: |
|---|
| 837 | lxvb16x 15, 0, 14 # load block |
|---|
| 838 | |
|---|
| 839 | Loop_aes_middle_1x |
|---|
| 840 | |
|---|
| 841 | xxlor 23+32, 10, 10 |
|---|
| 842 | |
|---|
| 843 | cmpdi 10, 10 |
|---|
| 844 | beq Do_next_1x |
|---|
| 845 | |
|---|
| 846 | # 192 bits |
|---|
| 847 | xxlor 24+32, 11, 11 |
|---|
| 848 | |
|---|
| 849 | vcipher 15, 15, 23 |
|---|
| 850 | vcipher 15, 15, 24 |
|---|
| 851 | |
|---|
| 852 | xxlor 23+32, 12, 12 |
|---|
| 853 | |
|---|
| 854 | cmpdi 10, 12 |
|---|
| 855 | beq Do_next_1x |
|---|
| 856 | |
|---|
| 857 | # 256 bits |
|---|
| 858 | xxlor 24+32, 13, 13 |
|---|
| 859 | |
|---|
| 860 | vcipher 15, 15, 23 |
|---|
| 861 | vcipher 15, 15, 24 |
|---|
| 862 | |
|---|
| 863 | xxlor 23+32, 14, 14 |
|---|
| 864 | |
|---|
| 865 | cmpdi 10, 14 |
|---|
| 866 | beq Do_next_1x |
|---|
| 867 | |
|---|
| 868 | Do_next_1x: |
|---|
| 869 | vcipherlast 15, 15, 23 |
|---|
| 870 | |
|---|
| 871 | xxlxor 47, 47, 15 |
|---|
| 872 | stxvb16x 47, 0, 9 # store output |
|---|
| 873 | addi 14, 14, 16 |
|---|
| 874 | addi 9, 9, 16 |
|---|
| 875 | |
|---|
| 876 | vmr 28, 15 |
|---|
| 877 | ppc_update_hash_1x |
|---|
| 878 | |
|---|
| 879 | addi 12, 12, -16 |
|---|
| 880 | addi 11, 11, 16 |
|---|
| 881 | xxlor 19+32, 0, 0 |
|---|
| 882 | vaddudm 30, 30, 31 # IV + counter |
|---|
| 883 | vxor 15, 30, 19 # add round key |
|---|
| 884 | |
|---|
| 885 | bdnz Next_rem_block |
|---|
| 886 | |
|---|
| 887 | li 15, 0 |
|---|
| 888 | std 15, 56(7) # clear partial? |
|---|
| 889 | stxvb16x 30+32, 0, 7 # update IV |
|---|
| 890 | cmpdi 12, 0 |
|---|
| 891 | beq aes_gcm_out |
|---|
| 892 | |
|---|
| 893 | Final_block: |
|---|
| 894 | lwz 10, 240(6) |
|---|
| 895 | Loop_aes_middle_1x |
|---|
| 896 | |
|---|
| 897 | xxlor 23+32, 10, 10 |
|---|
| 898 | |
|---|
| 899 | cmpdi 10, 10 |
|---|
| 900 | beq Do_final_1x |
|---|
| 901 | |
|---|
| 902 | # 192 bits |
|---|
| 903 | xxlor 24+32, 11, 11 |
|---|
| 904 | |
|---|
| 905 | vcipher 15, 15, 23 |
|---|
| 906 | vcipher 15, 15, 24 |
|---|
| 907 | |
|---|
| 908 | xxlor 23+32, 12, 12 |
|---|
| 909 | |
|---|
| 910 | cmpdi 10, 12 |
|---|
| 911 | beq Do_final_1x |
|---|
| 912 | |
|---|
| 913 | # 256 bits |
|---|
| 914 | xxlor 24+32, 13, 13 |
|---|
| 915 | |
|---|
| 916 | vcipher 15, 15, 23 |
|---|
| 917 | vcipher 15, 15, 24 |
|---|
| 918 | |
|---|
| 919 | xxlor 23+32, 14, 14 |
|---|
| 920 | |
|---|
| 921 | cmpdi 10, 14 |
|---|
| 922 | beq Do_final_1x |
|---|
| 923 | |
|---|
| 924 | Do_final_1x: |
|---|
| 925 | vcipherlast 15, 15, 23 |
|---|
| 926 | |
|---|
| 927 | # check partial block |
|---|
| 928 | li 21, 0 # encrypt |
|---|
| 929 | ld 15, 56(7) # partial? |
|---|
| 930 | cmpdi 15, 0 |
|---|
| 931 | beq Normal_block |
|---|
| 932 | bl Do_partial_block |
|---|
| 933 | |
|---|
| 934 | cmpdi 12, 0 |
|---|
| 935 | ble aes_gcm_out |
|---|
| 936 | |
|---|
| 937 | b Continue_partial_check |
|---|
| 938 | |
|---|
| 939 | Normal_block: |
|---|
| 940 | lxvb16x 15, 0, 14 # load last block |
|---|
| 941 | xxlxor 47, 47, 15 |
|---|
| 942 | |
|---|
| 943 | # create partial block mask |
|---|
| 944 | li 15, 16 |
|---|
| 945 | sub 15, 15, 12 # index to the mask |
|---|
| 946 | |
|---|
| 947 | vspltisb 16, -1 # first 16 bytes - 0xffff...ff |
|---|
| 948 | vspltisb 17, 0 # second 16 bytes - 0x0000...00 |
|---|
| 949 | li 10, 192 |
|---|
| 950 | stvx 16, 10, 1 |
|---|
| 951 | addi 10, 10, 16 |
|---|
| 952 | stvx 17, 10, 1 |
|---|
| 953 | |
|---|
| 954 | addi 10, 1, 192 |
|---|
| 955 | lxvb16x 16, 15, 10 # load partial block mask |
|---|
| 956 | xxland 47, 47, 16 |
|---|
| 957 | |
|---|
| 958 | vmr 28, 15 |
|---|
| 959 | ppc_update_hash_1x |
|---|
| 960 | |
|---|
| 961 | # * should store only the remaining bytes. |
|---|
| 962 | bl Write_partial_block |
|---|
| 963 | |
|---|
| 964 | stxvb16x 30+32, 0, 7 # update IV |
|---|
| 965 | std 12, 56(7) # update partial? |
|---|
| 966 | li 16, 16 |
|---|
| 967 | |
|---|
| 968 | stxvb16x 32, 0, 8 # write out Xi |
|---|
| 969 | stxvb16x 32, 16, 8 # write out Xi |
|---|
| 970 | b aes_gcm_out |
|---|
| 971 | |
|---|
| 972 | # |
|---|
| 973 | # Compute data mask |
|---|
| 974 | # |
|---|
| 975 | .macro GEN_MASK _mask _start _end |
|---|
| 976 | vspltisb 16, -1 # first 16 bytes - 0xffff...ff |
|---|
| 977 | vspltisb 17, 0 # second 16 bytes - 0x0000...00 |
|---|
| 978 | li 10, 192 |
|---|
| 979 | stxvb16x 17+32, 10, 1 |
|---|
| 980 | add 10, 10, \_start |
|---|
| 981 | stxvb16x 16+32, 10, 1 |
|---|
| 982 | add 10, 10, \_end |
|---|
| 983 | stxvb16x 17+32, 10, 1 |
|---|
| 984 | |
|---|
| 985 | addi 10, 1, 192 |
|---|
| 986 | lxvb16x \_mask, 0, 10 # load partial block mask |
|---|
| 987 | .endm |
|---|
| 988 | |
|---|
| 989 | # |
|---|
| 990 | # Handle multiple partial blocks for encrypt and decrypt |
|---|
| 991 | # operations. |
|---|
| 992 | # |
|---|
| 993 | SYM_FUNC_START_LOCAL(Do_partial_block) |
|---|
| 994 | add 17, 15, 5 |
|---|
| 995 | cmpdi 17, 16 |
|---|
| 996 | bgt Big_block |
|---|
| 997 | GEN_MASK 18, 15, 5 |
|---|
| 998 | b _Partial |
|---|
| 999 | SYM_FUNC_END(Do_partial_block) |
|---|
| 1000 | Big_block: |
|---|
| 1001 | li 16, 16 |
|---|
| 1002 | GEN_MASK 18, 15, 16 |
|---|
| 1003 | |
|---|
| 1004 | _Partial: |
|---|
| 1005 | lxvb16x 17+32, 0, 14 # load last block |
|---|
| 1006 | sldi 16, 15, 3 |
|---|
| 1007 | mtvsrdd 32+16, 0, 16 |
|---|
| 1008 | vsro 17, 17, 16 |
|---|
| 1009 | xxlxor 47, 47, 17+32 |
|---|
| 1010 | xxland 47, 47, 18 |
|---|
| 1011 | |
|---|
| 1012 | vxor 0, 0, 0 # clear Xi |
|---|
| 1013 | vmr 28, 15 |
|---|
| 1014 | |
|---|
| 1015 | cmpdi 21, 0 # encrypt/decrypt ops? |
|---|
| 1016 | beq Skip_decrypt |
|---|
| 1017 | xxland 32+28, 32+17, 18 |
|---|
| 1018 | |
|---|
| 1019 | Skip_decrypt: |
|---|
| 1020 | |
|---|
| 1021 | ppc_update_hash_1x |
|---|
| 1022 | |
|---|
| 1023 | li 16, 16 |
|---|
| 1024 | lxvb16x 32+29, 16, 8 |
|---|
| 1025 | vxor 0, 0, 29 |
|---|
| 1026 | stxvb16x 32, 0, 8 # save Xi |
|---|
| 1027 | stxvb16x 32, 16, 8 # save Xi |
|---|
| 1028 | |
|---|
| 1029 | # store partial block |
|---|
| 1030 | # loop the rest of the stream if any |
|---|
| 1031 | sldi 16, 15, 3 |
|---|
| 1032 | mtvsrdd 32+16, 0, 16 |
|---|
| 1033 | vslo 15, 15, 16 |
|---|
| 1034 | #stxvb16x 15+32, 0, 9 # last block |
|---|
| 1035 | |
|---|
| 1036 | li 16, 16 |
|---|
| 1037 | sub 17, 16, 15 # 16 - partial |
|---|
| 1038 | |
|---|
| 1039 | add 16, 15, 5 |
|---|
| 1040 | cmpdi 16, 16 |
|---|
| 1041 | bgt Larger_16 |
|---|
| 1042 | mr 17, 5 |
|---|
| 1043 | Larger_16: |
|---|
| 1044 | |
|---|
| 1045 | # write partial |
|---|
| 1046 | li 10, 192 |
|---|
| 1047 | stxvb16x 15+32, 10, 1 # save current block |
|---|
| 1048 | |
|---|
| 1049 | addi 10, 9, -1 |
|---|
| 1050 | addi 16, 1, 191 |
|---|
| 1051 | mtctr 17 # move partial byte count |
|---|
| 1052 | |
|---|
| 1053 | Write_last_partial: |
|---|
| 1054 | lbzu 18, 1(16) |
|---|
| 1055 | stbu 18, 1(10) |
|---|
| 1056 | bdnz Write_last_partial |
|---|
| 1057 | # Complete loop partial |
|---|
| 1058 | |
|---|
| 1059 | add 14, 14, 17 |
|---|
| 1060 | add 9, 9, 17 |
|---|
| 1061 | sub 12, 12, 17 |
|---|
| 1062 | add 11, 11, 17 |
|---|
| 1063 | |
|---|
| 1064 | add 15, 15, 5 |
|---|
| 1065 | cmpdi 15, 16 |
|---|
| 1066 | blt Save_partial |
|---|
| 1067 | |
|---|
| 1068 | vaddudm 30, 30, 31 |
|---|
| 1069 | stxvb16x 30+32, 0, 7 # update IV |
|---|
| 1070 | xxlor 32+29, 0, 0 |
|---|
| 1071 | vxor 15, 30, 29 # IV + round key - add round key 0 |
|---|
| 1072 | li 15, 0 |
|---|
| 1073 | std 15, 56(7) # partial done - clear |
|---|
| 1074 | b Partial_done |
|---|
| 1075 | Save_partial: |
|---|
| 1076 | std 15, 56(7) # partial |
|---|
| 1077 | |
|---|
| 1078 | Partial_done: |
|---|
| 1079 | blr |
|---|
| 1080 | |
|---|
| 1081 | # |
|---|
| 1082 | # Write partial block |
|---|
| 1083 | # r9 - output |
|---|
| 1084 | # r12 - remaining bytes |
|---|
| 1085 | # v15 - partial input data |
|---|
| 1086 | # |
|---|
| 1087 | SYM_FUNC_START_LOCAL(Write_partial_block) |
|---|
| 1088 | li 10, 192 |
|---|
| 1089 | stxvb16x 15+32, 10, 1 # last block |
|---|
| 1090 | |
|---|
| 1091 | addi 10, 9, -1 |
|---|
| 1092 | addi 16, 1, 191 |
|---|
| 1093 | |
|---|
| 1094 | mtctr 12 # remaining bytes |
|---|
| 1095 | li 15, 0 |
|---|
| 1096 | |
|---|
| 1097 | Write_last_byte: |
|---|
| 1098 | lbzu 14, 1(16) |
|---|
| 1099 | stbu 14, 1(10) |
|---|
| 1100 | bdnz Write_last_byte |
|---|
| 1101 | blr |
|---|
| 1102 | SYM_FUNC_END(Write_partial_block) |
|---|
| 1103 | |
|---|
| 1104 | aes_gcm_out: |
|---|
| 1105 | # out = state |
|---|
| 1106 | stxvb16x 32, 0, 8 # write out Xi |
|---|
| 1107 | add 3, 11, 12 # return count |
|---|
| 1108 | |
|---|
| 1109 | RESTORE_REGS |
|---|
| 1110 | blr |
|---|
| 1111 | |
|---|
| 1112 | # |
|---|
| 1113 | # 8x Decrypt |
|---|
| 1114 | # |
|---|
| 1115 | _GLOBAL(aes_p10_gcm_decrypt) |
|---|
| 1116 | .align 5 |
|---|
| 1117 | |
|---|
| 1118 | SAVE_REGS |
|---|
| 1119 | |
|---|
| 1120 | LOAD_HASH_TABLE |
|---|
| 1121 | |
|---|
| 1122 | # initialize ICB: GHASH( IV ), IV - r7 |
|---|
| 1123 | lxvb16x 30+32, 0, 7 # load IV - v30 |
|---|
| 1124 | |
|---|
| 1125 | mr 12, 5 # length |
|---|
| 1126 | li 11, 0 # block index |
|---|
| 1127 | |
|---|
| 1128 | # counter 1 |
|---|
| 1129 | vxor 31, 31, 31 |
|---|
| 1130 | vspltisb 22, 1 |
|---|
| 1131 | vsldoi 31, 31, 22,1 # counter 1 |
|---|
| 1132 | |
|---|
| 1133 | # load round key to VSR |
|---|
| 1134 | lxv 0, 0(6) |
|---|
| 1135 | lxv 1, 0x10(6) |
|---|
| 1136 | lxv 2, 0x20(6) |
|---|
| 1137 | lxv 3, 0x30(6) |
|---|
| 1138 | lxv 4, 0x40(6) |
|---|
| 1139 | lxv 5, 0x50(6) |
|---|
| 1140 | lxv 6, 0x60(6) |
|---|
| 1141 | lxv 7, 0x70(6) |
|---|
| 1142 | lxv 8, 0x80(6) |
|---|
| 1143 | lxv 9, 0x90(6) |
|---|
| 1144 | lxv 10, 0xa0(6) |
|---|
| 1145 | |
|---|
| 1146 | # load rounds - 10 (128), 12 (192), 14 (256) |
|---|
| 1147 | lwz 9,240(6) |
|---|
| 1148 | |
|---|
| 1149 | # |
|---|
| 1150 | # vxor state, state, w # addroundkey |
|---|
| 1151 | xxlor 32+29, 0, 0 |
|---|
| 1152 | vxor 15, 30, 29 # IV + round key - add round key 0 |
|---|
| 1153 | |
|---|
| 1154 | cmpdi 9, 10 |
|---|
| 1155 | beq Loop_aes_gcm_8x_dec |
|---|
| 1156 | |
|---|
| 1157 | # load 2 more round keys (v11, v12) |
|---|
| 1158 | lxv 11, 0xb0(6) |
|---|
| 1159 | lxv 12, 0xc0(6) |
|---|
| 1160 | |
|---|
| 1161 | cmpdi 9, 12 |
|---|
| 1162 | beq Loop_aes_gcm_8x_dec |
|---|
| 1163 | |
|---|
| 1164 | # load 2 more round keys (v11, v12, v13, v14) |
|---|
| 1165 | lxv 13, 0xd0(6) |
|---|
| 1166 | lxv 14, 0xe0(6) |
|---|
| 1167 | cmpdi 9, 14 |
|---|
| 1168 | beq Loop_aes_gcm_8x_dec |
|---|
| 1169 | |
|---|
| 1170 | b aes_gcm_out |
|---|
| 1171 | |
|---|
| 1172 | .align 5 |
|---|
| 1173 | Loop_aes_gcm_8x_dec: |
|---|
| 1174 | mr 14, 3 |
|---|
| 1175 | mr 9, 4 |
|---|
| 1176 | |
|---|
| 1177 | # |
|---|
| 1178 | # check partial block |
|---|
| 1179 | # |
|---|
| 1180 | Continue_partial_check_dec: |
|---|
| 1181 | ld 15, 56(7) |
|---|
| 1182 | cmpdi 15, 0 |
|---|
| 1183 | beq Continue_dec |
|---|
| 1184 | bgt Final_block_dec |
|---|
| 1185 | cmpdi 15, 16 |
|---|
| 1186 | blt Final_block_dec |
|---|
| 1187 | |
|---|
| 1188 | Continue_dec: |
|---|
| 1189 | # n blcoks |
|---|
| 1190 | li 10, 128 |
|---|
| 1191 | divdu 10, 12, 10 # n 128 bytes-blocks |
|---|
| 1192 | cmpdi 10, 0 |
|---|
| 1193 | beq Loop_last_block_dec |
|---|
| 1194 | |
|---|
| 1195 | vaddudm 30, 30, 31 # IV + counter |
|---|
| 1196 | vxor 16, 30, 29 |
|---|
| 1197 | vaddudm 30, 30, 31 |
|---|
| 1198 | vxor 17, 30, 29 |
|---|
| 1199 | vaddudm 30, 30, 31 |
|---|
| 1200 | vxor 18, 30, 29 |
|---|
| 1201 | vaddudm 30, 30, 31 |
|---|
| 1202 | vxor 19, 30, 29 |
|---|
| 1203 | vaddudm 30, 30, 31 |
|---|
| 1204 | vxor 20, 30, 29 |
|---|
| 1205 | vaddudm 30, 30, 31 |
|---|
| 1206 | vxor 21, 30, 29 |
|---|
| 1207 | vaddudm 30, 30, 31 |
|---|
| 1208 | vxor 22, 30, 29 |
|---|
| 1209 | |
|---|
| 1210 | mtctr 10 |
|---|
| 1211 | |
|---|
| 1212 | li 15, 16 |
|---|
| 1213 | li 16, 32 |
|---|
| 1214 | li 17, 48 |
|---|
| 1215 | li 18, 64 |
|---|
| 1216 | li 19, 80 |
|---|
| 1217 | li 20, 96 |
|---|
| 1218 | li 21, 112 |
|---|
| 1219 | |
|---|
| 1220 | lwz 10, 240(6) |
|---|
| 1221 | |
|---|
| 1222 | Loop_8x_block_dec: |
|---|
| 1223 | |
|---|
| 1224 | lxvb16x 15, 0, 14 # load block |
|---|
| 1225 | lxvb16x 16, 15, 14 # load block |
|---|
| 1226 | lxvb16x 17, 16, 14 # load block |
|---|
| 1227 | lxvb16x 18, 17, 14 # load block |
|---|
| 1228 | lxvb16x 19, 18, 14 # load block |
|---|
| 1229 | lxvb16x 20, 19, 14 # load block |
|---|
| 1230 | lxvb16x 21, 20, 14 # load block |
|---|
| 1231 | lxvb16x 22, 21, 14 # load block |
|---|
| 1232 | addi 14, 14, 128 |
|---|
| 1233 | |
|---|
| 1234 | Loop_aes_middle8x |
|---|
| 1235 | |
|---|
| 1236 | xxlor 23+32, 10, 10 |
|---|
| 1237 | |
|---|
| 1238 | cmpdi 10, 10 |
|---|
| 1239 | beq Do_next_ghash_dec |
|---|
| 1240 | |
|---|
| 1241 | # 192 bits |
|---|
| 1242 | xxlor 24+32, 11, 11 |
|---|
| 1243 | |
|---|
| 1244 | vcipher 15, 15, 23 |
|---|
| 1245 | vcipher 16, 16, 23 |
|---|
| 1246 | vcipher 17, 17, 23 |
|---|
| 1247 | vcipher 18, 18, 23 |
|---|
| 1248 | vcipher 19, 19, 23 |
|---|
| 1249 | vcipher 20, 20, 23 |
|---|
| 1250 | vcipher 21, 21, 23 |
|---|
| 1251 | vcipher 22, 22, 23 |
|---|
| 1252 | |
|---|
| 1253 | vcipher 15, 15, 24 |
|---|
| 1254 | vcipher 16, 16, 24 |
|---|
| 1255 | vcipher 17, 17, 24 |
|---|
| 1256 | vcipher 18, 18, 24 |
|---|
| 1257 | vcipher 19, 19, 24 |
|---|
| 1258 | vcipher 20, 20, 24 |
|---|
| 1259 | vcipher 21, 21, 24 |
|---|
| 1260 | vcipher 22, 22, 24 |
|---|
| 1261 | |
|---|
| 1262 | xxlor 23+32, 12, 12 |
|---|
| 1263 | |
|---|
| 1264 | cmpdi 10, 12 |
|---|
| 1265 | beq Do_next_ghash_dec |
|---|
| 1266 | |
|---|
| 1267 | # 256 bits |
|---|
| 1268 | xxlor 24+32, 13, 13 |
|---|
| 1269 | |
|---|
| 1270 | vcipher 15, 15, 23 |
|---|
| 1271 | vcipher 16, 16, 23 |
|---|
| 1272 | vcipher 17, 17, 23 |
|---|
| 1273 | vcipher 18, 18, 23 |
|---|
| 1274 | vcipher 19, 19, 23 |
|---|
| 1275 | vcipher 20, 20, 23 |
|---|
| 1276 | vcipher 21, 21, 23 |
|---|
| 1277 | vcipher 22, 22, 23 |
|---|
| 1278 | |
|---|
| 1279 | vcipher 15, 15, 24 |
|---|
| 1280 | vcipher 16, 16, 24 |
|---|
| 1281 | vcipher 17, 17, 24 |
|---|
| 1282 | vcipher 18, 18, 24 |
|---|
| 1283 | vcipher 19, 19, 24 |
|---|
| 1284 | vcipher 20, 20, 24 |
|---|
| 1285 | vcipher 21, 21, 24 |
|---|
| 1286 | vcipher 22, 22, 24 |
|---|
| 1287 | |
|---|
| 1288 | xxlor 23+32, 14, 14 |
|---|
| 1289 | |
|---|
| 1290 | cmpdi 10, 14 |
|---|
| 1291 | beq Do_next_ghash_dec |
|---|
| 1292 | b aes_gcm_out |
|---|
| 1293 | |
|---|
| 1294 | Do_next_ghash_dec: |
|---|
| 1295 | |
|---|
| 1296 | # |
|---|
| 1297 | # last round |
|---|
| 1298 | vcipherlast 15, 15, 23 |
|---|
| 1299 | vcipherlast 16, 16, 23 |
|---|
| 1300 | |
|---|
| 1301 | xxlxor 47, 47, 15 |
|---|
| 1302 | stxvb16x 47, 0, 9 # store output |
|---|
| 1303 | xxlxor 48, 48, 16 |
|---|
| 1304 | stxvb16x 48, 15, 9 # store output |
|---|
| 1305 | |
|---|
| 1306 | vcipherlast 17, 17, 23 |
|---|
| 1307 | vcipherlast 18, 18, 23 |
|---|
| 1308 | |
|---|
| 1309 | xxlxor 49, 49, 17 |
|---|
| 1310 | stxvb16x 49, 16, 9 # store output |
|---|
| 1311 | xxlxor 50, 50, 18 |
|---|
| 1312 | stxvb16x 50, 17, 9 # store output |
|---|
| 1313 | |
|---|
| 1314 | vcipherlast 19, 19, 23 |
|---|
| 1315 | vcipherlast 20, 20, 23 |
|---|
| 1316 | |
|---|
| 1317 | xxlxor 51, 51, 19 |
|---|
| 1318 | stxvb16x 51, 18, 9 # store output |
|---|
| 1319 | xxlxor 52, 52, 20 |
|---|
| 1320 | stxvb16x 52, 19, 9 # store output |
|---|
| 1321 | |
|---|
| 1322 | vcipherlast 21, 21, 23 |
|---|
| 1323 | vcipherlast 22, 22, 23 |
|---|
| 1324 | |
|---|
| 1325 | xxlxor 53, 53, 21 |
|---|
| 1326 | stxvb16x 53, 20, 9 # store output |
|---|
| 1327 | xxlxor 54, 54, 22 |
|---|
| 1328 | stxvb16x 54, 21, 9 # store output |
|---|
| 1329 | |
|---|
| 1330 | addi 9, 9, 128 |
|---|
| 1331 | |
|---|
| 1332 | xxlor 15+32, 15, 15 |
|---|
| 1333 | xxlor 16+32, 16, 16 |
|---|
| 1334 | xxlor 17+32, 17, 17 |
|---|
| 1335 | xxlor 18+32, 18, 18 |
|---|
| 1336 | xxlor 19+32, 19, 19 |
|---|
| 1337 | xxlor 20+32, 20, 20 |
|---|
| 1338 | xxlor 21+32, 21, 21 |
|---|
| 1339 | xxlor 22+32, 22, 22 |
|---|
| 1340 | |
|---|
| 1341 | # ghash here |
|---|
| 1342 | ppc_aes_gcm_ghash2_4x |
|---|
| 1343 | |
|---|
| 1344 | xxlor 27+32, 0, 0 |
|---|
| 1345 | vaddudm 30, 30, 31 # IV + counter |
|---|
| 1346 | vmr 29, 30 |
|---|
| 1347 | vxor 15, 30, 27 # add round key |
|---|
| 1348 | vaddudm 30, 30, 31 |
|---|
| 1349 | vxor 16, 30, 27 |
|---|
| 1350 | vaddudm 30, 30, 31 |
|---|
| 1351 | vxor 17, 30, 27 |
|---|
| 1352 | vaddudm 30, 30, 31 |
|---|
| 1353 | vxor 18, 30, 27 |
|---|
| 1354 | vaddudm 30, 30, 31 |
|---|
| 1355 | vxor 19, 30, 27 |
|---|
| 1356 | vaddudm 30, 30, 31 |
|---|
| 1357 | vxor 20, 30, 27 |
|---|
| 1358 | vaddudm 30, 30, 31 |
|---|
| 1359 | vxor 21, 30, 27 |
|---|
| 1360 | vaddudm 30, 30, 31 |
|---|
| 1361 | vxor 22, 30, 27 |
|---|
| 1362 | |
|---|
| 1363 | addi 12, 12, -128 |
|---|
| 1364 | addi 11, 11, 128 |
|---|
| 1365 | |
|---|
| 1366 | bdnz Loop_8x_block_dec |
|---|
| 1367 | |
|---|
| 1368 | vmr 30, 29 |
|---|
| 1369 | stxvb16x 30+32, 0, 7 # update IV |
|---|
| 1370 | |
|---|
| 1371 | Loop_last_block_dec: |
|---|
| 1372 | cmpdi 12, 0 |
|---|
| 1373 | beq aes_gcm_out |
|---|
| 1374 | |
|---|
| 1375 | # loop last few blocks |
|---|
| 1376 | li 10, 16 |
|---|
| 1377 | divdu 10, 12, 10 |
|---|
| 1378 | |
|---|
| 1379 | mtctr 10 |
|---|
| 1380 | |
|---|
| 1381 | lwz 10, 240(6) |
|---|
| 1382 | |
|---|
| 1383 | cmpdi 12, 16 |
|---|
| 1384 | blt Final_block_dec |
|---|
| 1385 | |
|---|
| 1386 | Next_rem_block_dec: |
|---|
| 1387 | lxvb16x 15, 0, 14 # load block |
|---|
| 1388 | |
|---|
| 1389 | Loop_aes_middle_1x |
|---|
| 1390 | |
|---|
| 1391 | xxlor 23+32, 10, 10 |
|---|
| 1392 | |
|---|
| 1393 | cmpdi 10, 10 |
|---|
| 1394 | beq Do_next_1x_dec |
|---|
| 1395 | |
|---|
| 1396 | # 192 bits |
|---|
| 1397 | xxlor 24+32, 11, 11 |
|---|
| 1398 | |
|---|
| 1399 | vcipher 15, 15, 23 |
|---|
| 1400 | vcipher 15, 15, 24 |
|---|
| 1401 | |
|---|
| 1402 | xxlor 23+32, 12, 12 |
|---|
| 1403 | |
|---|
| 1404 | cmpdi 10, 12 |
|---|
| 1405 | beq Do_next_1x_dec |
|---|
| 1406 | |
|---|
| 1407 | # 256 bits |
|---|
| 1408 | xxlor 24+32, 13, 13 |
|---|
| 1409 | |
|---|
| 1410 | vcipher 15, 15, 23 |
|---|
| 1411 | vcipher 15, 15, 24 |
|---|
| 1412 | |
|---|
| 1413 | xxlor 23+32, 14, 14 |
|---|
| 1414 | |
|---|
| 1415 | cmpdi 10, 14 |
|---|
| 1416 | beq Do_next_1x_dec |
|---|
| 1417 | |
|---|
| 1418 | Do_next_1x_dec: |
|---|
| 1419 | vcipherlast 15, 15, 23 |
|---|
| 1420 | |
|---|
| 1421 | xxlxor 47, 47, 15 |
|---|
| 1422 | stxvb16x 47, 0, 9 # store output |
|---|
| 1423 | addi 14, 14, 16 |
|---|
| 1424 | addi 9, 9, 16 |
|---|
| 1425 | |
|---|
| 1426 | xxlor 28+32, 15, 15 |
|---|
| 1427 | #vmr 28, 15 |
|---|
| 1428 | ppc_update_hash_1x |
|---|
| 1429 | |
|---|
| 1430 | addi 12, 12, -16 |
|---|
| 1431 | addi 11, 11, 16 |
|---|
| 1432 | xxlor 19+32, 0, 0 |
|---|
| 1433 | vaddudm 30, 30, 31 # IV + counter |
|---|
| 1434 | vxor 15, 30, 19 # add round key |
|---|
| 1435 | |
|---|
| 1436 | bdnz Next_rem_block_dec |
|---|
| 1437 | |
|---|
| 1438 | li 15, 0 |
|---|
| 1439 | std 15, 56(7) # clear partial? |
|---|
| 1440 | stxvb16x 30+32, 0, 7 # update IV |
|---|
| 1441 | cmpdi 12, 0 |
|---|
| 1442 | beq aes_gcm_out |
|---|
| 1443 | |
|---|
| 1444 | Final_block_dec: |
|---|
| 1445 | lwz 10, 240(6) |
|---|
| 1446 | Loop_aes_middle_1x |
|---|
| 1447 | |
|---|
| 1448 | xxlor 23+32, 10, 10 |
|---|
| 1449 | |
|---|
| 1450 | cmpdi 10, 10 |
|---|
| 1451 | beq Do_final_1x_dec |
|---|
| 1452 | |
|---|
| 1453 | # 192 bits |
|---|
| 1454 | xxlor 24+32, 11, 11 |
|---|
| 1455 | |
|---|
| 1456 | vcipher 15, 15, 23 |
|---|
| 1457 | vcipher 15, 15, 24 |
|---|
| 1458 | |
|---|
| 1459 | xxlor 23+32, 12, 12 |
|---|
| 1460 | |
|---|
| 1461 | cmpdi 10, 12 |
|---|
| 1462 | beq Do_final_1x_dec |
|---|
| 1463 | |
|---|
| 1464 | # 256 bits |
|---|
| 1465 | xxlor 24+32, 13, 13 |
|---|
| 1466 | |
|---|
| 1467 | vcipher 15, 15, 23 |
|---|
| 1468 | vcipher 15, 15, 24 |
|---|
| 1469 | |
|---|
| 1470 | xxlor 23+32, 14, 14 |
|---|
| 1471 | |
|---|
| 1472 | cmpdi 10, 14 |
|---|
| 1473 | beq Do_final_1x_dec |
|---|
| 1474 | |
|---|
| 1475 | Do_final_1x_dec: |
|---|
| 1476 | vcipherlast 15, 15, 23 |
|---|
| 1477 | |
|---|
| 1478 | # check partial block |
|---|
| 1479 | li 21, 1 # decrypt |
|---|
| 1480 | ld 15, 56(7) # partial? |
|---|
| 1481 | cmpdi 15, 0 |
|---|
| 1482 | beq Normal_block_dec |
|---|
| 1483 | bl Do_partial_block |
|---|
| 1484 | cmpdi 12, 0 |
|---|
| 1485 | ble aes_gcm_out |
|---|
| 1486 | |
|---|
| 1487 | b Continue_partial_check_dec |
|---|
| 1488 | |
|---|
| 1489 | Normal_block_dec: |
|---|
| 1490 | lxvb16x 15, 0, 14 # load last block |
|---|
| 1491 | xxlxor 47, 47, 15 |
|---|
| 1492 | |
|---|
| 1493 | # create partial block mask |
|---|
| 1494 | li 15, 16 |
|---|
| 1495 | sub 15, 15, 12 # index to the mask |
|---|
| 1496 | |
|---|
| 1497 | vspltisb 16, -1 # first 16 bytes - 0xffff...ff |
|---|
| 1498 | vspltisb 17, 0 # second 16 bytes - 0x0000...00 |
|---|
| 1499 | li 10, 192 |
|---|
| 1500 | stvx 16, 10, 1 |
|---|
| 1501 | addi 10, 10, 16 |
|---|
| 1502 | stvx 17, 10, 1 |
|---|
| 1503 | |
|---|
| 1504 | addi 10, 1, 192 |
|---|
| 1505 | lxvb16x 16, 15, 10 # load partial block mask |
|---|
| 1506 | xxland 47, 47, 16 |
|---|
| 1507 | |
|---|
| 1508 | xxland 32+28, 15, 16 |
|---|
| 1509 | #vmr 28, 15 |
|---|
| 1510 | ppc_update_hash_1x |
|---|
| 1511 | |
|---|
| 1512 | # * should store only the remaining bytes. |
|---|
| 1513 | bl Write_partial_block |
|---|
| 1514 | |
|---|
| 1515 | stxvb16x 30+32, 0, 7 # update IV |
|---|
| 1516 | std 12, 56(7) # update partial? |
|---|
| 1517 | li 16, 16 |
|---|
| 1518 | |
|---|
| 1519 | stxvb16x 32, 0, 8 # write out Xi |
|---|
| 1520 | stxvb16x 32, 16, 8 # write out Xi |
|---|
| 1521 | b aes_gcm_out |
|---|
| 1522 | |
|---|
