mirror of https://github.com/openssl/openssl.git
2873 lines
78 KiB
Perl
2873 lines
78 KiB
Perl
#! /usr/bin/env perl
|
|
# Copyright (C) 2023 Intel Corporation
|
|
#
|
|
# Licensed under the OpenSSL license (the "License"). You may not use
|
|
# this file except in compliance with the License. You can obtain a copy
|
|
# in the file LICENSE in the source distribution or at
|
|
# https://www.openssl.org/source/license.html
|
|
|
|
# This implementation is based on the AES-XTS code (AVX512VAES + VPCLMULQDQ)
|
|
# from Intel(R) Intelligent Storage Acceleration Library Crypto Version
|
|
# (https://github.com/intel/isa-l_crypto).
|
|
#
|
|
######################################################################
|
|
# The main building block of the loop is code that encrypts/decrypts
|
|
# 8/16 blocks of data stitching with generation of tweak for the next
|
|
# 8/16 blocks, utilizing VAES and VPCLMULQDQ instructions with full width
|
|
# of ZMM registers. The main loop is selected based on the input length.
|
|
# main_loop_run_16 encrypts/decrypts 16 blocks in parallel and it's selected
|
|
# when input length >= 256 bytes (16 blocks)
|
|
# main_loop_run_8 encrypts/decrypts 8 blocks in parallel and it's selected
|
|
# when 128 bytes <= input length < 256 bytes (8-15 blocks)
|
|
# Input length < 128 bytes (8 blocks) is handled by do_n_blocks.
|
|
#
|
|
# This implementation mainly uses vpshrdq from AVX-512-VBMI2 family and vaesenc,
|
|
# vaesdec, vpclmulqdq from AVX-512F family.
|
|
$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
|
|
$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
|
|
|
|
$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
|
|
$avx512vaes=0;
|
|
|
|
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
|
|
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
|
|
( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
|
|
die "can't locate x86_64-xlate.pl";
|
|
|
|
if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
|
|
=~ /GNU assembler version ([2-9]\.[0-9]+)/) {
|
|
$avx512vaes = ($1>=2.30);
|
|
}
|
|
|
|
if (!$avx512vaes && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
|
|
`nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) {
|
|
$avx512vaes = ($1==2.11 && $2>=8) + ($1>=2.12);
|
|
}
|
|
|
|
if (!$avx512vaes && `$ENV{CC} -v 2>&1`
|
|
=~ /(Apple)?\s*((?:clang|LLVM) version|.*based on LLVM) ([0-9]+)\.([0-9]+)\.([0-9]+)?/) {
|
|
my $ver = $3 + $4/100.0 + $5/10000.0; # 3.1.0->3.01, 3.10.1->3.1001
|
|
if ($1) {
|
|
# Apple conditions, they use a different version series, see
|
|
# https://en.wikipedia.org/wiki/Xcode#Xcode_7.0_-_10.x_(since_Free_On-Device_Development)_2
|
|
# clang 7.0.0 is Apple clang 10.0.1
|
|
$avx512vaes = ($ver>=10.0001)
|
|
} else {
|
|
$avx512vaes = ($ver>=7.0);
|
|
}
|
|
}
|
|
|
|
open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""
|
|
or die "can't call $xlate: $!";
|
|
*STDOUT=*OUT;
|
|
|
|
#======================================================================
|
|
|
|
if ($avx512vaes) {
|
|
|
|
my $GP_STORAGE = $win64 ? (16 * 18) : (16 * 8); # store rbx
|
|
my $XMM_STORAGE = $win64 ? (16 * 8) : 0; # store xmm6:xmm15
|
|
my $VARIABLE_OFFSET = $win64 ? (16*8 + 16*10 + 8*3) :
|
|
(16*8 + 8*1);
|
|
|
|
# right now, >= 0x80 (128) is used for expanded keys. all usages of
|
|
# rsp should be invoked via $TW, not shadowed by any other name or
|
|
# used directly.
|
|
my $TW = "%rsp";
|
|
my $TEMPHIGH = "%rbx";
|
|
my $TEMPLOW = "%rax";
|
|
my $ZPOLY = "%zmm25";
|
|
|
|
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
# ;;; Function arguments abstraction
|
|
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
my ($key2, $key1, $tweak, $length, $input, $output);
|
|
|
|
|
|
$input = "%rdi";
|
|
$output = "%rsi";
|
|
$length = "%rdx";
|
|
$key1 = "%rcx";
|
|
$key2 = "%r8";
|
|
$tweak = "%r9";
|
|
|
|
# arguments for temp parameters
|
|
my ($tmp1, $gf_poly_8b, $gf_poly_8b_temp);
|
|
$tmp1 = "%r8";
|
|
$gf_poly_8b = "%r10";
|
|
$gf_poly_8b_temp = "%r11";
|
|
|
|
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
# ;;; Helper functions
|
|
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
# Generates "random" local labels
|
|
sub random_string() {
|
|
my @chars = ('a' .. 'z', 'A' .. 'Z', '0' .. '9', '_');
|
|
my $length = 15;
|
|
my $str;
|
|
map { $str .= $chars[rand(33)] } 1 .. $length;
|
|
return $str;
|
|
}
|
|
|
|
# ; Seed the RNG so the labels are generated deterministically
|
|
srand(12345);
|
|
|
|
sub encrypt_tweak {
|
|
my $state_tweak = $_[0];
|
|
my $is_128 = $_[1];
|
|
|
|
$code.=<<___;
|
|
vpxor ($key2), $state_tweak, $state_tweak
|
|
vaesenc 0x10($key2), $state_tweak, $state_tweak
|
|
vaesenc 0x20($key2), $state_tweak, $state_tweak
|
|
vaesenc 0x30($key2), $state_tweak, $state_tweak
|
|
vaesenc 0x40($key2), $state_tweak, $state_tweak
|
|
vaesenc 0x50($key2), $state_tweak, $state_tweak
|
|
vaesenc 0x60($key2), $state_tweak, $state_tweak
|
|
vaesenc 0x70($key2), $state_tweak, $state_tweak
|
|
vaesenc 0x80($key2), $state_tweak, $state_tweak
|
|
vaesenc 0x90($key2), $state_tweak, $state_tweak
|
|
___
|
|
|
|
if ($is_128) {
|
|
$code .= "vaesenclast 0xa0($key2), $state_tweak, $state_tweak\n";
|
|
} else {
|
|
$code .= "vaesenc 0xa0($key2), $state_tweak, $state_tweak\n";
|
|
$code .= "vaesenc 0xb0($key2), $state_tweak, $state_tweak\n";
|
|
$code .= "vaesenc 0xc0($key2), $state_tweak, $state_tweak\n";
|
|
$code .= "vaesenc 0xd0($key2), $state_tweak, $state_tweak\n";
|
|
$code .= "vaesenclast 0xe0($key2), $state_tweak, $state_tweak\n";
|
|
}
|
|
$code .= "vmovdqa $state_tweak, ($TW)\n";
|
|
}
|
|
|
|
sub encrypt_final {
|
|
my $st = $_[0];
|
|
my $tw = $_[1];
|
|
my $is_128 = $_[2];
|
|
|
|
# xor Tweak value
|
|
$code .= "vpxor $tw, $st, $st\n";
|
|
$code .= "vpxor ($key1), $st, $st\n";
|
|
|
|
my $rounds = $is_128 ? 10 : 14;
|
|
for (my $i = 1; $i < $rounds; $i++) {
|
|
$code .= "vaesenc 16*$i($key1), $st, $st\n";
|
|
}
|
|
|
|
$code .=<<___;
|
|
vaesenclast 16*$rounds($key1), $st, $st
|
|
vpxor $tw, $st, $st
|
|
___
|
|
}
|
|
|
|
# decrypt initial blocks of AES
|
|
# 1, 2, 3, 4, 5, 6 or 7 blocks are encrypted
|
|
# next 8 Tweak values are generated
|
|
sub decrypt_initial {
|
|
my @st;
|
|
$st[0] = $_[0];
|
|
$st[1] = $_[1];
|
|
$st[2] = $_[2];
|
|
$st[3] = $_[3];
|
|
$st[4] = $_[4];
|
|
$st[5] = $_[5];
|
|
$st[6] = $_[6];
|
|
$st[7] = $_[7];
|
|
|
|
my @tw;
|
|
$tw[0] = $_[8];
|
|
$tw[1] = $_[9];
|
|
$tw[2] = $_[10];
|
|
$tw[3] = $_[11];
|
|
$tw[4] = $_[12];
|
|
$tw[5] = $_[13];
|
|
$tw[6] = $_[14];
|
|
my $t0 = $_[15];
|
|
my $num_blocks = $_[16];
|
|
my $lt128 = $_[17];
|
|
my $is_128 = $_[18];
|
|
|
|
# num_blocks blocks encrypted
|
|
# num_blocks can be 1, 2, 3, 4, 5, 6, 7
|
|
|
|
# xor Tweak value
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vpxor $tw[$i], $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
$code .= "vmovdqu ($key1), $t0\n";
|
|
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vpxor $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $lt128) {
|
|
$code .= <<___;
|
|
xor $gf_poly_8b_temp, $gf_poly_8b_temp
|
|
shl \$1, $TEMPLOW
|
|
adc $TEMPHIGH, $TEMPHIGH
|
|
___
|
|
}
|
|
# round 1
|
|
$code .= "vmovdqu 0x10($key1), $t0\n";
|
|
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $lt128) {
|
|
$code .= <<___;
|
|
cmovc $gf_poly_8b, $gf_poly_8b_temp
|
|
xor $gf_poly_8b_temp, $TEMPLOW
|
|
mov $TEMPLOW, ($TW) # next Tweak1 generated
|
|
mov $TEMPLOW, 0x08($TW)
|
|
xor $gf_poly_8b_temp, $gf_poly_8b_temp
|
|
___
|
|
}
|
|
|
|
# round 2
|
|
$code .= "vmovdqu 0x20($key1), $t0\n";
|
|
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $lt128) {
|
|
$code .= <<___;
|
|
shl \$1, $TEMPLOW
|
|
adc $TEMPHIGH, $TEMPHIGH
|
|
cmovc $gf_poly_8b, $gf_poly_8b_temp
|
|
xor $gf_poly_8b_temp, $TEMPLOW
|
|
mov $TEMPLOW, 0x10($TW) # next Tweak2 generated
|
|
___
|
|
}
|
|
|
|
# round 3
|
|
$code .= "vmovdqu 0x30($key1), $t0\n";
|
|
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $lt128) {
|
|
$code .= <<___;
|
|
mov $TEMPHIGH, 0x18($TW)
|
|
xor $gf_poly_8b_temp, $gf_poly_8b_temp
|
|
shl \$1, $TEMPLOW
|
|
adc $TEMPHIGH, $TEMPHIGH
|
|
cmovc $gf_poly_8b, $gf_poly_8b_temp
|
|
___
|
|
}
|
|
|
|
# round 4
|
|
$code .= "vmovdqu 0x40($key1), $t0\n";
|
|
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $lt128) {
|
|
$code .= <<___;
|
|
xor $gf_poly_8b_temp, $TEMPLOW
|
|
mov $TEMPLOW, 0x20($TW) # next Tweak3 generated
|
|
mov $TEMPHIGH, 0x28($TW)
|
|
xor $gf_poly_8b_temp, $gf_poly_8b_temp
|
|
shl \$1, $TEMPLOW
|
|
___
|
|
}
|
|
|
|
# round 5
|
|
$code .= "vmovdqu 0x50($key1), $t0\n";
|
|
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $lt128) {
|
|
$code .= <<___;
|
|
adc $TEMPHIGH, $TEMPHIGH
|
|
cmovc $gf_poly_8b, $gf_poly_8b_temp
|
|
xor $gf_poly_8b_temp, $TEMPLOW
|
|
mov $TEMPLOW, 0x30($TW) # next Tweak4 generated
|
|
mov $TEMPHIGH, 0x38($TW)
|
|
___
|
|
}
|
|
|
|
# round 6
|
|
$code .= "vmovdqu 0x60($key1), $t0\n";
|
|
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $lt128) {
|
|
$code .= <<___;
|
|
xor $gf_poly_8b_temp, $gf_poly_8b_temp
|
|
shl \$1, $TEMPLOW
|
|
adc $TEMPHIGH, $TEMPHIGH
|
|
cmovc $gf_poly_8b, $gf_poly_8b_temp
|
|
xor $gf_poly_8b_temp, $TEMPLOW
|
|
mov $TEMPLOW, 0x40($TW) # next Tweak5 generated
|
|
mov $TEMPHIGH, 0x48($TW)
|
|
___
|
|
}
|
|
|
|
# round 7
|
|
$code .= "vmovdqu 0x70($key1), $t0\n";
|
|
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $lt128) {
|
|
$code .= <<___;
|
|
xor $gf_poly_8b_temp, $gf_poly_8b_temp
|
|
shl \$1, $TEMPLOW
|
|
adc $TEMPHIGH, $TEMPHIGH
|
|
cmovc $gf_poly_8b, $gf_poly_8b_temp
|
|
xor $gf_poly_8b_temp, $TEMPLOW
|
|
mov $TEMPLOW, 0x50($TW) # next Tweak6 generated
|
|
mov $TEMPHIGH, 0x58($TW)
|
|
___
|
|
}
|
|
|
|
# round 8
|
|
$code .= "vmovdqu 0x80($key1), $t0\n";
|
|
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $lt128) {
|
|
$code .= <<___;
|
|
xor $gf_poly_8b_temp, $gf_poly_8b_temp
|
|
shl \$1, $TEMPLOW
|
|
adc $TEMPHIGH, $TEMPHIGH
|
|
cmovc $gf_poly_8b, $gf_poly_8b_temp
|
|
xor $gf_poly_8b_temp, $TEMPLOW
|
|
mov $TEMPLOW, 0x60($TW) # next Tweak7 generated
|
|
mov $TEMPHIGH, 0x68($TW)
|
|
___
|
|
}
|
|
|
|
# round 9
|
|
$code .= "vmovdqu 0x90($key1), $t0\n";
|
|
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $lt128) {
|
|
$code .= <<___;
|
|
xor $gf_poly_8b_temp, $gf_poly_8b_temp
|
|
shl \$1, $TEMPLOW
|
|
adc $TEMPHIGH, $TEMPHIGH
|
|
cmovc $gf_poly_8b, $gf_poly_8b_temp
|
|
xor $gf_poly_8b_temp, $TEMPLOW
|
|
mov $TEMPLOW, 0x70($TW) # next Tweak8 generated
|
|
mov $TEMPHIGH, 0x78($TW)
|
|
___
|
|
}
|
|
|
|
if ($is_128) {
|
|
# round 10
|
|
$code .= "vmovdqu 0xa0($key1), $t0\n";
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vaesdeclast $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
} else {
|
|
# round 10
|
|
$code .= "vmovdqu 0xa0($key1), $t0\n";
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 11
|
|
$code .= "vmovdqu 0xb0($key1), $t0\n";
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 12
|
|
$code .= "vmovdqu 0xc0($key1), $t0\n";
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 13
|
|
$code .= "vmovdqu 0xd0($key1), $t0\n";
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 14
|
|
$code .= "vmovdqu 0xe0($key1), $t0\n";
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vaesdeclast $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
}
|
|
|
|
# xor Tweak values
|
|
for (my $i = 0; $i < $num_blocks; $i++) {
|
|
$code .= "vpxor $tw[$i], $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $lt128) {
|
|
# load next Tweak values
|
|
$code .= <<___;
|
|
vmovdqa ($TW), $tw1
|
|
vmovdqa 0x10($TW), $tw2
|
|
vmovdqa 0x20($TW), $tw3
|
|
vmovdqa 0x30($TW), $tw4
|
|
vmovdqa 0x40($TW), $tw5
|
|
vmovdqa 0x50($TW), $tw6
|
|
vmovdqa 0x60($TW), $tw7
|
|
___
|
|
}
|
|
}
|
|
|
|
sub initialize {
|
|
my @st;
|
|
$st[0] = $_[0];
|
|
$st[1] = $_[1];
|
|
$st[2] = $_[2];
|
|
$st[3] = $_[3];
|
|
$st[4] = $_[4];
|
|
$st[5] = $_[5];
|
|
$st[6] = $_[6];
|
|
$st[7] = $_[7];
|
|
|
|
my @tw;
|
|
$tw[0] = $_[8];
|
|
$tw[1] = $_[9];
|
|
$tw[2] = $_[10];
|
|
$tw[3] = $_[11];
|
|
$tw[4] = $_[12];
|
|
$tw[5] = $_[13];
|
|
$tw[6] = $_[14];
|
|
my $num_initial_blocks = $_[15];
|
|
|
|
$code .= <<___;
|
|
vmovdqa 0x0($TW), $tw[0]
|
|
mov 0x0($TW), $TEMPLOW
|
|
mov 0x08($TW), $TEMPHIGH
|
|
vmovdqu 0x0($input), $st[0]
|
|
___
|
|
|
|
if ($num_initial_blocks >= 2) {
|
|
for (my $i = 1; $i < $num_initial_blocks; $i++) {
|
|
$code .= "xor $gf_poly_8b_temp, $gf_poly_8b_temp\n";
|
|
$code .= "shl \$1, $TEMPLOW\n";
|
|
$code .= "adc $TEMPHIGH, $TEMPHIGH\n";
|
|
$code .= "cmovc $gf_poly_8b, $gf_poly_8b_temp\n";
|
|
$code .= "xor $gf_poly_8b_temp, $TEMPLOW\n";
|
|
my $offset = $i * 16;
|
|
$code .= "mov $TEMPLOW, $offset($TW)\n";
|
|
$code .= "mov $TEMPHIGH, $offset + 8($TW)\n";
|
|
$code .= "vmovdqa $offset($TW), $tw[$i]\n";
|
|
$code .= "vmovdqu $offset($input), $st[$i]\n";
|
|
}
|
|
}
|
|
}
|
|
|
|
# Encrypt 4 blocks in parallel
|
|
sub encrypt_by_four {
|
|
my $st1 = $_[0]; # state 1
|
|
my $tw1 = $_[1]; # tweak 1
|
|
my $tmp = $_[2];
|
|
my $is_128 = $_[3];
|
|
|
|
$code .= "vbroadcasti32x4 ($key1), $tmp\n";
|
|
$code .= "vpternlogq \$0x96, $tmp, $tw1, $st1\n";
|
|
|
|
my $rounds = $is_128 ? 10 : 14;
|
|
for (my $i = 1; $i < $rounds; $i++) {
|
|
$code .= "vbroadcasti32x4 16*$i($key1), $tmp\n";
|
|
$code .= "vaesenc $tmp, $st1, $st1\n";
|
|
}
|
|
|
|
$code .= "vbroadcasti32x4 16*$rounds($key1), $tmp\n";
|
|
$code .= "vaesenclast $tmp, $st1, $st1\n";
|
|
|
|
$code .= "vpxorq $tw1, $st1, $st1\n";
|
|
}
|
|
|
|
# Encrypt 8 blocks in parallel
|
|
# generate next 8 tweak values
|
|
sub encrypt_by_eight_zmm {
|
|
my $st1 = $_[0];
|
|
my $st2 = $_[1];
|
|
my $tw1 = $_[2];
|
|
my $tw2 = $_[3];
|
|
my $t0 = $_[4];
|
|
my $last_eight = $_[5];
|
|
my $is_128 = $_[6];
|
|
|
|
$code .= <<___;
|
|
vbroadcasti32x4 ($key1), $t0
|
|
vpternlogq \$0x96, $t0, $tw1, $st1
|
|
vpternlogq \$0x96, $t0, $tw2, $st2
|
|
___
|
|
|
|
if (0 == $last_eight) {
|
|
$code .= <<___;
|
|
vpsrldq \$0xf, $tw1, %zmm13
|
|
vpclmulqdq \$0x0, $ZPOLY, %zmm13, %zmm14
|
|
vpslldq \$0x1, $tw1, %zmm15
|
|
vpxord %zmm14, %zmm15, %zmm15
|
|
___
|
|
}
|
|
# round 1
|
|
$code .= <<___;
|
|
vbroadcasti32x4 0x10($key1), $t0
|
|
vaesenc $t0, $st1, $st1
|
|
vaesenc $t0, $st2, $st2
|
|
|
|
# round 2
|
|
vbroadcasti32x4 0x20($key1), $t0
|
|
vaesenc $t0, $st1, $st1
|
|
vaesenc $t0, $st2, $st2
|
|
|
|
# round 3
|
|
vbroadcasti32x4 0x30($key1), $t0
|
|
vaesenc $t0, $st1, $st1
|
|
vaesenc $t0, $st2, $st2
|
|
___
|
|
|
|
if (0 == $last_eight) {
|
|
$code .= <<___;
|
|
vpsrldq \$0xf, $tw2, %zmm13
|
|
vpclmulqdq \$0x0, $ZPOLY, %zmm13, %zmm14
|
|
vpslldq \$0x1, $tw2, %zmm16
|
|
vpxord %zmm14, %zmm16, %zmm16
|
|
___
|
|
}
|
|
|
|
$code .= <<___;
|
|
# round 4
|
|
vbroadcasti32x4 0x40($key1), $t0
|
|
vaesenc $t0, $st1, $st1
|
|
vaesenc $t0, $st2, $st2
|
|
|
|
# round 5
|
|
vbroadcasti32x4 0x50($key1), $t0
|
|
vaesenc $t0, $st1, $st1
|
|
vaesenc $t0, $st2, $st2
|
|
|
|
# round 6
|
|
vbroadcasti32x4 0x60($key1), $t0
|
|
vaesenc $t0, $st1, $st1
|
|
vaesenc $t0, $st2, $st2
|
|
|
|
# round 7
|
|
vbroadcasti32x4 0x70($key1), $t0
|
|
vaesenc $t0, $st1, $st1
|
|
vaesenc $t0, $st2, $st2
|
|
|
|
# round 8
|
|
vbroadcasti32x4 0x80($key1), $t0
|
|
vaesenc $t0, $st1, $st1
|
|
vaesenc $t0, $st2, $st2
|
|
|
|
# round 9
|
|
vbroadcasti32x4 0x90($key1), $t0
|
|
vaesenc $t0, $st1, $st1
|
|
vaesenc $t0, $st2, $st2
|
|
___
|
|
|
|
if ($is_128) {
|
|
$code .= <<___;
|
|
# round 10
|
|
vbroadcasti32x4 0xa0($key1), $t0
|
|
vaesenclast $t0, $st1, $st1
|
|
vaesenclast $t0, $st2, $st2
|
|
___
|
|
} else {
|
|
$code .= <<___;
|
|
# round 10
|
|
vbroadcasti32x4 0xa0($key1), $t0
|
|
vaesenc $t0, $st1, $st1
|
|
vaesenc $t0, $st2, $st2
|
|
|
|
# round 11
|
|
vbroadcasti32x4 0xb0($key1), $t0
|
|
vaesenc $t0, $st1, $st1
|
|
vaesenc $t0, $st2, $st2
|
|
|
|
# round 12
|
|
vbroadcasti32x4 0xc0($key1), $t0
|
|
vaesenc $t0, $st1, $st1
|
|
vaesenc $t0, $st2, $st2
|
|
|
|
# round 13
|
|
vbroadcasti32x4 0xd0($key1), $t0
|
|
vaesenc $t0, $st1, $st1
|
|
vaesenc $t0, $st2, $st2
|
|
|
|
# round 14
|
|
vbroadcasti32x4 0xe0($key1), $t0
|
|
vaesenclast $t0, $st1, $st1
|
|
vaesenclast $t0, $st2, $st2
|
|
___
|
|
}
|
|
|
|
# xor Tweak values
|
|
$code .= "vpxorq $tw1, $st1, $st1\n";
|
|
$code .= "vpxorq $tw2, $st2, $st2\n";
|
|
|
|
if (0 == $last_eight) {
|
|
# load next Tweak values
|
|
$code .= <<___;
|
|
vmovdqa32 %zmm15, $tw1
|
|
vmovdqa32 %zmm16, $tw2
|
|
___
|
|
}
|
|
}
|
|
|
|
# Decrypt 8 blocks in parallel
|
|
# generate next 8 tweak values
|
|
sub decrypt_by_eight_zmm {
|
|
my $st1 = $_[0];
|
|
my $st2 = $_[1];
|
|
my $tw1 = $_[2];
|
|
my $tw2 = $_[3];
|
|
my $t0 = $_[4];
|
|
my $last_eight = $_[5];
|
|
my $is_128 = $_[6];
|
|
|
|
$code .= <<___;
|
|
# xor Tweak values
|
|
vpxorq $tw1, $st1, $st1
|
|
vpxorq $tw2, $st2, $st2
|
|
|
|
# ARK
|
|
vbroadcasti32x4 ($key1), $t0
|
|
vpxorq $t0, $st1, $st1
|
|
vpxorq $t0, $st2, $st2
|
|
___
|
|
|
|
if (0 == $last_eight) {
|
|
$code .= <<___;
|
|
vpsrldq \$0xf, $tw1, %zmm13
|
|
vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14
|
|
vpslldq \$0x1, $tw1, %zmm15
|
|
vpxord %zmm14, %zmm15, %zmm15
|
|
___
|
|
}
|
|
# round 1
|
|
$code .= <<___;
|
|
vbroadcasti32x4 0x10($key1), $t0
|
|
vaesdec $t0, $st1, $st1
|
|
vaesdec $t0, $st2, $st2
|
|
|
|
# round 2
|
|
vbroadcasti32x4 0x20($key1), $t0
|
|
vaesdec $t0, $st1, $st1
|
|
vaesdec $t0, $st2, $st2
|
|
|
|
# round 3
|
|
vbroadcasti32x4 0x30($key1), $t0
|
|
vaesdec $t0, $st1, $st1
|
|
vaesdec $t0, $st2, $st2
|
|
___
|
|
|
|
if (0 == $last_eight) {
|
|
$code .= <<___;
|
|
vpsrldq \$0xf, $tw2, %zmm13
|
|
vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14
|
|
vpslldq \$0x1, $tw2, %zmm16
|
|
vpxord %zmm14, %zmm16, %zmm16
|
|
___
|
|
}
|
|
|
|
$code .= <<___;
|
|
# round 4
|
|
vbroadcasti32x4 0x40($key1), $t0
|
|
vaesdec $t0, $st1, $st1
|
|
vaesdec $t0, $st2, $st2
|
|
|
|
# round 5
|
|
vbroadcasti32x4 0x50($key1), $t0
|
|
vaesdec $t0, $st1, $st1
|
|
vaesdec $t0, $st2, $st2
|
|
|
|
# round 6
|
|
vbroadcasti32x4 0x60($key1), $t0
|
|
vaesdec $t0, $st1, $st1
|
|
vaesdec $t0, $st2, $st2
|
|
|
|
# round 7
|
|
vbroadcasti32x4 0x70($key1), $t0
|
|
vaesdec $t0, $st1, $st1
|
|
vaesdec $t0, $st2, $st2
|
|
|
|
# round 8
|
|
vbroadcasti32x4 0x80($key1), $t0
|
|
vaesdec $t0, $st1, $st1
|
|
vaesdec $t0, $st2, $st2
|
|
|
|
# round 9
|
|
vbroadcasti32x4 0x90($key1), $t0
|
|
vaesdec $t0, $st1, $st1
|
|
vaesdec $t0, $st2, $st2
|
|
|
|
___
|
|
if ($is_128) {
|
|
$code .= <<___;
|
|
# round 10
|
|
vbroadcasti32x4 0xa0($key1), $t0
|
|
vaesdeclast $t0, $st1, $st1
|
|
vaesdeclast $t0, $st2, $st2
|
|
___
|
|
} else {
|
|
$code .= <<___;
|
|
# round 10
|
|
vbroadcasti32x4 0xa0($key1), $t0
|
|
vaesdec $t0, $st1, $st1
|
|
vaesdec $t0, $st2, $st2
|
|
|
|
# round 11
|
|
vbroadcasti32x4 0xb0($key1), $t0
|
|
vaesdec $t0, $st1, $st1
|
|
vaesdec $t0, $st2, $st2
|
|
|
|
# round 12
|
|
vbroadcasti32x4 0xc0($key1), $t0
|
|
vaesdec $t0, $st1, $st1
|
|
vaesdec $t0, $st2, $st2
|
|
|
|
# round 13
|
|
vbroadcasti32x4 0xd0($key1), $t0
|
|
vaesdec $t0, $st1, $st1
|
|
vaesdec $t0, $st2, $st2
|
|
|
|
# round 14
|
|
vbroadcasti32x4 0xe0($key1), $t0
|
|
vaesdeclast $t0, $st1, $st1
|
|
vaesdeclast $t0, $st2, $st2
|
|
___
|
|
}
|
|
|
|
$code .= <<___;
|
|
# xor Tweak values
|
|
vpxorq $tw1, $st1, $st1
|
|
vpxorq $tw2, $st2, $st2
|
|
|
|
# load next Tweak values
|
|
vmovdqa32 %zmm15, $tw1
|
|
vmovdqa32 %zmm16, $tw2
|
|
___
|
|
}
|
|
|
|
# Encrypt 16 blocks in parallel
|
|
# generate next 16 tweak values
|
|
sub encrypt_by_16_zmm {
|
|
my @st;
|
|
$st[0] = $_[0];
|
|
$st[1] = $_[1];
|
|
$st[2] = $_[2];
|
|
$st[3] = $_[3];
|
|
|
|
my @tw;
|
|
$tw[0] = $_[4];
|
|
$tw[1] = $_[5];
|
|
$tw[2] = $_[6];
|
|
$tw[3] = $_[7];
|
|
|
|
my $t0 = $_[8];
|
|
my $last_eight = $_[9];
|
|
my $is_128 = $_[10];
|
|
|
|
# xor Tweak values
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vpxorq $tw[$i], $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# ARK
|
|
$code .= "vbroadcasti32x4 ($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vpxorq $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $last_eight) {
|
|
$code .= <<___;
|
|
vpsrldq \$0xf, $tw[2], %zmm13
|
|
vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14
|
|
vpslldq \$0x1, $tw[2], %zmm15
|
|
vpxord %zmm14, %zmm15, %zmm15
|
|
___
|
|
}
|
|
|
|
# round 1
|
|
$code .= "vbroadcasti32x4 0x10($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesenc $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 2
|
|
$code .= "vbroadcasti32x4 0x20($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesenc $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 3
|
|
$code .= "vbroadcasti32x4 0x30($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesenc $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $last_eight) {
|
|
$code .= <<___;
|
|
vpsrldq \$0xf, $tw[3], %zmm13
|
|
vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14
|
|
vpslldq \$0x1, $tw[3], %zmm16
|
|
vpxord %zmm14, %zmm16, %zmm16
|
|
___
|
|
}
|
|
# round 4
|
|
$code .= "vbroadcasti32x4 0x40($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesenc $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 5
|
|
$code .= "vbroadcasti32x4 0x50($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesenc $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 6
|
|
$code .= "vbroadcasti32x4 0x60($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesenc $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $last_eight) {
|
|
$code .= <<___;
|
|
vpsrldq \$0xf, %zmm15, %zmm13
|
|
vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14
|
|
vpslldq \$0x1, %zmm15, %zmm17
|
|
vpxord %zmm14, %zmm17, %zmm17
|
|
___
|
|
}
|
|
# round 7
|
|
$code .= "vbroadcasti32x4 0x70($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesenc $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 8
|
|
$code .= "vbroadcasti32x4 0x80($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesenc $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 9
|
|
$code .= "vbroadcasti32x4 0x90($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesenc $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $last_eight) {
|
|
$code .= <<___;
|
|
vpsrldq \$0xf, %zmm16, %zmm13
|
|
vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14
|
|
vpslldq \$0x1, %zmm16, %zmm18
|
|
vpxord %zmm14, %zmm18, %zmm18
|
|
___
|
|
}
|
|
if ($is_128) {
|
|
# round 10
|
|
$code .= "vbroadcasti32x4 0xa0($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesenclast $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
} else {
|
|
# round 10
|
|
$code .= "vbroadcasti32x4 0xa0($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesenc $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
# round 11
|
|
$code .= "vbroadcasti32x4 0xb0($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesenc $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
# round 12
|
|
$code .= "vbroadcasti32x4 0xc0($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesenc $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
# round 13
|
|
$code .= "vbroadcasti32x4 0xd0($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesenc $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
# round 14
|
|
$code .= "vbroadcasti32x4 0xe0($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesenclast $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
}
|
|
|
|
# xor Tweak values
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vpxorq $tw[$i], $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
$code .= <<___;
|
|
# load next Tweak values
|
|
vmovdqa32 %zmm15, $tw[0]
|
|
vmovdqa32 %zmm16, $tw[1]
|
|
vmovdqa32 %zmm17, $tw[2]
|
|
vmovdqa32 %zmm18, $tw[3]
|
|
___
|
|
}
|
|
|
|
# Decrypt 16 blocks in parallel
|
|
# generate next 8 tweak values
|
|
sub decrypt_by_16_zmm {
|
|
my @st;
|
|
$st[0] = $_[0];
|
|
$st[1] = $_[1];
|
|
$st[2] = $_[2];
|
|
$st[3] = $_[3];
|
|
|
|
my @tw;
|
|
$tw[0] = $_[4];
|
|
$tw[1] = $_[5];
|
|
$tw[2] = $_[6];
|
|
$tw[3] = $_[7];
|
|
|
|
my $t0 = $_[8];
|
|
my $last_eight = $_[9];
|
|
my $is_128 = $_[10];
|
|
|
|
# xor Tweak values
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vpxorq $tw[$i], $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# ARK
|
|
$code .= "vbroadcasti32x4 ($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vpxorq $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $last_eight) {
|
|
$code .= <<___;
|
|
vpsrldq \$0xf, $tw[2], %zmm13
|
|
vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14
|
|
vpslldq \$0x1, $tw[2], %zmm15
|
|
vpxord %zmm14, %zmm15, %zmm15
|
|
___
|
|
}
|
|
|
|
# round 1
|
|
$code .= "vbroadcasti32x4 0x10($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 2
|
|
$code .= "vbroadcasti32x4 0x20($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 3
|
|
$code .= "vbroadcasti32x4 0x30($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $last_eight) {
|
|
$code .= <<___;
|
|
vpsrldq \$0xf, $tw[3], %zmm13
|
|
vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14
|
|
vpslldq \$0x1, $tw[3], %zmm16
|
|
vpxord %zmm14, %zmm16, %zmm16
|
|
___
|
|
}
|
|
# round 4
|
|
$code .= "vbroadcasti32x4 0x40($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 5
|
|
$code .= "vbroadcasti32x4 0x50($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 6
|
|
$code .= "vbroadcasti32x4 0x60($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $last_eight) {
|
|
$code .= <<___;
|
|
vpsrldq \$0xf, %zmm15, %zmm13
|
|
vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14
|
|
vpslldq \$0x1, %zmm15, %zmm17
|
|
vpxord %zmm14, %zmm17, %zmm17
|
|
___
|
|
}
|
|
# round 7
|
|
$code .= "vbroadcasti32x4 0x70($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 8
|
|
$code .= "vbroadcasti32x4 0x80($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 9
|
|
$code .= "vbroadcasti32x4 0x90($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
if (0 == $last_eight) {
|
|
$code .= <<___;
|
|
vpsrldq \$0xf, %zmm16, %zmm13
|
|
vpclmulqdq \$0x0,$ZPOLY, %zmm13, %zmm14
|
|
vpslldq \$0x1, %zmm16, %zmm18
|
|
vpxord %zmm14, %zmm18, %zmm18
|
|
___
|
|
}
|
|
if ($is_128) {
|
|
# round 10
|
|
$code .= "vbroadcasti32x4 0xa0($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesdeclast $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
} else {
|
|
# round 10
|
|
$code .= "vbroadcasti32x4 0xa0($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 11
|
|
$code .= "vbroadcasti32x4 0xb0($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 12
|
|
$code .= "vbroadcasti32x4 0xc0($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 13
|
|
$code .= "vbroadcasti32x4 0xd0($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesdec $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
# round 14
|
|
$code .= "vbroadcasti32x4 0xe0($key1), $t0\n";
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vaesdeclast $t0, $st[$i], $st[$i]\n";
|
|
}
|
|
}
|
|
|
|
# xor Tweak values
|
|
for (my $i = 0; $i < 4; $i++) {
|
|
$code .= "vpxorq $tw[$i], $st[$i], $st[$i]\n";
|
|
}
|
|
|
|
$code .= <<___;
|
|
# load next Tweak values
|
|
vmovdqa32 %zmm15, $tw[0]
|
|
vmovdqa32 %zmm16, $tw[1]
|
|
vmovdqa32 %zmm17, $tw[2]
|
|
vmovdqa32 %zmm18, $tw[3]
|
|
___
|
|
}
|
|
|
|
$code .= ".text\n";
|
|
|
|
{
|
|
$code.=<<"___";
|
|
.extern OPENSSL_ia32cap_P
|
|
.globl aesni_xts_avx512_eligible
|
|
.type aesni_xts_avx512_eligible,\@abi-omnipotent
|
|
.align 32
|
|
aesni_xts_avx512_eligible:
|
|
mov OPENSSL_ia32cap_P+8(%rip), %ecx
|
|
xor %eax,%eax
|
|
# 1<<31|1<<30|1<<17|1<<16 avx512vl + avx512bw + avx512dq + avx512f
|
|
and \$0xc0030000, %ecx
|
|
cmp \$0xc0030000, %ecx
|
|
jne .L_done
|
|
mov OPENSSL_ia32cap_P+12(%rip), %ecx
|
|
# 1<<10|1<<9|1<<6 vaes + vpclmulqdq + vbmi2
|
|
and \$0x640, %ecx
|
|
cmp \$0x640, %ecx
|
|
cmove %ecx,%eax
|
|
.L_done:
|
|
ret
|
|
.size aesni_xts_avx512_eligible, .-aesni_xts_avx512_eligible
|
|
___
|
|
}
|
|
|
|
|
|
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
# ;void aesni_xts_[128|256]_encrypt_avx512(
|
|
# ; const uint8_t *in, // input data
|
|
# ; uint8_t *out, // output data
|
|
# ; size_t length, // sector size, in bytes
|
|
# ; const AES_KEY *key1, // key used for "ECB" encryption
|
|
# ; const AES_KEY *key2, // key used for tweaking
|
|
# ; const uint8_t iv[16]) // initial tweak value, 16 bytes
|
|
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
sub enc {
|
|
my $is_128 = $_[0];
|
|
my $rndsuffix = &random_string();
|
|
|
|
if ($is_128) {
|
|
$code.=<<___;
|
|
.globl aesni_xts_128_encrypt_avx512
|
|
.hidden aesni_xts_128_encrypt_avx512
|
|
.type aesni_xts_128_encrypt_avx512,\@function,6
|
|
.align 32
|
|
aesni_xts_128_encrypt_avx512:
|
|
.cfi_startproc
|
|
endbranch
|
|
___
|
|
} else {
|
|
$code.=<<___;
|
|
.globl aesni_xts_256_encrypt_avx512
|
|
.hidden aesni_xts_256_encrypt_avx512
|
|
.type aesni_xts_256_encrypt_avx512,\@function,6
|
|
.align 32
|
|
aesni_xts_256_encrypt_avx512:
|
|
.cfi_startproc
|
|
endbranch
|
|
___
|
|
}
|
|
$code .= "push %rbp\n";
|
|
$code .= "mov $TW,%rbp\n";
|
|
$code .= "sub \$$VARIABLE_OFFSET,$TW\n";
|
|
$code .= "and \$0xffffffffffffffc0,$TW\n";
|
|
$code .= "mov %rbx,$GP_STORAGE($TW)\n";
|
|
|
|
if ($win64) {
|
|
$code .= "mov %rdi,$GP_STORAGE + 8*1($TW)\n";
|
|
$code .= "mov %rsi,$GP_STORAGE + 8*2($TW)\n";
|
|
$code .= "vmovdqa %xmm6, $XMM_STORAGE + 16*0($TW)\n";
|
|
$code .= "vmovdqa %xmm7, $XMM_STORAGE + 16*1($TW)\n";
|
|
$code .= "vmovdqa %xmm8, $XMM_STORAGE + 16*2($TW)\n";
|
|
$code .= "vmovdqa %xmm9, $XMM_STORAGE + 16*3($TW)\n";
|
|
$code .= "vmovdqa %xmm10, $XMM_STORAGE + 16*4($TW)\n";
|
|
$code .= "vmovdqa %xmm11, $XMM_STORAGE + 16*5($TW)\n";
|
|
$code .= "vmovdqa %xmm12, $XMM_STORAGE + 16*6($TW)\n";
|
|
$code .= "vmovdqa %xmm13, $XMM_STORAGE + 16*7($TW)\n";
|
|
$code .= "vmovdqa %xmm14, $XMM_STORAGE + 16*8($TW)\n";
|
|
$code .= "vmovdqa %xmm15, $XMM_STORAGE + 16*9($TW)\n";
|
|
}
|
|
|
|
$code .= "mov \$0x87, $gf_poly_8b\n";
|
|
$code .= "vmovdqu ($tweak),%xmm1\n"; # read initial tweak values
|
|
|
|
encrypt_tweak("%xmm1", $is_128);
|
|
|
|
if ($win64) {
|
|
$code .= "mov $input, 8 + 8*5(%rbp)\n"; # ciphertext pointer
|
|
$code .= "mov $output, 8 + 8*6(%rbp)\n"; # plaintext pointer
|
|
}
|
|
|
|
{
|
|
$code.=<<___;
|
|
|
|
cmp \$0x80,$length
|
|
jl .L_less_than_128_bytes_${rndsuffix}
|
|
vpbroadcastq $gf_poly_8b,$ZPOLY
|
|
cmp \$0x100,$length
|
|
jge .L_start_by16_${rndsuffix}
|
|
cmp \$0x80,$length
|
|
jge .L_start_by8_${rndsuffix}
|
|
|
|
.L_do_n_blocks_${rndsuffix}:
|
|
cmp \$0x0,$length
|
|
je .L_ret_${rndsuffix}
|
|
cmp \$0x70,$length
|
|
jge .L_remaining_num_blocks_is_7_${rndsuffix}
|
|
cmp \$0x60,$length
|
|
jge .L_remaining_num_blocks_is_6_${rndsuffix}
|
|
cmp \$0x50,$length
|
|
jge .L_remaining_num_blocks_is_5_${rndsuffix}
|
|
cmp \$0x40,$length
|
|
jge .L_remaining_num_blocks_is_4_${rndsuffix}
|
|
cmp \$0x30,$length
|
|
jge .L_remaining_num_blocks_is_3_${rndsuffix}
|
|
cmp \$0x20,$length
|
|
jge .L_remaining_num_blocks_is_2_${rndsuffix}
|
|
cmp \$0x10,$length
|
|
jge .L_remaining_num_blocks_is_1_${rndsuffix}
|
|
vmovdqa %xmm0,%xmm8
|
|
vmovdqa %xmm9,%xmm0
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
|
|
.L_remaining_num_blocks_is_7_${rndsuffix}:
|
|
mov \$0x0000ffffffffffff,$tmp1
|
|
kmovq $tmp1,%k1
|
|
vmovdqu8 ($input),%zmm1
|
|
vmovdqu8 0x40($input),%zmm2{%k1}
|
|
add \$0x70,$input
|
|
___
|
|
}
|
|
|
|
encrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1,($output)
|
|
vmovdqu8 %zmm2,0x40($output){%k1}
|
|
add \$0x70,$output
|
|
vextracti32x4 \$0x2,%zmm2,%xmm8
|
|
vextracti32x4 \$0x3,%zmm10,%xmm0
|
|
and \$0xf,$length
|
|
je .L_ret_${rndsuffix}
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
|
|
.L_remaining_num_blocks_is_6_${rndsuffix}:
|
|
vmovdqu8 ($input),%zmm1
|
|
vmovdqu8 0x40($input),%ymm2
|
|
add \$0x60,$input
|
|
___
|
|
}
|
|
|
|
encrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1,($output)
|
|
vmovdqu8 %ymm2,0x40($output)
|
|
add \$0x60,$output
|
|
vextracti32x4 \$0x1,%zmm2,%xmm8
|
|
vextracti32x4 \$0x2,%zmm10,%xmm0
|
|
and \$0xf,$length
|
|
je .L_ret_${rndsuffix}
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
|
|
.L_remaining_num_blocks_is_5_${rndsuffix}:
|
|
vmovdqu8 ($input),%zmm1
|
|
vmovdqu 0x40($input),%xmm2
|
|
add \$0x50,$input
|
|
___
|
|
}
|
|
|
|
encrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1,($output)
|
|
vmovdqu %xmm2,0x40($output)
|
|
add \$0x50,$output
|
|
vmovdqa %xmm2,%xmm8
|
|
vextracti32x4 \$0x1,%zmm10,%xmm0
|
|
and \$0xf,$length
|
|
je .L_ret_${rndsuffix}
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
|
|
.L_remaining_num_blocks_is_4_${rndsuffix}:
|
|
vmovdqu8 ($input),%zmm1
|
|
add \$0x40,$input
|
|
___
|
|
}
|
|
|
|
encrypt_by_four("%zmm1", "%zmm9", "%zmm0", $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1,($output)
|
|
add \$0x40,$output
|
|
vextracti32x4 \$0x3,%zmm1,%xmm8
|
|
vmovdqa64 %xmm10, %xmm0
|
|
and \$0xf,$length
|
|
je .L_ret_${rndsuffix}
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
{
|
|
$code .= <<___;
|
|
.L_remaining_num_blocks_is_3_${rndsuffix}:
|
|
mov \$-1, $tmp1
|
|
shr \$0x10, $tmp1
|
|
kmovq $tmp1, %k1
|
|
vmovdqu8 ($input), %zmm1{%k1}
|
|
add \$0x30, $input
|
|
___
|
|
}
|
|
|
|
encrypt_by_four("%zmm1", "%zmm9", "%zmm0", $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1, ($output){%k1}
|
|
add \$0x30, $output
|
|
vextracti32x4 \$0x2, %zmm1, %xmm8
|
|
vextracti32x4 \$0x3, %zmm9, %xmm0
|
|
and \$0xf, $length
|
|
je .L_ret_${rndsuffix}
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
{
|
|
$code .= <<___;
|
|
.L_remaining_num_blocks_is_2_${rndsuffix}:
|
|
vmovdqu8 ($input), %ymm1
|
|
add \$0x20, $input
|
|
___
|
|
}
|
|
|
|
encrypt_by_four("%ymm1", "%ymm9", "%ymm0", $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %ymm1,($output)
|
|
add \$0x20,$output
|
|
vextracti32x4 \$0x1, %zmm1, %xmm8
|
|
vextracti32x4 \$0x2,%zmm9,%xmm0
|
|
and \$0xf,$length
|
|
je .L_ret_${rndsuffix}
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
{
|
|
$code .= <<___;
|
|
.L_remaining_num_blocks_is_1_${rndsuffix}:
|
|
vmovdqu ($input),%xmm1
|
|
add \$0x10,$input
|
|
___
|
|
}
|
|
|
|
encrypt_final("%xmm1", "%xmm9", $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
add \$0x10,$output
|
|
vmovdqa %xmm1,%xmm8
|
|
vextracti32x4 \$0x1,%zmm9,%xmm0
|
|
and \$0xf,$length
|
|
je .L_ret_${rndsuffix}
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
|
|
|
|
.L_start_by16_${rndsuffix}:
|
|
vbroadcasti32x4 ($TW),%zmm0
|
|
vbroadcasti32x4 shufb_15_7(%rip),%zmm8
|
|
mov \$0xaa,$tmp1
|
|
kmovq $tmp1,%k2
|
|
vpshufb %zmm8,%zmm0,%zmm1
|
|
vpsllvq const_dq3210(%rip),%zmm0,%zmm4
|
|
vpsrlvq const_dq5678(%rip),%zmm1,%zmm2
|
|
vpclmulqdq \$0x0,%zmm25,%zmm2,%zmm3
|
|
vpxorq %zmm2,%zmm4,%zmm4{%k2}
|
|
vpxord %zmm4,%zmm3,%zmm9
|
|
vpsllvq const_dq7654(%rip),%zmm0,%zmm5
|
|
vpsrlvq const_dq1234(%rip),%zmm1,%zmm6
|
|
vpclmulqdq \$0x0,%zmm25,%zmm6,%zmm7
|
|
vpxorq %zmm6,%zmm5,%zmm5{%k2}
|
|
vpxord %zmm5,%zmm7,%zmm10
|
|
vpsrldq \$0xf,%zmm9,%zmm13
|
|
vpclmulqdq \$0x0,%zmm25,%zmm13,%zmm14
|
|
vpslldq \$0x1,%zmm9,%zmm11
|
|
vpxord %zmm14,%zmm11,%zmm11
|
|
vpsrldq \$0xf,%zmm10,%zmm15
|
|
vpclmulqdq \$0x0,%zmm25,%zmm15,%zmm16
|
|
vpslldq \$0x1,%zmm10,%zmm12
|
|
vpxord %zmm16,%zmm12,%zmm12
|
|
|
|
.L_main_loop_run_16_${rndsuffix}:
|
|
vmovdqu8 ($input),%zmm1
|
|
vmovdqu8 0x40($input),%zmm2
|
|
vmovdqu8 0x80($input),%zmm3
|
|
vmovdqu8 0xc0($input),%zmm4
|
|
add \$0x100,$input
|
|
___
|
|
}
|
|
|
|
encrypt_by_16_zmm("%zmm1", "%zmm2", "%zmm3", "%zmm4", "%zmm9",
|
|
"%zmm10", "%zmm11", "%zmm12", "%zmm0", 0, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1,($output)
|
|
vmovdqu8 %zmm2,0x40($output)
|
|
vmovdqu8 %zmm3,0x80($output)
|
|
vmovdqu8 %zmm4,0xc0($output)
|
|
add \$0x100,$output
|
|
sub \$0x100,$length
|
|
cmp \$0x100,$length
|
|
jae .L_main_loop_run_16_${rndsuffix}
|
|
cmp \$0x80,$length
|
|
jae .L_main_loop_run_8_${rndsuffix}
|
|
vextracti32x4 \$0x3,%zmm4,%xmm0
|
|
jmp .L_do_n_blocks_${rndsuffix}
|
|
|
|
.L_start_by8_${rndsuffix}:
|
|
vbroadcasti32x4 ($TW),%zmm0
|
|
vbroadcasti32x4 shufb_15_7(%rip),%zmm8
|
|
mov \$0xaa,$tmp1
|
|
kmovq $tmp1,%k2
|
|
vpshufb %zmm8,%zmm0,%zmm1
|
|
vpsllvq const_dq3210(%rip),%zmm0,%zmm4
|
|
vpsrlvq const_dq5678(%rip),%zmm1,%zmm2
|
|
vpclmulqdq \$0x0,%zmm25,%zmm2,%zmm3
|
|
vpxorq %zmm2,%zmm4,%zmm4{%k2}
|
|
vpxord %zmm4,%zmm3,%zmm9
|
|
vpsllvq const_dq7654(%rip),%zmm0,%zmm5
|
|
vpsrlvq const_dq1234(%rip),%zmm1,%zmm6
|
|
vpclmulqdq \$0x0,%zmm25,%zmm6,%zmm7
|
|
vpxorq %zmm6,%zmm5,%zmm5{%k2}
|
|
vpxord %zmm5,%zmm7,%zmm10
|
|
|
|
.L_main_loop_run_8_${rndsuffix}:
|
|
vmovdqu8 ($input),%zmm1
|
|
vmovdqu8 0x40($input),%zmm2
|
|
add \$0x80,$input
|
|
___
|
|
}
|
|
|
|
encrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 0, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1,($output)
|
|
vmovdqu8 %zmm2,0x40($output)
|
|
add \$0x80,$output
|
|
sub \$0x80,$length
|
|
cmp \$0x80,$length
|
|
jae .L_main_loop_run_8_${rndsuffix}
|
|
vextracti32x4 \$0x3,%zmm2,%xmm0
|
|
jmp .L_do_n_blocks_${rndsuffix}
|
|
|
|
.L_steal_cipher_${rndsuffix}:
|
|
vmovdqa %xmm8,%xmm2
|
|
lea vpshufb_shf_table(%rip),$TEMPLOW
|
|
vmovdqu ($TEMPLOW,$length,1),%xmm10
|
|
vpshufb %xmm10,%xmm8,%xmm8
|
|
vmovdqu -0x10($input,$length,1),%xmm3
|
|
vmovdqu %xmm8,-0x10($output,$length,1)
|
|
lea vpshufb_shf_table(%rip),$TEMPLOW
|
|
add \$16, $TEMPLOW
|
|
sub $length,$TEMPLOW
|
|
vmovdqu ($TEMPLOW),%xmm10
|
|
vpxor mask1(%rip),%xmm10,%xmm10
|
|
vpshufb %xmm10,%xmm3,%xmm3
|
|
vpblendvb %xmm10,%xmm2,%xmm3,%xmm3
|
|
vpxor %xmm0,%xmm3,%xmm8
|
|
vpxor ($key1),%xmm8,%xmm8
|
|
vaesenc 0x10($key1),%xmm8,%xmm8
|
|
vaesenc 0x20($key1),%xmm8,%xmm8
|
|
vaesenc 0x30($key1),%xmm8,%xmm8
|
|
vaesenc 0x40($key1),%xmm8,%xmm8
|
|
vaesenc 0x50($key1),%xmm8,%xmm8
|
|
vaesenc 0x60($key1),%xmm8,%xmm8
|
|
vaesenc 0x70($key1),%xmm8,%xmm8
|
|
vaesenc 0x80($key1),%xmm8,%xmm8
|
|
vaesenc 0x90($key1),%xmm8,%xmm8
|
|
___
|
|
if ($is_128) {
|
|
$code .= "vaesenclast 0xa0($key1),%xmm8,%xmm8\n";
|
|
} else {
|
|
$code .= <<___
|
|
vaesenc 0xa0($key1),%xmm8,%xmm8
|
|
vaesenc 0xb0($key1),%xmm8,%xmm8
|
|
vaesenc 0xc0($key1),%xmm8,%xmm8
|
|
vaesenc 0xd0($key1),%xmm8,%xmm8
|
|
vaesenclast 0xe0($key1),%xmm8,%xmm8
|
|
___
|
|
}
|
|
$code .= "vpxor %xmm0,%xmm8,%xmm8\n";
|
|
$code .= "vmovdqu %xmm8,-0x10($output)\n";
|
|
}
|
|
|
|
{
|
|
$code .= <<___;
|
|
.L_ret_${rndsuffix}:
|
|
mov $GP_STORAGE($TW),%rbx
|
|
xor $tmp1,$tmp1
|
|
mov $tmp1,$GP_STORAGE($TW)
|
|
# Zero-out the whole of `%zmm0`.
|
|
vpxorq %zmm0,%zmm0,%zmm0
|
|
___
|
|
}
|
|
|
|
if ($win64) {
|
|
$code .= <<___;
|
|
mov $GP_STORAGE + 8*1($TW),%rdi
|
|
mov $tmp1,$GP_STORAGE + 8*1($TW)
|
|
mov $GP_STORAGE + 8*2($TW),%rsi
|
|
mov $tmp1,$GP_STORAGE + 8*2($TW)
|
|
|
|
vmovdqa $XMM_STORAGE + 16 * 0($TW), %xmm6
|
|
vmovdqa $XMM_STORAGE + 16 * 1($TW), %xmm7
|
|
vmovdqa $XMM_STORAGE + 16 * 2($TW), %xmm8
|
|
vmovdqa $XMM_STORAGE + 16 * 3($TW), %xmm9
|
|
|
|
# Zero the 64 bytes we just restored to the xmm registers.
|
|
vmovdqa64 %zmm0,$XMM_STORAGE($TW)
|
|
|
|
vmovdqa $XMM_STORAGE + 16 * 4($TW), %xmm10
|
|
vmovdqa $XMM_STORAGE + 16 * 5($TW), %xmm11
|
|
vmovdqa $XMM_STORAGE + 16 * 6($TW), %xmm12
|
|
vmovdqa $XMM_STORAGE + 16 * 7($TW), %xmm13
|
|
|
|
# And again.
|
|
vmovdqa64 %zmm0,$XMM_STORAGE + 16 * 4($TW)
|
|
|
|
vmovdqa $XMM_STORAGE + 16 * 8($TW), %xmm14
|
|
vmovdqa $XMM_STORAGE + 16 * 9($TW), %xmm15
|
|
|
|
# Last round is only 32 bytes (256-bits), so we use `%ymm` as the
|
|
# source operand.
|
|
vmovdqa %ymm0,$XMM_STORAGE + 16 * 8($TW)
|
|
___
|
|
}
|
|
|
|
{
|
|
$code .= <<___;
|
|
mov %rbp,$TW
|
|
pop %rbp
|
|
vzeroupper
|
|
ret
|
|
|
|
.L_less_than_128_bytes_${rndsuffix}:
|
|
vpbroadcastq $gf_poly_8b, $ZPOLY
|
|
cmp \$0x10,$length
|
|
jb .L_ret_${rndsuffix}
|
|
vbroadcasti32x4 ($TW), %zmm0
|
|
vbroadcasti32x4 shufb_15_7(%rip), %zmm8
|
|
movl \$0xaa, %r8d
|
|
kmovq %r8, %k2
|
|
mov $length,$tmp1
|
|
and \$0x70,$tmp1
|
|
cmp \$0x60,$tmp1
|
|
je .L_num_blocks_is_6_${rndsuffix}
|
|
cmp \$0x50,$tmp1
|
|
je .L_num_blocks_is_5_${rndsuffix}
|
|
cmp \$0x40,$tmp1
|
|
je .L_num_blocks_is_4_${rndsuffix}
|
|
cmp \$0x30,$tmp1
|
|
je .L_num_blocks_is_3_${rndsuffix}
|
|
cmp \$0x20,$tmp1
|
|
je .L_num_blocks_is_2_${rndsuffix}
|
|
cmp \$0x10,$tmp1
|
|
je .L_num_blocks_is_1_${rndsuffix}
|
|
|
|
.L_num_blocks_is_7_${rndsuffix}:
|
|
vpshufb %zmm8, %zmm0, %zmm1
|
|
vpsllvq const_dq3210(%rip), %zmm0, %zmm4
|
|
vpsrlvq const_dq5678(%rip), %zmm1, %zmm2
|
|
vpclmulqdq \$0x00, $ZPOLY, %zmm2, %zmm3
|
|
vpxorq %zmm2, %zmm4, %zmm4{%k2}
|
|
vpxord %zmm4, %zmm3, %zmm9
|
|
vpsllvq const_dq7654(%rip), %zmm0, %zmm5
|
|
vpsrlvq const_dq1234(%rip), %zmm1, %zmm6
|
|
vpclmulqdq \$0x00, $ZPOLY, %zmm6, %zmm7
|
|
vpxorq %zmm6, %zmm5, %zmm5{%k2}
|
|
vpxord %zmm5, %zmm7, %zmm10
|
|
mov \$0x0000ffffffffffff, $tmp1
|
|
kmovq $tmp1, %k1
|
|
vmovdqu8 16*0($input), %zmm1
|
|
vmovdqu8 16*4($input), %zmm2{%k1}
|
|
|
|
add \$0x70,$input
|
|
___
|
|
}
|
|
|
|
encrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1, 16*0($output)
|
|
vmovdqu8 %zmm2, 16*4($output){%k1}
|
|
add \$0x70,$output
|
|
vextracti32x4 \$0x2, %zmm2, %xmm8
|
|
vextracti32x4 \$0x3, %zmm10, %xmm0
|
|
and \$0xf,$length
|
|
je .L_ret_${rndsuffix}
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
{
|
|
$code .= <<___;
|
|
.L_num_blocks_is_6_${rndsuffix}:
|
|
vpshufb %zmm8, %zmm0, %zmm1
|
|
vpsllvq const_dq3210(%rip), %zmm0, %zmm4
|
|
vpsrlvq const_dq5678(%rip), %zmm1, %zmm2
|
|
vpclmulqdq \$0x00, $ZPOLY, %zmm2, %zmm3
|
|
vpxorq %zmm2, %zmm4, %zmm4{%k2}
|
|
vpxord %zmm4, %zmm3, %zmm9
|
|
vpsllvq const_dq7654(%rip), %zmm0, %zmm5
|
|
vpsrlvq const_dq1234(%rip), %zmm1, %zmm6
|
|
vpclmulqdq \$0x00, $ZPOLY, %zmm6, %zmm7
|
|
vpxorq %zmm6, %zmm5, %zmm5{%k2}
|
|
vpxord %zmm5, %zmm7, %zmm10
|
|
vmovdqu8 16*0($input), %zmm1
|
|
vmovdqu8 16*4($input), %ymm2
|
|
add \$96, $input
|
|
___
|
|
}
|
|
|
|
encrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1, 16*0($output)
|
|
vmovdqu8 %ymm2, 16*4($output)
|
|
add \$96, $output
|
|
|
|
vextracti32x4 \$0x1, %ymm2, %xmm8
|
|
vextracti32x4 \$0x2, %zmm10, %xmm0
|
|
and \$0xf,$length
|
|
je .L_ret_${rndsuffix}
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
{
|
|
$code .= <<___;
|
|
.L_num_blocks_is_5_${rndsuffix}:
|
|
vpshufb %zmm8, %zmm0, %zmm1
|
|
vpsllvq const_dq3210(%rip), %zmm0, %zmm4
|
|
vpsrlvq const_dq5678(%rip), %zmm1, %zmm2
|
|
vpclmulqdq \$0x00, $ZPOLY, %zmm2, %zmm3
|
|
vpxorq %zmm2, %zmm4, %zmm4{%k2}
|
|
vpxord %zmm4, %zmm3, %zmm9
|
|
vpsllvq const_dq7654(%rip), %zmm0, %zmm5
|
|
vpsrlvq const_dq1234(%rip), %zmm1, %zmm6
|
|
vpclmulqdq \$0x00, $ZPOLY, %zmm6, %zmm7
|
|
vpxorq %zmm6, %zmm5, %zmm5{%k2}
|
|
vpxord %zmm5, %zmm7, %zmm10
|
|
vmovdqu8 16*0($input), %zmm1
|
|
vmovdqu8 16*4($input), %xmm2
|
|
add \$80, $input
|
|
___
|
|
}
|
|
|
|
encrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1, 16*0($output)
|
|
vmovdqu8 %xmm2, 16*4($output)
|
|
add \$80, $output
|
|
|
|
vmovdqa %xmm2, %xmm8
|
|
vextracti32x4 \$0x1, %zmm10, %xmm0
|
|
and \$0xf,$length
|
|
je .L_ret_${rndsuffix}
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
{
|
|
$code .= <<___;
|
|
.L_num_blocks_is_4_${rndsuffix}:
|
|
vpshufb %zmm8, %zmm0, %zmm1
|
|
vpsllvq const_dq3210(%rip), %zmm0, %zmm4
|
|
vpsrlvq const_dq5678(%rip), %zmm1, %zmm2
|
|
vpclmulqdq \$0x00, $ZPOLY, %zmm2, %zmm3
|
|
vpxorq %zmm2, %zmm4, %zmm4{%k2}
|
|
vpxord %zmm4, %zmm3, %zmm9
|
|
vpsllvq const_dq7654(%rip), %zmm0, %zmm5
|
|
vpsrlvq const_dq1234(%rip), %zmm1, %zmm6
|
|
vpclmulqdq \$0x00, $ZPOLY, %zmm6, %zmm7
|
|
vpxorq %zmm6, %zmm5, %zmm5{%k2}
|
|
vpxord %zmm5, %zmm7, %zmm10
|
|
vmovdqu8 16*0($input), %zmm1
|
|
add \$64, $input
|
|
___
|
|
}
|
|
|
|
encrypt_by_four("%zmm1", "%zmm9", "%zmm0", $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1, 16*0($output)
|
|
add \$64, $output
|
|
vextracti32x4 \$0x3, %zmm1, %xmm8
|
|
vmovdqa %xmm10, %xmm0
|
|
and \$0xf,$length
|
|
je .L_ret_${rndsuffix}
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
{
|
|
$code .= <<___;
|
|
.L_num_blocks_is_3_${rndsuffix}:
|
|
vpshufb %zmm8, %zmm0, %zmm1
|
|
vpsllvq const_dq3210(%rip), %zmm0, %zmm4
|
|
vpsrlvq const_dq5678(%rip), %zmm1, %zmm2
|
|
vpclmulqdq \$0x00, $ZPOLY, %zmm2, %zmm3
|
|
vpxorq %zmm2, %zmm4, %zmm4{%k2}
|
|
vpxord %zmm4, %zmm3, %zmm9
|
|
mov \$0x0000ffffffffffff, $tmp1
|
|
kmovq $tmp1, %k1
|
|
vmovdqu8 16*0($input), %zmm1{%k1}
|
|
add \$48, $input
|
|
___
|
|
}
|
|
|
|
encrypt_by_four("%zmm1", "%zmm9", "%zmm0", $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1, 16*0($output){%k1}
|
|
add \$48, $output
|
|
vextracti32x4 \$2, %zmm1, %xmm8
|
|
vextracti32x4 \$3, %zmm9, %xmm0
|
|
and \$0xf,$length
|
|
je .L_ret_${rndsuffix}
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
{
|
|
$code .= <<___;
|
|
.L_num_blocks_is_2_${rndsuffix}:
|
|
vpshufb %zmm8, %zmm0, %zmm1
|
|
vpsllvq const_dq3210(%rip), %zmm0, %zmm4
|
|
vpsrlvq const_dq5678(%rip), %zmm1, %zmm2
|
|
vpclmulqdq \$0x00, $ZPOLY, %zmm2, %zmm3
|
|
vpxorq %zmm2, %zmm4, %zmm4{%k2}
|
|
vpxord %zmm4, %zmm3, %zmm9
|
|
|
|
vmovdqu8 16*0($input), %ymm1
|
|
add \$32, $input
|
|
___
|
|
}
|
|
|
|
encrypt_by_four("%ymm1", "%ymm9", "%ymm0", $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %ymm1, 16*0($output)
|
|
add \$32, $output
|
|
|
|
vextracti32x4 \$1, %ymm1, %xmm8
|
|
vextracti32x4 \$2, %zmm9, %xmm0
|
|
and \$0xf,$length
|
|
je .L_ret_${rndsuffix}
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
{
|
|
$code .= <<___;
|
|
.L_num_blocks_is_1_${rndsuffix}:
|
|
vpshufb %zmm8, %zmm0, %zmm1
|
|
vpsllvq const_dq3210(%rip), %zmm0, %zmm4
|
|
vpsrlvq const_dq5678(%rip), %zmm1, %zmm2
|
|
vpclmulqdq \$0x00, $ZPOLY, %zmm2, %zmm3
|
|
vpxorq %zmm2, %zmm4, %zmm4{%k2}
|
|
vpxord %zmm4, %zmm3, %zmm9
|
|
|
|
vmovdqu8 16*0($input), %xmm1
|
|
add \$16, $input
|
|
___
|
|
}
|
|
|
|
encrypt_by_four("%ymm1", "%ymm9", "%ymm0", $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %xmm1, 16*0($output)
|
|
add \$16, $output
|
|
|
|
vmovdqa %xmm1, %xmm8
|
|
vextracti32x4 \$1, %zmm9, %xmm0
|
|
and \$0xf,$length
|
|
je .L_ret_${rndsuffix}
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
.cfi_endproc
|
|
___
|
|
}
|
|
}
|
|
|
|
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
# ;void aesni_xts_[128|256]_decrypt_avx512(
|
|
# ; const uint8_t *in, // input data
|
|
# ; uint8_t *out, // output data
|
|
# ; size_t length, // sector size, in bytes
|
|
# ; const AES_KEY *key1, // key used for "ECB" encryption, 16*2 bytes
|
|
# ; const AES_KEY *key2, // key used for tweaking, 16*2 bytes
|
|
# ; const uint8_t iv[16]) // initial tweak value, 16 bytes
|
|
# ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
sub dec {
|
|
my $is_128 = $_[0];
|
|
my $rndsuffix = &random_string();
|
|
|
|
if ($is_128) {
|
|
$code.=<<___;
|
|
.globl aesni_xts_128_decrypt_avx512
|
|
.hidden aesni_xts_128_decrypt_avx512
|
|
.type aesni_xts_128_decrypt_avx512,\@function,6
|
|
.align 32
|
|
aesni_xts_128_decrypt_avx512:
|
|
.cfi_startproc
|
|
endbranch
|
|
___
|
|
} else {
|
|
$code.=<<___;
|
|
.globl aesni_xts_256_decrypt_avx512
|
|
.hidden aesni_xts_256_decrypt_avx512
|
|
.type aesni_xts_256_decrypt_avx512,\@function,6
|
|
.align 32
|
|
aesni_xts_256_decrypt_avx512:
|
|
.cfi_startproc
|
|
endbranch
|
|
___
|
|
}
|
|
$code .= "push %rbp\n";
|
|
$code .= "mov $TW,%rbp\n";
|
|
$code .= "sub \$$VARIABLE_OFFSET,$TW\n";
|
|
$code .= "and \$0xffffffffffffffc0,$TW\n";
|
|
$code .= "mov %rbx,$GP_STORAGE($TW)\n";
|
|
|
|
if ($win64) {
|
|
$code .= "mov %rdi,$GP_STORAGE + 8*1($TW)\n";
|
|
$code .= "mov %rsi,$GP_STORAGE + 8*2($TW)\n";
|
|
$code .= "vmovdqa %xmm6, $XMM_STORAGE + 16*0($TW)\n";
|
|
$code .= "vmovdqa %xmm7, $XMM_STORAGE + 16*1($TW)\n";
|
|
$code .= "vmovdqa %xmm8, $XMM_STORAGE + 16*2($TW)\n";
|
|
$code .= "vmovdqa %xmm9, $XMM_STORAGE + 16*3($TW)\n";
|
|
$code .= "vmovdqa %xmm10, $XMM_STORAGE + 16*4($TW)\n";
|
|
$code .= "vmovdqa %xmm11, $XMM_STORAGE + 16*5($TW)\n";
|
|
$code .= "vmovdqa %xmm12, $XMM_STORAGE + 16*6($TW)\n";
|
|
$code .= "vmovdqa %xmm13, $XMM_STORAGE + 16*7($TW)\n";
|
|
$code .= "vmovdqa %xmm14, $XMM_STORAGE + 16*8($TW)\n";
|
|
$code .= "vmovdqa %xmm15, $XMM_STORAGE + 16*9($TW)\n";
|
|
}
|
|
|
|
$code .= "mov \$0x87, $gf_poly_8b\n";
|
|
$code .= "vmovdqu ($tweak),%xmm1\n"; # read initial tweak values
|
|
|
|
encrypt_tweak("%xmm1", $is_128);
|
|
|
|
if ($win64) {
|
|
$code .= "mov $input, 8 + 8*5(%rbp)\n"; # ciphertext pointer
|
|
$code .= "mov $output, 8 + 8*6(%rbp)\n"; # plaintext pointer
|
|
}
|
|
|
|
{
|
|
$code.=<<___;
|
|
|
|
cmp \$0x80,$length
|
|
jb .L_less_than_128_bytes_${rndsuffix}
|
|
vpbroadcastq $gf_poly_8b,$ZPOLY
|
|
cmp \$0x100,$length
|
|
jge .L_start_by16_${rndsuffix}
|
|
jmp .L_start_by8_${rndsuffix}
|
|
|
|
.L_do_n_blocks_${rndsuffix}:
|
|
cmp \$0x0,$length
|
|
je .L_ret_${rndsuffix}
|
|
cmp \$0x70,$length
|
|
jge .L_remaining_num_blocks_is_7_${rndsuffix}
|
|
cmp \$0x60,$length
|
|
jge .L_remaining_num_blocks_is_6_${rndsuffix}
|
|
cmp \$0x50,$length
|
|
jge .L_remaining_num_blocks_is_5_${rndsuffix}
|
|
cmp \$0x40,$length
|
|
jge .L_remaining_num_blocks_is_4_${rndsuffix}
|
|
cmp \$0x30,$length
|
|
jge .L_remaining_num_blocks_is_3_${rndsuffix}
|
|
cmp \$0x20,$length
|
|
jge .L_remaining_num_blocks_is_2_${rndsuffix}
|
|
cmp \$0x10,$length
|
|
jge .L_remaining_num_blocks_is_1_${rndsuffix}
|
|
|
|
# _remaining_num_blocks_is_0:
|
|
vmovdqu %xmm5, %xmm1
|
|
# xmm5 contains last full block to decrypt with next teawk
|
|
___
|
|
}
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 1, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1, -0x10($output)
|
|
vmovdqa %xmm1, %xmm8
|
|
|
|
# Calc previous tweak
|
|
mov \$0x1,$tmp1
|
|
kmovq $tmp1, %k1
|
|
vpsllq \$0x3f,%xmm9,%xmm13
|
|
vpsraq \$0x3f,%xmm13,%xmm14
|
|
vpandq %xmm25,%xmm14,%xmm5
|
|
vpxorq %xmm5,%xmm9,%xmm9{%k1}
|
|
vpsrldq \$0x8,%xmm9,%xmm10
|
|
.byte 98, 211, 181, 8, 115, 194, 1 #vpshrdq \$0x1,%xmm10,%xmm9,%xmm0
|
|
vpslldq \$0x8,%xmm13,%xmm13
|
|
vpxorq %xmm13,%xmm0,%xmm0
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
|
|
.L_remaining_num_blocks_is_7_${rndsuffix}:
|
|
mov \$0xffffffffffffffff,$tmp1
|
|
shr \$0x10,$tmp1
|
|
kmovq $tmp1,%k1
|
|
vmovdqu8 ($input),%zmm1
|
|
vmovdqu8 0x40($input),%zmm2{%k1}
|
|
add \$0x70,$input
|
|
and \$0xf,$length
|
|
je .L_done_7_remain_${rndsuffix}
|
|
vextracti32x4 \$0x2,%zmm10,%xmm12
|
|
vextracti32x4 \$0x3,%zmm10,%xmm13
|
|
vinserti32x4 \$0x2,%xmm13,%zmm10,%zmm10
|
|
___
|
|
}
|
|
|
|
decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1, ($output)
|
|
vmovdqu8 %zmm2, 0x40($output){%k1}
|
|
add \$0x70, $output
|
|
vextracti32x4 \$0x2,%zmm2,%xmm8
|
|
vmovdqa %xmm12,%xmm0
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
$code .= "\n.L_done_7_remain_${rndsuffix}:\n";
|
|
decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1, ($output)
|
|
vmovdqu8 %zmm2, 0x40($output){%k1}
|
|
jmp .L_ret_${rndsuffix}
|
|
|
|
.L_remaining_num_blocks_is_6_${rndsuffix}:
|
|
vmovdqu8 ($input),%zmm1
|
|
vmovdqu8 0x40($input),%ymm2
|
|
add \$0x60,$input
|
|
and \$0xf, $length
|
|
je .L_done_6_remain_${rndsuffix}
|
|
vextracti32x4 \$0x1,%zmm10,%xmm12
|
|
vextracti32x4 \$0x2,%zmm10,%xmm13
|
|
vinserti32x4 \$0x1,%xmm13,%zmm10,%zmm10
|
|
___
|
|
}
|
|
|
|
decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1, ($output)
|
|
vmovdqu8 %ymm2, 0x40($output)
|
|
add \$0x60,$output
|
|
vextracti32x4 \$0x1,%zmm2,%xmm8
|
|
vmovdqa %xmm12,%xmm0
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
$code .= "\n.L_done_6_remain_${rndsuffix}:\n";
|
|
decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1, ($output)
|
|
vmovdqu8 %ymm2,0x40($output)
|
|
jmp .L_ret_${rndsuffix}
|
|
|
|
.L_remaining_num_blocks_is_5_${rndsuffix}:
|
|
vmovdqu8 ($input),%zmm1
|
|
vmovdqu 0x40($input),%xmm2
|
|
add \$0x50,$input
|
|
and \$0xf,$length
|
|
je .L_done_5_remain_${rndsuffix}
|
|
vmovdqa %xmm10,%xmm12
|
|
vextracti32x4 \$0x1,%zmm10,%xmm10
|
|
___
|
|
}
|
|
|
|
decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1, ($output)
|
|
vmovdqu %xmm2, 0x40($output)
|
|
add \$0x50, $output
|
|
vmovdqa %xmm2,%xmm8
|
|
vmovdqa %xmm12,%xmm0
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
$code .= "\n.L_done_5_remain_${rndsuffix}:\n";
|
|
decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1, ($output)
|
|
vmovdqu8 %xmm2, 0x40($output)
|
|
jmp .L_ret_${rndsuffix}
|
|
|
|
.L_remaining_num_blocks_is_4_${rndsuffix}:
|
|
vmovdqu8 ($input),%zmm1
|
|
add \$0x40,$input
|
|
and \$0xf, $length
|
|
je .L_done_4_remain_${rndsuffix}
|
|
vextracti32x4 \$0x3,%zmm9,%xmm12
|
|
vinserti32x4 \$0x3,%xmm10,%zmm9,%zmm9
|
|
___
|
|
}
|
|
|
|
decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1,($output)
|
|
add \$0x40,$output
|
|
vextracti32x4 \$0x3,%zmm1,%xmm8
|
|
vmovdqa %xmm12,%xmm0
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
$code .= "\n.L_done_4_remain_${rndsuffix}:\n";
|
|
decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1, ($output)
|
|
jmp .L_ret_${rndsuffix}
|
|
|
|
.L_remaining_num_blocks_is_3_${rndsuffix}:
|
|
vmovdqu ($input),%xmm1
|
|
vmovdqu 0x10($input),%xmm2
|
|
vmovdqu 0x20($input),%xmm3
|
|
add \$0x30,$input
|
|
and \$0xf,$length
|
|
je .L_done_3_remain_${rndsuffix}
|
|
vextracti32x4 \$0x2,%zmm9,%xmm13
|
|
vextracti32x4 \$0x1,%zmm9,%xmm10
|
|
vextracti32x4 \$0x3,%zmm9,%xmm11
|
|
___
|
|
}
|
|
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 3, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
vmovdqu %xmm2,0x10($output)
|
|
vmovdqu %xmm3,0x20($output)
|
|
add \$0x30,$output
|
|
vmovdqa %xmm3,%xmm8
|
|
vmovdqa %xmm13,%xmm0
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
$code .= "\n.L_done_3_remain_${rndsuffix}:\n";
|
|
$code .= "vextracti32x4 \$0x1,%zmm9,%xmm10\n";
|
|
$code .= "vextracti32x4 \$0x2,%zmm9,%xmm11\n";
|
|
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 3, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
vmovdqu %xmm2,0x10($output)
|
|
vmovdqu %xmm3,0x20($output)
|
|
jmp .L_ret_${rndsuffix}
|
|
|
|
.L_remaining_num_blocks_is_2_${rndsuffix}:
|
|
vmovdqu ($input),%xmm1
|
|
vmovdqu 0x10($input),%xmm2
|
|
add \$0x20,$input
|
|
and \$0xf,$length
|
|
je .L_done_2_remain_${rndsuffix}
|
|
vextracti32x4 \$0x2,%zmm9,%xmm10
|
|
vextracti32x4 \$0x1,%zmm9,%xmm12
|
|
___
|
|
}
|
|
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 2, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
vmovdqu %xmm2,0x10($output)
|
|
add \$0x20,$output
|
|
vmovdqa %xmm2,%xmm8
|
|
vmovdqa %xmm12,%xmm0
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
$code .= "\n.L_done_2_remain_${rndsuffix}:\n";
|
|
$code .= "vextracti32x4 \$0x1,%zmm9,%xmm10\n";
|
|
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 2, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
vmovdqu %xmm2,0x10($output)
|
|
jmp .L_ret_${rndsuffix}
|
|
|
|
.L_remaining_num_blocks_is_1_${rndsuffix}:
|
|
vmovdqu ($input),%xmm1
|
|
add \$0x10,$input
|
|
and \$0xf,$length
|
|
je .L_done_1_remain_${rndsuffix}
|
|
vextracti32x4 \$0x1,%zmm9,%xmm11
|
|
___
|
|
}
|
|
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm11", "%xmm10", "%xmm9", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 1, 1, $is_128);
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
add \$0x10,$output
|
|
vmovdqa %xmm1,%xmm8
|
|
vmovdqa %xmm9,%xmm0
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
$code .= "\n.L_done_1_remain_${rndsuffix}:\n";
|
|
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 1, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1, ($output)
|
|
jmp .L_ret_${rndsuffix}
|
|
|
|
.L_start_by16_${rndsuffix}:
|
|
vbroadcasti32x4 ($TW),%zmm0
|
|
vbroadcasti32x4 shufb_15_7(%rip),%zmm8
|
|
mov \$0xaa,$tmp1
|
|
kmovq $tmp1,%k2
|
|
|
|
# Mult tweak by 2^{3, 2, 1, 0}
|
|
vpshufb %zmm8,%zmm0,%zmm1
|
|
vpsllvq const_dq3210(%rip),%zmm0,%zmm4
|
|
vpsrlvq const_dq5678(%rip),%zmm1,%zmm2
|
|
vpclmulqdq \$0x0,$ZPOLY,%zmm2,%zmm3
|
|
vpxorq %zmm2,%zmm4,%zmm4{%k2}
|
|
vpxord %zmm4,%zmm3,%zmm9
|
|
|
|
# Mult tweak by 2^{7, 6, 5, 4}
|
|
vpsllvq const_dq7654(%rip),%zmm0,%zmm5
|
|
vpsrlvq const_dq1234(%rip),%zmm1,%zmm6
|
|
vpclmulqdq \$0x0,%zmm25,%zmm6,%zmm7
|
|
vpxorq %zmm6,%zmm5,%zmm5{%k2}
|
|
vpxord %zmm5,%zmm7,%zmm10
|
|
|
|
# Make next 8 tweak values by all x 2^8
|
|
vpsrldq \$0xf,%zmm9,%zmm13
|
|
vpclmulqdq \$0x0,%zmm25,%zmm13,%zmm14
|
|
vpslldq \$0x1,%zmm9,%zmm11
|
|
vpxord %zmm14,%zmm11,%zmm11
|
|
|
|
vpsrldq \$0xf,%zmm10,%zmm15
|
|
vpclmulqdq \$0x0,%zmm25,%zmm15,%zmm16
|
|
vpslldq \$0x1,%zmm10,%zmm12
|
|
vpxord %zmm16,%zmm12,%zmm12
|
|
|
|
.L_main_loop_run_16_${rndsuffix}:
|
|
vmovdqu8 ($input),%zmm1
|
|
vmovdqu8 0x40($input),%zmm2
|
|
vmovdqu8 0x80($input),%zmm3
|
|
vmovdqu8 0xc0($input),%zmm4
|
|
vmovdqu8 0xf0($input),%xmm5
|
|
add \$0x100,$input
|
|
___
|
|
}
|
|
|
|
decrypt_by_16_zmm("%zmm1", "%zmm2", "%zmm3", "%zmm4", "%zmm9",
|
|
"%zmm10", "%zmm11", "%zmm12", "%zmm0", 0, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1,($output)
|
|
vmovdqu8 %zmm2,0x40($output)
|
|
vmovdqu8 %zmm3,0x80($output)
|
|
vmovdqu8 %zmm4,0xc0($output)
|
|
add \$0x100,$output
|
|
sub \$0x100,$length
|
|
cmp \$0x100,$length
|
|
jge .L_main_loop_run_16_${rndsuffix}
|
|
|
|
cmp \$0x80,$length
|
|
jge .L_main_loop_run_8_${rndsuffix}
|
|
jmp .L_do_n_blocks_${rndsuffix}
|
|
|
|
.L_start_by8_${rndsuffix}:
|
|
# Make first 7 tweak values
|
|
vbroadcasti32x4 ($TW),%zmm0
|
|
vbroadcasti32x4 shufb_15_7(%rip),%zmm8
|
|
mov \$0xaa,$tmp1
|
|
kmovq $tmp1,%k2
|
|
|
|
# Mult tweak by 2^{3, 2, 1, 0}
|
|
vpshufb %zmm8,%zmm0,%zmm1
|
|
vpsllvq const_dq3210(%rip),%zmm0,%zmm4
|
|
vpsrlvq const_dq5678(%rip),%zmm1,%zmm2
|
|
vpclmulqdq \$0x0,%zmm25,%zmm2,%zmm3
|
|
vpxorq %zmm2,%zmm4,%zmm4{%k2}
|
|
vpxord %zmm4,%zmm3,%zmm9
|
|
|
|
# Mult tweak by 2^{7, 6, 5, 4}
|
|
vpsllvq const_dq7654(%rip),%zmm0,%zmm5
|
|
vpsrlvq const_dq1234(%rip),%zmm1,%zmm6
|
|
vpclmulqdq \$0x0,%zmm25,%zmm6,%zmm7
|
|
vpxorq %zmm6,%zmm5,%zmm5{%k2}
|
|
vpxord %zmm5,%zmm7,%zmm10
|
|
|
|
.L_main_loop_run_8_${rndsuffix}:
|
|
vmovdqu8 ($input),%zmm1
|
|
vmovdqu8 0x40($input),%zmm2
|
|
vmovdqu8 0x70($input),%xmm5
|
|
add \$0x80,$input
|
|
___
|
|
}
|
|
|
|
|
|
decrypt_by_eight_zmm("%zmm1", "%zmm2", "%zmm9", "%zmm10", "%zmm0", 0, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu8 %zmm1,($output)
|
|
vmovdqu8 %zmm2,0x40($output)
|
|
add \$0x80,$output
|
|
sub \$0x80,$length
|
|
cmp \$0x80,$length
|
|
jge .L_main_loop_run_8_${rndsuffix}
|
|
jmp .L_do_n_blocks_${rndsuffix}
|
|
|
|
.L_steal_cipher_${rndsuffix}:
|
|
# start cipher stealing simplified: xmm8-last cipher block, xmm0-next tweak
|
|
vmovdqa %xmm8,%xmm2
|
|
|
|
# shift xmm8 to the left by 16-N_val bytes
|
|
lea vpshufb_shf_table(%rip),$TEMPLOW
|
|
vmovdqu ($TEMPLOW,$length,1),%xmm10
|
|
vpshufb %xmm10,%xmm8,%xmm8
|
|
|
|
|
|
vmovdqu -0x10($input,$length,1),%xmm3
|
|
vmovdqu %xmm8,-0x10($output,$length,1)
|
|
|
|
# shift xmm3 to the right by 16-N_val bytes
|
|
lea vpshufb_shf_table(%rip), $TEMPLOW
|
|
add \$16, $TEMPLOW
|
|
sub $length,$TEMPLOW
|
|
vmovdqu ($TEMPLOW),%xmm10
|
|
vpxor mask1(%rip),%xmm10,%xmm10
|
|
vpshufb %xmm10,%xmm3,%xmm3
|
|
|
|
vpblendvb %xmm10,%xmm2,%xmm3,%xmm3
|
|
|
|
# xor Tweak value
|
|
vpxor %xmm0,%xmm3,%xmm8
|
|
|
|
# decrypt last block with cipher stealing
|
|
vpxor ($key1),%xmm8,%xmm8
|
|
vaesdec 0x10($key1),%xmm8,%xmm8
|
|
vaesdec 0x20($key1),%xmm8,%xmm8
|
|
vaesdec 0x30($key1),%xmm8,%xmm8
|
|
vaesdec 0x40($key1),%xmm8,%xmm8
|
|
vaesdec 0x50($key1),%xmm8,%xmm8
|
|
vaesdec 0x60($key1),%xmm8,%xmm8
|
|
vaesdec 0x70($key1),%xmm8,%xmm8
|
|
vaesdec 0x80($key1),%xmm8,%xmm8
|
|
vaesdec 0x90($key1),%xmm8,%xmm8
|
|
___
|
|
if ($is_128) {
|
|
$code .= "vaesdeclast 0xa0($key1),%xmm8,%xmm8\n";
|
|
} else {
|
|
$code .= <<___;
|
|
vaesdec 0xa0($key1),%xmm8,%xmm8
|
|
vaesdec 0xb0($key1),%xmm8,%xmm8
|
|
vaesdec 0xc0($key1),%xmm8,%xmm8
|
|
vaesdec 0xd0($key1),%xmm8,%xmm8
|
|
vaesdeclast 0xe0($key1),%xmm8,%xmm8
|
|
___
|
|
}
|
|
$code .= <<___
|
|
# xor Tweak value
|
|
vpxor %xmm0,%xmm8,%xmm8
|
|
|
|
.L_done_${rndsuffix}:
|
|
# store last ciphertext value
|
|
vmovdqu %xmm8,-0x10($output)
|
|
___
|
|
}
|
|
|
|
{
|
|
$code .= <<___;
|
|
.L_ret_${rndsuffix}:
|
|
mov $GP_STORAGE($TW),%rbx
|
|
xor $tmp1,$tmp1
|
|
mov $tmp1,$GP_STORAGE($TW)
|
|
# Zero-out the whole of `%zmm0`.
|
|
vpxorq %zmm0,%zmm0,%zmm0
|
|
___
|
|
}
|
|
|
|
if ($win64) {
|
|
$code .= <<___;
|
|
mov $GP_STORAGE + 8*1($TW),%rdi
|
|
mov $tmp1,$GP_STORAGE + 8*1($TW)
|
|
mov $GP_STORAGE + 8*2($TW),%rsi
|
|
mov $tmp1,$GP_STORAGE + 8*2($TW)
|
|
|
|
vmovdqa $XMM_STORAGE + 16 * 0($TW), %xmm6
|
|
vmovdqa $XMM_STORAGE + 16 * 1($TW), %xmm7
|
|
vmovdqa $XMM_STORAGE + 16 * 2($TW), %xmm8
|
|
vmovdqa $XMM_STORAGE + 16 * 3($TW), %xmm9
|
|
|
|
# Zero the 64 bytes we just restored to the xmm registers.
|
|
vmovdqa64 %zmm0,$XMM_STORAGE($TW)
|
|
|
|
vmovdqa $XMM_STORAGE + 16 * 4($TW), %xmm10
|
|
vmovdqa $XMM_STORAGE + 16 * 5($TW), %xmm11
|
|
vmovdqa $XMM_STORAGE + 16 * 6($TW), %xmm12
|
|
vmovdqa $XMM_STORAGE + 16 * 7($TW), %xmm13
|
|
|
|
# And again.
|
|
vmovdqa64 %zmm0,$XMM_STORAGE + 16 * 4($TW)
|
|
|
|
vmovdqa $XMM_STORAGE + 16 * 8($TW), %xmm14
|
|
vmovdqa $XMM_STORAGE + 16 * 9($TW), %xmm15
|
|
|
|
# Last round is only 32 bytes (256-bits), so we use `%ymm` as the
|
|
# source operand.
|
|
vmovdqa %ymm0,$XMM_STORAGE + 16 * 8($TW)
|
|
___
|
|
}
|
|
|
|
{
|
|
$code .= <<___;
|
|
mov %rbp,$TW
|
|
pop %rbp
|
|
vzeroupper
|
|
ret
|
|
|
|
.L_less_than_128_bytes_${rndsuffix}:
|
|
cmp \$0x10,$length
|
|
jb .L_ret_${rndsuffix}
|
|
|
|
mov $length,$tmp1
|
|
and \$0x70,$tmp1
|
|
cmp \$0x60,$tmp1
|
|
je .L_num_blocks_is_6_${rndsuffix}
|
|
cmp \$0x50,$tmp1
|
|
je .L_num_blocks_is_5_${rndsuffix}
|
|
cmp \$0x40,$tmp1
|
|
je .L_num_blocks_is_4_${rndsuffix}
|
|
cmp \$0x30,$tmp1
|
|
je .L_num_blocks_is_3_${rndsuffix}
|
|
cmp \$0x20,$tmp1
|
|
je .L_num_blocks_is_2_${rndsuffix}
|
|
cmp \$0x10,$tmp1
|
|
je .L_num_blocks_is_1_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
$code .= "\n.L_num_blocks_is_7_${rndsuffix}:\n";
|
|
initialize("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", 7);
|
|
|
|
{
|
|
$code .= <<___;
|
|
add \$0x70,$input
|
|
and \$0xf,$length
|
|
je .L_done_7_${rndsuffix}
|
|
|
|
.L_steal_cipher_7_${rndsuffix}:
|
|
xor $gf_poly_8b_temp, $gf_poly_8b_temp
|
|
shl \$1, $TEMPLOW
|
|
adc $TEMPHIGH, $TEMPHIGH
|
|
cmovc $gf_poly_8b, $gf_poly_8b_temp
|
|
xor $gf_poly_8b_temp, $TEMPLOW
|
|
mov $TEMPLOW,0x10($TW)
|
|
mov $TEMPHIGH,0x18($TW)
|
|
vmovdqa64 %xmm15,%xmm16
|
|
vmovdqa 0x10($TW),%xmm15
|
|
___
|
|
}
|
|
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 7, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
vmovdqu %xmm2,0x10($output)
|
|
vmovdqu %xmm3,0x20($output)
|
|
vmovdqu %xmm4,0x30($output)
|
|
vmovdqu %xmm5,0x40($output)
|
|
vmovdqu %xmm6,0x50($output)
|
|
add \$0x70,$output
|
|
vmovdqa64 %xmm16,%xmm0
|
|
vmovdqa %xmm7,%xmm8
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
$code .= "\n.L_done_7_${rndsuffix}:\n";
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 7, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
vmovdqu %xmm2,0x10($output)
|
|
vmovdqu %xmm3,0x20($output)
|
|
vmovdqu %xmm4,0x30($output)
|
|
vmovdqu %xmm5,0x40($output)
|
|
vmovdqu %xmm6,0x50($output)
|
|
add \$0x70,$output
|
|
vmovdqa %xmm7,%xmm8
|
|
jmp .L_done_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
$code .= "\n.L_num_blocks_is_6_${rndsuffix}:\n";
|
|
initialize("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", 6);
|
|
|
|
{
|
|
$code .= <<___;
|
|
add \$0x60,$input
|
|
and \$0xf,$length
|
|
je .L_done_6_${rndsuffix}
|
|
|
|
.L_steal_cipher_6_${rndsuffix}:
|
|
xor $gf_poly_8b_temp, $gf_poly_8b_temp
|
|
shl \$1, $TEMPLOW
|
|
adc $TEMPHIGH, $TEMPHIGH
|
|
cmovc $gf_poly_8b, $gf_poly_8b_temp
|
|
xor $gf_poly_8b_temp, $TEMPLOW
|
|
mov $TEMPLOW,0x10($TW)
|
|
mov $TEMPHIGH,0x18($TW)
|
|
vmovdqa64 %xmm14,%xmm15
|
|
vmovdqa 0x10($TW),%xmm14
|
|
___
|
|
}
|
|
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 6, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
vmovdqu %xmm2,0x10($output)
|
|
vmovdqu %xmm3,0x20($output)
|
|
vmovdqu %xmm4,0x30($output)
|
|
vmovdqu %xmm5,0x40($output)
|
|
add \$0x60,$output
|
|
vmovdqa %xmm15,%xmm0
|
|
vmovdqa %xmm6,%xmm8
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
$code .= "\n.L_done_6_${rndsuffix}:\n";
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 6, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
vmovdqu %xmm2,0x10($output)
|
|
vmovdqu %xmm3,0x20($output)
|
|
vmovdqu %xmm4,0x30($output)
|
|
vmovdqu %xmm5,0x40($output)
|
|
add \$0x60,$output
|
|
vmovdqa %xmm6,%xmm8
|
|
jmp .L_done_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
$code .= "\n.L_num_blocks_is_5_${rndsuffix}:\n";
|
|
initialize("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", 5);
|
|
|
|
{
|
|
$code .= <<___;
|
|
add \$0x50,$input
|
|
and \$0xf,$length
|
|
je .L_done_5_${rndsuffix}
|
|
|
|
.L_steal_cipher_5_${rndsuffix}:
|
|
xor $gf_poly_8b_temp, $gf_poly_8b_temp
|
|
shl \$1, $TEMPLOW
|
|
adc $TEMPHIGH, $TEMPHIGH
|
|
cmovc $gf_poly_8b, $gf_poly_8b_temp
|
|
xor $gf_poly_8b_temp, $TEMPLOW
|
|
mov $TEMPLOW,0x10($TW)
|
|
mov $TEMPHIGH,0x18($TW)
|
|
vmovdqa64 %xmm13,%xmm14
|
|
vmovdqa 0x10($TW),%xmm13
|
|
___
|
|
}
|
|
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 5, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
vmovdqu %xmm2,0x10($output)
|
|
vmovdqu %xmm3,0x20($output)
|
|
vmovdqu %xmm4,0x30($output)
|
|
add \$0x50,$output
|
|
vmovdqa %xmm14,%xmm0
|
|
vmovdqa %xmm5,%xmm8
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
$code .= "\n.L_done_5_${rndsuffix}:\n";
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 5, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
vmovdqu %xmm2,0x10($output)
|
|
vmovdqu %xmm3,0x20($output)
|
|
vmovdqu %xmm4,0x30($output)
|
|
add \$0x50,$output
|
|
vmovdqa %xmm5,%xmm8
|
|
jmp .L_done_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
$code .= "\n.L_num_blocks_is_4_${rndsuffix}:\n";
|
|
|
|
initialize("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", 4);
|
|
|
|
{
|
|
$code .= <<___;
|
|
add \$0x40,$input
|
|
and \$0xf,$length
|
|
je .L_done_4_${rndsuffix}
|
|
|
|
.L_steal_cipher_4_${rndsuffix}:
|
|
xor $gf_poly_8b_temp, $gf_poly_8b_temp
|
|
shl \$1, $TEMPLOW
|
|
adc $TEMPHIGH, $TEMPHIGH
|
|
cmovc $gf_poly_8b, $gf_poly_8b_temp
|
|
xor $gf_poly_8b_temp, $TEMPLOW
|
|
mov $TEMPLOW,0x10($TW)
|
|
mov $TEMPHIGH,0x18($TW)
|
|
vmovdqa64 %xmm12,%xmm13
|
|
vmovdqa 0x10($TW),%xmm12
|
|
___
|
|
}
|
|
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 4, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
vmovdqu %xmm2,0x10($output)
|
|
vmovdqu %xmm3,0x20($output)
|
|
add \$0x40,$output
|
|
vmovdqa %xmm13,%xmm0
|
|
vmovdqa %xmm4,%xmm8
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
$code .= "\n.L_done_4_${rndsuffix}:\n";
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 4, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
vmovdqu %xmm2,0x10($output)
|
|
vmovdqu %xmm3,0x20($output)
|
|
add \$0x40,$output
|
|
vmovdqa %xmm4,%xmm8
|
|
jmp .L_done_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
$code .= "\n.L_num_blocks_is_3_${rndsuffix}:\n";
|
|
|
|
initialize("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", 3);
|
|
|
|
{
|
|
$code .= <<___;
|
|
add \$0x30,$input
|
|
and \$0xf,$length
|
|
je .L_done_3_${rndsuffix}
|
|
|
|
.L_steal_cipher_3_${rndsuffix}:
|
|
xor $gf_poly_8b_temp, $gf_poly_8b_temp
|
|
shl \$1, $TEMPLOW
|
|
adc $TEMPHIGH, $TEMPHIGH
|
|
cmovc $gf_poly_8b, $gf_poly_8b_temp
|
|
xor $gf_poly_8b_temp, $TEMPLOW
|
|
mov $TEMPLOW,0x10($TW)
|
|
mov $TEMPHIGH,0x18($TW)
|
|
vmovdqa64 %xmm11,%xmm12
|
|
vmovdqa 0x10($TW),%xmm11
|
|
___
|
|
}
|
|
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 3, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
vmovdqu %xmm2,0x10($output)
|
|
add \$0x30,$output
|
|
vmovdqa %xmm12,%xmm0
|
|
vmovdqa %xmm3,%xmm8
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
$code .= "\n.L_done_3_${rndsuffix}:\n";
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 3, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
vmovdqu %xmm2,0x10($output)
|
|
add \$0x30,$output
|
|
vmovdqa %xmm3,%xmm8
|
|
jmp .L_done_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
$code .= "\n.L_num_blocks_is_2_${rndsuffix}:\n";
|
|
|
|
initialize("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", 2);
|
|
|
|
{
|
|
$code .= <<___;
|
|
add \$0x20,$input
|
|
and \$0xf,$length
|
|
je .L_done_2_${rndsuffix}
|
|
|
|
.L_steal_cipher_2_${rndsuffix}:
|
|
xor $gf_poly_8b_temp, $gf_poly_8b_temp
|
|
shl \$1, $TEMPLOW
|
|
adc $TEMPHIGH, $TEMPHIGH
|
|
cmovc $gf_poly_8b, $gf_poly_8b_temp
|
|
xor $gf_poly_8b_temp, $TEMPLOW
|
|
mov $TEMPLOW,0x10($TW)
|
|
mov $TEMPHIGH,0x18($TW)
|
|
vmovdqa64 %xmm10,%xmm11
|
|
vmovdqa 0x10($TW),%xmm10
|
|
___
|
|
}
|
|
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 2, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
add \$0x20,$output
|
|
vmovdqa %xmm11,%xmm0
|
|
vmovdqa %xmm2,%xmm8
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
$code .= "\n.L_done_2_${rndsuffix}:\n";
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 2, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
vmovdqu %xmm1,($output)
|
|
add \$0x20,$output
|
|
vmovdqa %xmm2,%xmm8
|
|
jmp .L_done_${rndsuffix}
|
|
___
|
|
}
|
|
|
|
$code .= "\n.L_num_blocks_is_1_${rndsuffix}:\n";
|
|
|
|
initialize("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", 1);
|
|
|
|
{
|
|
$code .= <<___;
|
|
add \$0x10,$input
|
|
and \$0xf,$length
|
|
je .L_done_1_${rndsuffix}
|
|
|
|
.L_steal_cipher_1_${rndsuffix}:
|
|
xor $gf_poly_8b_temp, $gf_poly_8b_temp
|
|
shl \$1, $TEMPLOW
|
|
adc $TEMPHIGH, $TEMPHIGH
|
|
cmovc $gf_poly_8b, $gf_poly_8b_temp
|
|
xor $gf_poly_8b_temp, $TEMPLOW
|
|
mov $TEMPLOW,0x10($TW)
|
|
mov $TEMPHIGH,0x18($TW)
|
|
vmovdqa64 %xmm9,%xmm10
|
|
vmovdqa 0x10($TW),%xmm9
|
|
___
|
|
}
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 1, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
add \$0x10,$output
|
|
vmovdqa %xmm10,%xmm0
|
|
vmovdqa %xmm1,%xmm8
|
|
jmp .L_steal_cipher_${rndsuffix}
|
|
___
|
|
}
|
|
$code .= "\n.L_done_1_${rndsuffix}:\n";
|
|
decrypt_initial("%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6",
|
|
"%xmm7", "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12",
|
|
"%xmm13", "%xmm14", "%xmm15", "%xmm0", 1, 1, $is_128);
|
|
|
|
{
|
|
$code .= <<___;
|
|
add \$0x10,$output
|
|
vmovdqa %xmm1,%xmm8
|
|
jmp .L_done_${rndsuffix}
|
|
.cfi_endproc
|
|
___
|
|
}
|
|
|
|
}
|
|
|
|
# The only difference between AES-XTS-128 and -256 is the number of rounds,
|
|
# so we generate from the same perlasm base, extending to 14 rounds when
|
|
# `$is_128' is 0.
|
|
|
|
enc(1);
|
|
dec(1);
|
|
|
|
enc(0);
|
|
dec(0);
|
|
|
|
$code .= <<___;
|
|
.section .rodata
|
|
.align 16
|
|
|
|
vpshufb_shf_table:
|
|
.quad 0x8786858483828100, 0x8f8e8d8c8b8a8988
|
|
.quad 0x0706050403020100, 0x000e0d0c0b0a0908
|
|
|
|
mask1:
|
|
.quad 0x8080808080808080, 0x8080808080808080
|
|
|
|
const_dq3210:
|
|
.quad 0, 0, 1, 1, 2, 2, 3, 3
|
|
const_dq5678:
|
|
.quad 8, 8, 7, 7, 6, 6, 5, 5
|
|
const_dq7654:
|
|
.quad 4, 4, 5, 5, 6, 6, 7, 7
|
|
const_dq1234:
|
|
.quad 4, 4, 3, 3, 2, 2, 1, 1
|
|
|
|
shufb_15_7:
|
|
.byte 15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 7, 0xff, 0xff
|
|
.byte 0xff, 0xff, 0xff, 0xff, 0xff
|
|
|
|
.text
|
|
___
|
|
|
|
} else {
|
|
$code .= <<___;
|
|
.text
|
|
.globl aesni_xts_128_encrypt_avx512
|
|
.globl aesni_xts_128_decrypt_avx512
|
|
|
|
aesni_xts_128_encrypt_avx512:
|
|
aesni_xts_128_decrypt_avx512:
|
|
.byte 0x0f,0x0b # ud2
|
|
ret
|
|
|
|
.globl aesni_xts_256_encrypt_avx512
|
|
.globl aesni_xts_256_decrypt_avx512
|
|
|
|
aesni_xts_256_encrypt_avx512:
|
|
aesni_xts_256_decrypt_avx512:
|
|
.byte 0x0f,0x0b # ud2
|
|
ret
|
|
|
|
.globl aesni_xts_avx512_eligible
|
|
.type aesni_xts_avx512_eligible,\@abi-omnipotent
|
|
aesni_xts_avx512_eligible:
|
|
xor %eax,%eax
|
|
ret
|
|
.size aesni_xts_avx512_eligible, .-aesni_xts_avx512_eligible
|
|
|
|
___
|
|
}
|
|
|
|
print $code;
|
|
|
|
close STDOUT or die "error closing STDOUT: $!";
|