Linaro Git Browser
Code Review Sign In
review.linaro.org / arm / vixl.git / c40e288158b6437d86f025928d44605298aa3cf4 / . / src / aarch64 / logic-aarch64.cc
blob: 3df6d2a087bf364a196c5dffc89623cb35ce8405 [file] [log] [blame]
// Copyright 2015, VIXL authors
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
// * Neither the name of ARM Limited nor the names of its contributors may be
// used to endorse or promote products derived from this software without
// specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifdef VIXL_INCLUDE_SIMULATOR_AARCH64
#include <cmath>
#include "simulator-aarch64.h"
namespace vixl {
namespace aarch64 {
using vixl::internal::SimFloat16;
template <typename T>
constexpr bool IsFloat64() {
return false;
}
template <>
constexpr bool IsFloat64<double>() {
return true;
}
template <typename T>
constexpr bool IsFloat32() {
return false;
}
template <>
constexpr bool IsFloat32<float>() {
return true;
}
template <typename T>
constexpr bool IsFloat16() {
return false;
}
template <>
constexpr bool IsFloat16<Float16>() {
return true;
}
template <>
constexpr bool IsFloat16<SimFloat16>() {
return true;
}
template <>
double Simulator::FPDefaultNaN<double>() {
return kFP64DefaultNaN;
}
template <>
float Simulator::FPDefaultNaN<float>() {
return kFP32DefaultNaN;
}
template <>
SimFloat16 Simulator::FPDefaultNaN<SimFloat16>() {
return SimFloat16(kFP16DefaultNaN);
}
double Simulator::FixedToDouble(int64_t src, int fbits, FPRounding round) {
if (src >= 0) {
return UFixedToDouble(src, fbits, round);
} else if (src == INT64_MIN) {
return -UFixedToDouble(src, fbits, round);
} else {
return -UFixedToDouble(-src, fbits, round);
}
}
double Simulator::UFixedToDouble(uint64_t src, int fbits, FPRounding round) {
// An input of 0 is a special case because the result is effectively
// subnormal: The exponent is encoded as 0 and there is no implicit 1 bit.
if (src == 0) {
return 0.0;
}
// Calculate the exponent. The highest significant bit will have the value
// 2^exponent.
const int highest_significant_bit = 63 - CountLeadingZeros(src);
const int64_t exponent = highest_significant_bit - fbits;
return FPRoundToDouble(0, exponent, src, round);
}
float Simulator::FixedToFloat(int64_t src, int fbits, FPRounding round) {
if (src >= 0) {
return UFixedToFloat(src, fbits, round);
} else if (src == INT64_MIN) {
return -UFixedToFloat(src, fbits, round);
} else {
return -UFixedToFloat(-src, fbits, round);
}
}
float Simulator::UFixedToFloat(uint64_t src, int fbits, FPRounding round) {
// An input of 0 is a special case because the result is effectively
// subnormal: The exponent is encoded as 0 and there is no implicit 1 bit.
if (src == 0) {
return 0.0f;
}
// Calculate the exponent. The highest significant bit will have the value
// 2^exponent.
const int highest_significant_bit = 63 - CountLeadingZeros(src);
const int32_t exponent = highest_significant_bit - fbits;
return FPRoundToFloat(0, exponent, src, round);
}
SimFloat16 Simulator::FixedToFloat16(int64_t src, int fbits, FPRounding round) {
if (src >= 0) {
return UFixedToFloat16(src, fbits, round);
} else if (src == INT64_MIN) {
return -UFixedToFloat16(src, fbits, round);
} else {
return -UFixedToFloat16(-src, fbits, round);
}
}
SimFloat16 Simulator::UFixedToFloat16(uint64_t src,
int fbits,
FPRounding round) {
// An input of 0 is a special case because the result is effectively
// subnormal: The exponent is encoded as 0 and there is no implicit 1 bit.
if (src == 0) {
return 0.0f;
}
// Calculate the exponent. The highest significant bit will have the value
// 2^exponent.
const int highest_significant_bit = 63 - CountLeadingZeros(src);
const int16_t exponent = highest_significant_bit - fbits;
return FPRoundToFloat16(0, exponent, src, round);
}
uint64_t Simulator::GenerateRandomTag(uint16_t exclude) {
// Generate a 4 bit integer from a 48bit random number
uint64_t rtag = rand_gen_() >> 44;
VIXL_ASSERT(IsUint4(rtag));
if (exclude == 0) {
exclude = static_cast<uint16_t>(rand_gen_() >> 44);
}
// TODO: implement this to better match the specification, which calls for a
// true random mode, and a pseudo-random mode with state (EL1.TAG) modified by
// PRNG.
return ChooseNonExcludedTag(rtag, 0, exclude);
}
bool Simulator::ld1(VectorFormat vform, LogicVRegister dst, uint64_t addr) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!LoadLane(dst, vform, i, addr)) {
return false;
}
addr += LaneSizeInBytesFromFormat(vform);
}
return true;
}
bool Simulator::ld1(VectorFormat vform,
LogicVRegister dst,
int index,
uint64_t addr) {
dst.ClearForWrite(vform);
return LoadLane(dst, vform, index, addr);
}
bool Simulator::ld1r(VectorFormat vform,
VectorFormat unpack_vform,
LogicVRegister dst,
uint64_t addr,
bool is_signed) {
unsigned unpack_size = LaneSizeInBytesFromFormat(unpack_vform);
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (is_signed) {
if (!LoadIntToLane(dst, vform, unpack_size, i, addr)) {
return false;
}
} else {
if (!LoadUintToLane(dst, vform, unpack_size, i, addr)) {
return false;
}
}
}
return true;
}
bool Simulator::ld1r(VectorFormat vform, LogicVRegister dst, uint64_t addr) {
return ld1r(vform, vform, dst, addr);
}
bool Simulator::ld2(VectorFormat vform,
LogicVRegister dst1,
LogicVRegister dst2,
uint64_t addr1) {
dst1.ClearForWrite(vform);
dst2.ClearForWrite(vform);
int esize = LaneSizeInBytesFromFormat(vform);
uint64_t addr2 = addr1 + esize;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!LoadLane(dst1, vform, i, addr1) || !LoadLane(dst2, vform, i, addr2)) {
return false;
}
addr1 += 2 * esize;
addr2 += 2 * esize;
}
return true;
}
bool Simulator::ld2(VectorFormat vform,
LogicVRegister dst1,
LogicVRegister dst2,
int index,
uint64_t addr1) {
dst1.ClearForWrite(vform);
dst2.ClearForWrite(vform);
uint64_t addr2 = addr1 + LaneSizeInBytesFromFormat(vform);
return (LoadLane(dst1, vform, index, addr1) &&
LoadLane(dst2, vform, index, addr2));
}
bool Simulator::ld2r(VectorFormat vform,
LogicVRegister dst1,
LogicVRegister dst2,
uint64_t addr) {
dst1.ClearForWrite(vform);
dst2.ClearForWrite(vform);
uint64_t addr2 = addr + LaneSizeInBytesFromFormat(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!LoadLane(dst1, vform, i, addr) || !LoadLane(dst2, vform, i, addr2)) {
return false;
}
}
return true;
}
bool Simulator::ld3(VectorFormat vform,
LogicVRegister dst1,
LogicVRegister dst2,
LogicVRegister dst3,
uint64_t addr1) {
dst1.ClearForWrite(vform);
dst2.ClearForWrite(vform);
dst3.ClearForWrite(vform);
int esize = LaneSizeInBytesFromFormat(vform);
uint64_t addr2 = addr1 + esize;
uint64_t addr3 = addr2 + esize;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!LoadLane(dst1, vform, i, addr1) || !LoadLane(dst2, vform, i, addr2) ||
!LoadLane(dst3, vform, i, addr3)) {
return false;
}
addr1 += 3 * esize;
addr2 += 3 * esize;
addr3 += 3 * esize;
}
return true;
}
bool Simulator::ld3(VectorFormat vform,
LogicVRegister dst1,
LogicVRegister dst2,
LogicVRegister dst3,
int index,
uint64_t addr1) {
dst1.ClearForWrite(vform);
dst2.ClearForWrite(vform);
dst3.ClearForWrite(vform);
uint64_t addr2 = addr1 + LaneSizeInBytesFromFormat(vform);
uint64_t addr3 = addr2 + LaneSizeInBytesFromFormat(vform);
return (LoadLane(dst1, vform, index, addr1) &&
LoadLane(dst2, vform, index, addr2) &&
LoadLane(dst3, vform, index, addr3));
}
bool Simulator::ld3r(VectorFormat vform,
LogicVRegister dst1,
LogicVRegister dst2,
LogicVRegister dst3,
uint64_t addr) {
dst1.ClearForWrite(vform);
dst2.ClearForWrite(vform);
dst3.ClearForWrite(vform);
uint64_t addr2 = addr + LaneSizeInBytesFromFormat(vform);
uint64_t addr3 = addr2 + LaneSizeInBytesFromFormat(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!LoadLane(dst1, vform, i, addr) || !LoadLane(dst2, vform, i, addr2) ||
!LoadLane(dst3, vform, i, addr3)) {
return false;
}
}
return true;
}
bool Simulator::ld4(VectorFormat vform,
LogicVRegister dst1,
LogicVRegister dst2,
LogicVRegister dst3,
LogicVRegister dst4,
uint64_t addr1) {
dst1.ClearForWrite(vform);
dst2.ClearForWrite(vform);
dst3.ClearForWrite(vform);
dst4.ClearForWrite(vform);
int esize = LaneSizeInBytesFromFormat(vform);
uint64_t addr2 = addr1 + esize;
uint64_t addr3 = addr2 + esize;
uint64_t addr4 = addr3 + esize;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!LoadLane(dst1, vform, i, addr1) || !LoadLane(dst2, vform, i, addr2) ||
!LoadLane(dst3, vform, i, addr3) || !LoadLane(dst4, vform, i, addr4)) {
return false;
}
addr1 += 4 * esize;
addr2 += 4 * esize;
addr3 += 4 * esize;
addr4 += 4 * esize;
}
return true;
}
bool Simulator::ld4(VectorFormat vform,
LogicVRegister dst1,
LogicVRegister dst2,
LogicVRegister dst3,
LogicVRegister dst4,
int index,
uint64_t addr1) {
dst1.ClearForWrite(vform);
dst2.ClearForWrite(vform);
dst3.ClearForWrite(vform);
dst4.ClearForWrite(vform);
uint64_t addr2 = addr1 + LaneSizeInBytesFromFormat(vform);
uint64_t addr3 = addr2 + LaneSizeInBytesFromFormat(vform);
uint64_t addr4 = addr3 + LaneSizeInBytesFromFormat(vform);
return (LoadLane(dst1, vform, index, addr1) &&
LoadLane(dst2, vform, index, addr2) &&
LoadLane(dst3, vform, index, addr3) &&
LoadLane(dst4, vform, index, addr4));
}
bool Simulator::ld4r(VectorFormat vform,
LogicVRegister dst1,
LogicVRegister dst2,
LogicVRegister dst3,
LogicVRegister dst4,
uint64_t addr) {
dst1.ClearForWrite(vform);
dst2.ClearForWrite(vform);
dst3.ClearForWrite(vform);
dst4.ClearForWrite(vform);
uint64_t addr2 = addr + LaneSizeInBytesFromFormat(vform);
uint64_t addr3 = addr2 + LaneSizeInBytesFromFormat(vform);
uint64_t addr4 = addr3 + LaneSizeInBytesFromFormat(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!LoadLane(dst1, vform, i, addr) || !LoadLane(dst2, vform, i, addr2) ||
!LoadLane(dst3, vform, i, addr3) || !LoadLane(dst4, vform, i, addr4)) {
return false;
}
}
return true;
}
bool Simulator::st1(VectorFormat vform, LogicVRegister src, uint64_t addr) {
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!StoreLane(src, vform, i, addr)) return false;
addr += LaneSizeInBytesFromFormat(vform);
}
return true;
}
bool Simulator::st1(VectorFormat vform,
LogicVRegister src,
int index,
uint64_t addr) {
return StoreLane(src, vform, index, addr);
}
bool Simulator::st2(VectorFormat vform,
LogicVRegister src,
LogicVRegister src2,
uint64_t addr) {
int esize = LaneSizeInBytesFromFormat(vform);
uint64_t addr2 = addr + esize;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!StoreLane(src, vform, i, addr) || !StoreLane(src2, vform, i, addr2)) {
return false;
}
addr += 2 * esize;
addr2 += 2 * esize;
}
return true;
}
bool Simulator::st2(VectorFormat vform,
LogicVRegister src,
LogicVRegister src2,
int index,
uint64_t addr) {
int esize = LaneSizeInBytesFromFormat(vform);
return (StoreLane(src, vform, index, addr) &&
StoreLane(src2, vform, index, addr + 1 * esize));
}
bool Simulator::st3(VectorFormat vform,
LogicVRegister src,
LogicVRegister src2,
LogicVRegister src3,
uint64_t addr) {
int esize = LaneSizeInBytesFromFormat(vform);
uint64_t addr2 = addr + esize;
uint64_t addr3 = addr2 + esize;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!StoreLane(src, vform, i, addr) || !StoreLane(src2, vform, i, addr2) ||
!StoreLane(src3, vform, i, addr3)) {
return false;
}
addr += 3 * esize;
addr2 += 3 * esize;
addr3 += 3 * esize;
}
return true;
}
bool Simulator::st3(VectorFormat vform,
LogicVRegister src,
LogicVRegister src2,
LogicVRegister src3,
int index,
uint64_t addr) {
int esize = LaneSizeInBytesFromFormat(vform);
return (StoreLane(src, vform, index, addr) &&
StoreLane(src2, vform, index, addr + 1 * esize) &&
StoreLane(src3, vform, index, addr + 2 * esize));
}
bool Simulator::st4(VectorFormat vform,
LogicVRegister src,
LogicVRegister src2,
LogicVRegister src3,
LogicVRegister src4,
uint64_t addr) {
int esize = LaneSizeInBytesFromFormat(vform);
uint64_t addr2 = addr + esize;
uint64_t addr3 = addr2 + esize;
uint64_t addr4 = addr3 + esize;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!StoreLane(src, vform, i, addr) || !StoreLane(src2, vform, i, addr2) ||
!StoreLane(src3, vform, i, addr3) ||
!StoreLane(src4, vform, i, addr4)) {
return false;
}
addr += 4 * esize;
addr2 += 4 * esize;
addr3 += 4 * esize;
addr4 += 4 * esize;
}
return true;
}
bool Simulator::st4(VectorFormat vform,
LogicVRegister src,
LogicVRegister src2,
LogicVRegister src3,
LogicVRegister src4,
int index,
uint64_t addr) {
int esize = LaneSizeInBytesFromFormat(vform);
return (StoreLane(src, vform, index, addr) &&
StoreLane(src2, vform, index, addr + 1 * esize) &&
StoreLane(src3, vform, index, addr + 2 * esize) &&
StoreLane(src4, vform, index, addr + 3 * esize));
}
LogicVRegister Simulator::cmp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
Condition cond) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
int64_t sa = src1.Int(vform, i);
int64_t sb = src2.Int(vform, i);
uint64_t ua = src1.Uint(vform, i);
uint64_t ub = src2.Uint(vform, i);
bool result = false;
switch (cond) {
case eq:
result = (ua == ub);
break;
case ge:
result = (sa >= sb);
break;
case gt:
result = (sa > sb);
break;
case hi:
result = (ua > ub);
break;
case hs:
result = (ua >= ub);
break;
case lt:
result = (sa < sb);
break;
case le:
result = (sa <= sb);
break;
default:
VIXL_UNREACHABLE();
break;
}
dst.SetUint(vform, i, result ? MaxUintFromFormat(vform) : 0);
}
return dst;
}
LogicVRegister Simulator::cmp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
int imm,
Condition cond) {
SimVRegister temp;
LogicVRegister imm_reg = dup_immediate(vform, temp, imm);
return cmp(vform, dst, src1, imm_reg, cond);
}
LogicVRegister Simulator::cmptst(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
uint64_t ua = src1.Uint(vform, i);
uint64_t ub = src2.Uint(vform, i);
dst.SetUint(vform, i, ((ua & ub) != 0) ? MaxUintFromFormat(vform) : 0);
}
return dst;
}
LogicVRegister Simulator::add(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
int lane_size = LaneSizeInBitsFromFormat(vform);
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
// Test for unsigned saturation.
uint64_t ua = src1.UintLeftJustified(vform, i);
uint64_t ub = src2.UintLeftJustified(vform, i);
uint64_t ur = ua + ub;
if (ur < ua) {
dst.SetUnsignedSat(i, true);
}
// Test for signed saturation.
bool pos_a = (ua >> 63) == 0;
bool pos_b = (ub >> 63) == 0;
bool pos_r = (ur >> 63) == 0;
// If the signs of the operands are the same, but different from the result,
// there was an overflow.
if ((pos_a == pos_b) && (pos_a != pos_r)) {
dst.SetSignedSat(i, pos_a);
}
dst.SetInt(vform, i, ur >> (64 - lane_size));
}
return dst;
}
LogicVRegister Simulator::add_uint(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
uint64_t value) {
int lane_size = LaneSizeInBitsFromFormat(vform);
VIXL_ASSERT(IsUintN(lane_size, value));
dst.ClearForWrite(vform);
// Left-justify `value`.
uint64_t ub = value << (64 - lane_size);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
// Test for unsigned saturation.
uint64_t ua = src1.UintLeftJustified(vform, i);
uint64_t ur = ua + ub;
if (ur < ua) {
dst.SetUnsignedSat(i, true);
}
// Test for signed saturation.
// `value` is always positive, so we have an overflow if the (signed) result
// is smaller than the first operand.
if (RawbitsToInt64(ur) < RawbitsToInt64(ua)) {
dst.SetSignedSat(i, true);
}
dst.SetInt(vform, i, ur >> (64 - lane_size));
}
return dst;
}
LogicVRegister Simulator::addp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp1, temp2;
uzp1(vform, temp1, src1, src2);
uzp2(vform, temp2, src1, src2);
add(vform, dst, temp1, temp2);
if (IsSVEFormat(vform)) {
interleave_top_bottom(vform, dst, dst);
}
return dst;
}
LogicVRegister Simulator::sdiv(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
VIXL_ASSERT((vform == kFormatVnS) || (vform == kFormatVnD));
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
int64_t val1 = src1.Int(vform, i);
int64_t val2 = src2.Int(vform, i);
int64_t min_int = (vform == kFormatVnD) ? kXMinInt : kWMinInt;
int64_t quotient = 0;
if ((val1 == min_int) && (val2 == -1)) {
quotient = min_int;
} else if (val2 != 0) {
quotient = val1 / val2;
}
dst.SetInt(vform, i, quotient);
}
return dst;
}
LogicVRegister Simulator::udiv(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
VIXL_ASSERT((vform == kFormatVnS) || (vform == kFormatVnD));
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
uint64_t val1 = src1.Uint(vform, i);
uint64_t val2 = src2.Uint(vform, i);
uint64_t quotient = 0;
if (val2 != 0) {
quotient = val1 / val2;
}
dst.SetUint(vform, i, quotient);
}
return dst;
}
LogicVRegister Simulator::mla(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& srca,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
mul(vform, temp, src1, src2);
add(vform, dst, srca, temp);
return dst;
}
LogicVRegister Simulator::mls(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& srca,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
mul(vform, temp, src1, src2);
sub(vform, dst, srca, temp);
return dst;
}
LogicVRegister Simulator::mul(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
dst.SetUint(vform, i, src1.Uint(vform, i) * src2.Uint(vform, i));
}
return dst;
}
LogicVRegister Simulator::mul(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
SimVRegister temp;
VectorFormat indexform = VectorFormatFillQ(vform);
return mul(vform, dst, src1, dup_element(indexform, temp, src2, index));
}
LogicVRegister Simulator::smulh(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
int64_t dst_val = 0xbadbeef;
int64_t val1 = src1.Int(vform, i);
int64_t val2 = src2.Int(vform, i);
switch (LaneSizeInBitsFromFormat(vform)) {
case 8:
dst_val = internal::MultiplyHigh<8>(val1, val2);
break;
case 16:
dst_val = internal::MultiplyHigh<16>(val1, val2);
break;
case 32:
dst_val = internal::MultiplyHigh<32>(val1, val2);
break;
case 64:
dst_val = internal::MultiplyHigh<64>(val1, val2);
break;
default:
VIXL_UNREACHABLE();
break;
}
dst.SetInt(vform, i, dst_val);
}
return dst;
}
LogicVRegister Simulator::umulh(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
uint64_t dst_val = 0xbadbeef;
uint64_t val1 = src1.Uint(vform, i);
uint64_t val2 = src2.Uint(vform, i);
switch (LaneSizeInBitsFromFormat(vform)) {
case 8:
dst_val = internal::MultiplyHigh<8>(val1, val2);
break;
case 16:
dst_val = internal::MultiplyHigh<16>(val1, val2);
break;
case 32:
dst_val = internal::MultiplyHigh<32>(val1, val2);
break;
case 64:
dst_val = internal::MultiplyHigh<64>(val1, val2);
break;
default:
VIXL_UNREACHABLE();
break;
}
dst.SetUint(vform, i, dst_val);
}
return dst;
}
LogicVRegister Simulator::mla(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
SimVRegister temp;
VectorFormat indexform = VectorFormatFillQ(vform);
return mla(vform, dst, dst, src1, dup_element(indexform, temp, src2, index));
}
LogicVRegister Simulator::mls(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
SimVRegister temp;
VectorFormat indexform = VectorFormatFillQ(vform);
return mls(vform, dst, dst, src1, dup_element(indexform, temp, src2, index));
}
LogicVRegister Simulator::sqdmull(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
SimVRegister temp;
VectorFormat indexform =
VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
return sqdmull(vform, dst, src1, dup_element(indexform, temp, src2, index));
}
LogicVRegister Simulator::sqdmlal(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
SimVRegister temp;
VectorFormat indexform =
VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
return sqdmlal(vform, dst, src1, dup_element(indexform, temp, src2, index));
}
LogicVRegister Simulator::sqdmlsl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
SimVRegister temp;
VectorFormat indexform =
VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
return sqdmlsl(vform, dst, src1, dup_element(indexform, temp, src2, index));
}
LogicVRegister Simulator::sqdmulh(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
SimVRegister temp;
VectorFormat indexform = VectorFormatFillQ(vform);
return sqdmulh(vform, dst, src1, dup_element(indexform, temp, src2, index));
}
LogicVRegister Simulator::sqrdmulh(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
SimVRegister temp;
VectorFormat indexform = VectorFormatFillQ(vform);
return sqrdmulh(vform, dst, src1, dup_element(indexform, temp, src2, index));
}
LogicVRegister Simulator::sqrdmlah(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
SimVRegister temp;
VectorFormat indexform = VectorFormatFillQ(vform);
return sqrdmlah(vform, dst, src1, dup_element(indexform, temp, src2, index));
}
LogicVRegister Simulator::sqrdmlsh(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
SimVRegister temp;
VectorFormat indexform = VectorFormatFillQ(vform);
return sqrdmlsh(vform, dst, src1, dup_element(indexform, temp, src2, index));
}
uint64_t Simulator::PolynomialMult(uint64_t op1,
uint64_t op2,
int lane_size_in_bits) const {
return PolynomialMult128(op1, op2, lane_size_in_bits).second;
}
LogicVRegister Simulator::pmul(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
dst.SetUint(vform,
i,
PolynomialMult(src1.Uint(vform, i),
src2.Uint(vform, i),
LaneSizeInBitsFromFormat(vform)));
}
return dst;
}
LogicVRegister Simulator::pmull(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
VectorFormat vform_src = VectorFormatHalfWidth(vform);
// Process the elements in reverse to avoid problems when the destination
// register is the same as a source.
for (int i = LaneCountFromFormat(vform) - 1; i >= 0; i--) {
dst.SetUint(vform,
i,
PolynomialMult128(src1.Uint(vform_src, i),
src2.Uint(vform_src, i),
LaneSizeInBitsFromFormat(vform_src)));
}
return dst;
}
LogicVRegister Simulator::pmull2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
VectorFormat vform_src = VectorFormatHalfWidthDoubleLanes(vform);
int lane_count = LaneCountFromFormat(vform);
for (int i = 0; i < lane_count; i++) {
dst.SetUint(vform,
i,
PolynomialMult128(src1.Uint(vform_src, lane_count + i),
src2.Uint(vform_src, lane_count + i),
LaneSizeInBitsFromFormat(vform_src)));
}
return dst;
}
LogicVRegister Simulator::sub(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
int lane_size = LaneSizeInBitsFromFormat(vform);
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
// Test for unsigned saturation.
uint64_t ua = src1.UintLeftJustified(vform, i);
uint64_t ub = src2.UintLeftJustified(vform, i);
uint64_t ur = ua - ub;
if (ub > ua) {
dst.SetUnsignedSat(i, false);
}
// Test for signed saturation.
bool pos_a = (ua >> 63) == 0;
bool pos_b = (ub >> 63) == 0;
bool pos_r = (ur >> 63) == 0;
// If the signs of the operands are different, and the sign of the first
// operand doesn't match the result, there was an overflow.
if ((pos_a != pos_b) && (pos_a != pos_r)) {
dst.SetSignedSat(i, pos_a);
}
dst.SetInt(vform, i, ur >> (64 - lane_size));
}
return dst;
}
LogicVRegister Simulator::sub_uint(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
uint64_t value) {
int lane_size = LaneSizeInBitsFromFormat(vform);
VIXL_ASSERT(IsUintN(lane_size, value));
dst.ClearForWrite(vform);
// Left-justify `value`.
uint64_t ub = value << (64 - lane_size);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
// Test for unsigned saturation.
uint64_t ua = src1.UintLeftJustified(vform, i);
uint64_t ur = ua - ub;
if (ub > ua) {
dst.SetUnsignedSat(i, false);
}
// Test for signed saturation.
// `value` is always positive, so we have an overflow if the (signed) result
// is greater than the first operand.
if (RawbitsToInt64(ur) > RawbitsToInt64(ua)) {
dst.SetSignedSat(i, false);
}
dst.SetInt(vform, i, ur >> (64 - lane_size));
}
return dst;
}
LogicVRegister Simulator::and_(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
dst.SetUint(vform, i, src1.Uint(vform, i) & src2.Uint(vform, i));
}
return dst;
}
LogicVRegister Simulator::orr(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
dst.SetUint(vform, i, src1.Uint(vform, i) | src2.Uint(vform, i));
}
return dst;
}
LogicVRegister Simulator::orn(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
dst.SetUint(vform, i, src1.Uint(vform, i) | ~src2.Uint(vform, i));
}
return dst;
}
LogicVRegister Simulator::eor(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
dst.SetUint(vform, i, src1.Uint(vform, i) ^ src2.Uint(vform, i));
}
return dst;
}
LogicVRegister Simulator::bic(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
dst.SetUint(vform, i, src1.Uint(vform, i) & ~src2.Uint(vform, i));
}
return dst;
}
LogicVRegister Simulator::bic(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
uint64_t imm) {
uint64_t result[16];
int lane_count = LaneCountFromFormat(vform);
for (int i = 0; i < lane_count; ++i) {
result[i] = src.Uint(vform, i) & ~imm;
}
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, result[i]);
}
return dst;
}
LogicVRegister Simulator::bif(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
uint64_t operand1 = dst.Uint(vform, i);
uint64_t operand2 = ~src2.Uint(vform, i);
uint64_t operand3 = src1.Uint(vform, i);
uint64_t result = operand1 ^ ((operand1 ^ operand3) & operand2);
dst.SetUint(vform, i, result);
}
return dst;
}
LogicVRegister Simulator::bit(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
uint64_t operand1 = dst.Uint(vform, i);
uint64_t operand2 = src2.Uint(vform, i);
uint64_t operand3 = src1.Uint(vform, i);
uint64_t result = operand1 ^ ((operand1 ^ operand3) & operand2);
dst.SetUint(vform, i, result);
}
return dst;
}
LogicVRegister Simulator::bsl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src_mask,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
uint64_t operand1 = src2.Uint(vform, i);
uint64_t operand2 = src_mask.Uint(vform, i);
uint64_t operand3 = src1.Uint(vform, i);
uint64_t result = operand1 ^ ((operand1 ^ operand3) & operand2);
dst.SetUint(vform, i, result);
}
return dst;
}
LogicVRegister Simulator::sminmax(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool max) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
int64_t src1_val = src1.Int(vform, i);
int64_t src2_val = src2.Int(vform, i);
int64_t dst_val;
if (max) {
dst_val = (src1_val > src2_val) ? src1_val : src2_val;
} else {
dst_val = (src1_val < src2_val) ? src1_val : src2_val;
}
dst.SetInt(vform, i, dst_val);
}
return dst;
}
LogicVRegister Simulator::smax(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return sminmax(vform, dst, src1, src2, true);
}
LogicVRegister Simulator::smin(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return sminmax(vform, dst, src1, src2, false);
}
LogicVRegister Simulator::sminmaxp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool max) {
unsigned lanes = LaneCountFromFormat(vform);
int64_t result[kZRegMaxSizeInBytes];
const LogicVRegister* src = &src1;
for (unsigned j = 0; j < 2; j++) {
for (unsigned i = 0; i < lanes; i += 2) {
int64_t first_val = src->Int(vform, i);
int64_t second_val = src->Int(vform, i + 1);
int64_t dst_val;
if (max) {
dst_val = (first_val > second_val) ? first_val : second_val;
} else {
dst_val = (first_val < second_val) ? first_val : second_val;
}
VIXL_ASSERT(((i >> 1) + (j * lanes / 2)) < ArrayLength(result));
result[(i >> 1) + (j * lanes / 2)] = dst_val;
}
src = &src2;
}
dst.SetIntArray(vform, result);
if (IsSVEFormat(vform)) {
interleave_top_bottom(vform, dst, dst);
}
return dst;
}
LogicVRegister Simulator::smaxp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return sminmaxp(vform, dst, src1, src2, true);
}
LogicVRegister Simulator::sminp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return sminmaxp(vform, dst, src1, src2, false);
}
LogicVRegister Simulator::addp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
VIXL_ASSERT(vform == kFormatD);
uint64_t dst_val = src.Uint(kFormat2D, 0) + src.Uint(kFormat2D, 1);
dst.ClearForWrite(vform);
dst.SetUint(vform, 0, dst_val);
return dst;
}
LogicVRegister Simulator::addv(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
VectorFormat vform_dst =
ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform));
int64_t dst_val = 0;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
dst_val += src.Int(vform, i);
}
dst.ClearForWrite(vform_dst);
dst.SetInt(vform_dst, 0, dst_val);
return dst;
}
LogicVRegister Simulator::saddlv(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
VectorFormat vform_dst =
ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform) * 2);
int64_t dst_val = 0;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
dst_val += src.Int(vform, i);
}
dst.ClearForWrite(vform_dst);
dst.SetInt(vform_dst, 0, dst_val);
return dst;
}
LogicVRegister Simulator::uaddlv(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
VectorFormat vform_dst =
ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform) * 2);
uint64_t dst_val = 0;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
dst_val += src.Uint(vform, i);
}
dst.ClearForWrite(vform_dst);
dst.SetUint(vform_dst, 0, dst_val);
return dst;
}
LogicVRegister Simulator::sminmaxv(VectorFormat vform,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src,
bool max) {
int64_t dst_val = max ? INT64_MIN : INT64_MAX;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!pg.IsActive(vform, i)) continue;
int64_t src_val = src.Int(vform, i);
if (max) {
dst_val = (src_val > dst_val) ? src_val : dst_val;
} else {
dst_val = (src_val < dst_val) ? src_val : dst_val;
}
}
dst.ClearForWrite(ScalarFormatFromFormat(vform));
dst.SetInt(vform, 0, dst_val);
return dst;
}
LogicVRegister Simulator::smaxv(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
sminmaxv(vform, dst, GetPTrue(), src, true);
return dst;
}
LogicVRegister Simulator::sminv(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
sminmaxv(vform, dst, GetPTrue(), src, false);
return dst;
}
LogicVRegister Simulator::smaxv(VectorFormat vform,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src) {
VIXL_ASSERT(IsSVEFormat(vform));
sminmaxv(vform, dst, pg, src, true);
return dst;
}
LogicVRegister Simulator::sminv(VectorFormat vform,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src) {
VIXL_ASSERT(IsSVEFormat(vform));
sminmaxv(vform, dst, pg, src, false);
return dst;
}
LogicVRegister Simulator::uminmax(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool max) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
uint64_t src1_val = src1.Uint(vform, i);
uint64_t src2_val = src2.Uint(vform, i);
uint64_t dst_val;
if (max) {
dst_val = (src1_val > src2_val) ? src1_val : src2_val;
} else {
dst_val = (src1_val < src2_val) ? src1_val : src2_val;
}
dst.SetUint(vform, i, dst_val);
}
return dst;
}
LogicVRegister Simulator::umax(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return uminmax(vform, dst, src1, src2, true);
}
LogicVRegister Simulator::umin(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return uminmax(vform, dst, src1, src2, false);
}
LogicVRegister Simulator::uminmaxp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool max) {
unsigned lanes = LaneCountFromFormat(vform);
uint64_t result[kZRegMaxSizeInBytes];
const LogicVRegister* src = &src1;
for (unsigned j = 0; j < 2; j++) {
for (unsigned i = 0; i < lanes; i += 2) {
uint64_t first_val = src->Uint(vform, i);
uint64_t second_val = src->Uint(vform, i + 1);
uint64_t dst_val;
if (max) {
dst_val = (first_val > second_val) ? first_val : second_val;
} else {
dst_val = (first_val < second_val) ? first_val : second_val;
}
VIXL_ASSERT(((i >> 1) + (j * lanes / 2)) < ArrayLength(result));
result[(i >> 1) + (j * lanes / 2)] = dst_val;
}
src = &src2;
}
dst.SetUintArray(vform, result);
if (IsSVEFormat(vform)) {
interleave_top_bottom(vform, dst, dst);
}
return dst;
}
LogicVRegister Simulator::umaxp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return uminmaxp(vform, dst, src1, src2, true);
}
LogicVRegister Simulator::uminp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return uminmaxp(vform, dst, src1, src2, false);
}
LogicVRegister Simulator::uminmaxv(VectorFormat vform,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src,
bool max) {
uint64_t dst_val = max ? 0 : UINT64_MAX;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!pg.IsActive(vform, i)) continue;
uint64_t src_val = src.Uint(vform, i);
if (max) {
dst_val = (src_val > dst_val) ? src_val : dst_val;
} else {
dst_val = (src_val < dst_val) ? src_val : dst_val;
}
}
dst.ClearForWrite(ScalarFormatFromFormat(vform));
dst.SetUint(vform, 0, dst_val);
return dst;
}
LogicVRegister Simulator::umaxv(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
uminmaxv(vform, dst, GetPTrue(), src, true);
return dst;
}
LogicVRegister Simulator::uminv(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
uminmaxv(vform, dst, GetPTrue(), src, false);
return dst;
}
LogicVRegister Simulator::umaxv(VectorFormat vform,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src) {
VIXL_ASSERT(IsSVEFormat(vform));
uminmaxv(vform, dst, pg, src, true);
return dst;
}
LogicVRegister Simulator::uminv(VectorFormat vform,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src) {
VIXL_ASSERT(IsSVEFormat(vform));
uminmaxv(vform, dst, pg, src, false);
return dst;
}
LogicVRegister Simulator::shl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
VIXL_ASSERT(shift >= 0);
SimVRegister temp;
LogicVRegister shiftreg = dup_immediate(vform, temp, shift);
return ushl(vform, dst, src, shiftreg);
}
LogicVRegister Simulator::sshll(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
VIXL_ASSERT(shift >= 0);
SimVRegister temp1, temp2;
LogicVRegister shiftreg = dup_immediate(vform, temp1, shift);
LogicVRegister extendedreg = sxtl(vform, temp2, src);
return sshl(vform, dst, extendedreg, shiftreg);
}
LogicVRegister Simulator::sshll2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
VIXL_ASSERT(shift >= 0);
SimVRegister temp1, temp2;
LogicVRegister shiftreg = dup_immediate(vform, temp1, shift);
LogicVRegister extendedreg = sxtl2(vform, temp2, src);
return sshl(vform, dst, extendedreg, shiftreg);
}
LogicVRegister Simulator::shll(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
int shift = LaneSizeInBitsFromFormat(vform) / 2;
return sshll(vform, dst, src, shift);
}
LogicVRegister Simulator::shll2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
int shift = LaneSizeInBitsFromFormat(vform) / 2;
return sshll2(vform, dst, src, shift);
}
LogicVRegister Simulator::ushll(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
VIXL_ASSERT(shift >= 0);
SimVRegister temp1, temp2;
LogicVRegister shiftreg = dup_immediate(vform, temp1, shift);
LogicVRegister extendedreg = uxtl(vform, temp2, src);
return ushl(vform, dst, extendedreg, shiftreg);
}
LogicVRegister Simulator::ushll2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
VIXL_ASSERT(shift >= 0);
SimVRegister temp1, temp2;
LogicVRegister shiftreg = dup_immediate(vform, temp1, shift);
LogicVRegister extendedreg = uxtl2(vform, temp2, src);
return ushl(vform, dst, extendedreg, shiftreg);
}
std::pair<bool, uint64_t> Simulator::clast(VectorFormat vform,
const LogicPRegister& pg,
const LogicVRegister& src,
int offset_from_last_active) {
// Untested for any other values.
VIXL_ASSERT((offset_from_last_active == 0) || (offset_from_last_active == 1));
int last_active = GetLastActive(vform, pg);
int lane_count = LaneCountFromFormat(vform);
int index =
((last_active + offset_from_last_active) + lane_count) % lane_count;
return std::make_pair(last_active >= 0, src.Uint(vform, index));
}
LogicVRegister Simulator::compact(VectorFormat vform,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src) {
int j = 0;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (pg.IsActive(vform, i)) {
dst.SetUint(vform, j++, src.Uint(vform, i));
}
}
for (; j < LaneCountFromFormat(vform); j++) {
dst.SetUint(vform, j, 0);
}
return dst;
}
LogicVRegister Simulator::splice(VectorFormat vform,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src1,
const LogicVRegister& src2) {
int lane_count = LaneCountFromFormat(vform);
int first_active = GetFirstActive(vform, pg);
int last_active = GetLastActive(vform, pg);
int dst_idx = 0;
uint64_t result[kZRegMaxSizeInBytes];
if (first_active >= 0) {
VIXL_ASSERT(last_active >= first_active);
VIXL_ASSERT(last_active < lane_count);
for (int i = first_active; i <= last_active; i++) {
result[dst_idx++] = src1.Uint(vform, i);
}
}
VIXL_ASSERT(dst_idx <= lane_count);
for (int i = dst_idx; i < lane_count; i++) {
result[i] = src2.Uint(vform, i - dst_idx);
}
dst.SetUintArray(vform, result);
return dst;
}
LogicVRegister Simulator::sel(VectorFormat vform,
LogicVRegister dst,
const SimPRegister& pg,
const LogicVRegister& src1,
const LogicVRegister& src2) {
int p_reg_bits_per_lane =
LaneSizeInBitsFromFormat(vform) / kZRegBitsPerPRegBit;
for (int lane = 0; lane < LaneCountFromFormat(vform); lane++) {
uint64_t lane_value = pg.GetBit(lane * p_reg_bits_per_lane)
? src1.Uint(vform, lane)
: src2.Uint(vform, lane);
dst.SetUint(vform, lane, lane_value);
}
return dst;
}
LogicPRegister Simulator::sel(LogicPRegister dst,
const LogicPRegister& pg,
const LogicPRegister& src1,
const LogicPRegister& src2) {
for (int i = 0; i < dst.GetChunkCount(); i++) {
LogicPRegister::ChunkType mask = pg.GetChunk(i);
LogicPRegister::ChunkType result =
(mask & src1.GetChunk(i)) | (~mask & src2.GetChunk(i));
dst.SetChunk(i, result);
}
return dst;
}
LogicVRegister Simulator::sli(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
dst.ClearForWrite(vform);
int lane_count = LaneCountFromFormat(vform);
for (int i = 0; i < lane_count; i++) {
uint64_t src_lane = src.Uint(vform, i);
uint64_t dst_lane = dst.Uint(vform, i);
uint64_t shifted = src_lane << shift;
uint64_t mask = MaxUintFromFormat(vform) << shift;
dst.SetUint(vform, i, (dst_lane & ~mask) | shifted);
}
return dst;
}
LogicVRegister Simulator::sqshl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
VIXL_ASSERT(shift >= 0);
SimVRegister temp;
LogicVRegister shiftreg = dup_immediate(vform, temp, shift);
return sshl(vform, dst, src, shiftreg).SignedSaturate(vform);
}
LogicVRegister Simulator::uqshl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
VIXL_ASSERT(shift >= 0);
SimVRegister temp;
LogicVRegister shiftreg = dup_immediate(vform, temp, shift);
return ushl(vform, dst, src, shiftreg).UnsignedSaturate(vform);
}
LogicVRegister Simulator::sqshlu(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
VIXL_ASSERT(shift >= 0);
SimVRegister temp;
LogicVRegister shiftreg = dup_immediate(vform, temp, shift);
return sshl(vform, dst, src, shiftreg).UnsignedSaturate(vform);
}
LogicVRegister Simulator::sri(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
dst.ClearForWrite(vform);
int lane_count = LaneCountFromFormat(vform);
VIXL_ASSERT((shift > 0) &&
(shift <= static_cast<int>(LaneSizeInBitsFromFormat(vform))));
for (int i = 0; i < lane_count; i++) {
uint64_t src_lane = src.Uint(vform, i);
uint64_t dst_lane = dst.Uint(vform, i);
uint64_t shifted;
uint64_t mask;
if (shift == 64) {
shifted = 0;
mask = 0;
} else {
shifted = src_lane >> shift;
mask = MaxUintFromFormat(vform) >> shift;
}
dst.SetUint(vform, i, (dst_lane & ~mask) | shifted);
}
return dst;
}
LogicVRegister Simulator::ushr(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
VIXL_ASSERT(shift >= 0);
SimVRegister temp;
LogicVRegister shiftreg = dup_immediate(vform, temp, -shift);
return ushl(vform, dst, src, shiftreg);
}
LogicVRegister Simulator::sshr(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
VIXL_ASSERT(shift >= 0);
SimVRegister temp;
LogicVRegister shiftreg = dup_immediate(vform, temp, -shift);
return sshl(vform, dst, src, shiftreg);
}
LogicVRegister Simulator::ssra(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
SimVRegister temp;
LogicVRegister shifted_reg = sshr(vform, temp, src, shift);
return add(vform, dst, dst, shifted_reg);
}
LogicVRegister Simulator::usra(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
SimVRegister temp;
LogicVRegister shifted_reg = ushr(vform, temp, src, shift);
return add(vform, dst, dst, shifted_reg);
}
LogicVRegister Simulator::srsra(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
SimVRegister temp;
LogicVRegister shifted_reg = sshr(vform, temp, src, shift).Round(vform);
return add(vform, dst, dst, shifted_reg);
}
LogicVRegister Simulator::ursra(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
SimVRegister temp;
LogicVRegister shifted_reg = ushr(vform, temp, src, shift).Round(vform);
return add(vform, dst, dst, shifted_reg);
}
LogicVRegister Simulator::cls(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
int lane_size_in_bits = LaneSizeInBitsFromFormat(vform);
int lane_count = LaneCountFromFormat(vform);
// Ensure that we can store one result per lane.
int result[kZRegMaxSizeInBytes];
for (int i = 0; i < lane_count; i++) {
result[i] = CountLeadingSignBits(src.Int(vform, i), lane_size_in_bits);
}
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, result[i]);
}
return dst;
}
LogicVRegister Simulator::clz(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
int lane_size_in_bits = LaneSizeInBitsFromFormat(vform);
int lane_count = LaneCountFromFormat(vform);
// Ensure that we can store one result per lane.
int result[kZRegMaxSizeInBytes];
for (int i = 0; i < lane_count; i++) {
result[i] = CountLeadingZeros(src.Uint(vform, i), lane_size_in_bits);
}
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, result[i]);
}
return dst;
}
LogicVRegister Simulator::cnot(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
uint64_t value = (src.Uint(vform, i) == 0) ? 1 : 0;
dst.SetUint(vform, i, value);
}
return dst;
}
LogicVRegister Simulator::cnt(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
int lane_size_in_bits = LaneSizeInBitsFromFormat(vform);
int lane_count = LaneCountFromFormat(vform);
// Ensure that we can store one result per lane.
int result[kZRegMaxSizeInBytes];
for (int i = 0; i < lane_count; i++) {
result[i] = CountSetBits(src.Uint(vform, i), lane_size_in_bits);
}
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, result[i]);
}
return dst;
}
static int64_t CalculateSignedShiftDistance(int64_t shift_val,
int esize,
bool shift_in_ls_byte) {
if (shift_in_ls_byte) {
// Neon uses the least-significant byte of the lane as the shift distance.
shift_val = ExtractSignedBitfield64(7, 0, shift_val);
} else {
// SVE uses a saturated shift distance in the range
// -(esize + 1) ... (esize + 1).
if (shift_val > (esize + 1)) shift_val = esize + 1;
if (shift_val < -(esize + 1)) shift_val = -(esize + 1);
}
return shift_val;
}
LogicVRegister Simulator::sshl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool shift_in_ls_byte) {
dst.ClearForWrite(vform);
int esize = LaneSizeInBitsFromFormat(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
int64_t shift_val = CalculateSignedShiftDistance(src2.Int(vform, i),
esize,
shift_in_ls_byte);
int64_t lj_src_val = src1.IntLeftJustified(vform, i);
// Set signed saturation state.
if ((shift_val > CountLeadingSignBits(lj_src_val)) && (lj_src_val != 0)) {
dst.SetSignedSat(i, lj_src_val >= 0);
}
// Set unsigned saturation state.
if (lj_src_val < 0) {
dst.SetUnsignedSat(i, false);
} else if ((shift_val > CountLeadingZeros(lj_src_val)) &&
(lj_src_val != 0)) {
dst.SetUnsignedSat(i, true);
}
int64_t src_val = src1.Int(vform, i);
bool src_is_negative = src_val < 0;
if (shift_val > 63) {
dst.SetInt(vform, i, 0);
} else if (shift_val < -63) {
dst.SetRounding(i, src_is_negative);
dst.SetInt(vform, i, src_is_negative ? -1 : 0);
} else {
// Use unsigned types for shifts, as behaviour is undefined for signed
// lhs.
uint64_t usrc_val = static_cast<uint64_t>(src_val);
if (shift_val < 0) {
// Convert to right shift.
shift_val = -shift_val;
// Set rounding state by testing most-significant bit shifted out.
// Rounding only needed on right shifts.
if (((usrc_val >> (shift_val - 1)) & 1) == 1) {
dst.SetRounding(i, true);
}
usrc_val >>= shift_val;
if (src_is_negative) {
// Simulate sign-extension.
usrc_val |= (~UINT64_C(0) << (64 - shift_val));
}
} else {
usrc_val <<= shift_val;
}
dst.SetUint(vform, i, usrc_val);
}
}
return dst;
}
LogicVRegister Simulator::ushl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool shift_in_ls_byte) {
dst.ClearForWrite(vform);
int esize = LaneSizeInBitsFromFormat(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
int64_t shift_val = CalculateSignedShiftDistance(src2.Int(vform, i),
esize,
shift_in_ls_byte);
uint64_t lj_src_val = src1.UintLeftJustified(vform, i);
// Set saturation state.
if ((shift_val > CountLeadingZeros(lj_src_val)) && (lj_src_val != 0)) {
dst.SetUnsignedSat(i, true);
}
uint64_t src_val = src1.Uint(vform, i);
if ((shift_val > 63) || (shift_val < -64)) {
dst.SetUint(vform, i, 0);
} else {
if (shift_val < 0) {
// Set rounding state. Rounding only needed on right shifts.
if (((src_val >> (-shift_val - 1)) & 1) == 1) {
dst.SetRounding(i, true);
}
if (shift_val == -64) {
src_val = 0;
} else {
src_val >>= -shift_val;
}
} else {
src_val <<= shift_val;
}
dst.SetUint(vform, i, src_val);
}
}
return dst;
}
LogicVRegister Simulator::sshr(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
// Saturate to sidestep the min-int problem.
neg(vform, temp, src2).SignedSaturate(vform);
sshl(vform, dst, src1, temp, false);
return dst;
}
LogicVRegister Simulator::ushr(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
// Saturate to sidestep the min-int problem.
neg(vform, temp, src2).SignedSaturate(vform);
ushl(vform, dst, src1, temp, false);
return dst;
}
LogicVRegister Simulator::neg(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
// Test for signed saturation.
int64_t sa = src.Int(vform, i);
if (sa == MinIntFromFormat(vform)) {
dst.SetSignedSat(i, true);
}
dst.SetInt(vform, i, (sa == INT64_MIN) ? sa : -sa);
}
return dst;
}
LogicVRegister Simulator::suqadd(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
int64_t sa = src1.IntLeftJustified(vform, i);
uint64_t ub = src2.UintLeftJustified(vform, i);
uint64_t ur = sa + ub;
int64_t sr;
memcpy(&sr, &ur, sizeof(sr));
if (sr < sa) { // Test for signed positive saturation.
dst.SetInt(vform, i, MaxIntFromFormat(vform));
} else {
dst.SetUint(vform, i, src1.Int(vform, i) + src2.Uint(vform, i));
}
}
return dst;
}
LogicVRegister Simulator::usqadd(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
uint64_t ua = src1.UintLeftJustified(vform, i);
int64_t sb = src2.IntLeftJustified(vform, i);
uint64_t ur = ua + sb;
if ((sb > 0) && (ur <= ua)) {
dst.SetUint(vform, i, MaxUintFromFormat(vform)); // Positive saturation.
} else if ((sb < 0) && (ur >= ua)) {
dst.SetUint(vform, i, 0); // Negative saturation.
} else {
dst.SetUint(vform, i, src1.Uint(vform, i) + src2.Int(vform, i));
}
}
return dst;
}
LogicVRegister Simulator::abs(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
// Test for signed saturation.
int64_t sa = src.Int(vform, i);
if (sa == MinIntFromFormat(vform)) {
dst.SetSignedSat(i, true);
}
if (sa < 0) {
dst.SetInt(vform, i, (sa == INT64_MIN) ? sa : -sa);
} else {
dst.SetInt(vform, i, sa);
}
}
return dst;
}
LogicVRegister Simulator::andv(VectorFormat vform,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src) {
VIXL_ASSERT(IsSVEFormat(vform));
uint64_t result = GetUintMask(LaneSizeInBitsFromFormat(vform));
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!pg.IsActive(vform, i)) continue;
result &= src.Uint(vform, i);
}
VectorFormat vform_dst =
ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform));
dst.ClearForWrite(vform_dst);
dst.SetUint(vform_dst, 0, result);
return dst;
}
LogicVRegister Simulator::eorv(VectorFormat vform,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src) {
VIXL_ASSERT(IsSVEFormat(vform));
uint64_t result = 0;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!pg.IsActive(vform, i)) continue;
result ^= src.Uint(vform, i);
}
VectorFormat vform_dst =
ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform));
dst.ClearForWrite(vform_dst);
dst.SetUint(vform_dst, 0, result);
return dst;
}
LogicVRegister Simulator::orv(VectorFormat vform,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src) {
VIXL_ASSERT(IsSVEFormat(vform));
uint64_t result = 0;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!pg.IsActive(vform, i)) continue;
result |= src.Uint(vform, i);
}
VectorFormat vform_dst =
ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform));
dst.ClearForWrite(vform_dst);
dst.SetUint(vform_dst, 0, result);
return dst;
}
LogicVRegister Simulator::saddv(VectorFormat vform,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src) {
VIXL_ASSERT(IsSVEFormat(vform));
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) <= kSRegSize);
int64_t result = 0;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!pg.IsActive(vform, i)) continue;
// The destination register always has D-lane sizes and the source register
// always has S-lanes or smaller, so signed integer overflow -- undefined
// behaviour -- can't occur.
result += src.Int(vform, i);
}
dst.ClearForWrite(kFormatD);
dst.SetInt(kFormatD, 0, result);
return dst;
}
LogicVRegister Simulator::uaddv(VectorFormat vform,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src) {
VIXL_ASSERT(IsSVEFormat(vform));
uint64_t result = 0;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!pg.IsActive(vform, i)) continue;
result += src.Uint(vform, i);
}
dst.ClearForWrite(kFormatD);
dst.SetUint(kFormatD, 0, result);
return dst;
}
LogicVRegister Simulator::extractnarrow(VectorFormat dstform,
LogicVRegister dst,
bool dst_is_signed,
const LogicVRegister& src,
bool src_is_signed) {
bool upperhalf = false;
VectorFormat srcform = dstform;
if ((dstform == kFormat16B) || (dstform == kFormat8H) ||
(dstform == kFormat4S)) {
upperhalf = true;
srcform = VectorFormatHalfLanes(srcform);
}
srcform = VectorFormatDoubleWidth(srcform);
LogicVRegister src_copy = src;
int offset;
if (upperhalf) {
offset = LaneCountFromFormat(dstform) / 2;
} else {
offset = 0;
}
for (int i = 0; i < LaneCountFromFormat(srcform); i++) {
int64_t ssrc = src_copy.Int(srcform, i);
uint64_t usrc = src_copy.Uint(srcform, i);
// Test for signed saturation
if (ssrc > MaxIntFromFormat(dstform)) {
dst.SetSignedSat(offset + i, true);
} else if (ssrc < MinIntFromFormat(dstform)) {
dst.SetSignedSat(offset + i, false);
}
// Test for unsigned saturation
if (src_is_signed) {
if (ssrc > static_cast<int64_t>(MaxUintFromFormat(dstform))) {
dst.SetUnsignedSat(offset + i, true);
} else if (ssrc < 0) {
dst.SetUnsignedSat(offset + i, false);
}
} else {
if (usrc > MaxUintFromFormat(dstform)) {
dst.SetUnsignedSat(offset + i, true);
}
}
int64_t result;
if (src_is_signed) {
result = ssrc & MaxUintFromFormat(dstform);
} else {
result = usrc & MaxUintFromFormat(dstform);
}
if (dst_is_signed) {
dst.SetInt(dstform, offset + i, result);
} else {
dst.SetUint(dstform, offset + i, result);
}
}
if (upperhalf) {
// Clear any bits beyond a Q register.
dst.ClearForWrite(kFormat16B);
} else {
dst.ClearForWrite(dstform);
}
return dst;
}
LogicVRegister Simulator::xtn(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
return extractnarrow(vform, dst, true, src, true);
}
LogicVRegister Simulator::sqxtn(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
return extractnarrow(vform, dst, true, src, true).SignedSaturate(vform);
}
LogicVRegister Simulator::sqxtun(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
return extractnarrow(vform, dst, false, src, true).UnsignedSaturate(vform);
}
LogicVRegister Simulator::uqxtn(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
return extractnarrow(vform, dst, false, src, false).UnsignedSaturate(vform);
}
LogicVRegister Simulator::absdiff(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool is_signed) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
bool src1_gt_src2 = is_signed ? (src1.Int(vform, i) > src2.Int(vform, i))
: (src1.Uint(vform, i) > src2.Uint(vform, i));
// Always calculate the answer using unsigned arithmetic, to avoid
// implementation-defined signed overflow.
if (src1_gt_src2) {
dst.SetUint(vform, i, src1.Uint(vform, i) - src2.Uint(vform, i));
} else {
dst.SetUint(vform, i, src2.Uint(vform, i) - src1.Uint(vform, i));
}
}
return dst;
}
LogicVRegister Simulator::saba(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
dst.ClearForWrite(vform);
absdiff(vform, temp, src1, src2, true);
add(vform, dst, dst, temp);
return dst;
}
LogicVRegister Simulator::uaba(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
dst.ClearForWrite(vform);
absdiff(vform, temp, src1, src2, false);
add(vform, dst, dst, temp);
return dst;
}
LogicVRegister Simulator::not_(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
dst.SetUint(vform, i, ~src.Uint(vform, i));
}
return dst;
}
LogicVRegister Simulator::rbit(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
uint64_t result[kZRegMaxSizeInBytes];
int lane_count = LaneCountFromFormat(vform);
int lane_size_in_bits = LaneSizeInBitsFromFormat(vform);
uint64_t reversed_value;
uint64_t value;
for (int i = 0; i < lane_count; i++) {
value = src.Uint(vform, i);
reversed_value = 0;
for (int j = 0; j < lane_size_in_bits; j++) {
reversed_value = (reversed_value << 1) | (value & 1);
value >>= 1;
}
result[i] = reversed_value;
}
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, result[i]);
}
return dst;
}
LogicVRegister Simulator::rev(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
VIXL_ASSERT(IsSVEFormat(vform));
int lane_count = LaneCountFromFormat(vform);
for (int i = 0; i < lane_count / 2; i++) {
uint64_t t = src.Uint(vform, i);
dst.SetUint(vform, i, src.Uint(vform, lane_count - i - 1));
dst.SetUint(vform, lane_count - i - 1, t);
}
return dst;
}
LogicVRegister Simulator::rev_byte(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int rev_size) {
uint64_t result[kZRegMaxSizeInBytes] = {};
int lane_count = LaneCountFromFormat(vform);
int lane_size = LaneSizeInBytesFromFormat(vform);
int lanes_per_loop = rev_size / lane_size;
for (int i = 0; i < lane_count; i += lanes_per_loop) {
for (int j = 0; j < lanes_per_loop; j++) {
result[i + lanes_per_loop - 1 - j] = src.Uint(vform, i + j);
}
}
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, result[i]);
}
return dst;
}
LogicVRegister Simulator::rev16(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
return rev_byte(vform, dst, src, 2);
}
LogicVRegister Simulator::rev32(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
return rev_byte(vform, dst, src, 4);
}
LogicVRegister Simulator::rev64(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
return rev_byte(vform, dst, src, 8);
}
LogicVRegister Simulator::addlp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
bool is_signed,
bool do_accumulate) {
VectorFormat vformsrc = VectorFormatHalfWidthDoubleLanes(vform);
VIXL_ASSERT(LaneSizeInBitsFromFormat(vformsrc) <= kSRegSize);
uint64_t result[kZRegMaxSizeInBytes];
int lane_count = LaneCountFromFormat(vform);
for (int i = 0; i < lane_count; i++) {
if (is_signed) {
result[i] = static_cast<uint64_t>(src.Int(vformsrc, 2 * i) +
src.Int(vformsrc, 2 * i + 1));
} else {
result[i] = src.Uint(vformsrc, 2 * i) + src.Uint(vformsrc, 2 * i + 1);
}
}
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
if (do_accumulate) {
result[i] += dst.Uint(vform, i);
}
dst.SetUint(vform, i, result[i]);
}
return dst;
}
LogicVRegister Simulator::saddlp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
return addlp(vform, dst, src, true, false);
}
LogicVRegister Simulator::uaddlp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
return addlp(vform, dst, src, false, false);
}
LogicVRegister Simulator::sadalp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
return addlp(vform, dst, src, true, true);
}
LogicVRegister Simulator::uadalp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
return addlp(vform, dst, src, false, true);
}
LogicVRegister Simulator::ror(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int rotation) {
dst.ClearForWrite(vform);
int width = LaneSizeInBitsFromFormat(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
uint64_t value = src.Uint(vform, i);
dst.SetUint(vform, i, RotateRight(value, rotation, width));
}
return dst;
}
LogicVRegister Simulator::rol(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int rotation) {
int ror_equivalent = LaneSizeInBitsFromFormat(vform) - rotation;
return ror(vform, dst, src, ror_equivalent);
}
LogicVRegister Simulator::ext(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
uint8_t result[kZRegMaxSizeInBytes] = {};
int lane_count = LaneCountFromFormat(vform);
for (int i = 0; i < lane_count - index; ++i) {
result[i] = static_cast<uint8_t>(src1.Uint(vform, i + index));
}
for (int i = 0; i < index; ++i) {
result[lane_count - index + i] = static_cast<uint8_t>(src2.Uint(vform, i));
}
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, result[i]);
}
return dst;
}
LogicVRegister Simulator::rotate_elements_right(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int index) {
if (index < 0) index += LaneCountFromFormat(vform);
VIXL_ASSERT((index >= 0) && (index < LaneCountFromFormat(vform)));
index *= LaneSizeInBytesFromFormat(vform);
return ext(kFormatVnB, dst, src, src, index);
}
template <typename T>
LogicVRegister Simulator::fadda(VectorFormat vform,
LogicVRegister acc,
const LogicPRegister& pg,
const LogicVRegister& src) {
T result = acc.Float<T>(0);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!pg.IsActive(vform, i)) continue;
result = FPAdd(result, src.Float<T>(i));
}
VectorFormat vform_dst =
ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform));
acc.ClearForWrite(vform_dst);
acc.SetFloat(0, result);
return acc;
}
LogicVRegister Simulator::fadda(VectorFormat vform,
LogicVRegister acc,
const LogicPRegister& pg,
const LogicVRegister& src) {
switch (LaneSizeInBitsFromFormat(vform)) {
case kHRegSize:
fadda<SimFloat16>(vform, acc, pg, src);
break;
case kSRegSize:
fadda<float>(vform, acc, pg, src);
break;
case kDRegSize:
fadda<double>(vform, acc, pg, src);
break;
default:
VIXL_UNREACHABLE();
}
return acc;
}
template <typename T>
LogicVRegister Simulator::fcadd(VectorFormat vform,
LogicVRegister dst, // d
const LogicVRegister& src1, // n
const LogicVRegister& src2, // m
int rot) {
int elements = LaneCountFromFormat(vform);
T element1, element3;
rot = (rot == 1) ? 270 : 90;
// Loop example:
// 2S --> (2/2 = 1 - 1 = 0) --> 1 x Complex Number (2x components: r+i)
// 4S --> (4/2 = 2) - 1 = 1) --> 2 x Complex Number (2x2 components: r+i)
for (int e = 0; e <= (elements / 2) - 1; e++) {
switch (rot) {
case 90:
element1 = FPNeg(src2.Float<T>(e * 2 + 1));
element3 = src2.Float<T>(e * 2);
break;
case 270:
element1 = src2.Float<T>(e * 2 + 1);
element3 = FPNeg(src2.Float<T>(e * 2));
break;
default:
VIXL_UNREACHABLE();
return dst; // prevents "element(n) may be unintialized" errors
}
dst.ClearForWrite(vform);
dst.SetFloat<T>(e * 2, FPAdd(src1.Float<T>(e * 2), element1));
dst.SetFloat<T>(e * 2 + 1, FPAdd(src1.Float<T>(e * 2 + 1), element3));
}
return dst;
}
LogicVRegister Simulator::fcadd(VectorFormat vform,
LogicVRegister dst, // d
const LogicVRegister& src1, // n
const LogicVRegister& src2, // m
int rot) {
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
fcadd<SimFloat16>(vform, dst, src1, src2, rot);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
fcadd<float>(vform, dst, src1, src2, rot);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
fcadd<double>(vform, dst, src1, src2, rot);
}
return dst;
}
template <typename T>
LogicVRegister Simulator::fcmla(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
const LogicVRegister& acc,
int index,
int rot) {
int elements = LaneCountFromFormat(vform);
T element1, element2, element3, element4;
rot *= 90;
// Loop example:
// 2S --> (2/2 = 1 - 1 = 0) --> 1 x Complex Number (2x components: r+i)
// 4S --> (4/2 = 2) - 1 = 1) --> 2 x Complex Number (2x2 components: r+i)
for (int e = 0; e <= (elements / 2) - 1; e++) {
// Index == -1 indicates a vector/vector rather than vector/indexed-element
// operation.
int f = (index < 0) ? e : index;
switch (rot) {
case 0:
element1 = src2.Float<T>(f * 2);
element2 = src1.Float<T>(e * 2);
element3 = src2.Float<T>(f * 2 + 1);
element4 = src1.Float<T>(e * 2);
break;
case 90:
element1 = FPNeg(src2.Float<T>(f * 2 + 1));
element2 = src1.Float<T>(e * 2 + 1);
element3 = src2.Float<T>(f * 2);
element4 = src1.Float<T>(e * 2 + 1);
break;
case 180:
element1 = FPNeg(src2.Float<T>(f * 2));
element2 = src1.Float<T>(e * 2);
element3 = FPNeg(src2.Float<T>(f * 2 + 1));
element4 = src1.Float<T>(e * 2);
break;
case 270:
element1 = src2.Float<T>(f * 2 + 1);
element2 = src1.Float<T>(e * 2 + 1);
element3 = FPNeg(src2.Float<T>(f * 2));
element4 = src1.Float<T>(e * 2 + 1);
break;
default:
VIXL_UNREACHABLE();
return dst; // prevents "element(n) may be unintialized" errors
}
dst.ClearForWrite(vform);
dst.SetFloat<T>(vform,
e * 2,
FPMulAdd(acc.Float<T>(e * 2), element2, element1));
dst.SetFloat<T>(vform,
e * 2 + 1,
FPMulAdd(acc.Float<T>(e * 2 + 1), element4, element3));
}
return dst;
}
LogicVRegister Simulator::fcmla(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
const LogicVRegister& acc,
int rot) {
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
fcmla<SimFloat16>(vform, dst, src1, src2, acc, -1, rot);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
fcmla<float>(vform, dst, src1, src2, acc, -1, rot);
} else {
fcmla<double>(vform, dst, src1, src2, acc, -1, rot);
}
return dst;
}
LogicVRegister Simulator::fcmla(VectorFormat vform,
LogicVRegister dst, // d
const LogicVRegister& src1, // n
const LogicVRegister& src2, // m
int index,
int rot) {
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
fcmla<SimFloat16>(vform, dst, src1, src2, dst, index, rot);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
fcmla<float>(vform, dst, src1, src2, dst, index, rot);
} else {
fcmla<double>(vform, dst, src1, src2, dst, index, rot);
}
return dst;
}
LogicVRegister Simulator::cadd(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int rot,
bool saturate) {
SimVRegister src1_r, src1_i;
SimVRegister src2_r, src2_i;
SimVRegister zero;
zero.Clear();
uzp1(vform, src1_r, src1, zero);
uzp2(vform, src1_i, src1, zero);
uzp1(vform, src2_r, src2, zero);
uzp2(vform, src2_i, src2, zero);
if (rot == 90) {
if (saturate) {
sub(vform, src1_r, src1_r, src2_i).SignedSaturate(vform);
add(vform, src1_i, src1_i, src2_r).SignedSaturate(vform);
} else {
sub(vform, src1_r, src1_r, src2_i);
add(vform, src1_i, src1_i, src2_r);
}
} else {
VIXL_ASSERT(rot == 270);
if (saturate) {
add(vform, src1_r, src1_r, src2_i).SignedSaturate(vform);
sub(vform, src1_i, src1_i, src2_r).SignedSaturate(vform);
} else {
add(vform, src1_r, src1_r, src2_i);
sub(vform, src1_i, src1_i, src2_r);
}
}
zip1(vform, dst, src1_r, src1_i);
return dst;
}
LogicVRegister Simulator::cmla(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& srca,
const LogicVRegister& src1,
const LogicVRegister& src2,
int rot) {
SimVRegister src1_a;
SimVRegister src2_a, src2_b;
SimVRegister srca_i, srca_r;
SimVRegister zero, temp;
zero.Clear();
if ((rot == 0) || (rot == 180)) {
uzp1(vform, src1_a, src1, zero);
uzp1(vform, src2_a, src2, zero);
uzp2(vform, src2_b, src2, zero);
} else {
uzp2(vform, src1_a, src1, zero);
uzp2(vform, src2_a, src2, zero);
uzp1(vform, src2_b, src2, zero);
}
uzp1(vform, srca_r, srca, zero);
uzp2(vform, srca_i, srca, zero);
bool sub_r = (rot == 90) || (rot == 180);
bool sub_i = (rot == 180) || (rot == 270);
mul(vform, temp, src1_a, src2_a);
if (sub_r) {
sub(vform, srca_r, srca_r, temp);
} else {
add(vform, srca_r, srca_r, temp);
}
mul(vform, temp, src1_a, src2_b);
if (sub_i) {
sub(vform, srca_i, srca_i, temp);
} else {
add(vform, srca_i, srca_i, temp);
}
zip1(vform, dst, srca_r, srca_i);
return dst;
}
LogicVRegister Simulator::cmla(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& srca,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index,
int rot) {
SimVRegister temp;
dup_elements_to_segments(VectorFormatDoubleWidth(vform), temp, src2, index);
return cmla(vform, dst, srca, src1, temp, rot);
}
LogicVRegister Simulator::bgrp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool do_bext) {
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
uint64_t value = src1.Uint(vform, i);
uint64_t mask = src2.Uint(vform, i);
int high_pos = 0;
int low_pos = 0;
uint64_t result_high = 0;
uint64_t result_low = 0;
for (unsigned j = 0; j < LaneSizeInBitsFromFormat(vform); j++) {
if ((mask & 1) == 0) {
result_high |= (value & 1) << high_pos;
high_pos++;
} else {
result_low |= (value & 1) << low_pos;
low_pos++;
}
mask >>= 1;
value >>= 1;
}
if (!do_bext) {
result_low |= result_high << low_pos;
}
dst.SetUint(vform, i, result_low);
}
return dst;
}
LogicVRegister Simulator::bdep(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
uint64_t value = src1.Uint(vform, i);
uint64_t mask = src2.Uint(vform, i);
uint64_t result = 0;
for (unsigned j = 0; j < LaneSizeInBitsFromFormat(vform); j++) {
if ((mask & 1) == 1) {
result |= (value & 1) << j;
value >>= 1;
}
mask >>= 1;
}
dst.SetUint(vform, i, result);
}
return dst;
}
LogicVRegister Simulator::histogram(VectorFormat vform,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool do_segmented) {
int elements_per_segment = kQRegSize / LaneSizeInBitsFromFormat(vform);
uint64_t result[kZRegMaxSizeInBytes];
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
uint64_t count = 0;
uint64_t value = src1.Uint(vform, i);
int segment = do_segmented ? (i / elements_per_segment) : 0;
int segment_offset = segment * elements_per_segment;
int hist_limit = do_segmented ? elements_per_segment : (i + 1);
for (int j = 0; j < hist_limit; j++) {
if (pg.IsActive(vform, j) &&
(value == src2.Uint(vform, j + segment_offset))) {
count++;
}
}
result[i] = count;
}
dst.SetUintArray(vform, result);
return dst;
}
LogicVRegister Simulator::dup_element(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int src_index) {
if ((vform == kFormatVnQ) || (vform == kFormatVnO)) {
// When duplicating an element larger than 64 bits, split the element into
// 64-bit parts, and duplicate the parts across the destination.
uint64_t d[4];
int count = (vform == kFormatVnQ) ? 2 : 4;
for (int i = 0; i < count; i++) {
d[i] = src.Uint(kFormatVnD, (src_index * count) + i);
}
dst.Clear();
for (int i = 0; i < LaneCountFromFormat(vform) * count; i++) {
dst.SetUint(kFormatVnD, i, d[i % count]);
}
} else {
int lane_count = LaneCountFromFormat(vform);
uint64_t value = src.Uint(vform, src_index);
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, value);
}
}
return dst;
}
LogicVRegister Simulator::dup_elements_to_segments(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int src_index) {
// In SVE, a segment is a 128-bit portion of a vector, like a Q register,
// whereas in NEON, the size of segment is equal to the size of register
// itself.
int segment_size = std::min(kQRegSize, RegisterSizeInBitsFromFormat(vform));
VIXL_ASSERT(IsMultiple(segment_size, LaneSizeInBitsFromFormat(vform)));
int lanes_per_segment = segment_size / LaneSizeInBitsFromFormat(vform);
VIXL_ASSERT(src_index >= 0);
VIXL_ASSERT(src_index < lanes_per_segment);
dst.ClearForWrite(vform);
for (int j = 0; j < LaneCountFromFormat(vform); j += lanes_per_segment) {
uint64_t value = src.Uint(vform, j + src_index);
for (int i = 0; i < lanes_per_segment; i++) {
dst.SetUint(vform, j + i, value);
}
}
return dst;
}
LogicVRegister Simulator::dup_elements_to_segments(
VectorFormat vform,
LogicVRegister dst,
const std::pair<int, int>& src_and_index) {
return dup_elements_to_segments(vform,
dst,
ReadVRegister(src_and_index.first),
src_and_index.second);
}
LogicVRegister Simulator::dup_immediate(VectorFormat vform,
LogicVRegister dst,
uint64_t imm) {
int lane_count = LaneCountFromFormat(vform);
uint64_t value = imm & MaxUintFromFormat(vform);
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, value);
}
return dst;
}
LogicVRegister Simulator::ins_element(VectorFormat vform,
LogicVRegister dst,
int dst_index,
const LogicVRegister& src,
int src_index) {
dst.SetUint(vform, dst_index, src.Uint(vform, src_index));
return dst;
}
LogicVRegister Simulator::ins_immediate(VectorFormat vform,
LogicVRegister dst,
int dst_index,
uint64_t imm) {
uint64_t value = imm & MaxUintFromFormat(vform);
dst.SetUint(vform, dst_index, value);
return dst;
}
LogicVRegister Simulator::index(VectorFormat vform,
LogicVRegister dst,
uint64_t start,
uint64_t step) {
VIXL_ASSERT(IsSVEFormat(vform));
uint64_t value = start;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
dst.SetUint(vform, i, value);
value += step;
}
return dst;
}
LogicVRegister Simulator::insr(VectorFormat vform,
LogicVRegister dst,
uint64_t imm) {
VIXL_ASSERT(IsSVEFormat(vform));
for (int i = LaneCountFromFormat(vform) - 1; i > 0; i--) {
dst.SetUint(vform, i, dst.Uint(vform, i - 1));
}
dst.SetUint(vform, 0, imm);
return dst;
}
LogicVRegister Simulator::mov(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
dst.ClearForWrite(vform);
for (int lane = 0; lane < LaneCountFromFormat(vform); lane++) {
dst.SetUint(vform, lane, src.Uint(vform, lane));
}
return dst;
}
LogicPRegister Simulator::mov(LogicPRegister dst, const LogicPRegister& src) {
// Avoid a copy if the registers already alias.
if (dst.Aliases(src)) return dst;
for (int i = 0; i < dst.GetChunkCount(); i++) {
dst.SetChunk(i, src.GetChunk(i));
}
return dst;
}
LogicVRegister Simulator::mov_merging(VectorFormat vform,
LogicVRegister dst,
const SimPRegister& pg,
const LogicVRegister& src) {
return sel(vform, dst, pg, src, dst);
}
LogicVRegister Simulator::mov_zeroing(VectorFormat vform,
LogicVRegister dst,
const SimPRegister& pg,
const LogicVRegister& src) {
SimVRegister zero;
dup_immediate(vform, zero, 0);
return sel(vform, dst, pg, src, zero);
}
LogicVRegister Simulator::mov_alternating(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int start_at) {
VIXL_ASSERT((start_at == 0) || (start_at == 1));
for (int i = start_at; i < LaneCountFromFormat(vform); i += 2) {
dst.SetUint(vform, i, src.Uint(vform, i));
}
return dst;
}
LogicPRegister Simulator::mov_merging(LogicPRegister dst,
const LogicPRegister& pg,
const LogicPRegister& src) {
return sel(dst, pg, src, dst);
}
LogicPRegister Simulator::mov_zeroing(LogicPRegister dst,
const LogicPRegister& pg,
const LogicPRegister& src) {
SimPRegister all_false;
return sel(dst, pg, src, pfalse(all_false));
}
LogicVRegister Simulator::movi(VectorFormat vform,
LogicVRegister dst,
uint64_t imm) {
int lane_count = LaneCountFromFormat(vform);
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, imm);
}
return dst;
}
LogicVRegister Simulator::mvni(VectorFormat vform,
LogicVRegister dst,
uint64_t imm) {
int lane_count = LaneCountFromFormat(vform);
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, ~imm);
}
return dst;
}
LogicVRegister Simulator::orr(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
uint64_t imm) {
uint64_t result[16];
int lane_count = LaneCountFromFormat(vform);
for (int i = 0; i < lane_count; ++i) {
result[i] = src.Uint(vform, i) | imm;
}
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, result[i]);
}
return dst;
}
LogicVRegister Simulator::uxtl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
bool is_2) {
VectorFormat vform_half = VectorFormatHalfWidth(vform);
int lane_count = LaneCountFromFormat(vform);
int src_offset = is_2 ? lane_count : 0;
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; i++) {
dst.SetUint(vform, i, src.Uint(vform_half, src_offset + i));
}
return dst;
}
LogicVRegister Simulator::sxtl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
bool is_2) {
VectorFormat vform_half = VectorFormatHalfWidth(vform);
int lane_count = LaneCountFromFormat(vform);
int src_offset = is_2 ? lane_count : 0;
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
dst.SetInt(vform, i, src.Int(vform_half, src_offset + i));
}
return dst;
}
LogicVRegister Simulator::uxtl2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
return uxtl(vform, dst, src, /* is_2 = */ true);
}
LogicVRegister Simulator::sxtl2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
return sxtl(vform, dst, src, /* is_2 = */ true);
}
LogicVRegister Simulator::uxt(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
unsigned from_size_in_bits) {
int lane_count = LaneCountFromFormat(vform);
uint64_t mask = GetUintMask(from_size_in_bits);
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; i++) {
dst.SetInt(vform, i, src.Uint(vform, i) & mask);
}
return dst;
}
LogicVRegister Simulator::sxt(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
unsigned from_size_in_bits) {
int lane_count = LaneCountFromFormat(vform);
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; i++) {
uint64_t value =
ExtractSignedBitfield64(from_size_in_bits - 1, 0, src.Uint(vform, i));
dst.SetInt(vform, i, value);
}
return dst;
}
LogicVRegister Simulator::shrn(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
SimVRegister temp;
VectorFormat vform_src = VectorFormatDoubleWidth(vform);
VectorFormat vform_dst = vform;
LogicVRegister shifted_src = ushr(vform_src, temp, src, shift);
return extractnarrow(vform_dst, dst, false, shifted_src, false);
}
LogicVRegister Simulator::shrn2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
SimVRegister temp;
VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
VectorFormat vformdst = vform;
LogicVRegister shifted_src = ushr(vformsrc, temp, src, shift);
return extractnarrow(vformdst, dst, false, shifted_src, false);
}
LogicVRegister Simulator::rshrn(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
SimVRegister temp;
VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
VectorFormat vformdst = vform;
LogicVRegister shifted_src = ushr(vformsrc, temp, src, shift).Round(vformsrc);
return extractnarrow(vformdst, dst, false, shifted_src, false);
}
LogicVRegister Simulator::rshrn2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
SimVRegister temp;
VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
VectorFormat vformdst = vform;
LogicVRegister shifted_src = ushr(vformsrc, temp, src, shift).Round(vformsrc);
return extractnarrow(vformdst, dst, false, shifted_src, false);
}
LogicVRegister Simulator::Table(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& ind,
bool zero_out_of_bounds,
const LogicVRegister* tab1,
const LogicVRegister* tab2,
const LogicVRegister* tab3,
const LogicVRegister* tab4) {
VIXL_ASSERT(tab1 != NULL);
int lane_count = LaneCountFromFormat(vform);
VIXL_ASSERT((tab3 == NULL) || (lane_count <= 16));
uint64_t table[kZRegMaxSizeInBytes * 2];
uint64_t result[kZRegMaxSizeInBytes];
// For Neon, the table source registers are always 16B, and Neon allows only
// 8B or 16B vform for the destination, so infer the table format from the
// destination.
VectorFormat vform_tab = (vform == kFormat8B) ? kFormat16B : vform;
uint64_t tab_size = tab1->UintArray(vform_tab, &table[0]);
if (tab2 != NULL) tab_size += tab2->UintArray(vform_tab, &table[tab_size]);
if (tab3 != NULL) tab_size += tab3->UintArray(vform_tab, &table[tab_size]);
if (tab4 != NULL) tab_size += tab4->UintArray(vform_tab, &table[tab_size]);
for (int i = 0; i < lane_count; i++) {
uint64_t index = ind.Uint(vform, i);
result[i] = zero_out_of_bounds ? 0 : dst.Uint(vform, i);
if (index < tab_size) result[i] = table[index];
}
dst.SetUintArray(vform, result);
return dst;
}
LogicVRegister Simulator::tbl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& tab,
const LogicVRegister& ind) {
return Table(vform, dst, ind, true, &tab);
}
LogicVRegister Simulator::tbl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& tab,
const LogicVRegister& tab2,
const LogicVRegister& ind) {
return Table(vform, dst, ind, true, &tab, &tab2);
}
LogicVRegister Simulator::tbl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& tab,
const LogicVRegister& tab2,
const LogicVRegister& tab3,
const LogicVRegister& ind) {
return Table(vform, dst, ind, true, &tab, &tab2, &tab3);
}
LogicVRegister Simulator::tbl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& tab,
const LogicVRegister& tab2,
const LogicVRegister& tab3,
const LogicVRegister& tab4,
const LogicVRegister& ind) {
return Table(vform, dst, ind, true, &tab, &tab2, &tab3, &tab4);
}
LogicVRegister Simulator::tbx(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& tab,
const LogicVRegister& ind) {
return Table(vform, dst, ind, false, &tab);
}
LogicVRegister Simulator::tbx(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& tab,
const LogicVRegister& tab2,
const LogicVRegister& ind) {
return Table(vform, dst, ind, false, &tab, &tab2);
}
LogicVRegister Simulator::tbx(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& tab,
const LogicVRegister& tab2,
const LogicVRegister& tab3,
const LogicVRegister& ind) {
return Table(vform, dst, ind, false, &tab, &tab2, &tab3);
}
LogicVRegister Simulator::tbx(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& tab,
const LogicVRegister& tab2,
const LogicVRegister& tab3,
const LogicVRegister& tab4,
const LogicVRegister& ind) {
return Table(vform, dst, ind, false, &tab, &tab2, &tab3, &tab4);
}
LogicVRegister Simulator::uqshrn(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
return shrn(vform, dst, src, shift).UnsignedSaturate(vform);
}
LogicVRegister Simulator::uqshrn2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
return shrn2(vform, dst, src, shift).UnsignedSaturate(vform);
}
LogicVRegister Simulator::uqrshrn(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
return rshrn(vform, dst, src, shift).UnsignedSaturate(vform);
}
LogicVRegister Simulator::uqrshrn2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
return rshrn2(vform, dst, src, shift).UnsignedSaturate(vform);
}
LogicVRegister Simulator::sqshrn(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
SimVRegister temp;
VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
VectorFormat vformdst = vform;
LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift);
return sqxtn(vformdst, dst, shifted_src);
}
LogicVRegister Simulator::sqshrn2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
SimVRegister temp;
VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
VectorFormat vformdst = vform;
LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift);
return sqxtn(vformdst, dst, shifted_src);
}
LogicVRegister Simulator::sqrshrn(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
SimVRegister temp;
VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
VectorFormat vformdst = vform;
LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift).Round(vformsrc);
return sqxtn(vformdst, dst, shifted_src);
}
LogicVRegister Simulator::sqrshrn2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
SimVRegister temp;
VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
VectorFormat vformdst = vform;
LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift).Round(vformsrc);
return sqxtn(vformdst, dst, shifted_src);
}
LogicVRegister Simulator::sqshrun(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
SimVRegister temp;
VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
VectorFormat vformdst = vform;
LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift);
return sqxtun(vformdst, dst, shifted_src);
}
LogicVRegister Simulator::sqshrun2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
SimVRegister temp;
VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
VectorFormat vformdst = vform;
LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift);
return sqxtun(vformdst, dst, shifted_src);
}
LogicVRegister Simulator::sqrshrun(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
SimVRegister temp;
VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
VectorFormat vformdst = vform;
LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift).Round(vformsrc);
return sqxtun(vformdst, dst, shifted_src);
}
LogicVRegister Simulator::sqrshrun2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int shift) {
SimVRegister temp;
VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
VectorFormat vformdst = vform;
LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift).Round(vformsrc);
return sqxtun(vformdst, dst, shifted_src);
}
LogicVRegister Simulator::uaddl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp1, temp2;
uxtl(vform, temp1, src1);
uxtl(vform, temp2, src2);
add(vform, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::uaddl2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp1, temp2;
uxtl2(vform, temp1, src1);
uxtl2(vform, temp2, src2);
add(vform, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::uaddw(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
uxtl(vform, temp, src2);
add(vform, dst, src1, temp);
return dst;
}
LogicVRegister Simulator::uaddw2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
uxtl2(vform, temp, src2);
add(vform, dst, src1, temp);
return dst;
}
LogicVRegister Simulator::saddl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp1, temp2;
sxtl(vform, temp1, src1);
sxtl(vform, temp2, src2);
add(vform, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::saddl2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp1, temp2;
sxtl2(vform, temp1, src1);
sxtl2(vform, temp2, src2);
add(vform, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::saddw(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
sxtl(vform, temp, src2);
add(vform, dst, src1, temp);
return dst;
}
LogicVRegister Simulator::saddw2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
sxtl2(vform, temp, src2);
add(vform, dst, src1, temp);
return dst;
}
LogicVRegister Simulator::usubl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp1, temp2;
uxtl(vform, temp1, src1);
uxtl(vform, temp2, src2);
sub(vform, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::usubl2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp1, temp2;
uxtl2(vform, temp1, src1);
uxtl2(vform, temp2, src2);
sub(vform, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::usubw(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
uxtl(vform, temp, src2);
sub(vform, dst, src1, temp);
return dst;
}
LogicVRegister Simulator::usubw2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
uxtl2(vform, temp, src2);
sub(vform, dst, src1, temp);
return dst;
}
LogicVRegister Simulator::ssubl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp1, temp2;
sxtl(vform, temp1, src1);
sxtl(vform, temp2, src2);
sub(vform, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::ssubl2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp1, temp2;
sxtl2(vform, temp1, src1);
sxtl2(vform, temp2, src2);
sub(vform, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::ssubw(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
sxtl(vform, temp, src2);
sub(vform, dst, src1, temp);
return dst;
}
LogicVRegister Simulator::ssubw2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
sxtl2(vform, temp, src2);
sub(vform, dst, src1, temp);
return dst;
}
LogicVRegister Simulator::uabal(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp1, temp2;
uxtl(vform, temp1, src1);
uxtl(vform, temp2, src2);
uaba(vform, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::uabal2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp1, temp2;
uxtl2(vform, temp1, src1);
uxtl2(vform, temp2, src2);
uaba(vform, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::sabal(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp1, temp2;
sxtl(vform, temp1, src1);
sxtl(vform, temp2, src2);
saba(vform, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::sabal2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp1, temp2;
sxtl2(vform, temp1, src1);
sxtl2(vform, temp2, src2);
saba(vform, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::uabdl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp1, temp2;
uxtl(vform, temp1, src1);
uxtl(vform, temp2, src2);
absdiff(vform, dst, temp1, temp2, false);
return dst;
}
LogicVRegister Simulator::uabdl2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp1, temp2;
uxtl2(vform, temp1, src1);
uxtl2(vform, temp2, src2);
absdiff(vform, dst, temp1, temp2, false);
return dst;
}
LogicVRegister Simulator::sabdl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp1, temp2;
sxtl(vform, temp1, src1);
sxtl(vform, temp2, src2);
absdiff(vform, dst, temp1, temp2, true);
return dst;
}
LogicVRegister Simulator::sabdl2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp1, temp2;
sxtl2(vform, temp1, src1);
sxtl2(vform, temp2, src2);
absdiff(vform, dst, temp1, temp2, true);
return dst;
}
LogicVRegister Simulator::umull(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool is_2) {
SimVRegister temp1, temp2;
uxtl(vform, temp1, src1, is_2);
uxtl(vform, temp2, src2, is_2);
mul(vform, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::umull2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return umull(vform, dst, src1, src2, /* is_2 = */ true);
}
LogicVRegister Simulator::smull(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool is_2) {
SimVRegister temp1, temp2;
sxtl(vform, temp1, src1, is_2);
sxtl(vform, temp2, src2, is_2);
mul(vform, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::smull2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return smull(vform, dst, src1, src2, /* is_2 = */ true);
}
LogicVRegister Simulator::umlsl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool is_2) {
SimVRegister temp1, temp2;
uxtl(vform, temp1, src1, is_2);
uxtl(vform, temp2, src2, is_2);
mls(vform, dst, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::umlsl2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return umlsl(vform, dst, src1, src2, /* is_2 = */ true);
}
LogicVRegister Simulator::smlsl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool is_2) {
SimVRegister temp1, temp2;
sxtl(vform, temp1, src1, is_2);
sxtl(vform, temp2, src2, is_2);
mls(vform, dst, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::smlsl2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return smlsl(vform, dst, src1, src2, /* is_2 = */ true);
}
LogicVRegister Simulator::umlal(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool is_2) {
SimVRegister temp1, temp2;
uxtl(vform, temp1, src1, is_2);
uxtl(vform, temp2, src2, is_2);
mla(vform, dst, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::umlal2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return umlal(vform, dst, src1, src2, /* is_2 = */ true);
}
LogicVRegister Simulator::smlal(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool is_2) {
SimVRegister temp1, temp2;
sxtl(vform, temp1, src1, is_2);
sxtl(vform, temp2, src2, is_2);
mla(vform, dst, dst, temp1, temp2);
return dst;
}
LogicVRegister Simulator::smlal2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return smlal(vform, dst, src1, src2, /* is_2 = */ true);
}
LogicVRegister Simulator::sqdmlal(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool is_2) {
SimVRegister temp;
LogicVRegister product = sqdmull(vform, temp, src1, src2, is_2);
return add(vform, dst, dst, product).SignedSaturate(vform);
}
LogicVRegister Simulator::sqdmlal2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return sqdmlal(vform, dst, src1, src2, /* is_2 = */ true);
}
LogicVRegister Simulator::sqdmlsl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool is_2) {
SimVRegister temp;
LogicVRegister product = sqdmull(vform, temp, src1, src2, is_2);
return sub(vform, dst, dst, product).SignedSaturate(vform);
}
LogicVRegister Simulator::sqdmlsl2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return sqdmlsl(vform, dst, src1, src2, /* is_2 = */ true);
}
LogicVRegister Simulator::sqdmull(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool is_2) {
SimVRegister temp;
LogicVRegister product = smull(vform, temp, src1, src2, is_2);
return add(vform, dst, product, product).SignedSaturate(vform);
}
LogicVRegister Simulator::sqdmull2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return sqdmull(vform, dst, src1, src2, /* is_2 = */ true);
}
LogicVRegister Simulator::sqrdmulh(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool round) {
int esize = LaneSizeInBitsFromFormat(vform);
SimVRegister temp_lo, temp_hi;
// Compute low and high multiplication results.
mul(vform, temp_lo, src1, src2);
smulh(vform, temp_hi, src1, src2);
// Double by shifting high half, and adding in most-significant bit of low
// half.
shl(vform, temp_hi, temp_hi, 1);
usra(vform, temp_hi, temp_lo, esize - 1);
if (round) {
// Add the second (due to doubling) most-significant bit of the low half
// into the result.
shl(vform, temp_lo, temp_lo, 1);
usra(vform, temp_hi, temp_lo, esize - 1);
}
SimPRegister not_sat;
LogicPRegister ptemp(not_sat);
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
// Saturation only occurs when src1 = src2 = minimum representable value.
// Check this as a special case.
ptemp.SetActive(vform, i, true);
if ((src1.Int(vform, i) == MinIntFromFormat(vform)) &&
(src2.Int(vform, i) == MinIntFromFormat(vform))) {
ptemp.SetActive(vform, i, false);
}
dst.SetInt(vform, i, MaxIntFromFormat(vform));
}
mov_merging(vform, dst, not_sat, temp_hi);
return dst;
}
LogicVRegister Simulator::dot(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool is_src1_signed,
bool is_src2_signed) {
VectorFormat quarter_vform =
VectorFormatHalfWidthDoubleLanes(VectorFormatHalfWidthDoubleLanes(vform));
dst.ClearForWrite(vform);
for (int e = 0; e < LaneCountFromFormat(vform); e++) {
uint64_t result = 0;
int64_t element1, element2;
for (int i = 0; i < 4; i++) {
int index = 4 * e + i;
if (is_src1_signed) {
element1 = src1.Int(quarter_vform, index);
} else {
element1 = src1.Uint(quarter_vform, index);
}
if (is_src2_signed) {
element2 = src2.Int(quarter_vform, index);
} else {
element2 = src2.Uint(quarter_vform, index);
}
result += element1 * element2;
}
dst.SetUint(vform, e, result + dst.Uint(vform, e));
}
return dst;
}
LogicVRegister Simulator::sdot(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return dot(vform, dst, src1, src2, true, true);
}
LogicVRegister Simulator::udot(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return dot(vform, dst, src1, src2, false, false);
}
LogicVRegister Simulator::usdot(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return dot(vform, dst, src1, src2, false, true);
}
LogicVRegister Simulator::cdot(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& acc,
const LogicVRegister& src1,
const LogicVRegister& src2,
int rot) {
VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270));
VectorFormat quarter_vform =
VectorFormatHalfWidthDoubleLanes(VectorFormatHalfWidthDoubleLanes(vform));
int sel_a = ((rot == 0) || (rot == 180)) ? 0 : 1;
int sel_b = 1 - sel_a;
int sub_i = ((rot == 90) || (rot == 180)) ? 1 : -1;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
int64_t result = acc.Int(vform, i);
for (int j = 0; j < 2; j++) {
int64_t r1 = src1.Int(quarter_vform, (4 * i) + (2 * j) + 0);
int64_t i1 = src1.Int(quarter_vform, (4 * i) + (2 * j) + 1);
int64_t r2 = src2.Int(quarter_vform, (4 * i) + (2 * j) + sel_a);
int64_t i2 = src2.Int(quarter_vform, (4 * i) + (2 * j) + sel_b);
result += (r1 * r2) + (sub_i * i1 * i2);
}
dst.SetInt(vform, i, result);
}
return dst;
}
LogicVRegister Simulator::sqrdcmlah(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& srca,
const LogicVRegister& src1,
const LogicVRegister& src2,
int rot) {
SimVRegister src1_a, src1_b;
SimVRegister src2_a, src2_b;
SimVRegister srca_i, srca_r;
SimVRegister zero, temp;
zero.Clear();
if ((rot == 0) || (rot == 180)) {
uzp1(vform, src1_a, src1, zero);
uzp1(vform, src2_a, src2, zero);
uzp2(vform, src2_b, src2, zero);
} else {
uzp2(vform, src1_a, src1, zero);
uzp2(vform, src2_a, src2, zero);
uzp1(vform, src2_b, src2, zero);
}
uzp1(vform, srca_r, srca, zero);
uzp2(vform, srca_i, srca, zero);
bool sub_r = (rot == 90) || (rot == 180);
bool sub_i = (rot == 180) || (rot == 270);
const bool round = true;
sqrdmlash(vform, srca_r, src1_a, src2_a, round, sub_r);
sqrdmlash(vform, srca_i, src1_a, src2_b, round, sub_i);
zip1(vform, dst, srca_r, srca_i);
return dst;
}
LogicVRegister Simulator::sqrdcmlah(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& srca,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index,
int rot) {
SimVRegister temp;
dup_elements_to_segments(VectorFormatDoubleWidth(vform), temp, src2, index);
return sqrdcmlah(vform, dst, srca, src1, temp, rot);
}
LogicVRegister Simulator::sqrdmlash_d(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool round,
bool sub_op) {
// 2 * INT_64_MIN * INT_64_MIN causes INT_128 to overflow.
// To avoid this, we use:
// (dst << (esize - 1) + src1 * src2 + 1 << (esize - 2)) >> (esize - 1)
// which is same as:
// (dst << esize + 2 * src1 * src2 + 1 << (esize - 1)) >> esize.
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
int esize = kDRegSize;
vixl_uint128_t round_const, accum;
round_const.first = 0;
if (round) {
round_const.second = UINT64_C(1) << (esize - 2);
} else {
round_const.second = 0;
}
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
// Shift the whole value left by `esize - 1` bits.
accum.first = dst.Int(vform, i) >> 1;
accum.second = dst.Int(vform, i) << (esize - 1);
vixl_uint128_t product = Mul64(src1.Int(vform, i), src2.Int(vform, i));
if (sub_op) {
product = Neg128(product);
}
accum = Add128(accum, product);
// Perform rounding.
accum = Add128(accum, round_const);
// Arithmetic shift the whole value right by `esize - 1` bits.
accum.second = (accum.first << 1) | (accum.second >> (esize - 1));
accum.first = UnsignedNegate(accum.first >> (esize - 1));
// Perform saturation.
bool is_pos = (accum.first == 0) ? true : false;
if (is_pos &&
(accum.second > static_cast<uint64_t>(MaxIntFromFormat(vform)))) {
accum.second = MaxIntFromFormat(vform);
} else if (!is_pos && (accum.second <
static_cast<uint64_t>(MinIntFromFormat(vform)))) {
accum.second = MinIntFromFormat(vform);
}
dst.SetInt(vform, i, accum.second);
}
return dst;
}
LogicVRegister Simulator::sqrdmlash(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool round,
bool sub_op) {
// 2 * INT_32_MIN * INT_32_MIN causes int64_t to overflow.
// To avoid this, we use:
// (dst << (esize - 1) + src1 * src2 + 1 << (esize - 2)) >> (esize - 1)
// which is same as:
// (dst << esize + 2 * src1 * src2 + 1 << (esize - 1)) >> esize.
if (vform == kFormatVnD) {
return sqrdmlash_d(vform, dst, src1, src2, round, sub_op);
}
int esize = LaneSizeInBitsFromFormat(vform);
int round_const = round ? (1 << (esize - 2)) : 0;
int64_t accum;
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
accum = dst.Int(vform, i) << (esize - 1);
if (sub_op) {
accum -= src1.Int(vform, i) * src2.Int(vform, i);
} else {
accum += src1.Int(vform, i) * src2.Int(vform, i);
}
accum += round_const;
accum = accum >> (esize - 1);
if (accum > MaxIntFromFormat(vform)) {
accum = MaxIntFromFormat(vform);
} else if (accum < MinIntFromFormat(vform)) {
accum = MinIntFromFormat(vform);
}
dst.SetInt(vform, i, accum);
}
return dst;
}
LogicVRegister Simulator::sqrdmlah(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool round) {
return sqrdmlash(vform, dst, src1, src2, round, false);
}
LogicVRegister Simulator::sqrdmlsh(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool round) {
return sqrdmlash(vform, dst, src1, src2, round, true);
}
LogicVRegister Simulator::sqdmulh(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
return sqrdmulh(vform, dst, src1, src2, false);
}
LogicVRegister Simulator::addhn(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
add(VectorFormatDoubleWidth(vform), temp, src1, src2);
shrn(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
return dst;
}
LogicVRegister Simulator::addhn2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
add(VectorFormatDoubleWidth(VectorFormatHalfLanes(vform)), temp, src1, src2);
shrn2(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
return dst;
}
LogicVRegister Simulator::raddhn(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
add(VectorFormatDoubleWidth(vform), temp, src1, src2);
rshrn(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
return dst;
}
LogicVRegister Simulator::raddhn2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
add(VectorFormatDoubleWidth(VectorFormatHalfLanes(vform)), temp, src1, src2);
rshrn2(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
return dst;
}
LogicVRegister Simulator::subhn(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
sub(VectorFormatDoubleWidth(vform), temp, src1, src2);
shrn(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
return dst;
}
LogicVRegister Simulator::subhn2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
sub(VectorFormatDoubleWidth(VectorFormatHalfLanes(vform)), temp, src1, src2);
shrn2(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
return dst;
}
LogicVRegister Simulator::rsubhn(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
sub(VectorFormatDoubleWidth(vform), temp, src1, src2);
rshrn(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
return dst;
}
LogicVRegister Simulator::rsubhn2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
sub(VectorFormatDoubleWidth(VectorFormatHalfLanes(vform)), temp, src1, src2);
rshrn2(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
return dst;
}
LogicVRegister Simulator::trn1(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
uint64_t result[kZRegMaxSizeInBytes] = {};
int lane_count = LaneCountFromFormat(vform);
int pairs = lane_count / 2;
for (int i = 0; i < pairs; ++i) {
result[2 * i] = src1.Uint(vform, 2 * i);
result[(2 * i) + 1] = src2.Uint(vform, 2 * i);
}
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, result[i]);
}
return dst;
}
LogicVRegister Simulator::trn2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
uint64_t result[kZRegMaxSizeInBytes] = {};
int lane_count = LaneCountFromFormat(vform);
int pairs = lane_count / 2;
for (int i = 0; i < pairs; ++i) {
result[2 * i] = src1.Uint(vform, (2 * i) + 1);
result[(2 * i) + 1] = src2.Uint(vform, (2 * i) + 1);
}
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, result[i]);
}
return dst;
}
LogicVRegister Simulator::zip1(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
uint64_t result[kZRegMaxSizeInBytes] = {};
int lane_count = LaneCountFromFormat(vform);
int pairs = lane_count / 2;
for (int i = 0; i < pairs; ++i) {
result[2 * i] = src1.Uint(vform, i);
result[(2 * i) + 1] = src2.Uint(vform, i);
}
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, result[i]);
}
return dst;
}
LogicVRegister Simulator::zip2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
uint64_t result[kZRegMaxSizeInBytes] = {};
int lane_count = LaneCountFromFormat(vform);
int pairs = lane_count / 2;
for (int i = 0; i < pairs; ++i) {
result[2 * i] = src1.Uint(vform, pairs + i);
result[(2 * i) + 1] = src2.Uint(vform, pairs + i);
}
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, result[i]);
}
return dst;
}
LogicVRegister Simulator::uzp1(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
uint64_t result[kZRegMaxSizeInBytes * 2];
int lane_count = LaneCountFromFormat(vform);
for (int i = 0; i < lane_count; ++i) {
result[i] = src1.Uint(vform, i);
result[lane_count + i] = src2.Uint(vform, i);
}
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, result[2 * i]);
}
return dst;
}
LogicVRegister Simulator::uzp2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
uint64_t result[kZRegMaxSizeInBytes * 2];
int lane_count = LaneCountFromFormat(vform);
for (int i = 0; i < lane_count; ++i) {
result[i] = src1.Uint(vform, i);
result[lane_count + i] = src2.Uint(vform, i);
}
dst.ClearForWrite(vform);
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, result[(2 * i) + 1]);
}
return dst;
}
LogicVRegister Simulator::interleave_top_bottom(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
// Interleave the top and bottom half of a vector, ie. for a vector:
//
// [ ... | F | D | B | ... | E | C | A ]
//
// where B is the first element in the top half of the vector, produce a
// result vector:
//
// [ ... | ... | F | E | D | C | B | A ]
uint64_t result[kZRegMaxSizeInBytes] = {};
int lane_count = LaneCountFromFormat(vform);
for (int i = 0; i < lane_count; i += 2) {
result[i] = src.Uint(vform, i / 2);
result[i + 1] = src.Uint(vform, (lane_count / 2) + (i / 2));
}
dst.SetUintArray(vform, result);
return dst;
}
template <typename T>
T Simulator::FPNeg(T op) {
return -op;
}
template <typename T>
T Simulator::FPAdd(T op1, T op2) {
T result = FPProcessNaNs(op1, op2);
if (IsNaN(result)) {
return result;
}
if (IsInf(op1) && IsInf(op2) && (op1 != op2)) {
// inf + -inf returns the default NaN.
FPProcessException();
return FPDefaultNaN<T>();
} else {
// Other cases should be handled by standard arithmetic.
return op1 + op2;
}
}
template <typename T>
T Simulator::FPSub(T op1, T op2) {
// NaNs should be handled elsewhere.
VIXL_ASSERT(!IsNaN(op1) && !IsNaN(op2));
if (IsInf(op1) && IsInf(op2) && (op1 == op2)) {
// inf - inf returns the default NaN.
FPProcessException();
return FPDefaultNaN<T>();
} else {
// Other cases should be handled by standard arithmetic.
return op1 - op2;
}
}
template <typename T>
T Simulator::FPMulNaNs(T op1, T op2) {
T result = FPProcessNaNs(op1, op2);
return IsNaN(result) ? result : FPMul(op1, op2);
}
template <typename T>
T Simulator::FPMul(T op1, T op2) {
// NaNs should be handled elsewhere.
VIXL_ASSERT(!IsNaN(op1) && !IsNaN(op2));
if ((IsInf(op1) && (op2 == 0.0)) || (IsInf(op2) && (op1 == 0.0))) {
// inf * 0.0 returns the default NaN.
FPProcessException();
return FPDefaultNaN<T>();
} else {
// Other cases should be handled by standard arithmetic.
return op1 * op2;
}
}
template <typename T>
T Simulator::FPMulx(T op1, T op2) {
if ((IsInf(op1) && (op2 == 0.0)) || (IsInf(op2) && (op1 == 0.0))) {
// inf * 0.0 returns +/-2.0.
T two = 2.0;
return copysign(T(1.0), op1) * copysign(T(1.0), op2) * two;
}
return FPMul(op1, op2);
}
template <typename T>
T Simulator::FPMulAdd(T a, T op1, T op2) {
T result = FPProcessNaNs3(a, op1, op2);
T sign_a = copysign(T(1.0), a);
T sign_prod = copysign(T(1.0), op1) * copysign(T(1.0), op2);
bool isinf_prod = IsInf(op1) || IsInf(op2);
bool operation_generates_nan =
(IsInf(op1) && (op2 == 0.0)) || // inf * 0.0
(IsInf(op2) && (op1 == 0.0)) || // 0.0 * inf
(IsInf(a) && isinf_prod && (sign_a != sign_prod)); // inf - inf
if (IsNaN(result)) {
// Generated NaNs override quiet NaNs propagated from a.
if (operation_generates_nan && IsQuietNaN(a)) {
FPProcessException();
return FPDefaultNaN<T>();
} else {
return result;
}
}
// If the operation would produce a NaN, return the default NaN.
if (operation_generates_nan) {
FPProcessException();
return FPDefaultNaN<T>();
}
// Work around broken fma implementations for exact zero results: The sign of
// exact 0.0 results is positive unless both a and op1 * op2 are negative.
if (((op1 == 0.0) || (op2 == 0.0)) && (a == 0.0)) {
return ((sign_a < T(0.0)) && (sign_prod < T(0.0))) ? T(-0.0) : T(0.0);
}
result = FusedMultiplyAdd(op1, op2, a);
VIXL_ASSERT(!IsNaN(result));
// Work around broken fma implementations for rounded zero results: If a is
// 0.0, the sign of the result is the sign of op1 * op2 before rounding.
if ((a == 0.0) && (result == 0.0)) {
return copysign(T(0.0), sign_prod);
}
return result;
}
template <typename T>
T Simulator::FPDiv(T op1, T op2) {
// NaNs should be handled elsewhere.
VIXL_ASSERT(!IsNaN(op1) && !IsNaN(op2));
if ((IsInf(op1) && IsInf(op2)) || ((op1 == 0.0) && (op2 == 0.0))) {
// inf / inf and 0.0 / 0.0 return the default NaN.
FPProcessException();
return FPDefaultNaN<T>();
} else {
if (op2 == 0.0) {
FPProcessException();
if (!IsNaN(op1)) {
double op1_sign = copysign(1.0, op1);
double op2_sign = copysign(1.0, op2);
return static_cast<T>(op1_sign * op2_sign * kFP64PositiveInfinity);
}
}
// Other cases should be handled by standard arithmetic.
return op1 / op2;
}
}
template <typename T>
T Simulator::FPSqrt(T op) {
if (IsNaN(op)) {
return FPProcessNaN(op);
} else if (op < T(0.0)) {
FPProcessException();
return FPDefaultNaN<T>();
} else {
return sqrt(op);
}
}
template <typename T>
T Simulator::FPMax(T a, T b) {
T result = FPProcessNaNs(a, b);
if (IsNaN(result)) return result;
if ((a == 0.0) && (b == 0.0) && (copysign(1.0, a) != copysign(1.0, b))) {
// a and b are zero, and the sign differs: return +0.0.
return 0.0;
} else {
return (a > b) ? a : b;
}
}
template <typename T>
T Simulator::FPMaxNM(T a, T b) {
if (IsQuietNaN(a) && !IsQuietNaN(b)) {
a = T(kFP64NegativeInfinity);
} else if (!IsQuietNaN(a) && IsQuietNaN(b)) {
b = T(kFP64NegativeInfinity);
}
T result = FPProcessNaNs(a, b);
return IsNaN(result) ? result : FPMax(a, b);
}
template <typename T>
T Simulator::FPMin(T a, T b) {
T result = FPProcessNaNs(a, b);
if (IsNaN(result)) return result;
if ((a == 0.0) && (b == 0.0) && (copysign(1.0, a) != copysign(1.0, b))) {
// a and b are zero, and the sign differs: return -0.0.
return -0.0;
} else {
return (a < b) ? a : b;
}
}
template <typename T>
T Simulator::FPMinNM(T a, T b) {
if (IsQuietNaN(a) && !IsQuietNaN(b)) {
a = T(kFP64PositiveInfinity);
} else if (!IsQuietNaN(a) && IsQuietNaN(b)) {
b = T(kFP64PositiveInfinity);
}
T result = FPProcessNaNs(a, b);
return IsNaN(result) ? result : FPMin(a, b);
}
template <typename T>
T Simulator::FPRecipStepFused(T op1, T op2) {
const T two = 2.0;
if ((IsInf(op1) && (op2 == 0.0)) || ((op1 == 0.0) && (IsInf(op2)))) {
return two;
} else if (IsInf(op1) || IsInf(op2)) {
// Return +inf if signs match, otherwise -inf.
return ((op1 >= 0.0) == (op2 >= 0.0)) ? T(kFP64PositiveInfinity)
: T(kFP64NegativeInfinity);
} else {
return FusedMultiplyAdd(op1, op2, two);
}
}
template <typename T>
bool IsNormal(T value) {
return std::isnormal(value);
}
template <>
bool IsNormal(SimFloat16 value) {
uint16_t rawbits = Float16ToRawbits(value);
uint16_t exp_mask = 0x7c00;
// Check that the exponent is neither all zeroes or all ones.
return ((rawbits & exp_mask) != 0) && ((~rawbits & exp_mask) != 0);
}
template <typename T>
T Simulator::FPRSqrtStepFused(T op1, T op2) {
const T one_point_five = 1.5;
const T two = 2.0;
if ((IsInf(op1) && (op2 == 0.0)) || ((op1 == 0.0) && (IsInf(op2)))) {
return one_point_five;
} else if (IsInf(op1) || IsInf(op2)) {
// Return +inf if signs match, otherwise -inf.
return ((op1 >= 0.0) == (op2 >= 0.0)) ? T(kFP64PositiveInfinity)
: T(kFP64NegativeInfinity);
} else {
// The multiply-add-halve operation must be fully fused, so avoid interim
// rounding by checking which operand can be losslessly divided by two
// before doing the multiply-add.
if (IsNormal(op1 / two)) {
return FusedMultiplyAdd(op1 / two, op2, one_point_five);
} else if (IsNormal(op2 / two)) {
return FusedMultiplyAdd(op1, op2 / two, one_point_five);
} else {
// Neither operand is normal after halving: the result is dominated by
// the addition term, so just return that.
return one_point_five;
}
}
}
int32_t Simulator::FPToFixedJS(double value) {
// The Z-flag is set when the conversion from double precision floating-point
// to 32-bit integer is exact. If the source value is +/-Infinity, -0.0, NaN,
// outside the bounds of a 32-bit integer, or isn't an exact integer then the
// Z-flag is unset.
int Z = 1;
int32_t result;
if ((value == 0.0) || (value == kFP64PositiveInfinity) ||
(value == kFP64NegativeInfinity)) {
// +/- zero and infinity all return zero, however -0 and +/- Infinity also
// unset the Z-flag.
result = 0;
if ((value != 0.0) || std::signbit(value)) {
Z = 0;
}
} else if (std::isnan(value)) {
// NaN values unset the Z-flag and set the result to 0.
FPProcessNaN(value);
result = 0;
Z = 0;
} else {
// All other values are converted to an integer representation, rounded
// toward zero.
double int_result = std::floor(value);
double error = value - int_result;
if ((error != 0.0) && (int_result < 0.0)) {
int_result++;
}
// Constrain the value into the range [INT32_MIN, INT32_MAX]. We can almost
// write a one-liner with std::round, but the behaviour on ties is incorrect
// for our purposes.
double mod_const = static_cast<double>(UINT64_C(1) << 32);
double mod_error =
(int_result / mod_const) - std::floor(int_result / mod_const);
double constrained;
if (mod_error == 0.5) {
constrained = INT32_MIN;
} else {
constrained = int_result - mod_const * round(int_result / mod_const);
}
VIXL_ASSERT(std::floor(constrained) == constrained);
VIXL_ASSERT(constrained >= INT32_MIN);
VIXL_ASSERT(constrained <= INT32_MAX);
// Take the bottom 32 bits of the result as a 32-bit integer.
result = static_cast<int32_t>(constrained);
if ((int_result < INT32_MIN) || (int_result > INT32_MAX) ||
(error != 0.0)) {
// If the integer result is out of range or the conversion isn't exact,
// take exception and unset the Z-flag.
FPProcessException();
Z = 0;
}
}
ReadNzcv().SetN(0);
ReadNzcv().SetZ(Z);
ReadNzcv().SetC(0);
ReadNzcv().SetV(0);
return result;
}
double Simulator::FPRoundIntCommon(double value, FPRounding round_mode) {
VIXL_ASSERT((value != kFP64PositiveInfinity) &&
(value != kFP64NegativeInfinity));
VIXL_ASSERT(!IsNaN(value));
double int_result = std::floor(value);
double error = value - int_result;
switch (round_mode) {
case FPTieAway: {
// Take care of correctly handling the range ]-0.5, -0.0], which must
// yield -0.0.
if ((-0.5 < value) && (value < 0.0)) {
int_result = -0.0;
} else if ((error > 0.5) || ((error == 0.5) && (int_result >= 0.0))) {
// If the error is greater than 0.5, or is equal to 0.5 and the integer
// result is positive, round up.
int_result++;
}
break;
}
case FPTieEven: {
// Take care of correctly handling the range [-0.5, -0.0], which must
// yield -0.0.
if ((-0.5 <= value) && (value < 0.0)) {
int_result = -0.0;
// If the error is greater than 0.5, or is equal to 0.5 and the integer
// result is odd, round up.
} else if ((error > 0.5) ||
((error == 0.5) && (std::fmod(int_result, 2) != 0))) {
int_result++;
}
break;
}
case FPZero: {
// If value>0 then we take floor(value)
// otherwise, ceil(value).
if (value < 0) {
int_result = ceil(value);
}
break;
}
case FPNegativeInfinity: {
// We always use floor(value).
break;
}
case FPPositiveInfinity: {
// Take care of correctly handling the range ]-1.0, -0.0], which must
// yield -0.0.
if ((-1.0 < value) && (value < 0.0)) {
int_result = -0.0;
// If the error is non-zero, round up.
} else if (error > 0.0) {
int_result++;
}
break;
}
default:
VIXL_UNIMPLEMENTED();
}
return int_result;
}
double Simulator::FPRoundInt(double value, FPRounding round_mode) {
if ((value == 0.0) || (value == kFP64PositiveInfinity) ||
(value == kFP64NegativeInfinity)) {
return value;
} else if (IsNaN(value)) {
return FPProcessNaN(value);
}
return FPRoundIntCommon(value, round_mode);
}
double Simulator::FPRoundInt(double value,
FPRounding round_mode,
FrintMode frint_mode) {
if (frint_mode == kFrintToInteger) {
return FPRoundInt(value, round_mode);
}
VIXL_ASSERT((frint_mode == kFrintToInt32) || (frint_mode == kFrintToInt64));
if (value == 0.0) {
return value;
}
if ((value == kFP64PositiveInfinity) || (value == kFP64NegativeInfinity) ||
IsNaN(value)) {
if (frint_mode == kFrintToInt32) {
return INT32_MIN;
} else {
return INT64_MIN;
}
}
double result = FPRoundIntCommon(value, round_mode);
// We want to compare `result > INT64_MAX` below, but INT64_MAX isn't exactly
// representable as a double, and is rounded to (INT64_MAX + 1) when
// converted. To avoid this, we compare `result >= int64_max_plus_one`
// instead; this is safe because `result` is known to be integral, and
// `int64_max_plus_one` is exactly representable as a double.
constexpr uint64_t int64_max_plus_one = static_cast<uint64_t>(INT64_MAX) + 1;
VIXL_STATIC_ASSERT(static_cast<uint64_t>(static_cast<double>(
int64_max_plus_one)) == int64_max_plus_one);
if (frint_mode == kFrintToInt32) {
if ((result > INT32_MAX) || (result < INT32_MIN)) {
return INT32_MIN;
}
} else if ((result >= int64_max_plus_one) || (result < INT64_MIN)) {
return INT64_MIN;
}
return result;
}
int16_t Simulator::FPToInt16(double value, FPRounding rmode) {
value = FPRoundInt(value, rmode);
if (value >= kHMaxInt) {
return kHMaxInt;
} else if (value < kHMinInt) {
return kHMinInt;
}
return IsNaN(value) ? 0 : static_cast<int16_t>(value);
}
int32_t Simulator::FPToInt32(double value, FPRounding rmode) {
value = FPRoundInt(value, rmode);
if (value >= kWMaxInt) {
return kWMaxInt;
} else if (value < kWMinInt) {
return kWMinInt;
}
return IsNaN(value) ? 0 : static_cast<int32_t>(value);
}
int64_t Simulator::FPToInt64(double value, FPRounding rmode) {
value = FPRoundInt(value, rmode);
// This is equivalent to "if (value >= kXMaxInt)" but avoids rounding issues
// as a result of kMaxInt not being representable as a double.
if (value >= 9223372036854775808.) {
return kXMaxInt;
} else if (value < kXMinInt) {
return kXMinInt;
}
return IsNaN(value) ? 0 : static_cast<int64_t>(value);
}
uint16_t Simulator::FPToUInt16(double value, FPRounding rmode) {
value = FPRoundInt(value, rmode);
if (value >= kHMaxUInt) {
return kHMaxUInt;
} else if (value < 0.0) {
return 0;
}
return IsNaN(value) ? 0 : static_cast<uint16_t>(value);
}
uint32_t Simulator::FPToUInt32(double value, FPRounding rmode) {
value = FPRoundInt(value, rmode);
if (value >= kWMaxUInt) {
return kWMaxUInt;
} else if (value < 0.0) {
return 0;
}
return IsNaN(value) ? 0 : static_cast<uint32_t>(value);
}
uint64_t Simulator::FPToUInt64(double value, FPRounding rmode) {
value = FPRoundInt(value, rmode);
// This is equivalent to "if (value >= kXMaxUInt)" but avoids rounding issues
// as a result of kMaxUInt not being representable as a double.
if (value >= 18446744073709551616.) {
return kXMaxUInt;
} else if (value < 0.0) {
return 0;
}
return IsNaN(value) ? 0 : static_cast<uint64_t>(value);
}
#define DEFINE_NEON_FP_VECTOR_OP(FN, OP, PROCNAN) \
template <typename T> \
LogicVRegister Simulator::FN(VectorFormat vform, \
LogicVRegister dst, \
const LogicVRegister& src1, \
const LogicVRegister& src2) { \
dst.ClearForWrite(vform); \
for (int i = 0; i < LaneCountFromFormat(vform); i++) { \
T op1 = src1.Float<T>(i); \
T op2 = src2.Float<T>(i); \
T result; \
if (PROCNAN) { \
result = FPProcessNaNs(op1, op2); \
if (!IsNaN(result)) { \
result = OP(op1, op2); \
} \
} else { \
result = OP(op1, op2); \
} \
dst.SetFloat(vform, i, result); \
} \
return dst; \
} \
\
LogicVRegister Simulator::FN(VectorFormat vform, \
LogicVRegister dst, \
const LogicVRegister& src1, \
const LogicVRegister& src2) { \
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) { \
FN<SimFloat16>(vform, dst, src1, src2); \
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) { \
FN<float>(vform, dst, src1, src2); \
} else { \
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize); \
FN<double>(vform, dst, src1, src2); \
} \
return dst; \
}
NEON_FP3SAME_LIST(DEFINE_NEON_FP_VECTOR_OP)
#undef DEFINE_NEON_FP_VECTOR_OP
LogicVRegister Simulator::fnmul(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
LogicVRegister product = fmul(vform, temp, src1, src2);
return fneg(vform, dst, product);
}
template <typename T>
LogicVRegister Simulator::frecps(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
T op1 = -src1.Float<T>(i);
T op2 = src2.Float<T>(i);
T result = FPProcessNaNs(op1, op2);
dst.SetFloat(vform, i, IsNaN(result) ? result : FPRecipStepFused(op1, op2));
}
return dst;
}
LogicVRegister Simulator::frecps(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
frecps<SimFloat16>(vform, dst, src1, src2);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
frecps<float>(vform, dst, src1, src2);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
frecps<double>(vform, dst, src1, src2);
}
return dst;
}
template <typename T>
LogicVRegister Simulator::frsqrts(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
T op1 = -src1.Float<T>(i);
T op2 = src2.Float<T>(i);
T result = FPProcessNaNs(op1, op2);
dst.SetFloat(vform, i, IsNaN(result) ? result : FPRSqrtStepFused(op1, op2));
}
return dst;
}
LogicVRegister Simulator::frsqrts(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
frsqrts<SimFloat16>(vform, dst, src1, src2);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
frsqrts<float>(vform, dst, src1, src2);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
frsqrts<double>(vform, dst, src1, src2);
}
return dst;
}
template <typename T>
LogicVRegister Simulator::fcmp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
Condition cond) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
bool result = false;
T op1 = src1.Float<T>(i);
T op2 = src2.Float<T>(i);
bool unordered = IsNaN(FPProcessNaNs(op1, op2));
switch (cond) {
case eq:
result = (op1 == op2);
break;
case ge:
result = (op1 >= op2);
break;
case gt:
result = (op1 > op2);
break;
case le:
result = (op1 <= op2);
break;
case lt:
result = (op1 < op2);
break;
case ne:
result = (op1 != op2);
break;
case uo:
result = unordered;
break;
default:
// Other conditions are defined in terms of those above.
VIXL_UNREACHABLE();
break;
}
if (result && unordered) {
// Only `uo` and `ne` can be true for unordered comparisons.
VIXL_ASSERT((cond == uo) || (cond == ne));
}
dst.SetUint(vform, i, result ? MaxUintFromFormat(vform) : 0);
}
return dst;
}
LogicVRegister Simulator::fcmp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
Condition cond) {
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
fcmp<SimFloat16>(vform, dst, src1, src2, cond);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
fcmp<float>(vform, dst, src1, src2, cond);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
fcmp<double>(vform, dst, src1, src2, cond);
}
return dst;
}
LogicVRegister Simulator::fcmp_zero(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
Condition cond) {
SimVRegister temp;
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
LogicVRegister zero_reg =
dup_immediate(vform, temp, Float16ToRawbits(SimFloat16(0.0)));
fcmp<SimFloat16>(vform, dst, src, zero_reg, cond);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
LogicVRegister zero_reg = dup_immediate(vform, temp, FloatToRawbits(0.0));
fcmp<float>(vform, dst, src, zero_reg, cond);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
LogicVRegister zero_reg = dup_immediate(vform, temp, DoubleToRawbits(0.0));
fcmp<double>(vform, dst, src, zero_reg, cond);
}
return dst;
}
LogicVRegister Simulator::fabscmp(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
Condition cond) {
SimVRegister temp1, temp2;
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
LogicVRegister abs_src1 = fabs_<SimFloat16>(vform, temp1, src1);
LogicVRegister abs_src2 = fabs_<SimFloat16>(vform, temp2, src2);
fcmp<SimFloat16>(vform, dst, abs_src1, abs_src2, cond);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
LogicVRegister abs_src1 = fabs_<float>(vform, temp1, src1);
LogicVRegister abs_src2 = fabs_<float>(vform, temp2, src2);
fcmp<float>(vform, dst, abs_src1, abs_src2, cond);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
LogicVRegister abs_src1 = fabs_<double>(vform, temp1, src1);
LogicVRegister abs_src2 = fabs_<double>(vform, temp2, src2);
fcmp<double>(vform, dst, abs_src1, abs_src2, cond);
}
return dst;
}
template <typename T>
LogicVRegister Simulator::fmla(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& srca,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
T op1 = src1.Float<T>(i);
T op2 = src2.Float<T>(i);
T acc = srca.Float<T>(i);
T result = FPMulAdd(acc, op1, op2);
dst.SetFloat(vform, i, result);
}
return dst;
}
LogicVRegister Simulator::fmla(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& srca,
const LogicVRegister& src1,
const LogicVRegister& src2) {
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
fmla<SimFloat16>(vform, dst, srca, src1, src2);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
fmla<float>(vform, dst, srca, src1, src2);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
fmla<double>(vform, dst, srca, src1, src2);
}
return dst;
}
template <typename T>
LogicVRegister Simulator::fmls(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& srca,
const LogicVRegister& src1,
const LogicVRegister& src2) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
T op1 = -src1.Float<T>(i);
T op2 = src2.Float<T>(i);
T acc = srca.Float<T>(i);
T result = FPMulAdd(acc, op1, op2);
dst.SetFloat(i, result);
}
return dst;
}
LogicVRegister Simulator::fmls(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& srca,
const LogicVRegister& src1,
const LogicVRegister& src2) {
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
fmls<SimFloat16>(vform, dst, srca, src1, src2);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
fmls<float>(vform, dst, srca, src1, src2);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
fmls<double>(vform, dst, srca, src1, src2);
}
return dst;
}
LogicVRegister Simulator::fmlal(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
float op1 = FPToFloat(src1.Float<SimFloat16>(i), kIgnoreDefaultNaN);
float op2 = FPToFloat(src2.Float<SimFloat16>(i), kIgnoreDefaultNaN);
float acc = dst.Float<float>(i);
float result = FPMulAdd(acc, op1, op2);
dst.SetFloat(i, result);
}
return dst;
}
LogicVRegister Simulator::fmlal2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
int src = i + LaneCountFromFormat(vform);
float op1 = FPToFloat(src1.Float<SimFloat16>(src), kIgnoreDefaultNaN);
float op2 = FPToFloat(src2.Float<SimFloat16>(src), kIgnoreDefaultNaN);
float acc = dst.Float<float>(i);
float result = FPMulAdd(acc, op1, op2);
dst.SetFloat(i, result);
}
return dst;
}
LogicVRegister Simulator::fmlsl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
float op1 = -FPToFloat(src1.Float<SimFloat16>(i), kIgnoreDefaultNaN);
float op2 = FPToFloat(src2.Float<SimFloat16>(i), kIgnoreDefaultNaN);
float acc = dst.Float<float>(i);
float result = FPMulAdd(acc, op1, op2);
dst.SetFloat(i, result);
}
return dst;
}
LogicVRegister Simulator::fmlsl2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
int src = i + LaneCountFromFormat(vform);
float op1 = -FPToFloat(src1.Float<SimFloat16>(src), kIgnoreDefaultNaN);
float op2 = FPToFloat(src2.Float<SimFloat16>(src), kIgnoreDefaultNaN);
float acc = dst.Float<float>(i);
float result = FPMulAdd(acc, op1, op2);
dst.SetFloat(i, result);
}
return dst;
}
LogicVRegister Simulator::fmlal(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
dst.ClearForWrite(vform);
float op2 = FPToFloat(src2.Float<SimFloat16>(index), kIgnoreDefaultNaN);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
float op1 = FPToFloat(src1.Float<SimFloat16>(i), kIgnoreDefaultNaN);
float acc = dst.Float<float>(i);
float result = FPMulAdd(acc, op1, op2);
dst.SetFloat(i, result);
}
return dst;
}
LogicVRegister Simulator::fmlal2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
dst.ClearForWrite(vform);
float op2 = FPToFloat(src2.Float<SimFloat16>(index), kIgnoreDefaultNaN);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
int src = i + LaneCountFromFormat(vform);
float op1 = FPToFloat(src1.Float<SimFloat16>(src), kIgnoreDefaultNaN);
float acc = dst.Float<float>(i);
float result = FPMulAdd(acc, op1, op2);
dst.SetFloat(i, result);
}
return dst;
}
LogicVRegister Simulator::fmlsl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
dst.ClearForWrite(vform);
float op2 = FPToFloat(src2.Float<SimFloat16>(index), kIgnoreDefaultNaN);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
float op1 = -FPToFloat(src1.Float<SimFloat16>(i), kIgnoreDefaultNaN);
float acc = dst.Float<float>(i);
float result = FPMulAdd(acc, op1, op2);
dst.SetFloat(i, result);
}
return dst;
}
LogicVRegister Simulator::fmlsl2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
dst.ClearForWrite(vform);
float op2 = FPToFloat(src2.Float<SimFloat16>(index), kIgnoreDefaultNaN);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
int src = i + LaneCountFromFormat(vform);
float op1 = -FPToFloat(src1.Float<SimFloat16>(src), kIgnoreDefaultNaN);
float acc = dst.Float<float>(i);
float result = FPMulAdd(acc, op1, op2);
dst.SetFloat(i, result);
}
return dst;
}
template <typename T>
LogicVRegister Simulator::fneg(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
T op = src.Float<T>(i);
op = -op;
dst.SetFloat(i, op);
}
return dst;
}
LogicVRegister Simulator::fneg(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
fneg<SimFloat16>(vform, dst, src);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
fneg<float>(vform, dst, src);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
fneg<double>(vform, dst, src);
}
return dst;
}
template <typename T>
LogicVRegister Simulator::fabs_(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
T op = src.Float<T>(i);
if (copysign(1.0, op) < 0.0) {
op = -op;
}
dst.SetFloat(i, op);
}
return dst;
}
LogicVRegister Simulator::fabs_(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
fabs_<SimFloat16>(vform, dst, src);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
fabs_<float>(vform, dst, src);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
fabs_<double>(vform, dst, src);
}
return dst;
}
LogicVRegister Simulator::fabd(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister temp;
fsub(vform, temp, src1, src2);
fabs_(vform, dst, temp);
return dst;
}
LogicVRegister Simulator::fsqrt(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
dst.ClearForWrite(vform);
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
SimFloat16 result = FPSqrt(src.Float<SimFloat16>(i));
dst.SetFloat(i, result);
}
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
float result = FPSqrt(src.Float<float>(i));
dst.SetFloat(i, result);
}
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
double result = FPSqrt(src.Float<double>(i));
dst.SetFloat(i, result);
}
}
return dst;
}
#define DEFINE_NEON_FP_PAIR_OP(FNP, FN, OP) \
LogicVRegister Simulator::FNP(VectorFormat vform, \
LogicVRegister dst, \
const LogicVRegister& src1, \
const LogicVRegister& src2) { \
SimVRegister temp1, temp2; \
uzp1(vform, temp1, src1, src2); \
uzp2(vform, temp2, src1, src2); \
FN(vform, dst, temp1, temp2); \
if (IsSVEFormat(vform)) { \
interleave_top_bottom(vform, dst, dst); \
} \
return dst; \
} \
\
LogicVRegister Simulator::FNP(VectorFormat vform, \
LogicVRegister dst, \
const LogicVRegister& src) { \
if (vform == kFormatH) { \
SimFloat16 result(OP(SimFloat16(RawbitsToFloat16( \
static_cast<uint16_t>(src.Uint(vform, 0)))), \
SimFloat16(RawbitsToFloat16( \
static_cast<uint16_t>(src.Uint(vform, 1)))))); \
dst.SetUint(vform, 0, Float16ToRawbits(result)); \
} else if (vform == kFormatS) { \
float result = OP(src.Float<float>(0), src.Float<float>(1)); \
dst.SetFloat(0, result); \
} else { \
VIXL_ASSERT(vform == kFormatD); \
double result = OP(src.Float<double>(0), src.Float<double>(1)); \
dst.SetFloat(0, result); \
} \
dst.ClearForWrite(vform); \
return dst; \
}
NEON_FPPAIRWISE_LIST(DEFINE_NEON_FP_PAIR_OP)
#undef DEFINE_NEON_FP_PAIR_OP
template <typename T>
LogicVRegister Simulator::FPPairedAcrossHelper(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
typename TFPPairOp<T>::type fn,
uint64_t inactive_value) {
int lane_count = LaneCountFromFormat(vform);
T result[kZRegMaxSizeInBytes / sizeof(T)];
// Copy the source vector into a working array. Initialise the unused elements
// at the end of the array to the same value that a false predicate would set.
for (int i = 0; i < static_cast<int>(ArrayLength(result)); i++) {
result[i] = (i < lane_count)
? src.Float<T>(i)
: RawbitsWithSizeToFP<T>(sizeof(T) * 8, inactive_value);
}
// Pairwise reduce the elements to a single value, using the pair op function
// argument.
for (int step = 1; step < lane_count; step *= 2) {
for (int i = 0; i < lane_count; i += step * 2) {
result[i] = (this->*fn)(result[i], result[i + step]);
}
}
dst.ClearForWrite(ScalarFormatFromFormat(vform));
dst.SetFloat<T>(0, result[0]);
return dst;
}
LogicVRegister Simulator::FPPairedAcrossHelper(
VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
typename TFPPairOp<SimFloat16>::type fn16,
typename TFPPairOp<float>::type fn32,
typename TFPPairOp<double>::type fn64,
uint64_t inactive_value) {
switch (LaneSizeInBitsFromFormat(vform)) {
case kHRegSize:
return FPPairedAcrossHelper<SimFloat16>(vform,
dst,
src,
fn16,
inactive_value);
case kSRegSize:
return FPPairedAcrossHelper<float>(vform, dst, src, fn32, inactive_value);
default:
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
return FPPairedAcrossHelper<double>(vform,
dst,
src,
fn64,
inactive_value);
}
}
LogicVRegister Simulator::faddv(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
return FPPairedAcrossHelper(vform,
dst,
src,
&Simulator::FPAdd<SimFloat16>,
&Simulator::FPAdd<float>,
&Simulator::FPAdd<double>,
0);
}
LogicVRegister Simulator::fmaxv(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
int lane_size = LaneSizeInBitsFromFormat(vform);
uint64_t inactive_value =
FPToRawbitsWithSize(lane_size, kFP64NegativeInfinity);
return FPPairedAcrossHelper(vform,
dst,
src,
&Simulator::FPMax<SimFloat16>,
&Simulator::FPMax<float>,
&Simulator::FPMax<double>,
inactive_value);
}
LogicVRegister Simulator::fminv(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
int lane_size = LaneSizeInBitsFromFormat(vform);
uint64_t inactive_value =
FPToRawbitsWithSize(lane_size, kFP64PositiveInfinity);
return FPPairedAcrossHelper(vform,
dst,
src,
&Simulator::FPMin<SimFloat16>,
&Simulator::FPMin<float>,
&Simulator::FPMin<double>,
inactive_value);
}
LogicVRegister Simulator::fmaxnmv(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
int lane_size = LaneSizeInBitsFromFormat(vform);
uint64_t inactive_value = FPToRawbitsWithSize(lane_size, kFP64DefaultNaN);
return FPPairedAcrossHelper(vform,
dst,
src,
&Simulator::FPMaxNM<SimFloat16>,
&Simulator::FPMaxNM<float>,
&Simulator::FPMaxNM<double>,
inactive_value);
}
LogicVRegister Simulator::fminnmv(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
int lane_size = LaneSizeInBitsFromFormat(vform);
uint64_t inactive_value = FPToRawbitsWithSize(lane_size, kFP64DefaultNaN);
return FPPairedAcrossHelper(vform,
dst,
src,
&Simulator::FPMinNM<SimFloat16>,
&Simulator::FPMinNM<float>,
&Simulator::FPMinNM<double>,
inactive_value);
}
LogicVRegister Simulator::fmul(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
dst.ClearForWrite(vform);
SimVRegister temp;
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
LogicVRegister index_reg = dup_element(kFormat8H, temp, src2, index);
fmul<SimFloat16>(vform, dst, src1, index_reg);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
LogicVRegister index_reg = dup_element(kFormat4S, temp, src2, index);
fmul<float>(vform, dst, src1, index_reg);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
LogicVRegister index_reg = dup_element(kFormat2D, temp, src2, index);
fmul<double>(vform, dst, src1, index_reg);
}
return dst;
}
LogicVRegister Simulator::fmla(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
dst.ClearForWrite(vform);
SimVRegister temp;
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
LogicVRegister index_reg = dup_element(kFormat8H, temp, src2, index);
fmla<SimFloat16>(vform, dst, dst, src1, index_reg);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
LogicVRegister index_reg = dup_element(kFormat4S, temp, src2, index);
fmla<float>(vform, dst, dst, src1, index_reg);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
LogicVRegister index_reg = dup_element(kFormat2D, temp, src2, index);
fmla<double>(vform, dst, dst, src1, index_reg);
}
return dst;
}
LogicVRegister Simulator::fmls(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
dst.ClearForWrite(vform);
SimVRegister temp;
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
LogicVRegister index_reg = dup_element(kFormat8H, temp, src2, index);
fmls<SimFloat16>(vform, dst, dst, src1, index_reg);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
LogicVRegister index_reg = dup_element(kFormat4S, temp, src2, index);
fmls<float>(vform, dst, dst, src1, index_reg);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
LogicVRegister index_reg = dup_element(kFormat2D, temp, src2, index);
fmls<double>(vform, dst, dst, src1, index_reg);
}
return dst;
}
LogicVRegister Simulator::fmulx(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index) {
dst.ClearForWrite(vform);
SimVRegister temp;
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
LogicVRegister index_reg = dup_element(kFormat8H, temp, src2, index);
fmulx<SimFloat16>(vform, dst, src1, index_reg);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
LogicVRegister index_reg = dup_element(kFormat4S, temp, src2, index);
fmulx<float>(vform, dst, src1, index_reg);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
LogicVRegister index_reg = dup_element(kFormat2D, temp, src2, index);
fmulx<double>(vform, dst, src1, index_reg);
}
return dst;
}
LogicVRegister Simulator::frint(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
FPRounding rounding_mode,
bool inexact_exception,
FrintMode frint_mode) {
dst.ClearForWrite(vform);
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
VIXL_ASSERT(frint_mode == kFrintToInteger);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
SimFloat16 input = src.Float<SimFloat16>(i);
SimFloat16 rounded = FPRoundInt(input, rounding_mode);
if (inexact_exception && !IsNaN(input) && (input != rounded)) {
FPProcessException();
}
dst.SetFloat<SimFloat16>(i, rounded);
}
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
float input = src.Float<float>(i);
float rounded =
static_cast<float>(FPRoundInt(input, rounding_mode, frint_mode));
if (inexact_exception && !IsNaN(input) && (input != rounded)) {
FPProcessException();
}
dst.SetFloat<float>(i, rounded);
}
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
double input = src.Float<double>(i);
double rounded = FPRoundInt(input, rounding_mode, frint_mode);
if (inexact_exception && !IsNaN(input) && (input != rounded)) {
FPProcessException();
}
dst.SetFloat<double>(i, rounded);
}
}
return dst;
}
LogicVRegister Simulator::fcvt(VectorFormat dst_vform,
VectorFormat src_vform,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src) {
unsigned dst_data_size_in_bits = LaneSizeInBitsFromFormat(dst_vform);
unsigned src_data_size_in_bits = LaneSizeInBitsFromFormat(src_vform);
VectorFormat vform = SVEFormatFromLaneSizeInBits(
std::max(dst_data_size_in_bits, src_data_size_in_bits));
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!pg.IsActive(vform, i)) continue;
uint64_t src_raw_bits = ExtractUnsignedBitfield64(src_data_size_in_bits - 1,
0,
src.Uint(vform, i));
double dst_value =
RawbitsWithSizeToFP<double>(src_data_size_in_bits, src_raw_bits);
uint64_t dst_raw_bits =
FPToRawbitsWithSize(dst_data_size_in_bits, dst_value);
dst.SetUint(vform, i, dst_raw_bits);
}
return dst;
}
LogicVRegister Simulator::fcvts(VectorFormat vform,
unsigned dst_data_size_in_bits,
unsigned src_data_size_in_bits,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src,
FPRounding round,
int fbits) {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) >= dst_data_size_in_bits);
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) >= src_data_size_in_bits);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!pg.IsActive(vform, i)) continue;
uint64_t value = ExtractUnsignedBitfield64(src_data_size_in_bits - 1,
0,
src.Uint(vform, i));
double result = RawbitsWithSizeToFP<double>(src_data_size_in_bits, value) *
std::pow(2.0, fbits);
switch (dst_data_size_in_bits) {
case kHRegSize:
dst.SetInt(vform, i, FPToInt16(result, round));
break;
case kSRegSize:
dst.SetInt(vform, i, FPToInt32(result, round));
break;
case kDRegSize:
dst.SetInt(vform, i, FPToInt64(result, round));
break;
default:
VIXL_UNIMPLEMENTED();
break;
}
}
return dst;
}
LogicVRegister Simulator::fcvts(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
FPRounding round,
int fbits) {
dst.ClearForWrite(vform);
return fcvts(vform,
LaneSizeInBitsFromFormat(vform),
LaneSizeInBitsFromFormat(vform),
dst,
GetPTrue(),
src,
round,
fbits);
}
LogicVRegister Simulator::fcvtu(VectorFormat vform,
unsigned dst_data_size_in_bits,
unsigned src_data_size_in_bits,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src,
FPRounding round,
int fbits) {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) >= dst_data_size_in_bits);
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) >= src_data_size_in_bits);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!pg.IsActive(vform, i)) continue;
uint64_t value = ExtractUnsignedBitfield64(src_data_size_in_bits - 1,
0,
src.Uint(vform, i));
double result = RawbitsWithSizeToFP<double>(src_data_size_in_bits, value) *
std::pow(2.0, fbits);
switch (dst_data_size_in_bits) {
case kHRegSize:
dst.SetUint(vform, i, FPToUInt16(result, round));
break;
case kSRegSize:
dst.SetUint(vform, i, FPToUInt32(result, round));
break;
case kDRegSize:
dst.SetUint(vform, i, FPToUInt64(result, round));
break;
default:
VIXL_UNIMPLEMENTED();
break;
}
}
return dst;
}
LogicVRegister Simulator::fcvtu(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
FPRounding round,
int fbits) {
dst.ClearForWrite(vform);
return fcvtu(vform,
LaneSizeInBitsFromFormat(vform),
LaneSizeInBitsFromFormat(vform),
dst,
GetPTrue(),
src,
round,
fbits);
}
LogicVRegister Simulator::fcvtl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
dst.ClearForWrite(vform);
if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
for (int i = LaneCountFromFormat(vform) - 1; i >= 0; i--) {
// TODO: Full support for SimFloat16 in SimRegister(s).
dst.SetFloat(i,
FPToFloat(RawbitsToFloat16(src.Float<uint16_t>(i)),
ReadDN()));
}
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
for (int i = LaneCountFromFormat(vform) - 1; i >= 0; i--) {
dst.SetFloat(i, FPToDouble(src.Float<float>(i), ReadDN()));
}
}
return dst;
}
LogicVRegister Simulator::fcvtl2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
dst.ClearForWrite(vform);
int lane_count = LaneCountFromFormat(vform);
if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
for (int i = 0; i < lane_count; i++) {
// TODO: Full support for SimFloat16 in SimRegister(s).
dst.SetFloat(i,
FPToFloat(RawbitsToFloat16(
src.Float<uint16_t>(i + lane_count)),
ReadDN()));
}
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
for (int i = 0; i < lane_count; i++) {
dst.SetFloat(i, FPToDouble(src.Float<float>(i + lane_count), ReadDN()));
}
}
return dst;
}
LogicVRegister Simulator::fcvtn(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
SimVRegister tmp;
LogicVRegister srctmp = mov(kFormat2D, tmp, src);
dst.ClearForWrite(vform);
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
dst.SetFloat(i,
Float16ToRawbits(FPToFloat16(srctmp.Float<float>(i),
FPTieEven,
ReadDN())));
}
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
dst.SetFloat(i, FPToFloat(srctmp.Float<double>(i), FPTieEven, ReadDN()));
}
}
return dst;
}
LogicVRegister Simulator::fcvtn2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
dst.ClearForWrite(vform);
int lane_count = LaneCountFromFormat(vform) / 2;
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
for (int i = lane_count - 1; i >= 0; i--) {
dst.SetFloat(i + lane_count,
Float16ToRawbits(
FPToFloat16(src.Float<float>(i), FPTieEven, ReadDN())));
}
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
for (int i = lane_count - 1; i >= 0; i--) {
dst.SetFloat(i + lane_count,
FPToFloat(src.Float<double>(i), FPTieEven, ReadDN()));
}
}
return dst;
}
LogicVRegister Simulator::fcvtxn(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
SimVRegister tmp;
LogicVRegister srctmp = mov(kFormat2D, tmp, src);
int input_lane_count = LaneCountFromFormat(vform);
if (IsSVEFormat(vform)) {
mov(kFormatVnB, tmp, src);
input_lane_count /= 2;
}
dst.ClearForWrite(vform);
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
for (int i = 0; i < input_lane_count; i++) {
dst.SetFloat(i, FPToFloat(srctmp.Float<double>(i), FPRoundOdd, ReadDN()));
}
return dst;
}
LogicVRegister Simulator::fcvtxn2(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
dst.ClearForWrite(vform);
int lane_count = LaneCountFromFormat(vform) / 2;
for (int i = lane_count - 1; i >= 0; i--) {
dst.SetFloat(i + lane_count,
FPToFloat(src.Float<double>(i), FPRoundOdd, ReadDN()));
}
return dst;
}
// Based on reference C function recip_sqrt_estimate from ARM ARM.
double Simulator::recip_sqrt_estimate(double a) {
int quot0, quot1, s;
double r;
if (a < 0.5) {
quot0 = static_cast<int>(a * 512.0);
r = 1.0 / sqrt((static_cast<double>(quot0) + 0.5) / 512.0);
} else {
quot1 = static_cast<int>(a * 256.0);
r = 1.0 / sqrt((static_cast<double>(quot1) + 0.5) / 256.0);
}
s = static_cast<int>(256.0 * r + 0.5);
return static_cast<double>(s) / 256.0;
}
static inline uint64_t Bits(uint64_t val, int start_bit, int end_bit) {
return ExtractUnsignedBitfield64(start_bit, end_bit, val);
}
template <typename T>
T Simulator::FPRecipSqrtEstimate(T op) {
if (IsNaN(op)) {
return FPProcessNaN(op);
} else if (op == 0.0) {
if (copysign(1.0, op) < 0.0) {
return T(kFP64NegativeInfinity);
} else {
return T(kFP64PositiveInfinity);
}
} else if (copysign(1.0, op) < 0.0) {
FPProcessException();
return FPDefaultNaN<T>();
} else if (IsInf(op)) {
return 0.0;
} else {
uint64_t fraction;
int exp, result_exp;
if constexpr (IsFloat16<T>()) {
exp = Float16Exp(op);
fraction = Float16Mantissa(op);
fraction <<= 42;
} else if constexpr (IsFloat32<T>()) {
exp = FloatExp(op);
fraction = FloatMantissa(op);
fraction <<= 29;
} else {
VIXL_ASSERT(IsFloat64<T>());
exp = DoubleExp(op);
fraction = DoubleMantissa(op);
}
if (exp == 0) {
while (Bits(fraction, 51, 51) == 0) {
fraction = Bits(fraction, 50, 0) << 1;
exp -= 1;
}
fraction = Bits(fraction, 50, 0) << 1;
}
double scaled;
if (Bits(exp, 0, 0) == 0) {
scaled = DoublePack(0, 1022, Bits(fraction, 51, 44) << 44);
} else {
scaled = DoublePack(0, 1021, Bits(fraction, 51, 44) << 44);
}
if constexpr (IsFloat16<T>()) {
result_exp = (44 - exp) / 2;
} else if constexpr (IsFloat32<T>()) {
result_exp = (380 - exp) / 2;
} else {
VIXL_ASSERT(IsFloat64<T>());
result_exp = (3068 - exp) / 2;
}
uint64_t estimate = DoubleToRawbits(recip_sqrt_estimate(scaled));
if constexpr (IsFloat16<T>()) {
uint16_t exp_bits = static_cast<uint16_t>(Bits(result_exp, 4, 0));
uint16_t est_bits = static_cast<uint16_t>(Bits(estimate, 51, 42));
return Float16Pack(0, exp_bits, est_bits);
} else if constexpr (IsFloat32<T>()) {
uint32_t exp_bits = static_cast<uint32_t>(Bits(result_exp, 7, 0));
uint32_t est_bits = static_cast<uint32_t>(Bits(estimate, 51, 29));
return FloatPack(0, exp_bits, est_bits);
} else {
VIXL_ASSERT(IsFloat64<T>());
return DoublePack(0, Bits(result_exp, 10, 0), Bits(estimate, 51, 0));
}
}
}
LogicVRegister Simulator::frsqrte(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
dst.ClearForWrite(vform);
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
SimFloat16 input = src.Float<SimFloat16>(i);
dst.SetFloat(vform, i, FPRecipSqrtEstimate<SimFloat16>(input));
}
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
float input = src.Float<float>(i);
dst.SetFloat(vform, i, FPRecipSqrtEstimate<float>(input));
}
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
double input = src.Float<double>(i);
dst.SetFloat(vform, i, FPRecipSqrtEstimate<double>(input));
}
}
return dst;
}
template <typename T>
T Simulator::FPRecipEstimate(T op, FPRounding rounding) {
uint32_t sign;
if constexpr (IsFloat16<T>()) {
sign = Float16Sign(op);
} else if constexpr (IsFloat32<T>()) {
sign = FloatSign(op);
} else {
VIXL_ASSERT(IsFloat64<T>());
sign = DoubleSign(op);
}
if (IsNaN(op)) {
return FPProcessNaN(op);
} else if (IsInf(op)) {
return (sign == 1) ? T(-0.0) : T(0.0);
} else if (op == 0.0) {
FPProcessException(); // FPExc_DivideByZero exception.
return (sign == 1) ? T(kFP64NegativeInfinity) : T(kFP64PositiveInfinity);
} else if ((IsFloat16<T>() && (std::fabs(op) < std::pow(2.0, -16.0))) ||
(IsFloat32<T>() && (std::fabs(op) < std::pow(2.0, -128.0))) ||
(IsFloat64<T>() && (std::fabs(op) < std::pow(2.0, -1024.0)))) {
bool overflow_to_inf = false;
switch (rounding) {
case FPTieEven:
overflow_to_inf = true;
break;
case FPPositiveInfinity:
overflow_to_inf = (sign == 0);
break;
case FPNegativeInfinity:
overflow_to_inf = (sign == 1);
break;
case FPZero:
overflow_to_inf = false;
break;
default:
break;
}
FPProcessException(); // FPExc_Overflow and FPExc_Inexact.
if (overflow_to_inf) {
return (sign == 1) ? T(kFP64NegativeInfinity) : T(kFP64PositiveInfinity);
} else {
// Return FPMaxNormal(sign).
if constexpr (IsFloat16<T>()) {
return Float16Pack(sign, 0x1f, 0x3ff);
} else if constexpr (IsFloat32<T>()) {
return FloatPack(sign, 0xfe, 0x07fffff);
} else {
VIXL_ASSERT(IsFloat64<T>());
return DoublePack(sign, 0x7fe, 0x0fffffffffffffl);
}
}
} else {
uint64_t fraction;
int exp, result_exp;
if constexpr (IsFloat16<T>()) {
sign = Float16Sign(op);
exp = Float16Exp(op);
fraction = Float16Mantissa(op);
fraction <<= 42;
} else if constexpr (IsFloat32<T>()) {
sign = FloatSign(op);
exp = FloatExp(op);
fraction = FloatMantissa(op);
fraction <<= 29;
} else {
VIXL_ASSERT(IsFloat64<T>());
sign = DoubleSign(op);
exp = DoubleExp(op);
fraction = DoubleMantissa(op);
}
if (exp == 0) {
if (Bits(fraction, 51, 51) == 0) {
exp -= 1;
fraction = Bits(fraction, 49, 0) << 2;
} else {
fraction = Bits(fraction, 50, 0) << 1;
}
}
double scaled = DoublePack(0, 1022, Bits(fraction, 51, 44) << 44);
if constexpr (IsFloat16<T>()) {
result_exp = (29 - exp); // In range 29-30 = -1 to 29+1 = 30.
} else if constexpr (IsFloat32<T>()) {
result_exp = (253 - exp); // In range 253-254 = -1 to 253+1 = 254.
} else {
VIXL_ASSERT(IsFloat64<T>());
result_exp = (2045 - exp); // In range 2045-2046 = -1 to 2045+1 = 2046.
}
double estimate = recip_estimate(scaled);
fraction = DoubleMantissa(estimate);
if (result_exp == 0) {
fraction = (UINT64_C(1) << 51) | Bits(fraction, 51, 1);
} else if (result_exp == -1) {
fraction = (UINT64_C(1) << 50) | Bits(fraction, 51, 2);
result_exp = 0;
}
if constexpr (IsFloat16<T>()) {
uint16_t exp_bits = static_cast<uint16_t>(Bits(result_exp, 4, 0));
uint16_t frac_bits = static_cast<uint16_t>(Bits(fraction, 51, 42));
return Float16Pack(sign, exp_bits, frac_bits);
} else if constexpr (IsFloat32<T>()) {
uint32_t exp_bits = static_cast<uint32_t>(Bits(result_exp, 7, 0));
uint32_t frac_bits = static_cast<uint32_t>(Bits(fraction, 51, 29));
return FloatPack(sign, exp_bits, frac_bits);
} else {
VIXL_ASSERT(IsFloat64<T>());
return DoublePack(sign, Bits(result_exp, 10, 0), Bits(fraction, 51, 0));
}
}
}
LogicVRegister Simulator::frecpe(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
FPRounding round) {
dst.ClearForWrite(vform);
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
SimFloat16 input = src.Float<SimFloat16>(i);
dst.SetFloat(vform, i, FPRecipEstimate<SimFloat16>(input, round));
}
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
float input = src.Float<float>(i);
dst.SetFloat(vform, i, FPRecipEstimate<float>(input, round));
}
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
double input = src.Float<double>(i);
dst.SetFloat(vform, i, FPRecipEstimate<double>(input, round));
}
}
return dst;
}
LogicVRegister Simulator::ursqrte(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
dst.ClearForWrite(vform);
uint64_t operand;
uint32_t result;
double dp_operand, dp_result;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
operand = src.Uint(vform, i);
if (operand <= 0x3FFFFFFF) {
result = 0xFFFFFFFF;
} else {
dp_operand = operand * std::pow(2.0, -32);
dp_result = recip_sqrt_estimate(dp_operand) * std::pow(2.0, 31);
result = static_cast<uint32_t>(dp_result);
}
dst.SetUint(vform, i, result);
}
return dst;
}
// Based on reference C function recip_estimate from ARM ARM.
double Simulator::recip_estimate(double a) {
int q, s;
double r;
q = static_cast<int>(a * 512.0);
r = 1.0 / ((static_cast<double>(q) + 0.5) / 512.0);
s = static_cast<int>(256.0 * r + 0.5);
return static_cast<double>(s) / 256.0;
}
LogicVRegister Simulator::urecpe(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
dst.ClearForWrite(vform);
uint64_t operand;
uint32_t result;
double dp_operand, dp_result;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
operand = src.Uint(vform, i);
if (operand <= 0x7FFFFFFF) {
result = 0xFFFFFFFF;
} else {
dp_operand = operand * std::pow(2.0, -32);
dp_result = recip_estimate(dp_operand) * std::pow(2.0, 31);
result = static_cast<uint32_t>(dp_result);
}
dst.SetUint(vform, i, result);
}
return dst;
}
LogicPRegister Simulator::pfalse(LogicPRegister dst) {
dst.Clear();
return dst;
}
LogicPRegister Simulator::pfirst(LogicPRegister dst,
const LogicPRegister& pg,
const LogicPRegister& src) {
int first_pg = GetFirstActive(kFormatVnB, pg);
VIXL_ASSERT(first_pg < LaneCountFromFormat(kFormatVnB));
mov(dst, src);
if (first_pg >= 0) dst.SetActive(kFormatVnB, first_pg, true);
return dst;
}
LogicPRegister Simulator::ptrue(VectorFormat vform,
LogicPRegister dst,
int pattern) {
int count = GetPredicateConstraintLaneCount(vform, pattern);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
dst.SetActive(vform, i, i < count);
}
return dst;
}
LogicPRegister Simulator::pnext(VectorFormat vform,
LogicPRegister dst,
const LogicPRegister& pg,
const LogicPRegister& src) {
int next = GetLastActive(vform, src) + 1;
while (next < LaneCountFromFormat(vform)) {
if (pg.IsActive(vform, next)) break;
next++;
}
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
dst.SetActive(vform, i, (i == next));
}
return dst;
}
template <typename T>
LogicVRegister Simulator::frecpx(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
T op = src.Float<T>(i);
T result;
if (IsNaN(op)) {
result = FPProcessNaN(op);
} else {
int exp;
uint32_t sign;
if constexpr (IsFloat16<T>()) {
sign = Float16Sign(op);
exp = Float16Exp(op);
exp = (exp == 0) ? (0x1F - 1) : static_cast<int>(Bits(~exp, 4, 0));
result = Float16Pack(sign, exp, 0);
} else if constexpr (IsFloat32<T>()) {
sign = FloatSign(op);
exp = FloatExp(op);
exp = (exp == 0) ? (0xFF - 1) : static_cast<int>(Bits(~exp, 7, 0));
result = FloatPack(sign, exp, 0);
} else {
VIXL_ASSERT(IsFloat64<T>());
sign = DoubleSign(op);
exp = DoubleExp(op);
exp = (exp == 0) ? (0x7FF - 1) : static_cast<int>(Bits(~exp, 10, 0));
result = DoublePack(sign, exp, 0);
}
}
dst.SetFloat(i, result);
}
return dst;
}
LogicVRegister Simulator::frecpx(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
frecpx<SimFloat16>(vform, dst, src);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
frecpx<float>(vform, dst, src);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
frecpx<double>(vform, dst, src);
}
return dst;
}
LogicVRegister Simulator::flogb(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
double op = 0.0;
switch (vform) {
case kFormatVnH:
op = FPToDouble(src.Float<SimFloat16>(i), kIgnoreDefaultNaN);
break;
case kFormatVnS:
op = src.Float<float>(i);
break;
case kFormatVnD:
op = src.Float<double>(i);
break;
default:
VIXL_UNREACHABLE();
}
switch (std::fpclassify(op)) {
case FP_INFINITE:
dst.SetInt(vform, i, MaxIntFromFormat(vform));
break;
case FP_NAN:
case FP_ZERO:
dst.SetInt(vform, i, MinIntFromFormat(vform));
break;
case FP_SUBNORMAL: {
// DoubleMantissa returns the mantissa of its input, leaving 12 zero
// bits where the sign and exponent would be. We subtract 12 to
// find the number of leading zero bits in the mantissa itself.
int64_t mant_zero_count = CountLeadingZeros(DoubleMantissa(op)) - 12;
// Log2 of a subnormal is the lowest exponent a normal number can
// represent, together with the zeros in the mantissa.
dst.SetInt(vform, i, -1023 - mant_zero_count);
break;
}
case FP_NORMAL:
// Log2 of a normal number is the exponent minus the bias.
dst.SetInt(vform, i, static_cast<int64_t>(DoubleExp(op)) - 1023);
break;
}
}
return dst;
}
LogicVRegister Simulator::ftsmul(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
SimVRegister maybe_neg_src1;
// The bottom bit of src2 controls the sign of the result. Use it to
// conditionally invert the sign of one `fmul` operand.
shl(vform, maybe_neg_src1, src2, LaneSizeInBitsFromFormat(vform) - 1);
eor(vform, maybe_neg_src1, maybe_neg_src1, src1);
// Multiply src1 by the modified neg_src1, which is potentially its negation.
// In the case of NaNs, NaN * -NaN will return the first NaN intact, so src1,
// rather than neg_src1, must be the first source argument.
fmul(vform, dst, src1, maybe_neg_src1);
return dst;
}
LogicVRegister Simulator::ftssel(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
unsigned lane_bits = LaneSizeInBitsFromFormat(vform);
uint64_t sign_bit = UINT64_C(1) << (lane_bits - 1);
uint64_t one;
if (lane_bits == kHRegSize) {
one = Float16ToRawbits(Float16(1.0));
} else if (lane_bits == kSRegSize) {
one = FloatToRawbits(1.0);
} else {
VIXL_ASSERT(lane_bits == kDRegSize);
one = DoubleToRawbits(1.0);
}
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
// Use integer accessors for this operation, as this is a data manipulation
// task requiring no calculation.
uint64_t op = src1.Uint(vform, i);
// Only the bottom two bits of the src2 register are significant, indicating
// the quadrant. Bit 0 controls whether src1 or 1.0 is written to dst. Bit 1
// determines the sign of the value written to dst.
uint64_t q = src2.Uint(vform, i);
if ((q & 1) == 1) op = one;
if ((q & 2) == 2) op ^= sign_bit;
dst.SetUint(vform, i, op);
}
return dst;
}
template <typename T>
LogicVRegister Simulator::FTMaddHelper(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
uint64_t coeff_pos,
uint64_t coeff_neg) {
SimVRegister zero;
dup_immediate(kFormatVnB, zero, 0);
SimVRegister cf;
SimVRegister cfn;
dup_immediate(vform, cf, coeff_pos);
dup_immediate(vform, cfn, coeff_neg);
// The specification requires testing the top bit of the raw value, rather
// than the sign of the floating point number, so use an integer comparison
// here.
SimPRegister is_neg;
SVEIntCompareVectorsHelper(lt,
vform,
is_neg,
GetPTrue(),
src2,
zero,
false,
LeaveFlags);
mov_merging(vform, cf, is_neg, cfn);
SimVRegister temp;
fabs_<T>(vform, temp, src2);
fmla<T>(vform, cf, cf, src1, temp);
mov(vform, dst, cf);
return dst;
}
LogicVRegister Simulator::ftmad(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
unsigned index) {
static const uint64_t ftmad_coeff16[] = {0x3c00,
0xb155,
0x2030,
0x0000,
0x0000,
0x0000,
0x0000,
0x0000,
0x3c00,
0xb800,
0x293a,
0x0000,
0x0000,
0x0000,
0x0000,
0x0000};
static const uint64_t ftmad_coeff32[] = {0x3f800000,
0xbe2aaaab,
0x3c088886,
0xb95008b9,
0x36369d6d,
0x00000000,
0x00000000,
0x00000000,
0x3f800000,
0xbf000000,
0x3d2aaaa6,
0xbab60705,
0x37cd37cc,
0x00000000,
0x00000000,
0x00000000};
static const uint64_t ftmad_coeff64[] = {0x3ff0000000000000,
0xbfc5555555555543,
0x3f8111111110f30c,
0xbf2a01a019b92fc6,
0x3ec71de351f3d22b,
0xbe5ae5e2b60f7b91,
0x3de5d8408868552f,
0x0000000000000000,
0x3ff0000000000000,
0xbfe0000000000000,
0x3fa5555555555536,
0xbf56c16c16c13a0b,
0x3efa01a019b1e8d8,
0xbe927e4f7282f468,
0x3e21ee96d2641b13,
0xbda8f76380fbb401};
VIXL_ASSERT((index + 8) < ArrayLength(ftmad_coeff64));
VIXL_ASSERT(ArrayLength(ftmad_coeff16) == ArrayLength(ftmad_coeff64));
VIXL_ASSERT(ArrayLength(ftmad_coeff32) == ArrayLength(ftmad_coeff64));
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
FTMaddHelper<SimFloat16>(vform,
dst,
src1,
src2,
ftmad_coeff16[index],
ftmad_coeff16[index + 8]);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
FTMaddHelper<float>(vform,
dst,
src1,
src2,
ftmad_coeff32[index],
ftmad_coeff32[index + 8]);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
FTMaddHelper<double>(vform,
dst,
src1,
src2,
ftmad_coeff64[index],
ftmad_coeff64[index + 8]);
}
return dst;
}
LogicVRegister Simulator::fexpa(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
static const uint64_t fexpa_coeff16[] = {0x0000, 0x0016, 0x002d, 0x0045,
0x005d, 0x0075, 0x008e, 0x00a8,
0x00c2, 0x00dc, 0x00f8, 0x0114,
0x0130, 0x014d, 0x016b, 0x0189,
0x01a8, 0x01c8, 0x01e8, 0x0209,
0x022b, 0x024e, 0x0271, 0x0295,
0x02ba, 0x02e0, 0x0306, 0x032e,
0x0356, 0x037f, 0x03a9, 0x03d4};
static const uint64_t fexpa_coeff32[] =
{0x000000, 0x0164d2, 0x02cd87, 0x043a29, 0x05aac3, 0x071f62, 0x08980f,
0x0a14d5, 0x0b95c2, 0x0d1adf, 0x0ea43a, 0x1031dc, 0x11c3d3, 0x135a2b,
0x14f4f0, 0x16942d, 0x1837f0, 0x19e046, 0x1b8d3a, 0x1d3eda, 0x1ef532,
0x20b051, 0x227043, 0x243516, 0x25fed7, 0x27cd94, 0x29a15b, 0x2b7a3a,
0x2d583f, 0x2f3b79, 0x3123f6, 0x3311c4, 0x3504f3, 0x36fd92, 0x38fbaf,
0x3aff5b, 0x3d08a4, 0x3f179a, 0x412c4d, 0x4346cd, 0x45672a, 0x478d75,
0x49b9be, 0x4bec15, 0x4e248c, 0x506334, 0x52a81e, 0x54f35b, 0x5744fd,
0x599d16, 0x5bfbb8, 0x5e60f5, 0x60ccdf, 0x633f89, 0x65b907, 0x68396a,
0x6ac0c7, 0x6d4f30, 0x6fe4ba, 0x728177, 0x75257d, 0x77d0df, 0x7a83b3,
0x7d3e0c};
static const uint64_t fexpa_coeff64[] =
{0X0000000000000, 0X02c9a3e778061, 0X059b0d3158574, 0X0874518759bc8,
0X0b5586cf9890f, 0X0e3ec32d3d1a2, 0X11301d0125b51, 0X1429aaea92de0,
0X172b83c7d517b, 0X1a35beb6fcb75, 0X1d4873168b9aa, 0X2063b88628cd6,
0X2387a6e756238, 0X26b4565e27cdd, 0X29e9df51fdee1, 0X2d285a6e4030b,
0X306fe0a31b715, 0X33c08b26416ff, 0X371a7373aa9cb, 0X3a7db34e59ff7,
0X3dea64c123422, 0X4160a21f72e2a, 0X44e086061892d, 0X486a2b5c13cd0,
0X4bfdad5362a27, 0X4f9b2769d2ca7, 0X5342b569d4f82, 0X56f4736b527da,
0X5ab07dd485429, 0X5e76f15ad2148, 0X6247eb03a5585, 0X6623882552225,
0X6a09e667f3bcd, 0X6dfb23c651a2f, 0X71f75e8ec5f74, 0X75feb564267c9,
0X7a11473eb0187, 0X7e2f336cf4e62, 0X82589994cce13, 0X868d99b4492ed,
0X8ace5422aa0db, 0X8f1ae99157736, 0X93737b0cdc5e5, 0X97d829fde4e50,
0X9c49182a3f090, 0Xa0c667b5de565, 0Xa5503b23e255d, 0Xa9e6b5579fdbf,
0Xae89f995ad3ad, 0Xb33a2b84f15fb, 0Xb7f76f2fb5e47, 0Xbcc1e904bc1d2,
0Xc199bdd85529c, 0Xc67f12e57d14b, 0Xcb720dcef9069, 0Xd072d4a07897c,
0Xd5818dcfba487, 0Xda9e603db3285, 0Xdfc97337b9b5f, 0Xe502ee78b3ff6,
0Xea4afa2a490da, 0Xefa1bee615a27, 0Xf50765b6e4540, 0Xfa7c1819e90d8};
unsigned lane_size = LaneSizeInBitsFromFormat(vform);
int index_highbit = 5;
int op_highbit, op_shift;
const uint64_t* fexpa_coeff;
if (lane_size == kHRegSize) {
index_highbit = 4;
VIXL_ASSERT(ArrayLength(fexpa_coeff16) ==
(uint64_t{1} << (index_highbit + 1)));
fexpa_coeff = fexpa_coeff16;
op_highbit = 9;
op_shift = 10;
} else if (lane_size == kSRegSize) {
VIXL_ASSERT(ArrayLength(fexpa_coeff32) ==
(uint64_t{1} << (index_highbit + 1)));
fexpa_coeff = fexpa_coeff32;
op_highbit = 13;
op_shift = 23;
} else {
VIXL_ASSERT(lane_size == kDRegSize);
VIXL_ASSERT(ArrayLength(fexpa_coeff64) ==
(uint64_t{1} << (index_highbit + 1)));
fexpa_coeff = fexpa_coeff64;
op_highbit = 16;
op_shift = 52;
}
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
uint64_t op = src.Uint(vform, i);
uint64_t result = fexpa_coeff[Bits(op, index_highbit, 0)];
result |= (Bits(op, op_highbit, index_highbit + 1) << op_shift);
dst.SetUint(vform, i, result);
}
return dst;
}
template <typename T>
LogicVRegister Simulator::fscale(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
T two = T(2.0);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
T src1_val = src1.Float<T>(i);
if (!IsNaN(src1_val)) {
int64_t scale = src2.Int(vform, i);
// TODO: this is a low-performance implementation, but it's simple and
// less likely to be buggy. Consider replacing it with something faster.
// Scales outside of these bounds become infinity or zero, so there's no
// point iterating further.
scale = std::min<int64_t>(std::max<int64_t>(scale, -2048), 2048);
// Compute src1_val * 2 ^ scale. If scale is positive, multiply by two and
// decrement scale until it's zero.
while (scale-- > 0) {
src1_val = FPMul(src1_val, two);
}
// If scale is negative, divide by two and increment scale until it's
// zero. Initially, scale is (src2 - 1), so we pre-increment.
while (++scale < 0) {
src1_val = FPDiv(src1_val, two);
}
}
dst.SetFloat<T>(i, src1_val);
}
return dst;
}
LogicVRegister Simulator::fscale(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
fscale<SimFloat16>(vform, dst, src1, src2);
} else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
fscale<float>(vform, dst, src1, src2);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
fscale<double>(vform, dst, src1, src2);
}
return dst;
}
LogicVRegister Simulator::scvtf(VectorFormat vform,
unsigned dst_data_size_in_bits,
unsigned src_data_size_in_bits,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src,
FPRounding round,
int fbits) {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) >= dst_data_size_in_bits);
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) >= src_data_size_in_bits);
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!pg.IsActive(vform, i)) continue;
int64_t value = ExtractSignedBitfield64(src_data_size_in_bits - 1,
0,
src.Uint(vform, i));
switch (dst_data_size_in_bits) {
case kHRegSize: {
SimFloat16 result = FixedToFloat16(value, fbits, round);
dst.SetUint(vform, i, Float16ToRawbits(result));
break;
}
case kSRegSize: {
float result = FixedToFloat(value, fbits, round);
dst.SetUint(vform, i, FloatToRawbits(result));
break;
}
case kDRegSize: {
double result = FixedToDouble(value, fbits, round);
dst.SetUint(vform, i, DoubleToRawbits(result));
break;
}
default:
VIXL_UNIMPLEMENTED();
break;
}
}
return dst;
}
LogicVRegister Simulator::scvtf(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int fbits,
FPRounding round) {
return scvtf(vform,
LaneSizeInBitsFromFormat(vform),
LaneSizeInBitsFromFormat(vform),
dst,
GetPTrue(),
src,
round,
fbits);
}
LogicVRegister Simulator::ucvtf(VectorFormat vform,
unsigned dst_data_size_in_bits,
unsigned src_data_size_in_bits,
LogicVRegister dst,
const LogicPRegister& pg,
const LogicVRegister& src,
FPRounding round,
int fbits) {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) >= dst_data_size_in_bits);
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) >= src_data_size_in_bits);
dst.ClearForWrite(vform);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!pg.IsActive(vform, i)) continue;
uint64_t value = ExtractUnsignedBitfield64(src_data_size_in_bits - 1,
0,
src.Uint(vform, i));
switch (dst_data_size_in_bits) {
case kHRegSize: {
SimFloat16 result = UFixedToFloat16(value, fbits, round);
dst.SetUint(vform, i, Float16ToRawbits(result));
break;
}
case kSRegSize: {
float result = UFixedToFloat(value, fbits, round);
dst.SetUint(vform, i, FloatToRawbits(result));
break;
}
case kDRegSize: {
double result = UFixedToDouble(value, fbits, round);
dst.SetUint(vform, i, DoubleToRawbits(result));
break;
}
default:
VIXL_UNIMPLEMENTED();
break;
}
}
return dst;
}
LogicVRegister Simulator::ucvtf(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
int fbits,
FPRounding round) {
return ucvtf(vform,
LaneSizeInBitsFromFormat(vform),
LaneSizeInBitsFromFormat(vform),
dst,
GetPTrue(),
src,
round,
fbits);
}
LogicVRegister Simulator::unpk(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src,
UnpackType unpack_type,
ExtendType extend_type) {
VectorFormat vform_half = VectorFormatHalfWidth(vform);
const int lane_count = LaneCountFromFormat(vform);
const int src_start_lane = (unpack_type == kLoHalf) ? 0 : lane_count;
switch (extend_type) {
case kSignedExtend: {
int64_t result[kZRegMaxSizeInBytes];
for (int i = 0; i < lane_count; ++i) {
result[i] = src.Int(vform_half, i + src_start_lane);
}
for (int i = 0; i < lane_count; ++i) {
dst.SetInt(vform, i, result[i]);
}
break;
}
case kUnsignedExtend: {
uint64_t result[kZRegMaxSizeInBytes];
for (int i = 0; i < lane_count; ++i) {
result[i] = src.Uint(vform_half, i + src_start_lane);
}
for (int i = 0; i < lane_count; ++i) {
dst.SetUint(vform, i, result[i]);
}
break;
}
default:
VIXL_UNREACHABLE();
}
return dst;
}
LogicPRegister Simulator::SVEIntCompareVectorsHelper(Condition cond,
VectorFormat vform,
LogicPRegister dst,
const LogicPRegister& mask,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool is_wide_elements,
FlagsUpdate flags) {
for (int lane = 0; lane < LaneCountFromFormat(vform); lane++) {
bool result = false;
if (mask.IsActive(vform, lane)) {
int64_t op1 = 0xbadbeef;
int64_t op2 = 0xbadbeef;
int d_lane = (lane * LaneSizeInBitsFromFormat(vform)) / kDRegSize;
switch (cond) {
case eq:
case ge:
case gt:
case lt:
case le:
case ne:
op1 = src1.Int(vform, lane);
op2 = is_wide_elements ? src2.Int(kFormatVnD, d_lane)
: src2.Int(vform, lane);
break;
case hi:
case hs:
case ls:
case lo:
op1 = src1.Uint(vform, lane);
op2 = is_wide_elements ? src2.Uint(kFormatVnD, d_lane)
: src2.Uint(vform, lane);
break;
default:
VIXL_UNREACHABLE();
}
switch (cond) {
case eq:
result = (op1 == op2);
break;
case ne:
result = (op1 != op2);
break;
case ge:
result = (op1 >= op2);
break;
case gt:
result = (op1 > op2);
break;
case le:
result = (op1 <= op2);
break;
case lt:
result = (op1 < op2);
break;
case hs:
result = (static_cast<uint64_t>(op1) >= static_cast<uint64_t>(op2));
break;
case hi:
result = (static_cast<uint64_t>(op1) > static_cast<uint64_t>(op2));
break;
case ls:
result = (static_cast<uint64_t>(op1) <= static_cast<uint64_t>(op2));
break;
case lo:
result = (static_cast<uint64_t>(op1) < static_cast<uint64_t>(op2));
break;
default:
VIXL_UNREACHABLE();
}
}
dst.SetActive(vform, lane, result);
}
if (flags == SetFlags) PredTest(vform, mask, dst);
return dst;
}
LogicVRegister Simulator::SVEBitwiseShiftHelper(Shift shift_op,
VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool is_wide_elements) {
unsigned lane_size = LaneSizeInBitsFromFormat(vform);
VectorFormat shift_vform = is_wide_elements ? kFormatVnD : vform;
for (int lane = 0; lane < LaneCountFromFormat(vform); lane++) {
int shift_src_lane = lane;
if (is_wide_elements) {
// If the shift amount comes from wide elements, select the D-sized lane
// which occupies the corresponding lanes of the value to be shifted.
shift_src_lane = (lane * lane_size) / kDRegSize;
}
uint64_t shift_amount = src2.Uint(shift_vform, shift_src_lane);
// Saturate shift_amount to the size of the lane that will be shifted.
if (shift_amount > lane_size) shift_amount = lane_size;
uint64_t value = src1.Uint(vform, lane);
int64_t result = ShiftOperand(lane_size,
value,
shift_op,
static_cast<unsigned>(shift_amount));
dst.SetUint(vform, lane, result);
}
return dst;
}
LogicVRegister Simulator::asrd(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
int shift) {
VIXL_ASSERT((shift > 0) && (static_cast<unsigned>(shift) <=
LaneSizeInBitsFromFormat(vform)));
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
int64_t value = src1.Int(vform, i);
if (shift <= 63) {
if (value < 0) {
// The max possible mask is 0x7fff'ffff'ffff'ffff, which can be safely
// cast to int64_t, and cannot cause signed overflow in the result.
value = value + GetUintMask(shift);
}
value = ShiftOperand(kDRegSize, value, ASR, shift);
} else {
value = 0;
}
dst.SetInt(vform, i, value);
}
return dst;
}
LogicVRegister Simulator::SVEBitwiseLogicalUnpredicatedHelper(
LogicalOp logical_op,
VectorFormat vform,
LogicVRegister zd,
const LogicVRegister& zn,
const LogicVRegister& zm) {
VIXL_ASSERT(IsSVEFormat(vform));
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
uint64_t op1 = zn.Uint(vform, i);
uint64_t op2 = zm.Uint(vform, i);
uint64_t result = 0;
switch (logical_op) {
case AND:
result = op1 & op2;
break;
case BIC:
result = op1 & ~op2;
break;
case EOR:
result = op1 ^ op2;
break;
case ORR:
result = op1 | op2;
break;
default:
VIXL_UNIMPLEMENTED();
}
zd.SetUint(vform, i, result);
}
return zd;
}
LogicPRegister Simulator::SVEPredicateLogicalHelper(SVEPredicateLogicalOp op,
LogicPRegister pd,
const LogicPRegister& pn,
const LogicPRegister& pm) {
for (int i = 0; i < pn.GetChunkCount(); i++) {
LogicPRegister::ChunkType op1 = pn.GetChunk(i);
LogicPRegister::ChunkType op2 = pm.GetChunk(i);
LogicPRegister::ChunkType result = 0;
switch (op) {
case ANDS_p_p_pp_z:
case AND_p_p_pp_z:
result = op1 & op2;
break;
case BICS_p_p_pp_z:
case BIC_p_p_pp_z:
result = op1 & ~op2;
break;
case EORS_p_p_pp_z:
case EOR_p_p_pp_z:
result = op1 ^ op2;
break;
case NANDS_p_p_pp_z:
case NAND_p_p_pp_z:
result = ~(op1 & op2);
break;
case NORS_p_p_pp_z:
case NOR_p_p_pp_z:
result = ~(op1 | op2);
break;
case ORNS_p_p_pp_z:
case ORN_p_p_pp_z:
result = op1 | ~op2;
break;
case ORRS_p_p_pp_z:
case ORR_p_p_pp_z:
result = op1 | op2;
break;
default:
VIXL_UNIMPLEMENTED();
}
pd.SetChunk(i, result);
}
return pd;
}
LogicVRegister Simulator::SVEBitwiseImmHelper(
SVEBitwiseLogicalWithImm_UnpredicatedOp op,
VectorFormat vform,
LogicVRegister zd,
uint64_t imm) {
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
uint64_t op1 = zd.Uint(vform, i);
uint64_t result = 0;
switch (op) {
case AND_z_zi:
result = op1 & imm;
break;
case EOR_z_zi:
result = op1 ^ imm;
break;
case ORR_z_zi:
result = op1 | imm;
break;
default:
VIXL_UNIMPLEMENTED();
}
zd.SetUint(vform, i, result);
}
return zd;
}
void Simulator::SVEStructuredStoreHelper(VectorFormat vform,
const LogicPRegister& pg,
unsigned zt_code,
const LogicSVEAddressVector& addr) {
VIXL_ASSERT(zt_code < kNumberOfZRegisters);
int esize_in_bytes_log2 = LaneSizeInBytesLog2FromFormat(vform);
int msize_in_bytes_log2 = addr.GetMsizeInBytesLog2();
int msize_in_bytes = addr.GetMsizeInBytes();
int reg_count = addr.GetRegCount();
VIXL_ASSERT(esize_in_bytes_log2 >= msize_in_bytes_log2);
VIXL_ASSERT((reg_count >= 1) && (reg_count <= 4));
unsigned zt_codes[4] = {zt_code,
(zt_code + 1) % kNumberOfZRegisters,
(zt_code + 2) % kNumberOfZRegisters,
(zt_code + 3) % kNumberOfZRegisters};
LogicVRegister zt[4] = {
ReadVRegister(zt_codes[0]),
ReadVRegister(zt_codes[1]),
ReadVRegister(zt_codes[2]),
ReadVRegister(zt_codes[3]),
};
// For unpacked forms (e.g. `st1b { z0.h }, ...`, the upper parts of the lanes
// are ignored, so read the source register using the VectorFormat that
// corresponds with the storage format, and multiply the index accordingly.
VectorFormat unpack_vform =
SVEFormatFromLaneSizeInBytesLog2(msize_in_bytes_log2);
int unpack_shift = esize_in_bytes_log2 - msize_in_bytes_log2;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (!pg.IsActive(vform, i)) continue;
for (int r = 0; r < reg_count; r++) {
uint64_t element_address = addr.GetElementAddress(i, r);
if (!StoreLane(zt[r], unpack_vform, i << unpack_shift, element_address)) {
return;
}
}
}
if (ShouldTraceWrites()) {
PrintRegisterFormat format = GetPrintRegisterFormat(vform);
if (esize_in_bytes_log2 == msize_in_bytes_log2) {
// Use an FP format where it's likely that we're accessing FP data.
format = GetPrintRegisterFormatTryFP(format);
}
// Stores don't represent a change to the source register's value, so only
// print the relevant part of the value.
format = GetPrintRegPartial(format);
PrintZStructAccess(zt_code,
reg_count,
pg,
format,
msize_in_bytes,
"->",
addr);
}
}
bool Simulator::SVEStructuredLoadHelper(VectorFormat vform,
const LogicPRegister& pg,
unsigned zt_code,
const LogicSVEAddressVector& addr,
bool is_signed) {
int esize_in_bytes_log2 = LaneSizeInBytesLog2FromFormat(vform);
int msize_in_bytes_log2 = addr.GetMsizeInBytesLog2();
int msize_in_bytes = addr.GetMsizeInBytes();
int reg_count = addr.GetRegCount();
VIXL_ASSERT(zt_code < kNumberOfZRegisters);
VIXL_ASSERT(esize_in_bytes_log2 >= msize_in_bytes_log2);
VIXL_ASSERT((reg_count >= 1) && (reg_count <= 4));
unsigned zt_codes[4] = {zt_code,
(zt_code + 1) % kNumberOfZRegisters,
(zt_code + 2) % kNumberOfZRegisters,
(zt_code + 3) % kNumberOfZRegisters};
LogicVRegister zt[4] = {
ReadVRegister(zt_codes[0]),
ReadVRegister(zt_codes[1]),
ReadVRegister(zt_codes[2]),
ReadVRegister(zt_codes[3]),
};
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
for (int r = 0; r < reg_count; r++) {
uint64_t element_address = addr.GetElementAddress(i, r);
if (!pg.IsActive(vform, i)) {
zt[r].SetUint(vform, i, 0);
continue;
}
if (is_signed) {
if (!LoadIntToLane(zt[r], vform, msize_in_bytes, i, element_address)) {
return false;
}
} else {
if (!LoadUintToLane(zt[r], vform, msize_in_bytes, i, element_address)) {
return false;
}
}
}
}
if (ShouldTraceVRegs()) {
PrintRegisterFormat format = GetPrintRegisterFormat(vform);
if ((esize_in_bytes_log2 == msize_in_bytes_log2) && !is_signed) {
// Use an FP format where it's likely that we're accessing FP data.
format = GetPrintRegisterFormatTryFP(format);
}
PrintZStructAccess(zt_code,
reg_count,
pg,
format,
msize_in_bytes,
"<-",
addr);
}
return true;
}
LogicPRegister Simulator::brka(LogicPRegister pd,
const LogicPRegister& pg,
const LogicPRegister& pn) {
bool break_ = false;
for (int i = 0; i < LaneCountFromFormat(kFormatVnB); i++) {
if (pg.IsActive(kFormatVnB, i)) {
pd.SetActive(kFormatVnB, i, !break_);
break_ |= pn.IsActive(kFormatVnB, i);
}
}
return pd;
}
LogicPRegister Simulator::brkb(LogicPRegister pd,
const LogicPRegister& pg,
const LogicPRegister& pn) {
bool break_ = false;
for (int i = 0; i < LaneCountFromFormat(kFormatVnB); i++) {
if (pg.IsActive(kFormatVnB, i)) {
break_ |= pn.IsActive(kFormatVnB, i);
pd.SetActive(kFormatVnB, i, !break_);
}
}
return pd;
}
LogicPRegister Simulator::brkn(LogicPRegister pdm,
const LogicPRegister& pg,
const LogicPRegister& pn) {
if (!IsLastActive(kFormatVnB, pg, pn)) {
pfalse(pdm);
}
return pdm;
}
LogicPRegister Simulator::brkpa(LogicPRegister pd,
const LogicPRegister& pg,
const LogicPRegister& pn,
const LogicPRegister& pm) {
bool last_active = IsLastActive(kFormatVnB, pg, pn);
for (int i = 0; i < LaneCountFromFormat(kFormatVnB); i++) {
bool active = false;
if (pg.IsActive(kFormatVnB, i)) {
active = last_active;
last_active = last_active && !pm.IsActive(kFormatVnB, i);
}
pd.SetActive(kFormatVnB, i, active);
}
return pd;
}
LogicPRegister Simulator::brkpb(LogicPRegister pd,
const LogicPRegister& pg,
const LogicPRegister& pn,
const LogicPRegister& pm) {
bool last_active = IsLastActive(kFormatVnB, pg, pn);
for (int i = 0; i < LaneCountFromFormat(kFormatVnB); i++) {
bool active = false;
if (pg.IsActive(kFormatVnB, i)) {
last_active = last_active && !pm.IsActive(kFormatVnB, i);
active = last_active;
}
pd.SetActive(kFormatVnB, i, active);
}
return pd;
}
void Simulator::SVEFaultTolerantLoadHelper(VectorFormat vform,
const LogicPRegister& pg,
unsigned zt_code,
const LogicSVEAddressVector& addr,
SVEFaultTolerantLoadType type,
bool is_signed) {
int esize_in_bytes = LaneSizeInBytesFromFormat(vform);
int msize_in_bits = addr.GetMsizeInBits();
int msize_in_bytes = addr.GetMsizeInBytes();
VIXL_ASSERT(zt_code < kNumberOfZRegisters);
VIXL_ASSERT(esize_in_bytes >= msize_in_bytes);
VIXL_ASSERT(addr.GetRegCount() == 1);
LogicVRegister zt = ReadVRegister(zt_code);
LogicPRegister ffr = ReadFFR();
// Non-faulting loads are allowed to fail arbitrarily. To stress user
// code, fail a random element in roughly one in eight full-vector loads.
uint32_t rnd = static_cast<uint32_t>(rand_gen_());
int fake_fault_at_lane = rnd % (LaneCountFromFormat(vform) * 8);
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
uint64_t value = 0;
if (pg.IsActive(vform, i)) {
uint64_t element_address = addr.GetElementAddress(i, 0);
if (type == kSVEFirstFaultLoad) {
// First-faulting loads always load the first active element, regardless
// of FFR. The result will be discarded if its FFR lane is inactive, but
// it could still generate a fault.
VIXL_DEFINE_OR_RETURN(mem_result,
MemReadUint(msize_in_bytes, element_address));
value = mem_result;
// All subsequent elements have non-fault semantics.
type = kSVENonFaultLoad;
} else if (ffr.IsActive(vform, i)) {
// Simulation of fault-tolerant loads relies on system calls, and is
// likely to be relatively slow, so we only actually perform the load if
// its FFR lane is active.
bool can_read = (i < fake_fault_at_lane) &&
CanReadMemory(element_address, msize_in_bytes);
if (can_read) {
VIXL_DEFINE_OR_RETURN(mem_result,
MemReadUint(msize_in_bytes, element_address));
value = mem_result;
} else {
// Propagate the fault to the end of FFR.
for (int j = i; j < LaneCountFromFormat(vform); j++) {
ffr.SetActive(vform, j, false);
}
}
}
}
// The architecture permits a few possible results for inactive FFR lanes
// (including those caused by a fault in this instruction). We choose to
// leave the register value unchanged (like merging predication) because
// no other input to this instruction can have the same behaviour.
//
// Note that this behaviour takes precedence over pg's zeroing predication.
if (ffr.IsActive(vform, i)) {
int msb = msize_in_bits - 1;
if (is_signed) {
zt.SetInt(vform, i, ExtractSignedBitfield64(msb, 0, value));
} else {
zt.SetUint(vform, i, ExtractUnsignedBitfield64(msb, 0, value));
}
}
}
if (ShouldTraceVRegs()) {
PrintRegisterFormat format = GetPrintRegisterFormat(vform);
if ((esize_in_bytes == msize_in_bytes) && !is_signed) {
// Use an FP format where it's likely that we're accessing FP data.
format = GetPrintRegisterFormatTryFP(format);
}
// Log accessed lanes that are active in both pg and ffr. PrintZStructAccess
// expects a single mask, so combine the two predicates.
SimPRegister mask;
SVEPredicateLogicalHelper(AND_p_p_pp_z, mask, pg, ffr);
PrintZStructAccess(zt_code, 1, mask, format, msize_in_bytes, "<-", addr);
}
}
void Simulator::SVEGatherLoadScalarPlusVectorHelper(const Instruction* instr,
VectorFormat vform,
SVEOffsetModifier mod) {
bool is_signed = instr->ExtractBit(14) == 0;
bool is_ff = instr->ExtractBit(13) == 1;
// Note that these instructions don't use the Dtype encoding.
int msize_in_bytes_log2 = instr->ExtractBits(24, 23);
int scale = instr->ExtractBit(21) * msize_in_bytes_log2;
uint64_t base = ReadXRegister(instr->GetRn(), Reg31IsStackPointer);
LogicSVEAddressVector addr(base,
&ReadVRegister(instr->GetRm()),
vform,
mod,
scale);
addr.SetMsizeInBytesLog2(msize_in_bytes_log2);
if (is_ff) {
SVEFaultTolerantLoadHelper(vform,
ReadPRegister(instr->GetPgLow8()),
instr->GetRt(),
addr,
kSVEFirstFaultLoad,
is_signed);
} else {
SVEStructuredLoadHelper(vform,
ReadPRegister(instr->GetPgLow8()),
instr->GetRt(),
addr,
is_signed);
}
}
int Simulator::GetFirstActive(VectorFormat vform,
const LogicPRegister& pg) const {
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
if (pg.IsActive(vform, i)) return i;
}
return -1;
}
int Simulator::GetLastActive(VectorFormat vform,
const LogicPRegister& pg) const {
for (int i = LaneCountFromFormat(vform) - 1; i >= 0; i--) {
if (pg.IsActive(vform, i)) return i;
}
return -1;
}
int Simulator::CountActiveLanes(VectorFormat vform,
const LogicPRegister& pg) const {
int count = 0;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
count += pg.IsActive(vform, i) ? 1 : 0;
}
return count;
}
int Simulator::CountActiveAndTrueLanes(VectorFormat vform,
const LogicPRegister& pg,
const LogicPRegister& pn) const {
int count = 0;
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
count += (pg.IsActive(vform, i) && pn.IsActive(vform, i)) ? 1 : 0;
}
return count;
}
int Simulator::GetPredicateConstraintLaneCount(VectorFormat vform,
int pattern) const {
VIXL_ASSERT(IsSVEFormat(vform));
int all = LaneCountFromFormat(vform);
VIXL_ASSERT(all > 0);
switch (pattern) {
case SVE_VL1:
case SVE_VL2:
case SVE_VL3:
case SVE_VL4:
case SVE_VL5:
case SVE_VL6:
case SVE_VL7:
case SVE_VL8:
// VL1-VL8 are encoded directly.
VIXL_STATIC_ASSERT(SVE_VL1 == 1);
VIXL_STATIC_ASSERT(SVE_VL8 == 8);
return (pattern <= all) ? pattern : 0;
case SVE_VL16:
case SVE_VL32:
case SVE_VL64:
case SVE_VL128:
case SVE_VL256: {
// VL16-VL256 are encoded as log2(N) + c.
int min = 16 << (pattern - SVE_VL16);
return (min <= all) ? min : 0;
}
// Special cases.
case SVE_POW2:
return 1 << HighestSetBitPosition(all);
case SVE_MUL4:
return all - (all % 4);
case SVE_MUL3:
return all - (all % 3);
case SVE_ALL:
return all;
}
// Unnamed cases architecturally return 0.
return 0;
}
LogicPRegister Simulator::match(VectorFormat vform,
LogicPRegister dst,
const LogicVRegister& haystack,
const LogicVRegister& needles,
bool negate_match) {
SimVRegister ztemp;
SimPRegister ptemp;
pfalse(dst);
int lanes_per_segment = kQRegSize / LaneSizeInBitsFromFormat(vform);
for (int i = 0; i < lanes_per_segment; i++) {
dup_elements_to_segments(vform, ztemp, needles, i);
SVEIntCompareVectorsHelper(eq,
vform,
ptemp,
GetPTrue(),
haystack,
ztemp,
false,
LeaveFlags);
SVEPredicateLogicalHelper(ORR_p_p_pp_z, dst, dst, ptemp);
}
if (negate_match) {
ptrue(vform, ptemp, SVE_ALL);
SVEPredicateLogicalHelper(EOR_p_p_pp_z, dst, dst, ptemp);
}
return dst;
}
uint64_t LogicSVEAddressVector::GetStructAddress(int lane) const {
if (IsContiguous()) {
return base_ + (lane * GetRegCount()) * GetMsizeInBytes();
}
VIXL_ASSERT(IsScatterGather());
VIXL_ASSERT(vector_ != NULL);
// For scatter-gather accesses, we need to extract the offset from vector_,
// and apply modifiers.
uint64_t offset = 0;
switch (vector_form_) {
case kFormatVnS:
offset = vector_->GetLane<uint32_t>(lane);
break;
case kFormatVnD:
offset = vector_->GetLane<uint64_t>(lane);
break;
default:
VIXL_UNIMPLEMENTED();
break;
}
switch (vector_mod_) {
case SVE_MUL_VL:
VIXL_UNIMPLEMENTED();
break;
case SVE_LSL:
// We apply the shift below. There's nothing to do here.
break;
case NO_SVE_OFFSET_MODIFIER:
VIXL_ASSERT(vector_shift_ == 0);
break;
case SVE_UXTW:
offset = ExtractUnsignedBitfield64(kWRegSize - 1, 0, offset);
break;
case SVE_SXTW:
offset = ExtractSignedBitfield64(kWRegSize - 1, 0, offset);
break;
}
return base_ + (offset << vector_shift_);
}
LogicVRegister Simulator::pack_odd_elements(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
SimVRegister zero;
zero.Clear();
return uzp2(vform, dst, src, zero);
}
LogicVRegister Simulator::pack_even_elements(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src) {
SimVRegister zero;
zero.Clear();
return uzp1(vform, dst, src, zero);
}
LogicVRegister Simulator::adcl(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool top) {
unsigned reg_size = LaneSizeInBitsFromFormat(vform);
VIXL_ASSERT((reg_size == kSRegSize) || (reg_size == kDRegSize));
for (int i = 0; i < LaneCountFromFormat(vform); i += 2) {
uint64_t left = src1.Uint(vform, i + (top ? 1 : 0));
uint64_t right = dst.Uint(vform, i);
unsigned carry_in = src2.Uint(vform, i + 1) & 1;
std::pair<uint64_t, uint8_t> val_and_flags =
AddWithCarry(reg_size, left, right, carry_in);
// Set even lanes to the result of the addition.
dst.SetUint(vform, i, val_and_flags.first);
// Set odd lanes to the carry flag from the addition.
uint64_t carry_out = (val_and_flags.second >> 1) & 1;
dst.SetUint(vform, i + 1, carry_out);
}
return dst;
}
// Multiply the 2x8 8-bit matrix in src1 by the 8x2 8-bit matrix in src2, add
// the 2x2 32-bit result to the matrix in srcdst, and write back to srcdst.
//
// Matrices of the form:
//
// src1 = ( a b c d e f g h ) src2 = ( A B )
// ( i j k l m n o p ) ( C D )
// ( E F )
// ( G H )
// ( I J )
// ( K L )
// ( M N )
// ( O P )
//
// Are stored in the input vector registers as:
//
// 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0
// src1 = [ p | o | n | m | l | k | j | i | h | g | f | e | d | c | b | a ]
// src2 = [ P | N | L | J | H | F | D | B | O | M | K | I | G | E | C | A ]
//
LogicVRegister Simulator::matmul(VectorFormat vform_dst,
LogicVRegister srcdst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool src1_signed,
bool src2_signed) {
// Two destination forms are supported: Q register containing four S-sized
// elements (4S) and Z register containing n S-sized elements (VnS).
VIXL_ASSERT((vform_dst == kFormat4S) || (vform_dst == kFormatVnS));
VectorFormat vform_src = kFormatVnB;
int b_per_segment = kQRegSize / kBRegSize;
int s_per_segment = kQRegSize / kSRegSize;
int64_t result[kZRegMaxSizeInBytes / kSRegSizeInBytes] = {};
int segment_count = LaneCountFromFormat(vform_dst) / 4;
for (int seg = 0; seg < segment_count; seg++) {
for (int i = 0; i < 2; i++) {
for (int j = 0; j < 2; j++) {
int dstidx = (2 * i) + j + (seg * s_per_segment);
int64_t sum = srcdst.Int(vform_dst, dstidx);
for (int k = 0; k < 8; k++) {
int idx1 = (8 * i) + k + (seg * b_per_segment);
int idx2 = (8 * j) + k + (seg * b_per_segment);
int64_t e1 = src1_signed ? src1.Int(vform_src, idx1)
: src1.Uint(vform_src, idx1);
int64_t e2 = src2_signed ? src2.Int(vform_src, idx2)
: src2.Uint(vform_src, idx2);
sum += e1 * e2;
}
result[dstidx] = sum;
}
}
}
srcdst.SetIntArray(vform_dst, result);
return srcdst;
}
// Multiply the 2x2 FP matrix in src1 by the 2x2 FP matrix in src2, add the 2x2
// result to the matrix in srcdst, and write back to srcdst.
//
// Matrices of the form:
//
// src1 = ( a b ) src2 = ( A B )
// ( c d ) ( C D )
//
// Are stored in the input vector registers as:
//
// 3 2 1 0
// src1 = [ d | c | b | a ]
// src2 = [ D | B | C | A ] nb. transposition
//
// Giving:
// 3 2 1 0
// result = [ w | z | y | x ]
//
// Where:
//
// x = (a * A) + (b * C) + a
// y = (a * B) + (b * D) + b
// z = (c * A) + (d * C) + c
// w = (c * B) + (d * D) + d
template <typename T>
LogicVRegister Simulator::fmatmul(VectorFormat vform,
LogicVRegister srcdst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
T result[kZRegMaxSizeInBytes / sizeof(T)] = {};
int T_per_segment = 4;
int segment_count = GetVectorLengthInBytes() / (T_per_segment * sizeof(T));
for (int seg = 0; seg < segment_count; seg++) {
int segoff = seg * T_per_segment;
for (int i = 0; i < 2; i++) {
for (int j = 0; j < 2; j++) {
T prod0 = FPMulNaNs(src1.Float<T>(2 * i + 0 + segoff),
src2.Float<T>(2 * j + 0 + segoff));
T prod1 = FPMulNaNs(src1.Float<T>(2 * i + 1 + segoff),
src2.Float<T>(2 * j + 1 + segoff));
T sum = FPAdd(srcdst.Float<T>(2 * i + j + segoff), prod0);
result[2 * i + j + segoff] = FPAdd(sum, prod1);
}
}
}
for (int i = 0; i < LaneCountFromFormat(vform); i++) {
srcdst.SetFloat<T>(vform, i, result[i]);
}
return srcdst;
}
LogicVRegister Simulator::fmatmul(VectorFormat vform,
LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
fmatmul<float>(vform, dst, src1, src2);
} else {
VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
fmatmul<double>(vform, dst, src1, src2);
}
return dst;
}
template <>
uint64_t CryptoOp<"choose"_h>(uint64_t x, uint64_t y, uint64_t z) {
return ((y ^ z) & x) ^ z;
}
template <>
uint64_t CryptoOp<"majority"_h>(uint64_t x, uint64_t y, uint64_t z) {
return (x & y) | ((x | y) & z);
}
template <>
uint64_t CryptoOp<"parity"_h>(uint64_t x, uint64_t y, uint64_t z) {
return x ^ y ^ z;
}
template <typename T, unsigned A, unsigned B, unsigned C>
static uint64_t SHASigma(uint64_t x) {
return static_cast<T>(RotateRight(x, A, sizeof(T) * kBitsPerByte) ^
RotateRight(x, B, sizeof(T) * kBitsPerByte) ^
RotateRight(x, C, sizeof(T) * kBitsPerByte));
}
LogicVRegister Simulator::sha2h(LogicVRegister srcdst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool part1) {
uint64_t x[4] = {};
uint64_t y[4] = {};
if (part1) {
// Switch input order based on which part is being handled.
srcdst.UintArray(kFormat4S, x);
src1.UintArray(kFormat4S, y);
} else {
src1.UintArray(kFormat4S, x);
srcdst.UintArray(kFormat4S, y);
}
for (unsigned i = 0; i < ArrayLength(x); i++) {
uint64_t chs = CryptoOp<"choose"_h>(y[0], y[1], y[2]);
uint64_t maj = CryptoOp<"majority"_h>(x[0], x[1], x[2]);
uint64_t w = src2.Uint(kFormat4S, i);
uint64_t t = y[3] + SHASigma<uint32_t, 6, 11, 25>(y[0]) + chs + w;
x[3] += t;
y[3] = t + SHASigma<uint32_t, 2, 13, 22>(x[0]) + maj;
// y:x = ROL(y:x, 32)
SHARotateEltsLeftOne(x);
SHARotateEltsLeftOne(y);
std::swap(x[0], y[0]);
}
srcdst.SetUintArray(kFormat4S, part1 ? x : y);
return srcdst;
}
template <typename T, unsigned A, unsigned B, unsigned C>
static uint64_t SHASURotate(uint64_t x) {
return RotateRight(x, A, sizeof(T) * kBitsPerByte) ^
RotateRight(x, B, sizeof(T) * kBitsPerByte) ^
((x & ~static_cast<T>(0)) >> C);
}
LogicVRegister Simulator::sha2su0(LogicVRegister srcdst,
const LogicVRegister& src1) {
uint64_t w[4] = {};
uint64_t result[4];
srcdst.UintArray(kFormat4S, w);
uint64_t x = src1.Uint(kFormat4S, 0);
result[0] = SHASURotate<uint32_t, 7, 18, 3>(w[1]) + w[0];
result[1] = SHASURotate<uint32_t, 7, 18, 3>(w[2]) + w[1];
result[2] = SHASURotate<uint32_t, 7, 18, 3>(w[3]) + w[2];
result[3] = SHASURotate<uint32_t, 7, 18, 3>(x) + w[3];
srcdst.SetUintArray(kFormat4S, result);
return srcdst;
}
LogicVRegister Simulator::sha2su1(LogicVRegister srcdst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
uint64_t w[4] = {};
uint64_t x[4] = {};
uint64_t y[4] = {};
uint64_t result[4];
srcdst.UintArray(kFormat4S, w);
src1.UintArray(kFormat4S, x);
src2.UintArray(kFormat4S, y);
result[0] = SHASURotate<uint32_t, 17, 19, 10>(y[2]) + w[0] + x[1];
result[1] = SHASURotate<uint32_t, 17, 19, 10>(y[3]) + w[1] + x[2];
result[2] = SHASURotate<uint32_t, 17, 19, 10>(result[0]) + w[2] + x[3];
result[3] = SHASURotate<uint32_t, 17, 19, 10>(result[1]) + w[3] + y[0];
srcdst.SetUintArray(kFormat4S, result);
return srcdst;
}
LogicVRegister Simulator::sha512h(LogicVRegister srcdst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
uint64_t w[2] = {};
uint64_t x[2] = {};
uint64_t y[2] = {};
uint64_t result[2] = {};
srcdst.UintArray(kFormat2D, w);
src1.UintArray(kFormat2D, x);
src2.UintArray(kFormat2D, y);
result[1] = (y[1] & x[0]) ^ (~y[1] & x[1]);
result[1] += SHASigma<uint64_t, 14, 18, 41>(y[1]) + w[1];
uint64_t tmp = result[1] + y[0];
result[0] = (tmp & y[1]) ^ (~tmp & x[0]);
result[0] += SHASigma<uint64_t, 14, 18, 41>(tmp) + w[0];
srcdst.SetUintArray(kFormat2D, result);
return srcdst;
}
LogicVRegister Simulator::sha512h2(LogicVRegister srcdst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
uint64_t w[2] = {};
uint64_t x[2] = {};
uint64_t y[2] = {};
uint64_t result[2] = {};
srcdst.UintArray(kFormat2D, w);
src1.UintArray(kFormat2D, x);
src2.UintArray(kFormat2D, y);
result[1] = (x[0] & y[1]) ^ (x[0] & y[0]) ^ (y[1] & y[0]);
result[1] += SHASigma<uint64_t, 28, 34, 39>(y[0]) + w[1];
result[0] = (result[1] & y[0]) ^ (result[1] & y[1]) ^ (y[1] & y[0]);
result[0] += SHASigma<uint64_t, 28, 34, 39>(result[1]) + w[0];
srcdst.SetUintArray(kFormat2D, result);
return srcdst;
}
LogicVRegister Simulator::sha512su0(LogicVRegister srcdst,
const LogicVRegister& src1) {
uint64_t w[2] = {};
uint64_t x[2] = {};
uint64_t result[2] = {};
srcdst.UintArray(kFormat2D, w);
src1.UintArray(kFormat2D, x);
result[0] = SHASURotate<uint64_t, 1, 8, 7>(w[1]) + w[0];
result[1] = SHASURotate<uint64_t, 1, 8, 7>(x[0]) + w[1];
srcdst.SetUintArray(kFormat2D, result);
return srcdst;
}
LogicVRegister Simulator::sha512su1(LogicVRegister srcdst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
uint64_t w[2] = {};
uint64_t x[2] = {};
uint64_t y[2] = {};
uint64_t result[2] = {};
srcdst.UintArray(kFormat2D, w);
src1.UintArray(kFormat2D, x);
src2.UintArray(kFormat2D, y);
result[1] = w[1] + SHASURotate<uint64_t, 19, 61, 6>(x[1]) + y[1];
result[0] = w[0] + SHASURotate<uint64_t, 19, 61, 6>(x[0]) + y[0];
srcdst.SetUintArray(kFormat2D, result);
return srcdst;
}
static uint8_t GalMul(int table, uint64_t x) {
// Galois multiplication lookup tables.
static const uint8_t ffmul02[256] = {
0x00, 0x02, 0x04, 0x06, 0x08, 0x0a, 0x0c, 0x0e, 0x10, 0x12, 0x14, 0x16,
0x18, 0x1a, 0x1c, 0x1e, 0x20, 0x22, 0x24, 0x26, 0x28, 0x2a, 0x2c, 0x2e,
0x30, 0x32, 0x34, 0x36, 0x38, 0x3a, 0x3c, 0x3e, 0x40, 0x42, 0x44, 0x46,
0x48, 0x4a, 0x4c, 0x4e, 0x50, 0x52, 0x54, 0x56, 0x58, 0x5a, 0x5c, 0x5e,
0x60, 0x62, 0x64, 0x66, 0x68, 0x6a, 0x6c, 0x6e, 0x70, 0x72, 0x74, 0x76,
0x78, 0x7a, 0x7c, 0x7e, 0x80, 0x82, 0x84, 0x86, 0x88, 0x8a, 0x8c, 0x8e,
0x90, 0x92, 0x94, 0x96, 0x98, 0x9a, 0x9c, 0x9e, 0xa0, 0xa2, 0xa4, 0xa6,
0xa8, 0xaa, 0xac, 0xae, 0xb0, 0xb2, 0xb4, 0xb6, 0xb8, 0xba, 0xbc, 0xbe,
0xc0, 0xc2, 0xc4, 0xc6, 0xc8, 0xca, 0xcc, 0xce, 0xd0, 0xd2, 0xd4, 0xd6,
0xd8, 0xda, 0xdc, 0xde, 0xe0, 0xe2, 0xe4, 0xe6, 0xe8, 0xea, 0xec, 0xee,
0xf0, 0xf2, 0xf4, 0xf6, 0xf8, 0xfa, 0xfc, 0xfe, 0x1b, 0x19, 0x1f, 0x1d,
0x13, 0x11, 0x17, 0x15, 0x0b, 0x09, 0x0f, 0x0d, 0x03, 0x01, 0x07, 0x05,
0x3b, 0x39, 0x3f, 0x3d, 0x33, 0x31, 0x37, 0x35, 0x2b, 0x29, 0x2f, 0x2d,
0x23, 0x21, 0x27, 0x25, 0x5b, 0x59, 0x5f, 0x5d, 0x53, 0x51, 0x57, 0x55,
0x4b, 0x49, 0x4f, 0x4d, 0x43, 0x41, 0x47, 0x45, 0x7b, 0x79, 0x7f, 0x7d,
0x73, 0x71, 0x77, 0x75, 0x6b, 0x69, 0x6f, 0x6d, 0x63, 0x61, 0x67, 0x65,
0x9b, 0x99, 0x9f, 0x9d, 0x93, 0x91, 0x97, 0x95, 0x8b, 0x89, 0x8f, 0x8d,
0x83, 0x81, 0x87, 0x85, 0xbb, 0xb9, 0xbf, 0xbd, 0xb3, 0xb1, 0xb7, 0xb5,
0xab, 0xa9, 0xaf, 0xad, 0xa3, 0xa1, 0xa7, 0xa5, 0xdb, 0xd9, 0xdf, 0xdd,
0xd3, 0xd1, 0xd7, 0xd5, 0xcb, 0xc9, 0xcf, 0xcd, 0xc3, 0xc1, 0xc7, 0xc5,
0xfb, 0xf9, 0xff, 0xfd, 0xf3, 0xf1, 0xf7, 0xf5, 0xeb, 0xe9, 0xef, 0xed,
0xe3, 0xe1, 0xe7, 0xe5,
};
static const uint8_t ffmul03[256] = {
0x00, 0x03, 0x06, 0x05, 0x0c, 0x0f, 0x0a, 0x09, 0x18, 0x1b, 0x1e, 0x1d,
0x14, 0x17, 0x12, 0x11, 0x30, 0x33, 0x36, 0x35, 0x3c, 0x3f, 0x3a, 0x39,
0x28, 0x2b, 0x2e, 0x2d, 0x24, 0x27, 0x22, 0x21, 0x60, 0x63, 0x66, 0x65,
0x6c, 0x6f, 0x6a, 0x69, 0x78, 0x7b, 0x7e, 0x7d, 0x74, 0x77, 0x72, 0x71,
0x50, 0x53, 0x56, 0x55, 0x5c, 0x5f, 0x5a, 0x59, 0x48, 0x4b, 0x4e, 0x4d,
0x44, 0x47, 0x42, 0x41, 0xc0, 0xc3, 0xc6, 0xc5, 0xcc, 0xcf, 0xca, 0xc9,
0xd8, 0xdb, 0xde, 0xdd, 0xd4, 0xd7, 0xd2, 0xd1, 0xf0, 0xf3, 0xf6, 0xf5,
0xfc, 0xff, 0xfa, 0xf9, 0xe8, 0xeb, 0xee, 0xed, 0xe4, 0xe7, 0xe2, 0xe1,
0xa0, 0xa3, 0xa6, 0xa5, 0xac, 0xaf, 0xaa, 0xa9, 0xb8, 0xbb, 0xbe, 0xbd,
0xb4, 0xb7, 0xb2, 0xb1, 0x90, 0x93, 0x96, 0x95, 0x9c, 0x9f, 0x9a, 0x99,
0x88, 0x8b, 0x8e, 0x8d, 0x84, 0x87, 0x82, 0x81, 0x9b, 0x98, 0x9d, 0x9e,
0x97, 0x94, 0x91, 0x92, 0x83, 0x80, 0x85, 0x86, 0x8f, 0x8c, 0x89, 0x8a,
0xab, 0xa8, 0xad, 0xae, 0xa7, 0xa4, 0xa1, 0xa2, 0xb3, 0xb0, 0xb5, 0xb6,
0xbf, 0xbc, 0xb9, 0xba, 0xfb, 0xf8, 0xfd, 0xfe, 0xf7, 0xf4, 0xf1, 0xf2,
0xe3, 0xe0, 0xe5, 0xe6, 0xef, 0xec, 0xe9, 0xea, 0xcb, 0xc8, 0xcd, 0xce,
0xc7, 0xc4, 0xc1, 0xc2, 0xd3, 0xd0, 0xd5, 0xd6, 0xdf, 0xdc, 0xd9, 0xda,
0x5b, 0x58, 0x5d, 0x5e, 0x57, 0x54, 0x51, 0x52, 0x43, 0x40, 0x45, 0x46,
0x4f, 0x4c, 0x49, 0x4a, 0x6b, 0x68, 0x6d, 0x6e, 0x67, 0x64, 0x61, 0x62,
0x73, 0x70, 0x75, 0x76, 0x7f, 0x7c, 0x79, 0x7a, 0x3b, 0x38, 0x3d, 0x3e,
0x37, 0x34, 0x31, 0x32, 0x23, 0x20, 0x25, 0x26, 0x2f, 0x2c, 0x29, 0x2a,
0x0b, 0x08, 0x0d, 0x0e, 0x07, 0x04, 0x01, 0x02, 0x13, 0x10, 0x15, 0x16,
0x1f, 0x1c, 0x19, 0x1a,
};
static const uint8_t ffmul09[256] = {
0x00, 0x09, 0x12, 0x1b, 0x24, 0x2d, 0x36, 0x3f, 0x48, 0x41, 0x5a, 0x53,
0x6c, 0x65, 0x7e, 0x77, 0x90, 0x99, 0x82, 0x8b, 0xb4, 0xbd, 0xa6, 0xaf,
0xd8, 0xd1, 0xca, 0xc3, 0xfc, 0xf5, 0xee, 0xe7, 0x3b, 0x32, 0x29, 0x20,
0x1f, 0x16, 0x0d, 0x04, 0x73, 0x7a, 0x61, 0x68, 0x57, 0x5e, 0x45, 0x4c,
0xab, 0xa2, 0xb9, 0xb0, 0x8f, 0x86, 0x9d, 0x94, 0xe3, 0xea, 0xf1, 0xf8,
0xc7, 0xce, 0xd5, 0xdc, 0x76, 0x7f, 0x64, 0x6d, 0x52, 0x5b, 0x40, 0x49,
0x3e, 0x37, 0x2c, 0x25, 0x1a, 0x13, 0x08, 0x01, 0xe6, 0xef, 0xf4, 0xfd,
0xc2, 0xcb, 0xd0, 0xd9, 0xae, 0xa7, 0xbc, 0xb5, 0x8a, 0x83, 0x98, 0x91,
0x4d, 0x44, 0x5f, 0x56, 0x69, 0x60, 0x7b, 0x72, 0x05, 0x0c, 0x17, 0x1e,
0x21, 0x28, 0x33, 0x3a, 0xdd, 0xd4, 0xcf, 0xc6, 0xf9, 0xf0, 0xeb, 0xe2,
0x95, 0x9c, 0x87, 0x8e, 0xb1, 0xb8, 0xa3, 0xaa, 0xec, 0xe5, 0xfe, 0xf7,
0xc8, 0xc1, 0xda, 0xd3, 0xa4, 0xad, 0xb6, 0xbf, 0x80, 0x89, 0x92, 0x9b,
0x7c, 0x75, 0x6e, 0x67, 0x58, 0x51, 0x4a, 0x43, 0x34, 0x3d, 0x26, 0x2f,
0x10, 0x19, 0x02, 0x0b, 0xd7, 0xde, 0xc5, 0xcc, 0xf3, 0xfa, 0xe1, 0xe8,
0x9f, 0x96, 0x8d, 0x84, 0xbb, 0xb2, 0xa9, 0xa0, 0x47, 0x4e, 0x55, 0x5c,
0x63, 0x6a, 0x71, 0x78, 0x0f, 0x06, 0x1d, 0x14, 0x2b, 0x22, 0x39, 0x30,
0x9a, 0x93, 0x88, 0x81, 0xbe, 0xb7, 0xac, 0xa5, 0xd2, 0xdb, 0xc0, 0xc9,
0xf6, 0xff, 0xe4, 0xed, 0x0a, 0x03, 0x18, 0x11, 0x2e, 0x27, 0x3c, 0x35,
0x42, 0x4b, 0x50, 0x59, 0x66, 0x6f, 0x74, 0x7d, 0xa1, 0xa8, 0xb3, 0xba,
0x85, 0x8c, 0x97, 0x9e, 0xe9, 0xe0, 0xfb, 0xf2, 0xcd, 0xc4, 0xdf, 0xd6,
0x31, 0x38, 0x23, 0x2a, 0x15, 0x1c, 0x07, 0x0e, 0x79, 0x70, 0x6b, 0x62,
0x5d, 0x54, 0x4f, 0x46,
};
static const uint8_t ffmul0b[256] = {
0x00, 0x0b, 0x16, 0x1d, 0x2c, 0x27, 0x3a, 0x31, 0x58, 0x53, 0x4e, 0x45,
0x74, 0x7f, 0x62, 0x69, 0xb0, 0xbb, 0xa6, 0xad, 0x9c, 0x97, 0x8a, 0x81,
0xe8, 0xe3, 0xfe, 0xf5, 0xc4, 0xcf, 0xd2, 0xd9, 0x7b, 0x70, 0x6d, 0x66,
0x57, 0x5c, 0x41, 0x4a, 0x23, 0x28, 0x35, 0x3e, 0x0f, 0x04, 0x19, 0x12,
0xcb, 0xc0, 0xdd, 0xd6, 0xe7, 0xec, 0xf1, 0xfa, 0x93, 0x98, 0x85, 0x8e,
0xbf, 0xb4, 0xa9, 0xa2, 0xf6, 0xfd, 0xe0, 0xeb, 0xda, 0xd1, 0xcc, 0xc7,
0xae, 0xa5, 0xb8, 0xb3, 0x82, 0x89, 0x94, 0x9f, 0x46, 0x4d, 0x50, 0x5b,
0x6a, 0x61, 0x7c, 0x77, 0x1e, 0x15, 0x08, 0x03, 0x32, 0x39, 0x24, 0x2f,
0x8d, 0x86, 0x9b, 0x90, 0xa1, 0xaa, 0xb7, 0xbc, 0xd5, 0xde, 0xc3, 0xc8,
0xf9, 0xf2, 0xef, 0xe4, 0x3d, 0x36, 0x2b, 0x20, 0x11, 0x1a, 0x07, 0x0c,
0x65, 0x6e, 0x73, 0x78, 0x49, 0x42, 0x5f, 0x54, 0xf7, 0xfc, 0xe1, 0xea,
0xdb, 0xd0, 0xcd, 0xc6, 0xaf, 0xa4, 0xb9, 0xb2, 0x83, 0x88, 0x95, 0x9e,
0x47, 0x4c, 0x51, 0x5a, 0x6b, 0x60, 0x7d, 0x76, 0x1f, 0x14, 0x09, 0x02,
0x33, 0x38, 0x25, 0x2e, 0x8c, 0x87, 0x9a, 0x91, 0xa0, 0xab, 0xb6, 0xbd,
0xd4, 0xdf, 0xc2, 0xc9, 0xf8, 0xf3, 0xee, 0xe5, 0x3c, 0x37, 0x2a, 0x21,
0x10, 0x1b, 0x06, 0x0d, 0x64, 0x6f, 0x72, 0x79, 0x48, 0x43, 0x5e, 0x55,
0x01, 0x0a, 0x17, 0x1c, 0x2d, 0x26, 0x3b, 0x30, 0x59, 0x52, 0x4f, 0x44,
0x75, 0x7e, 0x63, 0x68, 0xb1, 0xba, 0xa7, 0xac, 0x9d, 0x96, 0x8b, 0x80,
0xe9, 0xe2, 0xff, 0xf4, 0xc5, 0xce, 0xd3, 0xd8, 0x7a, 0x71, 0x6c, 0x67,
0x56, 0x5d, 0x40, 0x4b, 0x22, 0x29, 0x34, 0x3f, 0x0e, 0x05, 0x18, 0x13,
0xca, 0xc1, 0xdc, 0xd7, 0xe6, 0xed, 0xf0, 0xfb, 0x92, 0x99, 0x84, 0x8f,
0xbe, 0xb5, 0xa8, 0xa3,
};
static const uint8_t ffmul0d[256] = {
0x00, 0x0d, 0x1a, 0x17, 0x34, 0x39, 0x2e, 0x23, 0x68, 0x65, 0x72, 0x7f,
0x5c, 0x51, 0x46, 0x4b, 0xd0, 0xdd, 0xca, 0xc7, 0xe4, 0xe9, 0xfe, 0xf3,
0xb8, 0xb5, 0xa2, 0xaf, 0x8c, 0x81, 0x96, 0x9b, 0xbb, 0xb6, 0xa1, 0xac,
0x8f, 0x82, 0x95, 0x98, 0xd3, 0xde, 0xc9, 0xc4, 0xe7, 0xea, 0xfd, 0xf0,
0x6b, 0x66, 0x71, 0x7c, 0x5f, 0x52, 0x45, 0x48, 0x03, 0x0e, 0x19, 0x14,
0x37, 0x3a, 0x2d, 0x20, 0x6d, 0x60, 0x77, 0x7a, 0x59, 0x54, 0x43, 0x4e,
0x05, 0x08, 0x1f, 0x12, 0x31, 0x3c, 0x2b, 0x26, 0xbd, 0xb0, 0xa7, 0xaa,
0x89, 0x84, 0x93, 0x9e, 0xd5, 0xd8, 0xcf, 0xc2, 0xe1, 0xec, 0xfb, 0xf6,
0xd6, 0xdb, 0xcc, 0xc1, 0xe2, 0xef, 0xf8, 0xf5, 0xbe, 0xb3, 0xa4, 0xa9,
0x8a, 0x87, 0x90, 0x9d, 0x06, 0x0b, 0x1c, 0x11, 0x32, 0x3f, 0x28, 0x25,
0x6e, 0x63, 0x74, 0x79, 0x5a, 0x57, 0x40, 0x4d, 0xda, 0xd7, 0xc0, 0xcd,
0xee, 0xe3, 0xf4, 0xf9, 0xb2, 0xbf, 0xa8, 0xa5, 0x86, 0x8b, 0x9c, 0x91,
0x0a, 0x07, 0x10, 0x1d, 0x3e, 0x33, 0x24, 0x29, 0x62, 0x6f, 0x78, 0x75,
0x56, 0x5b, 0x4c, 0x41, 0x61, 0x6c, 0x7b, 0x76, 0x55, 0x58, 0x4f, 0x42,
0x09, 0x04, 0x13, 0x1e, 0x3d, 0x30, 0x27, 0x2a, 0xb1, 0xbc, 0xab, 0xa6,
0x85, 0x88, 0x9f, 0x92, 0xd9, 0xd4, 0xc3, 0xce, 0xed, 0xe0, 0xf7, 0xfa,
0xb7, 0xba, 0xad, 0xa0, 0x83, 0x8e, 0x99, 0x94, 0xdf, 0xd2, 0xc5, 0xc8,
0xeb, 0xe6, 0xf1, 0xfc, 0x67, 0x6a, 0x7d, 0x70, 0x53, 0x5e, 0x49, 0x44,
0x0f, 0x02, 0x15, 0x18, 0x3b, 0x36, 0x21, 0x2c, 0x0c, 0x01, 0x16, 0x1b,
0x38, 0x35, 0x22, 0x2f, 0x64, 0x69, 0x7e, 0x73, 0x50, 0x5d, 0x4a, 0x47,
0xdc, 0xd1, 0xc6, 0xcb, 0xe8, 0xe5, 0xf2, 0xff, 0xb4, 0xb9, 0xae, 0xa3,
0x80, 0x8d, 0x9a, 0x97,
};
static const uint8_t ffmul0e[256] = {
0x00, 0x0e, 0x1c, 0x12, 0x38, 0x36, 0x24, 0x2a, 0x70, 0x7e, 0x6c, 0x62,
0x48, 0x46, 0x54, 0x5a, 0xe0, 0xee, 0xfc, 0xf2, 0xd8, 0xd6, 0xc4, 0xca,
0x90, 0x9e, 0x8c, 0x82, 0xa8, 0xa6, 0xb4, 0xba, 0xdb, 0xd5, 0xc7, 0xc9,
0xe3, 0xed, 0xff, 0xf1, 0xab, 0xa5, 0xb7, 0xb9, 0x93, 0x9d, 0x8f, 0x81,
0x3b, 0x35, 0x27, 0x29, 0x03, 0x0d, 0x1f, 0x11, 0x4b, 0x45, 0x57, 0x59,
0x73, 0x7d, 0x6f, 0x61, 0xad, 0xa3, 0xb1, 0xbf, 0x95, 0x9b, 0x89, 0x87,
0xdd, 0xd3, 0xc1, 0xcf, 0xe5, 0xeb, 0xf9, 0xf7, 0x4d, 0x43, 0x51, 0x5f,
0x75, 0x7b, 0x69, 0x67, 0x3d, 0x33, 0x21, 0x2f, 0x05, 0x0b, 0x19, 0x17,
0x76, 0x78, 0x6a, 0x64, 0x4e, 0x40, 0x52, 0x5c, 0x06, 0x08, 0x1a, 0x14,
0x3e, 0x30, 0x22, 0x2c, 0x96, 0x98, 0x8a, 0x84, 0xae, 0xa0, 0xb2, 0xbc,
0xe6, 0xe8, 0xfa, 0xf4, 0xde, 0xd0, 0xc2, 0xcc, 0x41, 0x4f, 0x5d, 0x53,
0x79, 0x77, 0x65, 0x6b, 0x31, 0x3f, 0x2d, 0x23, 0x09, 0x07, 0x15, 0x1b,
0xa1, 0xaf, 0xbd, 0xb3, 0x99, 0x97, 0x85, 0x8b, 0xd1, 0xdf, 0xcd, 0xc3,
0xe9, 0xe7, 0xf5, 0xfb, 0x9a, 0x94, 0x86, 0x88, 0xa2, 0xac, 0xbe, 0xb0,
0xea, 0xe4, 0xf6, 0xf8, 0xd2, 0xdc, 0xce, 0xc0, 0x7a, 0x74, 0x66, 0x68,
0x42, 0x4c, 0x5e, 0x50, 0x0a, 0x04, 0x16, 0x18, 0x32, 0x3c, 0x2e, 0x20,
0xec, 0xe2, 0xf0, 0xfe, 0xd4, 0xda, 0xc8, 0xc6, 0x9c, 0x92, 0x80, 0x8e,
0xa4, 0xaa, 0xb8, 0xb6, 0x0c, 0x02, 0x10, 0x1e, 0x34, 0x3a, 0x28, 0x26,
0x7c, 0x72, 0x60, 0x6e, 0x44, 0x4a, 0x58, 0x56, 0x37, 0x39, 0x2b, 0x25,
0x0f, 0x01, 0x13, 0x1d, 0x47, 0x49, 0x5b, 0x55, 0x7f, 0x71, 0x63, 0x6d,
0xd7, 0xd9, 0xcb, 0xc5, 0xef, 0xe1, 0xf3, 0xfd, 0xa7, 0xa9, 0xbb, 0xb5,
0x9f, 0x91, 0x83, 0x8d,
};
x &= 255;
switch (table) {
case 0x2:
return ffmul02[x];
case 0x3:
return ffmul03[x];
case 0x9:
return ffmul09[x];
case 0xb:
return ffmul0b[x];
case 0xd:
return ffmul0d[x];
case 0xe:
return ffmul0e[x];
case 0:
// Case 0 indicates no table lookup, used for some forward mix stages.
return static_cast<uint8_t>(x);
default:
VIXL_UNREACHABLE();
return static_cast<uint8_t>(x);
}
}
static uint8_t AESMixInner(uint64_t* x, int stage, bool inverse) {
VIXL_ASSERT(IsUint2(stage));
int imc_gm[7] = {0xb, 0xd, 0x9, 0xe};
int mc_gm[7] = {0x3, 0x0, 0x0, 0x2};
int* gm = inverse ? imc_gm : mc_gm;
int index = 3 - stage;
uint8_t result = 0;
for (int i = 0; i < 4; i++) {
result ^= GalMul(gm[(index + i) % 4], x[i]);
}
return result;
}
LogicVRegister Simulator::aesmix(LogicVRegister dst,
const LogicVRegister& src,
bool inverse) {
uint64_t in[16] = {};
src.UintArray(kFormat16B, in);
dst.ClearForWrite(kFormat16B);
for (int c = 0; c < 16; c++) {
int cmod4 = c % 4;
int d = c - cmod4;
VIXL_ASSERT((d == 0) || (d == 4) || (d == 8) || (d == 12));
dst.SetUint(kFormat16B, c, AESMixInner(&in[d], cmod4, inverse));
}
return dst;
}
LogicVRegister Simulator::aes(LogicVRegister dst,
const LogicVRegister& src,
bool decrypt) {
dst.ClearForWrite(kFormat16B);
// (Inverse) shift rows.
uint8_t shift[] = {0, 5, 10, 15, 4, 9, 14, 3, 8, 13, 2, 7, 12, 1, 6, 11};
uint8_t shift_inv[] = {0, 13, 10, 7, 4, 1, 14, 11, 8, 5, 2, 15, 12, 9, 6, 3};
for (int i = 0; i < LaneCountFromFormat(kFormat16B); i++) {
uint8_t index = decrypt ? shift_inv[i] : shift[i];
dst.SetUint(kFormat16B, i, src.Uint(kFormat16B, index));
}
// (Inverse) substitute bytes.
static const uint8_t gf2[256] = {
0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b,
0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0,
0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 0xb7, 0xfd, 0x93, 0x26,
0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15,
0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2,
0xeb, 0x27, 0xb2, 0x75, 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0,
0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 0x53, 0xd1, 0x00, 0xed,
0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf,
0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f,
0x50, 0x3c, 0x9f, 0xa8, 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5,
0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 0xcd, 0x0c, 0x13, 0xec,
0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73,
0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14,
0xde, 0x5e, 0x0b, 0xdb, 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c,
0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 0xe7, 0xc8, 0x37, 0x6d,
0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08,
0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f,
0x4b, 0xbd, 0x8b, 0x8a, 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e,
0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 0xe1, 0xf8, 0x98, 0x11,
0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf,
0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f,
0xb0, 0x54, 0xbb, 0x16,
};
static const uint8_t gf2_inv[256] = {
0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e,
0x81, 0xf3, 0xd7, 0xfb, 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87,
0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, 0x54, 0x7b, 0x94, 0x32,
0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e,
0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49,
0x6d, 0x8b, 0xd1, 0x25, 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16,
0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, 0x6c, 0x70, 0x48, 0x50,
0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84,
0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05,
0xb8, 0xb3, 0x45, 0x06, 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02,
0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, 0x3a, 0x91, 0x11, 0x41,
0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73,
0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8,
0x1c, 0x75, 0xdf, 0x6e, 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89,
0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, 0xfc, 0x56, 0x3e, 0x4b,
0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4,
0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59,
0x27, 0x80, 0xec, 0x5f, 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d,
0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, 0xa0, 0xe0, 0x3b, 0x4d,
0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61,
0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63,
0x55, 0x21, 0x0c, 0x7d,
};
for (int i = 0; i < LaneCountFromFormat(kFormat16B); i++) {
const uint8_t* table = decrypt ? gf2_inv : gf2;
dst.SetUint(kFormat16B, i, table[dst.Uint(kFormat16B, i)]);
}
return dst;
}
LogicVRegister Simulator::sm3partw1(LogicVRegister srcdst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
using namespace std::placeholders;
auto ROL = std::bind(RotateLeft, _1, _2, kSRegSize);
SimVRegister temp;
ext(kFormat16B, temp, src2, temp, 4);
rol(kFormat4S, temp, temp, 15);
eor(kFormat4S, temp, temp, src1);
LogicVRegister r = eor(kFormat4S, temp, temp, srcdst);
uint64_t result[4] = {};
r.UintArray(kFormat4S, result);
for (int i = 0; i < 4; i++) {
if (i == 3) {
// result[3] already contains srcdst[3] ^ src1[3] from the operations
// above.
result[i] ^= ROL(result[0], 15);
}
result[i] ^= ROL(result[i], 15) ^ ROL(result[i], 23);
}
srcdst.SetUintArray(kFormat4S, result);
return srcdst;
}
LogicVRegister Simulator::sm3partw2(LogicVRegister srcdst,
const LogicVRegister& src1,
const LogicVRegister& src2) {
using namespace std::placeholders;
auto ROL = std::bind(RotateLeft, _1, _2, kSRegSize);
SimVRegister temp;
VectorFormat vf = kFormat4S;
rol(vf, temp, src2, 7);
LogicVRegister r = eor(vf, temp, temp, src1);
eor(vf, srcdst, temp, srcdst);
uint64_t tmp2 = ROL(r.Uint(vf, 0), 15);
tmp2 ^= ROL(tmp2, 15) ^ ROL(tmp2, 23);
srcdst.SetUint(vf, 3, srcdst.Uint(vf, 3) ^ tmp2);
return srcdst;
}
LogicVRegister Simulator::sm3ss1(LogicVRegister dst,
const LogicVRegister& src1,
const LogicVRegister& src2,
const LogicVRegister& src3) {
using namespace std::placeholders;
auto ROL = std::bind(RotateLeft, _1, _2, kSRegSize);
VectorFormat vf = kFormat4S;
uint64_t result = ROL(src1.Uint(vf, 3), 12);
result += src2.Uint(vf, 3) + src3.Uint(vf, 3);
dst.Clear();
dst.SetUint(vf, 3, ROL(result, 7));
return dst;
}
LogicVRegister Simulator::sm3tt1(LogicVRegister srcdst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index,
bool is_a) {
VectorFormat vf = kFormat4S;
using namespace std::placeholders;
auto ROL = std::bind(RotateLeft, _1, _2, kSRegSize);
auto sd = std::bind(&LogicVRegister::Uint, srcdst, vf, _1);
VIXL_ASSERT(IsUint2(index));
uint64_t wjprime = src2.Uint(vf, index);
uint64_t ss2 = src1.Uint(vf, 3) ^ ROL(sd(3), 12);
uint64_t tt1;
if (is_a) {
tt1 = CryptoOp<"parity"_h>(sd(1), sd(2), sd(3));
} else {
tt1 = CryptoOp<"majority"_h>(sd(1), sd(2), sd(3));
}
tt1 += sd(0) + ss2 + wjprime;
ext(kFormat16B, srcdst, srcdst, srcdst, 4);
srcdst.SetUint(vf, 1, ROL(sd(1), 9));
srcdst.SetUint(vf, 3, tt1);
return srcdst;
}
LogicVRegister Simulator::sm3tt2(LogicVRegister srcdst,
const LogicVRegister& src1,
const LogicVRegister& src2,
int index,
bool is_a) {
VectorFormat vf = kFormat4S;
using namespace std::placeholders;
auto ROL = std::bind(RotateLeft, _1, _2, kSRegSize);
auto sd = std::bind(&LogicVRegister::Uint, srcdst, vf, _1);
VIXL_ASSERT(IsUint2(index));
uint64_t wj = src2.Uint(vf, index);
uint64_t tt2;
if (is_a) {
tt2 = CryptoOp<"parity"_h>(sd(1), sd(2), sd(3));
} else {
tt2 = CryptoOp<"choose"_h>(sd(3), sd(2), sd(1));
}
tt2 += sd(0) + src1.Uint(vf, 3) + wj;
ext(kFormat16B, srcdst, srcdst, srcdst, 4);
srcdst.SetUint(vf, 1, ROL(sd(1), 19));
tt2 ^= ROL(tt2, 9) ^ ROL(tt2, 17);
srcdst.SetUint(vf, 3, tt2);
return srcdst;
}
static uint64_t SM4SBox(uint64_t x) {
static const uint8_t sbox[256] = {
0x48, 0x39, 0xcb, 0xd7, 0x3e, 0x5f, 0xee, 0x79, 0x20, 0x4d, 0xdc, 0x3a,
0xec, 0x7d, 0xf0, 0x18, 0x84, 0xc6, 0x6e, 0xc5, 0x09, 0xf1, 0xb9, 0x65,
0x7e, 0x77, 0x96, 0x0c, 0x4a, 0x97, 0x69, 0x89, 0xb0, 0xb4, 0xe5, 0xb8,
0x12, 0xd0, 0x74, 0x2d, 0xbd, 0x7b, 0xcd, 0xa5, 0x88, 0x31, 0xc1, 0x0a,
0xd8, 0x5a, 0x10, 0x1f, 0x41, 0x5c, 0xd9, 0x11, 0x7f, 0xbc, 0xdd, 0xbb,
0x92, 0xaf, 0x1b, 0x8d, 0x51, 0x5b, 0x6c, 0x6d, 0x72, 0x6a, 0xff, 0x03,
0x2f, 0x8e, 0xfd, 0xde, 0x45, 0x37, 0xdb, 0xd5, 0x6f, 0x4e, 0x53, 0x0d,
0xab, 0x23, 0x29, 0xc0, 0x60, 0xca, 0x66, 0x82, 0x2e, 0xe2, 0xf6, 0x1d,
0xe3, 0xb1, 0x8c, 0xf5, 0x30, 0x32, 0x93, 0xad, 0x55, 0x1a, 0x34, 0x9b,
0xa4, 0x5d, 0xae, 0xe0, 0xa1, 0x15, 0x61, 0xf9, 0xce, 0xf2, 0xf7, 0xa3,
0xb5, 0x38, 0xc7, 0x40, 0xd2, 0x8a, 0xbf, 0xea, 0x9e, 0xc8, 0xc4, 0xa0,
0xe7, 0x02, 0x36, 0x4c, 0x52, 0x27, 0xd3, 0x9f, 0x57, 0x46, 0x00, 0xd4,
0x87, 0x78, 0x21, 0x01, 0x3b, 0x7c, 0x22, 0x25, 0xa2, 0xd1, 0x58, 0x63,
0x5e, 0x0e, 0x24, 0x1e, 0x35, 0x9d, 0x56, 0x70, 0x4b, 0x0f, 0xeb, 0xf8,
0x8b, 0xda, 0x64, 0x71, 0xb2, 0x81, 0x6b, 0x68, 0xa8, 0x4f, 0x85, 0xe6,
0x19, 0x3c, 0x59, 0x83, 0xba, 0x17, 0x73, 0xf3, 0xfc, 0xa7, 0x07, 0x47,
0xa6, 0x3f, 0x8f, 0x75, 0xfa, 0x94, 0xdf, 0x80, 0x95, 0xe8, 0x08, 0xc9,
0xa9, 0x1c, 0xb3, 0xe4, 0x62, 0xac, 0xcf, 0xed, 0x43, 0x0b, 0x54, 0x33,
0x7a, 0x98, 0xef, 0x91, 0xf4, 0x50, 0x42, 0x9c, 0x99, 0x06, 0x86, 0x49,
0x26, 0x13, 0x44, 0xaa, 0xc3, 0x04, 0xbe, 0x2a, 0x76, 0x9a, 0x67, 0x2b,
0x05, 0x2c, 0xfb, 0x28, 0xc2, 0x14, 0xb6, 0x16, 0xb7, 0x3d, 0xe1, 0xcc,
0xfe, 0xe9, 0x90, 0xd6,
};
uint64_t result = 0;
for (int j = 24; j >= 0; j -= 8) {
uint8_t s = 255 - ((x >> j) & 0xff);
result = (result << 8) | sbox[s];
}
return result;
}
LogicVRegister Simulator::sm4(LogicVRegister srcdst,
const LogicVRegister& src1,
const LogicVRegister& src2,
bool is_key) {
using namespace std::placeholders;
auto ROL = std::bind(RotateLeft, _1, _2, kSRegSize);
VectorFormat vf = kFormat4S;
uint64_t result[4] = {};
if (is_key) {
src1.UintArray(vf, result);
} else {
srcdst.UintArray(vf, result);
}
for (int i = 0; i < 4; i++) {
uint64_t k = is_key ? src2.Uint(vf, i) : src1.Uint(vf, i);
uint64_t intval = result[3] ^ result[2] ^ result[1] ^ k;
intval = SM4SBox(intval);
if (is_key) {
intval ^= ROL(intval, 13) ^ ROL(intval, 23);
} else {
intval ^=
ROL(intval, 2) ^ ROL(intval, 10) ^ ROL(intval, 18) ^ ROL(intval, 24);
}
intval ^= result[0];
result[0] = result[1];
result[1] = result[2];
result[2] = result[3];
result[3] = intval;
}
srcdst.SetUintArray(vf, result);
return srcdst;
}
} // namespace aarch64
} // namespace vixl
#endif // VIXL_INCLUDE_SIMULATOR_AARCH64