mirror of
https://github.com/lifting-bits/remill
synced 2026-06-21 13:56:07 +00:00
Peter Goodman
99df2e19d4
* Running clang-format on files with some additional custom scripts for my style * Fix missing unique_ptr in remill/BC/Optimizer.h * Fixes and selective disabling of clang-format
882 lines
28 KiB
C++
882 lines
28 KiB
C++
/*
|
|
* Copyright (c) 2017 Trail of Bits, Inc.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
namespace {
|
|
|
|
template <typename S>
|
|
DEF_SEM(ORR_Vec, V128W dst, S src1, S src2) {
|
|
UWriteV64(dst, UOrV64(UReadV64(src1), UReadV64(src2)));
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(AND_Vec, V128W dst, S src1, S src2) {
|
|
UWriteV64(dst, UAndV64(UReadV64(src1), UReadV64(src2)));
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(BIC_Vec, V128W dst, S src1, S src2) {
|
|
UWriteV64(dst, UAndV64(UReadV64(src1), UNotV64(UReadV64(src2))));
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(EOR_Vec, V128W dst, S src1, S src2) {
|
|
auto operand4 = UReadV64(src1);
|
|
auto operand1 = UReadV64(src2);
|
|
auto operand2 = UClearV64(operand4);
|
|
auto operand3 = UNotV64(operand2);
|
|
UWriteV64(dst,
|
|
UXorV64(operand1, UAndV64(UXorV64(operand2, operand4), operand3)));
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(BIT_Vec, V128W dst, S dst_src, S src1, S src2) {
|
|
auto operand4 = UReadV64(src1);
|
|
auto operand1 = UReadV64(dst_src);
|
|
auto operand3 = UReadV64(src2);
|
|
UWriteV64(dst,
|
|
UXorV64(operand1, UAndV64(UXorV64(operand1, operand4), operand3)));
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(BIF_Vec, V128W dst, S dst_src, S src1, S src2) {
|
|
auto operand4 = UReadV64(src1);
|
|
auto operand1 = UReadV64(dst_src);
|
|
auto operand3 = UNotV64(UReadV64(src2));
|
|
UWriteV64(dst,
|
|
UXorV64(operand1, UAndV64(UXorV64(operand1, operand4), operand3)));
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(BSL_Vec, V128W dst, S dst_src, S src1, S src2) {
|
|
auto operand4 = UReadV64(src1);
|
|
auto operand1 = UReadV64(src2);
|
|
auto operand3 = UReadV64(dst_src);
|
|
UWriteV64(dst,
|
|
UXorV64(operand1, UAndV64(UXorV64(operand1, operand4), operand3)));
|
|
return memory;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
DEF_ISEL(ORR_ASIMDSAME_ONLY_8B) = ORR_Vec<V64>;
|
|
DEF_ISEL(ORR_ASIMDSAME_ONLY_16B) = ORR_Vec<V128>;
|
|
|
|
DEF_ISEL(AND_ASIMDSAME_ONLY_8B) = AND_Vec<V64>;
|
|
DEF_ISEL(AND_ASIMDSAME_ONLY_16B) = AND_Vec<V128>;
|
|
|
|
DEF_ISEL(BIC_ASIMDSAME_ONLY_8B) = BIC_Vec<V64>;
|
|
DEF_ISEL(BIC_ASIMDSAME_ONLY_16B) = BIC_Vec<V128>;
|
|
|
|
DEF_ISEL(EOR_ASIMDSAME_ONLY_8B) = EOR_Vec<V64>;
|
|
DEF_ISEL(EOR_ASIMDSAME_ONLY_16B) = EOR_Vec<V128>;
|
|
|
|
DEF_ISEL(BIT_ASIMDSAME_ONLY_8B) = BIT_Vec<V64>;
|
|
DEF_ISEL(BIT_ASIMDSAME_ONLY_16B) = BIT_Vec<V128>;
|
|
|
|
DEF_ISEL(BIF_ASIMDSAME_ONLY_8B) = BIF_Vec<V64>;
|
|
DEF_ISEL(BIF_ASIMDSAME_ONLY_16B) = BIF_Vec<V128>;
|
|
|
|
DEF_ISEL(BSL_ASIMDSAME_ONLY_8B) = BSL_Vec<V64>;
|
|
DEF_ISEL(BSL_ASIMDSAME_ONLY_16B) = BSL_Vec<V128>;
|
|
|
|
namespace {
|
|
|
|
DEF_SEM(FMOV_VectorToUInt64, R64W dst, V128 src) {
|
|
auto val = UExtractV64(UReadV64(src), 1);
|
|
WriteZExt(dst, val);
|
|
return memory;
|
|
}
|
|
|
|
DEF_SEM(FMOV_UInt64ToVector, V128W dst, R64 src) {
|
|
auto val = Read(src);
|
|
uint64v2_t temp_vec = {};
|
|
temp_vec = UInsertV64(temp_vec, 0, UExtractV64(UReadV64(dst), 0));
|
|
temp_vec = UInsertV64(temp_vec, 1, val);
|
|
UWriteV64(dst, temp_vec);
|
|
return memory;
|
|
}
|
|
} // namespace
|
|
|
|
DEF_ISEL(FMOV_64VX_FLOAT2INT) = FMOV_VectorToUInt64;
|
|
DEF_ISEL(FMOV_V64I_FLOAT2INT) = FMOV_UInt64ToVector;
|
|
|
|
namespace {
|
|
|
|
#define MAKE_DUP(size) \
|
|
template <typename V> \
|
|
DEF_SEM(DUP_##size, V128W dst, R64 src) { \
|
|
auto val = TruncTo<uint##size##_t>(Read(src)); \
|
|
V vec = {}; \
|
|
_Pragma("unroll") for (auto &element : vec.elems) { \
|
|
element = val; \
|
|
} \
|
|
UWriteV##size(dst, vec); \
|
|
return memory; \
|
|
}
|
|
|
|
MAKE_DUP(8)
|
|
MAKE_DUP(16)
|
|
MAKE_DUP(32)
|
|
MAKE_DUP(64)
|
|
|
|
#undef MAKE_DUP
|
|
|
|
} // namespace
|
|
|
|
DEF_ISEL(DUP_ASIMDINS_DR_R_8B) = DUP_8<uint8v8_t>;
|
|
DEF_ISEL(DUP_ASIMDINS_DR_R_16B) = DUP_8<uint8v16_t>;
|
|
DEF_ISEL(DUP_ASIMDINS_DR_R_4H) = DUP_16<uint16v4_t>;
|
|
DEF_ISEL(DUP_ASIMDINS_DR_R_8H) = DUP_16<uint16v8_t>;
|
|
DEF_ISEL(DUP_ASIMDINS_DR_R_2S) = DUP_32<uint32v2_t>;
|
|
DEF_ISEL(DUP_ASIMDINS_DR_R_4S) = DUP_32<uint32v4_t>;
|
|
DEF_ISEL(DUP_ASIMDINS_DR_R_2D) = DUP_64<uint64v2_t>;
|
|
|
|
namespace {
|
|
|
|
template <typename T>
|
|
ALWAYS_INLINE static T UMin(T lhs, T rhs) {
|
|
return lhs < rhs ? lhs : rhs;
|
|
}
|
|
|
|
template <typename T>
|
|
ALWAYS_INLINE static T UMax(T lhs, T rhs) {
|
|
return lhs < rhs ? rhs : lhs;
|
|
}
|
|
|
|
#define SMin UMin
|
|
#define SMax UMax
|
|
|
|
#define MAKE_BROADCAST(op, prefix, binop, size) \
|
|
template <typename S, typename V> \
|
|
DEF_SEM(op##_##size, V128W dst, S src1, S src2) { \
|
|
auto vec1 = prefix##ReadV##size(src1); \
|
|
auto vec2 = prefix##ReadV##size(src2); \
|
|
V sum = {}; \
|
|
_Pragma("unroll") for (size_t i = 0, max_i = NumVectorElems(sum); \
|
|
i < max_i; ++i) { \
|
|
sum.elems[i] = prefix##binop(prefix##ExtractV##size(vec1, i), \
|
|
prefix##ExtractV##size(vec2, i)); \
|
|
} \
|
|
prefix##WriteV##size(dst, sum); \
|
|
return memory; \
|
|
}
|
|
|
|
MAKE_BROADCAST(ADD, U, Add, 8)
|
|
MAKE_BROADCAST(ADD, U, Add, 16)
|
|
MAKE_BROADCAST(ADD, U, Add, 32)
|
|
MAKE_BROADCAST(ADD, U, Add, 64)
|
|
|
|
MAKE_BROADCAST(SUB, U, Sub, 8)
|
|
MAKE_BROADCAST(SUB, U, Sub, 16)
|
|
MAKE_BROADCAST(SUB, U, Sub, 32)
|
|
MAKE_BROADCAST(SUB, U, Sub, 64)
|
|
|
|
MAKE_BROADCAST(UMIN, U, Min, 8)
|
|
MAKE_BROADCAST(UMIN, U, Min, 16)
|
|
MAKE_BROADCAST(UMIN, U, Min, 32)
|
|
|
|
MAKE_BROADCAST(SMIN, S, Min, 8)
|
|
MAKE_BROADCAST(SMIN, S, Min, 16)
|
|
MAKE_BROADCAST(SMIN, S, Min, 32)
|
|
|
|
MAKE_BROADCAST(UMAX, U, Max, 8)
|
|
MAKE_BROADCAST(UMAX, U, Max, 16)
|
|
MAKE_BROADCAST(UMAX, U, Max, 32)
|
|
|
|
MAKE_BROADCAST(SMAX, S, Max, 8)
|
|
MAKE_BROADCAST(SMAX, S, Max, 16)
|
|
MAKE_BROADCAST(SMAX, S, Max, 32)
|
|
|
|
#undef MAKE_BROADCAST
|
|
|
|
} // namespace
|
|
|
|
DEF_ISEL(ADD_ASIMDSAME_ONLY_8B) = ADD_8<V64, uint8v8_t>;
|
|
DEF_ISEL(ADD_ASIMDSAME_ONLY_16B) = ADD_8<V128, uint8v16_t>;
|
|
DEF_ISEL(ADD_ASIMDSAME_ONLY_4H) = ADD_16<V64, uint16v4_t>;
|
|
DEF_ISEL(ADD_ASIMDSAME_ONLY_8H) = ADD_16<V128, uint16v8_t>;
|
|
DEF_ISEL(ADD_ASIMDSAME_ONLY_2S) = ADD_32<V64, uint32v2_t>;
|
|
DEF_ISEL(ADD_ASIMDSAME_ONLY_4S) = ADD_32<V128, uint32v4_t>;
|
|
DEF_ISEL(ADD_ASIMDSAME_ONLY_2D) = ADD_64<V128, uint64v2_t>;
|
|
|
|
DEF_ISEL(SUB_ASIMDSAME_ONLY_8B) = SUB_8<V64, uint8v8_t>;
|
|
DEF_ISEL(SUB_ASIMDSAME_ONLY_16B) = SUB_8<V128, uint8v16_t>;
|
|
DEF_ISEL(SUB_ASIMDSAME_ONLY_4H) = SUB_16<V64, uint16v4_t>;
|
|
DEF_ISEL(SUB_ASIMDSAME_ONLY_8H) = SUB_16<V128, uint16v8_t>;
|
|
DEF_ISEL(SUB_ASIMDSAME_ONLY_2S) = SUB_32<V64, uint32v2_t>;
|
|
DEF_ISEL(SUB_ASIMDSAME_ONLY_4S) = SUB_32<V128, uint32v4_t>;
|
|
DEF_ISEL(SUB_ASIMDSAME_ONLY_2D) = SUB_64<V128, uint64v2_t>;
|
|
|
|
DEF_ISEL(UMIN_ASIMDSAME_ONLY_8B) = UMIN_8<V64, uint8v8_t>;
|
|
DEF_ISEL(UMIN_ASIMDSAME_ONLY_16B) = UMIN_8<V128, uint8v16_t>;
|
|
DEF_ISEL(UMIN_ASIMDSAME_ONLY_4H) = UMIN_16<V64, uint16v4_t>;
|
|
DEF_ISEL(UMIN_ASIMDSAME_ONLY_8H) = UMIN_16<V128, uint16v8_t>;
|
|
DEF_ISEL(UMIN_ASIMDSAME_ONLY_2S) = UMIN_32<V64, uint32v2_t>;
|
|
DEF_ISEL(UMIN_ASIMDSAME_ONLY_4S) = UMIN_32<V128, uint32v4_t>;
|
|
|
|
DEF_ISEL(UMAX_ASIMDSAME_ONLY_8B) = UMAX_8<V64, uint8v8_t>;
|
|
DEF_ISEL(UMAX_ASIMDSAME_ONLY_16B) = UMAX_8<V128, uint8v16_t>;
|
|
DEF_ISEL(UMAX_ASIMDSAME_ONLY_4H) = UMAX_16<V64, uint16v4_t>;
|
|
DEF_ISEL(UMAX_ASIMDSAME_ONLY_8H) = UMAX_16<V128, uint16v8_t>;
|
|
DEF_ISEL(UMAX_ASIMDSAME_ONLY_2S) = UMAX_32<V64, uint32v2_t>;
|
|
DEF_ISEL(UMAX_ASIMDSAME_ONLY_4S) = UMAX_32<V128, uint32v4_t>;
|
|
|
|
DEF_ISEL(SMIN_ASIMDSAME_ONLY_8B) = SMIN_8<V64, int8v8_t>;
|
|
DEF_ISEL(SMIN_ASIMDSAME_ONLY_16B) = SMIN_8<V128, int8v16_t>;
|
|
DEF_ISEL(SMIN_ASIMDSAME_ONLY_4H) = SMIN_16<V64, int16v4_t>;
|
|
DEF_ISEL(SMIN_ASIMDSAME_ONLY_8H) = SMIN_16<V128, int16v8_t>;
|
|
DEF_ISEL(SMIN_ASIMDSAME_ONLY_2S) = SMIN_32<V64, int32v2_t>;
|
|
DEF_ISEL(SMIN_ASIMDSAME_ONLY_4S) = SMIN_32<V128, int32v4_t>;
|
|
|
|
DEF_ISEL(SMAX_ASIMDSAME_ONLY_8B) = SMAX_8<V64, int8v8_t>;
|
|
DEF_ISEL(SMAX_ASIMDSAME_ONLY_16B) = SMAX_8<V128, int8v16_t>;
|
|
DEF_ISEL(SMAX_ASIMDSAME_ONLY_4H) = SMAX_16<V64, int16v4_t>;
|
|
DEF_ISEL(SMAX_ASIMDSAME_ONLY_8H) = SMAX_16<V128, int16v8_t>;
|
|
DEF_ISEL(SMAX_ASIMDSAME_ONLY_2S) = SMAX_32<V64, int32v2_t>;
|
|
DEF_ISEL(SMAX_ASIMDSAME_ONLY_4S) = SMAX_32<V128, int32v4_t>;
|
|
|
|
namespace {
|
|
|
|
#define MAKE_CMP_BROADCAST(op, prefix, binop, size) \
|
|
template <typename S, typename V> \
|
|
DEF_SEM(op##_##size, V128W dst, S src1, I##size imm) { \
|
|
auto vec1 = prefix##ReadV##size(src1); \
|
|
auto ucmp_val = Read(imm); \
|
|
auto cmp_val = Signed(ucmp_val); \
|
|
decltype(ucmp_val) zeros = 0; \
|
|
decltype(ucmp_val) ones = ~zeros; \
|
|
V res = {}; \
|
|
_Pragma("unroll") for (size_t i = 0, max_i = NumVectorElems(res); \
|
|
i < max_i; ++i) { \
|
|
res.elems[i] = \
|
|
Select(prefix##binop(prefix##ExtractV##size(vec1, i), cmp_val), \
|
|
ones, zeros); \
|
|
} \
|
|
UWriteV##size(dst, res); \
|
|
return memory; \
|
|
}
|
|
|
|
MAKE_CMP_BROADCAST(CMPEQ_IMM, S, CmpEq, 8)
|
|
MAKE_CMP_BROADCAST(CMPEQ_IMM, S, CmpEq, 16)
|
|
MAKE_CMP_BROADCAST(CMPEQ_IMM, S, CmpEq, 32)
|
|
MAKE_CMP_BROADCAST(CMPEQ_IMM, S, CmpEq, 64)
|
|
|
|
MAKE_CMP_BROADCAST(CMPLT_IMM, S, CmpLt, 8)
|
|
MAKE_CMP_BROADCAST(CMPLT_IMM, S, CmpLt, 16)
|
|
MAKE_CMP_BROADCAST(CMPLT_IMM, S, CmpLt, 32)
|
|
MAKE_CMP_BROADCAST(CMPLT_IMM, S, CmpLt, 64)
|
|
|
|
MAKE_CMP_BROADCAST(CMPLE_IMM, S, CmpLte, 8)
|
|
MAKE_CMP_BROADCAST(CMPLE_IMM, S, CmpLte, 16)
|
|
MAKE_CMP_BROADCAST(CMPLE_IMM, S, CmpLte, 32)
|
|
MAKE_CMP_BROADCAST(CMPLE_IMM, S, CmpLte, 64)
|
|
|
|
MAKE_CMP_BROADCAST(CMPGT_IMM, S, CmpGt, 8)
|
|
MAKE_CMP_BROADCAST(CMPGT_IMM, S, CmpGt, 16)
|
|
MAKE_CMP_BROADCAST(CMPGT_IMM, S, CmpGt, 32)
|
|
MAKE_CMP_BROADCAST(CMPGT_IMM, S, CmpGt, 64)
|
|
|
|
MAKE_CMP_BROADCAST(CMPGE_IMM, S, CmpGte, 8)
|
|
MAKE_CMP_BROADCAST(CMPGE_IMM, S, CmpGte, 16)
|
|
MAKE_CMP_BROADCAST(CMPGE_IMM, S, CmpGte, 32)
|
|
MAKE_CMP_BROADCAST(CMPGE_IMM, S, CmpGte, 64)
|
|
|
|
#undef MAKE_CMP_BROADCAST
|
|
|
|
} // namespace
|
|
|
|
DEF_ISEL(CMEQ_ASIMDMISC_Z_8B) = CMPEQ_IMM_8<V64, uint8v8_t>;
|
|
DEF_ISEL(CMLT_ASIMDMISC_Z_8B) = CMPLT_IMM_8<V64, uint8v8_t>;
|
|
DEF_ISEL(CMLE_ASIMDMISC_Z_8B) = CMPLE_IMM_8<V64, uint8v8_t>;
|
|
DEF_ISEL(CMGT_ASIMDMISC_Z_8B) = CMPGT_IMM_8<V64, uint8v8_t>;
|
|
DEF_ISEL(CMGE_ASIMDMISC_Z_8B) = CMPGE_IMM_8<V64, uint8v8_t>;
|
|
|
|
DEF_ISEL(CMEQ_ASIMDMISC_Z_16B) = CMPEQ_IMM_8<V128, uint8v16_t>;
|
|
DEF_ISEL(CMLT_ASIMDMISC_Z_16B) = CMPLT_IMM_8<V128, uint8v16_t>;
|
|
DEF_ISEL(CMLE_ASIMDMISC_Z_16B) = CMPLE_IMM_8<V128, uint8v16_t>;
|
|
DEF_ISEL(CMGT_ASIMDMISC_Z_16B) = CMPGT_IMM_8<V128, uint8v16_t>;
|
|
DEF_ISEL(CMGE_ASIMDMISC_Z_16B) = CMPGE_IMM_8<V128, uint8v16_t>;
|
|
|
|
DEF_ISEL(CMEQ_ASIMDMISC_Z_4H) = CMPEQ_IMM_16<V64, uint16v4_t>;
|
|
DEF_ISEL(CMLT_ASIMDMISC_Z_4H) = CMPLT_IMM_16<V64, uint16v4_t>;
|
|
DEF_ISEL(CMLE_ASIMDMISC_Z_4H) = CMPLE_IMM_16<V64, uint16v4_t>;
|
|
DEF_ISEL(CMGT_ASIMDMISC_Z_4H) = CMPGT_IMM_16<V64, uint16v4_t>;
|
|
DEF_ISEL(CMGE_ASIMDMISC_Z_4H) = CMPGE_IMM_16<V64, uint16v4_t>;
|
|
|
|
DEF_ISEL(CMEQ_ASIMDMISC_Z_8H) = CMPEQ_IMM_16<V128, uint16v8_t>;
|
|
DEF_ISEL(CMLT_ASIMDMISC_Z_8H) = CMPLT_IMM_16<V128, uint16v8_t>;
|
|
DEF_ISEL(CMLE_ASIMDMISC_Z_8H) = CMPLE_IMM_16<V128, uint16v8_t>;
|
|
DEF_ISEL(CMGT_ASIMDMISC_Z_8H) = CMPGT_IMM_16<V128, uint16v8_t>;
|
|
DEF_ISEL(CMGE_ASIMDMISC_Z_8H) = CMPGE_IMM_16<V128, uint16v8_t>;
|
|
|
|
DEF_ISEL(CMEQ_ASIMDMISC_Z_2S) = CMPEQ_IMM_32<V64, uint32v2_t>;
|
|
DEF_ISEL(CMLT_ASIMDMISC_Z_2S) = CMPLT_IMM_32<V64, uint32v2_t>;
|
|
DEF_ISEL(CMLE_ASIMDMISC_Z_2S) = CMPLE_IMM_32<V64, uint32v2_t>;
|
|
DEF_ISEL(CMGT_ASIMDMISC_Z_2S) = CMPGT_IMM_32<V64, uint32v2_t>;
|
|
DEF_ISEL(CMGE_ASIMDMISC_Z_2S) = CMPGE_IMM_32<V64, uint32v2_t>;
|
|
|
|
DEF_ISEL(CMEQ_ASIMDMISC_Z_4S) = CMPEQ_IMM_32<V128, uint32v4_t>;
|
|
DEF_ISEL(CMLT_ASIMDMISC_Z_4S) = CMPLT_IMM_32<V128, uint32v4_t>;
|
|
DEF_ISEL(CMLE_ASIMDMISC_Z_4S) = CMPLE_IMM_32<V128, uint32v4_t>;
|
|
DEF_ISEL(CMGT_ASIMDMISC_Z_4S) = CMPGT_IMM_32<V128, uint32v4_t>;
|
|
DEF_ISEL(CMGE_ASIMDMISC_Z_4S) = CMPGE_IMM_32<V128, uint32v4_t>;
|
|
|
|
DEF_ISEL(CMEQ_ASIMDMISC_Z_1D) = CMPEQ_IMM_64<V64, uint64v1_t>;
|
|
DEF_ISEL(CMLT_ASIMDMISC_Z_1D) = CMPLT_IMM_64<V64, uint64v1_t>;
|
|
DEF_ISEL(CMLE_ASIMDMISC_Z_1D) = CMPLE_IMM_64<V64, uint64v1_t>;
|
|
DEF_ISEL(CMGT_ASIMDMISC_Z_1D) = CMPGT_IMM_64<V64, uint64v1_t>;
|
|
DEF_ISEL(CMGE_ASIMDMISC_Z_1D) = CMPGE_IMM_64<V64, uint64v1_t>;
|
|
|
|
DEF_ISEL(CMEQ_ASIMDMISC_Z_2D) = CMPEQ_IMM_64<V128, uint64v2_t>;
|
|
DEF_ISEL(CMLT_ASIMDMISC_Z_2D) = CMPLT_IMM_64<V128, uint64v2_t>;
|
|
DEF_ISEL(CMLE_ASIMDMISC_Z_2D) = CMPLE_IMM_64<V128, uint64v2_t>;
|
|
DEF_ISEL(CMGT_ASIMDMISC_Z_2D) = CMPGT_IMM_64<V128, uint64v2_t>;
|
|
DEF_ISEL(CMGE_ASIMDMISC_Z_2D) = CMPGE_IMM_64<V128, uint64v2_t>;
|
|
|
|
namespace {
|
|
|
|
#define MAKE_CMP_BROADCAST(op, prefix, binop, size) \
|
|
template <typename S, typename V> \
|
|
DEF_SEM(op##_##size, V128W dst, S src1, S src2) { \
|
|
auto vec1 = prefix##ReadV##size(src1); \
|
|
auto vec2 = prefix##ReadV##size(src2); \
|
|
uint##size##_t zeros = 0; \
|
|
uint##size##_t ones = ~zeros; \
|
|
V res = {}; \
|
|
_Pragma("unroll") for (size_t i = 0, max_i = NumVectorElems(res); \
|
|
i < max_i; ++i) { \
|
|
res.elems[i] = Select(prefix##binop(prefix##ExtractV##size(vec1, i), \
|
|
prefix##ExtractV##size(vec2, i)), \
|
|
ones, zeros); \
|
|
} \
|
|
UWriteV##size(dst, res); \
|
|
return memory; \
|
|
}
|
|
|
|
template <typename T>
|
|
ALWAYS_INLINE static bool UCmpTst(T lhs, T rhs) {
|
|
return UCmpNeq(UAnd(lhs, rhs), T(0));
|
|
}
|
|
|
|
MAKE_CMP_BROADCAST(CMPEQ, S, CmpEq, 8)
|
|
MAKE_CMP_BROADCAST(CMPEQ, S, CmpEq, 16)
|
|
MAKE_CMP_BROADCAST(CMPEQ, S, CmpEq, 32)
|
|
MAKE_CMP_BROADCAST(CMPEQ, S, CmpEq, 64)
|
|
|
|
MAKE_CMP_BROADCAST(CMPTST, U, CmpTst, 8)
|
|
MAKE_CMP_BROADCAST(CMPTST, U, CmpTst, 16)
|
|
MAKE_CMP_BROADCAST(CMPTST, U, CmpTst, 32)
|
|
MAKE_CMP_BROADCAST(CMPTST, U, CmpTst, 64)
|
|
|
|
MAKE_CMP_BROADCAST(CMPGT, S, CmpGt, 8)
|
|
MAKE_CMP_BROADCAST(CMPGT, S, CmpGt, 16)
|
|
MAKE_CMP_BROADCAST(CMPGT, S, CmpGt, 32)
|
|
MAKE_CMP_BROADCAST(CMPGT, S, CmpGt, 64)
|
|
|
|
MAKE_CMP_BROADCAST(CMPGE, S, CmpGte, 8)
|
|
MAKE_CMP_BROADCAST(CMPGE, S, CmpGte, 16)
|
|
MAKE_CMP_BROADCAST(CMPGE, S, CmpGte, 32)
|
|
MAKE_CMP_BROADCAST(CMPGE, S, CmpGte, 64)
|
|
|
|
#undef MAKE_CMP_BROADCAST
|
|
|
|
} // namespace
|
|
|
|
DEF_ISEL(CMEQ_ASIMDSAME_ONLY_8B) = CMPEQ_8<V64, uint8v8_t>;
|
|
DEF_ISEL(CMGT_ASIMDSAME_ONLY_8B) = CMPGT_8<V64, uint8v8_t>;
|
|
DEF_ISEL(CMGE_ASIMDSAME_ONLY_8B) = CMPGE_8<V64, uint8v8_t>;
|
|
DEF_ISEL(CMTST_ASIMDSAME_ONLY_8B) = CMPTST_8<V64, uint8v8_t>;
|
|
|
|
DEF_ISEL(CMEQ_ASIMDSAME_ONLY_16B) = CMPEQ_8<V128, uint8v16_t>;
|
|
DEF_ISEL(CMGT_ASIMDSAME_ONLY_16B) = CMPGT_8<V128, uint8v16_t>;
|
|
DEF_ISEL(CMGE_ASIMDSAME_ONLY_16B) = CMPGE_8<V128, uint8v16_t>;
|
|
DEF_ISEL(CMTST_ASIMDSAME_ONLY_16B) = CMPTST_8<V128, uint8v16_t>;
|
|
|
|
DEF_ISEL(CMEQ_ASIMDSAME_ONLY_4H) = CMPEQ_16<V64, uint16v4_t>;
|
|
DEF_ISEL(CMGT_ASIMDSAME_ONLY_4H) = CMPGT_16<V64, uint16v4_t>;
|
|
DEF_ISEL(CMGE_ASIMDSAME_ONLY_4H) = CMPGE_16<V64, uint16v4_t>;
|
|
DEF_ISEL(CMTST_ASIMDSAME_ONLY_4H) = CMPTST_16<V64, uint16v4_t>;
|
|
|
|
DEF_ISEL(CMEQ_ASIMDSAME_ONLY_8H) = CMPEQ_16<V128, uint16v8_t>;
|
|
DEF_ISEL(CMGT_ASIMDSAME_ONLY_8H) = CMPGT_16<V128, uint16v8_t>;
|
|
DEF_ISEL(CMGE_ASIMDSAME_ONLY_8H) = CMPGE_16<V128, uint16v8_t>;
|
|
DEF_ISEL(CMTST_ASIMDSAME_ONLY_8H) = CMPTST_16<V128, uint16v8_t>;
|
|
|
|
DEF_ISEL(CMEQ_ASIMDSAME_ONLY_2S) = CMPEQ_32<V64, uint32v2_t>;
|
|
DEF_ISEL(CMGT_ASIMDSAME_ONLY_2S) = CMPGT_32<V64, uint32v2_t>;
|
|
DEF_ISEL(CMGE_ASIMDSAME_ONLY_2S) = CMPGE_32<V64, uint32v2_t>;
|
|
DEF_ISEL(CMTST_ASIMDSAME_ONLY_2S) = CMPTST_32<V64, uint32v2_t>;
|
|
|
|
DEF_ISEL(CMEQ_ASIMDSAME_ONLY_4S) = CMPEQ_32<V128, uint32v4_t>;
|
|
DEF_ISEL(CMGT_ASIMDSAME_ONLY_4S) = CMPGT_32<V128, uint32v4_t>;
|
|
DEF_ISEL(CMGE_ASIMDSAME_ONLY_4S) = CMPGE_32<V128, uint32v4_t>;
|
|
DEF_ISEL(CMTST_ASIMDSAME_ONLY_4S) = CMPTST_32<V128, uint32v4_t>;
|
|
|
|
DEF_ISEL(CMEQ_ASIMDSAME_ONLY_2D) = CMPEQ_64<V128, uint64v2_t>;
|
|
DEF_ISEL(CMGT_ASIMDSAME_ONLY_2D) = CMPGT_64<V128, uint64v2_t>;
|
|
DEF_ISEL(CMGE_ASIMDSAME_ONLY_2D) = CMPGE_64<V128, uint64v2_t>;
|
|
DEF_ISEL(CMTST_ASIMDSAME_ONLY_2D) = CMPTST_64<V128, uint64v2_t>;
|
|
|
|
namespace {
|
|
|
|
#define MAKE_PAIRWAISE_BROADCAST(op, prefix, binop, size) \
|
|
template <typename S, typename V> \
|
|
DEF_SEM(op##_##size, V128W dst, S src1, S src2) { \
|
|
auto vec1 = prefix##ReadV##size(src1); \
|
|
auto vec2 = prefix##ReadV##size(src2); \
|
|
V res = {}; \
|
|
size_t max_i = NumVectorElems(res); \
|
|
size_t j = 0; \
|
|
_Pragma("unroll") for (size_t i = 0; i < max_i; i += 2) { \
|
|
res.elems[j++] = prefix##binop(prefix##ExtractV##size(vec1, i), \
|
|
prefix##ExtractV##size(vec1, i + 1)); \
|
|
} \
|
|
_Pragma("unroll") for (size_t i = 0; i < max_i; i += 2) { \
|
|
res.elems[j++] = prefix##binop(prefix##ExtractV##size(vec2, i), \
|
|
prefix##ExtractV##size(vec2, i + 1)); \
|
|
} \
|
|
prefix##WriteV##size(dst, res); \
|
|
return memory; \
|
|
}
|
|
|
|
MAKE_PAIRWAISE_BROADCAST(ADDP, U, Add, 8)
|
|
MAKE_PAIRWAISE_BROADCAST(ADDP, U, Add, 16)
|
|
MAKE_PAIRWAISE_BROADCAST(ADDP, U, Add, 32)
|
|
MAKE_PAIRWAISE_BROADCAST(ADDP, U, Add, 64)
|
|
|
|
MAKE_PAIRWAISE_BROADCAST(UMAXP, U, Max, 8)
|
|
MAKE_PAIRWAISE_BROADCAST(UMAXP, U, Max, 16)
|
|
MAKE_PAIRWAISE_BROADCAST(UMAXP, U, Max, 32)
|
|
|
|
MAKE_PAIRWAISE_BROADCAST(SMAXP, S, Max, 8)
|
|
MAKE_PAIRWAISE_BROADCAST(SMAXP, S, Max, 16)
|
|
MAKE_PAIRWAISE_BROADCAST(SMAXP, S, Max, 32)
|
|
|
|
MAKE_PAIRWAISE_BROADCAST(UMINP, U, Min, 8)
|
|
MAKE_PAIRWAISE_BROADCAST(UMINP, U, Min, 16)
|
|
MAKE_PAIRWAISE_BROADCAST(UMINP, U, Min, 32)
|
|
|
|
MAKE_PAIRWAISE_BROADCAST(SMINP, S, Min, 8)
|
|
MAKE_PAIRWAISE_BROADCAST(SMINP, S, Min, 16)
|
|
MAKE_PAIRWAISE_BROADCAST(SMINP, S, Min, 32)
|
|
|
|
#undef MAKE_PAIRWAISE_BROADCAST
|
|
|
|
} // namespace
|
|
|
|
DEF_ISEL(ADDP_ASIMDSAME_ONLY_8B) = ADDP_8<V64, uint8v8_t>;
|
|
DEF_ISEL(ADDP_ASIMDSAME_ONLY_16B) = ADDP_8<V128, uint8v16_t>;
|
|
DEF_ISEL(ADDP_ASIMDSAME_ONLY_4H) = ADDP_16<V64, uint16v4_t>;
|
|
DEF_ISEL(ADDP_ASIMDSAME_ONLY_8H) = ADDP_16<V128, uint16v8_t>;
|
|
DEF_ISEL(ADDP_ASIMDSAME_ONLY_2S) = ADDP_32<V64, uint32v2_t>;
|
|
DEF_ISEL(ADDP_ASIMDSAME_ONLY_4S) = ADDP_32<V128, uint32v4_t>;
|
|
DEF_ISEL(ADDP_ASIMDSAME_ONLY_2D) = ADDP_64<V128, uint64v2_t>;
|
|
|
|
DEF_ISEL(UMINP_ASIMDSAME_ONLY_8B) = UMINP_8<V64, uint8v8_t>;
|
|
DEF_ISEL(UMINP_ASIMDSAME_ONLY_16B) = UMINP_8<V128, uint8v16_t>;
|
|
DEF_ISEL(UMINP_ASIMDSAME_ONLY_4H) = UMINP_16<V64, uint16v4_t>;
|
|
DEF_ISEL(UMINP_ASIMDSAME_ONLY_8H) = UMINP_16<V128, uint16v8_t>;
|
|
DEF_ISEL(UMINP_ASIMDSAME_ONLY_2S) = UMINP_32<V64, uint32v2_t>;
|
|
DEF_ISEL(UMINP_ASIMDSAME_ONLY_4S) = UMINP_32<V128, uint32v4_t>;
|
|
|
|
DEF_ISEL(UMAXP_ASIMDSAME_ONLY_8B) = UMAXP_8<V64, uint8v8_t>;
|
|
DEF_ISEL(UMAXP_ASIMDSAME_ONLY_16B) = UMAXP_8<V128, uint8v16_t>;
|
|
DEF_ISEL(UMAXP_ASIMDSAME_ONLY_4H) = UMAXP_16<V64, uint16v4_t>;
|
|
DEF_ISEL(UMAXP_ASIMDSAME_ONLY_8H) = UMAXP_16<V128, uint16v8_t>;
|
|
DEF_ISEL(UMAXP_ASIMDSAME_ONLY_2S) = UMAXP_32<V64, uint32v2_t>;
|
|
DEF_ISEL(UMAXP_ASIMDSAME_ONLY_4S) = UMAXP_32<V128, uint32v4_t>;
|
|
|
|
DEF_ISEL(SMINP_ASIMDSAME_ONLY_8B) = SMINP_8<V64, int8v8_t>;
|
|
DEF_ISEL(SMINP_ASIMDSAME_ONLY_16B) = SMINP_8<V128, int8v16_t>;
|
|
DEF_ISEL(SMINP_ASIMDSAME_ONLY_4H) = SMINP_16<V64, int16v4_t>;
|
|
DEF_ISEL(SMINP_ASIMDSAME_ONLY_8H) = SMINP_16<V128, int16v8_t>;
|
|
DEF_ISEL(SMINP_ASIMDSAME_ONLY_2S) = SMINP_32<V64, int32v2_t>;
|
|
DEF_ISEL(SMINP_ASIMDSAME_ONLY_4S) = SMINP_32<V128, int32v4_t>;
|
|
|
|
DEF_ISEL(SMAXP_ASIMDSAME_ONLY_8B) = SMAXP_8<V64, int8v8_t>;
|
|
DEF_ISEL(SMAXP_ASIMDSAME_ONLY_16B) = SMAXP_8<V128, int8v16_t>;
|
|
DEF_ISEL(SMAXP_ASIMDSAME_ONLY_4H) = SMAXP_16<V64, int16v4_t>;
|
|
DEF_ISEL(SMAXP_ASIMDSAME_ONLY_8H) = SMAXP_16<V128, int16v8_t>;
|
|
DEF_ISEL(SMAXP_ASIMDSAME_ONLY_2S) = SMAXP_32<V64, int32v2_t>;
|
|
DEF_ISEL(SMAXP_ASIMDSAME_ONLY_4S) = SMAXP_32<V128, int32v4_t>;
|
|
|
|
namespace {
|
|
|
|
template <typename V, typename B>
|
|
ALWAYS_INLINE static auto Reduce2(const V &vec, B binop, size_t base = 0)
|
|
-> decltype(binop(vec.elems[0], vec.elems[1])) {
|
|
return binop(vec.elems[base + 0], vec.elems[base + 1]);
|
|
}
|
|
|
|
template <typename V, typename B>
|
|
ALWAYS_INLINE static auto Reduce4(const V &vec, B binop, size_t base = 0)
|
|
-> decltype(binop(vec.elems[0], vec.elems[1])) {
|
|
auto lo = Reduce2(vec, binop, base + 0);
|
|
auto hi = Reduce2(vec, binop, base + 2);
|
|
return binop(lo, hi);
|
|
}
|
|
|
|
template <typename V, typename B>
|
|
ALWAYS_INLINE static auto Reduce8(const V &vec, B binop, size_t base = 0)
|
|
-> decltype(binop(vec.elems[0], vec.elems[1])) {
|
|
auto lo = Reduce4(vec, binop, base + 0);
|
|
auto hi = Reduce4(vec, binop, base + 4);
|
|
return binop(lo, hi);
|
|
}
|
|
|
|
template <typename V, typename B>
|
|
ALWAYS_INLINE static auto Reduce16(const V &vec, B binop, size_t base = 0)
|
|
-> decltype(binop(vec.elems[0], vec.elems[1])) {
|
|
auto lo = Reduce8(vec, binop, base + 0);
|
|
auto hi = Reduce8(vec, binop, base + 8);
|
|
return binop(lo, hi);
|
|
}
|
|
|
|
template <typename V, typename B>
|
|
ALWAYS_INLINE static auto Reduce(const V &vec, B binop)
|
|
-> decltype(Reduce2(vec, binop)) {
|
|
switch (NumVectorElems(vec)) {
|
|
case 2: return Reduce2(vec, binop);
|
|
case 4: return Reduce4(vec, binop);
|
|
case 8: return Reduce8(vec, binop);
|
|
case 16: return Reduce16(vec, binop);
|
|
default: __builtin_unreachable();
|
|
}
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(ADDV_8_Reduce, V128W dst, S src) {
|
|
auto vec = SReadV8(src);
|
|
UWriteV8(dst, Unsigned(Reduce(vec, SAdd8)));
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(ADDV_16_Reduce, V128W dst, S src) {
|
|
auto vec = SReadV16(src);
|
|
UWriteV16(dst, Unsigned(Reduce(vec, SAdd16)));
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(ADDV_32_Reduce, V128W dst, S src) {
|
|
auto vec = SReadV32(src);
|
|
UWriteV32(dst, Unsigned(Reduce(vec, SAdd32)));
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(UMINV_8, V128W dst, S src) {
|
|
auto vec = UReadV8(src);
|
|
auto val = std::numeric_limits<uint8_t>::max();
|
|
_Pragma("unroll") for (auto elem : vec.elems) {
|
|
val = UMin(elem, val);
|
|
}
|
|
UWriteV8(dst, val);
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(UMINV_16, V128W dst, S src) {
|
|
auto vec = UReadV16(src);
|
|
auto val = std::numeric_limits<uint16_t>::max();
|
|
_Pragma("unroll") for (auto elem : vec.elems) {
|
|
val = UMin(elem, val);
|
|
}
|
|
UWriteV16(dst, val);
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(UMINV_32, V128W dst, S src) {
|
|
auto vec = UReadV32(src);
|
|
auto val = std::numeric_limits<uint32_t>::max();
|
|
_Pragma("unroll") for (auto elem : vec.elems) {
|
|
val = UMin(elem, val);
|
|
}
|
|
UWriteV32(dst, val);
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(SMINV_8, V128W dst, S src) {
|
|
auto vec = SReadV8(src);
|
|
auto val = std::numeric_limits<int8_t>::max();
|
|
_Pragma("unroll") for (auto elem : vec.elems) {
|
|
val = SMin(elem, val);
|
|
}
|
|
SWriteV8(dst, val);
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(SMINV_16, V128W dst, S src) {
|
|
auto vec = SReadV16(src);
|
|
auto val = std::numeric_limits<int16_t>::max();
|
|
_Pragma("unroll") for (auto elem : vec.elems) {
|
|
val = SMin(elem, val);
|
|
}
|
|
SWriteV16(dst, val);
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(SMINV_32, V128W dst, S src) {
|
|
auto vec = SReadV32(src);
|
|
auto val = std::numeric_limits<int32_t>::max();
|
|
_Pragma("unroll") for (auto elem : vec.elems) {
|
|
val = SMin(elem, val);
|
|
}
|
|
SWriteV32(dst, val);
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(UMAXV_8, V128W dst, S src) {
|
|
auto vec = UReadV8(src);
|
|
auto val = std::numeric_limits<uint8_t>::min();
|
|
_Pragma("unroll") for (auto elem : vec.elems) {
|
|
val = UMax(elem, val);
|
|
}
|
|
UWriteV8(dst, val);
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(UMAXV_16, V128W dst, S src) {
|
|
auto vec = UReadV16(src);
|
|
auto val = std::numeric_limits<uint16_t>::min();
|
|
_Pragma("unroll") for (auto elem : vec.elems) {
|
|
val = UMax(elem, val);
|
|
}
|
|
UWriteV16(dst, val);
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(UMAXV_32, V128W dst, S src) {
|
|
auto vec = UReadV32(src);
|
|
auto val = std::numeric_limits<uint32_t>::min();
|
|
_Pragma("unroll") for (auto elem : vec.elems) {
|
|
val = UMax(elem, val);
|
|
}
|
|
UWriteV32(dst, val);
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(SMAXV_8, V128W dst, S src) {
|
|
auto vec = SReadV8(src);
|
|
auto val = std::numeric_limits<int8_t>::min();
|
|
_Pragma("unroll") for (auto elem : vec.elems) {
|
|
val = SMax(elem, val);
|
|
}
|
|
SWriteV8(dst, val);
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(SMAXV_16, V128W dst, S src) {
|
|
auto vec = SReadV16(src);
|
|
auto val = std::numeric_limits<int16_t>::min();
|
|
_Pragma("unroll") for (auto elem : vec.elems) {
|
|
val = SMax(elem, val);
|
|
}
|
|
SWriteV16(dst, val);
|
|
return memory;
|
|
}
|
|
|
|
template <typename S>
|
|
DEF_SEM(SMAXV_32, V128W dst, S src) {
|
|
auto vec = SReadV32(src);
|
|
auto val = std::numeric_limits<int32_t>::min();
|
|
_Pragma("unroll") for (auto elem : vec.elems) {
|
|
val = SMax(elem, val);
|
|
}
|
|
SWriteV32(dst, val);
|
|
return memory;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
DEF_ISEL(ADDV_ASIMDALL_ONLY_8B) = ADDV_8_Reduce<V64>;
|
|
DEF_ISEL(ADDV_ASIMDALL_ONLY_16B) = ADDV_8_Reduce<V128>;
|
|
DEF_ISEL(ADDV_ASIMDALL_ONLY_4H) = ADDV_16_Reduce<V64>;
|
|
DEF_ISEL(ADDV_ASIMDALL_ONLY_8H) = ADDV_16_Reduce<V128>;
|
|
DEF_ISEL(ADDV_ASIMDALL_ONLY_4S) = ADDV_32_Reduce<V128>;
|
|
|
|
DEF_ISEL(UMINV_ASIMDALL_ONLY_8B) = UMINV_8<V64>;
|
|
DEF_ISEL(UMINV_ASIMDALL_ONLY_16B) = UMINV_8<V128>;
|
|
DEF_ISEL(UMINV_ASIMDALL_ONLY_4H) = UMINV_16<V64>;
|
|
DEF_ISEL(UMINV_ASIMDALL_ONLY_8H) = UMINV_16<V128>;
|
|
DEF_ISEL(UMINV_ASIMDALL_ONLY_4S) = UMINV_32<V128>;
|
|
|
|
DEF_ISEL(SMINV_ASIMDALL_ONLY_8B) = SMINV_8<V64>;
|
|
DEF_ISEL(SMINV_ASIMDALL_ONLY_16B) = SMINV_8<V128>;
|
|
DEF_ISEL(SMINV_ASIMDALL_ONLY_4H) = SMINV_16<V64>;
|
|
DEF_ISEL(SMINV_ASIMDALL_ONLY_8H) = SMINV_16<V128>;
|
|
DEF_ISEL(SMINV_ASIMDALL_ONLY_4S) = SMINV_32<V128>;
|
|
|
|
DEF_ISEL(UMAXV_ASIMDALL_ONLY_8B) = UMAXV_8<V64>;
|
|
DEF_ISEL(UMAXV_ASIMDALL_ONLY_16B) = UMAXV_8<V128>;
|
|
DEF_ISEL(UMAXV_ASIMDALL_ONLY_4H) = UMAXV_16<V64>;
|
|
DEF_ISEL(UMAXV_ASIMDALL_ONLY_8H) = UMAXV_16<V128>;
|
|
DEF_ISEL(UMAXV_ASIMDALL_ONLY_4S) = UMAXV_32<V128>;
|
|
|
|
DEF_ISEL(SMAXV_ASIMDALL_ONLY_8B) = SMAXV_8<V64>;
|
|
DEF_ISEL(SMAXV_ASIMDALL_ONLY_16B) = SMAXV_8<V128>;
|
|
DEF_ISEL(SMAXV_ASIMDALL_ONLY_4H) = SMAXV_16<V64>;
|
|
DEF_ISEL(SMAXV_ASIMDALL_ONLY_8H) = SMAXV_16<V128>;
|
|
DEF_ISEL(SMAXV_ASIMDALL_ONLY_4S) = SMAXV_32<V128>;
|
|
|
|
namespace {
|
|
|
|
template <typename T, typename I>
|
|
ALWAYS_INLINE static T FloatMin(T lhs, T rhs) {
|
|
if (__builtin_isunordered(lhs, rhs)) {
|
|
return NAN;
|
|
} else if (__builtin_isless(lhs, rhs)) {
|
|
return lhs;
|
|
} else {
|
|
return rhs;
|
|
}
|
|
|
|
// if (lhs < rhs) {
|
|
// return lhs;
|
|
//
|
|
// } else if (lhs > rhs) {
|
|
// return rhs;
|
|
//
|
|
// // Use integer comparisons; we need to return the "most negative" value
|
|
// // (e.g. in the case of +0 and -0).
|
|
// } else {
|
|
// auto a = reinterpret_cast<I &>(lhs);
|
|
// auto b = reinterpret_cast<I &>(rhs);
|
|
// auto res = SMin(a, b);
|
|
// return reinterpret_cast<T &>(res);
|
|
// }
|
|
}
|
|
|
|
template <typename T, typename I>
|
|
ALWAYS_INLINE static T FloatMax(T lhs, T rhs) {
|
|
if (__builtin_isunordered(lhs, rhs)) {
|
|
return NAN;
|
|
} else if (__builtin_isgreater(lhs, rhs)) {
|
|
return lhs;
|
|
} else {
|
|
return rhs;
|
|
}
|
|
//
|
|
// if (lhs < rhs) {
|
|
// return rhs;
|
|
//
|
|
// } else if (lhs > rhs) {
|
|
// return lhs;
|
|
//
|
|
// // Use integer comparisons; we need to return the "most negative" value
|
|
// // (e.g. in the case of +0 and -0).
|
|
// } else {
|
|
// auto a = reinterpret_cast<I &>(lhs);
|
|
// auto b = reinterpret_cast<I &>(rhs);
|
|
// auto res = SMax(a, b);
|
|
// return reinterpret_cast<T &>(res);
|
|
// }
|
|
}
|
|
|
|
// NOTE(pag): These aren't quite right w.r.t. NaN propagation.
|
|
DEF_SEM(FMINV_32_Reduce, V128W dst, V128 src) {
|
|
auto vec = FReadV32(src);
|
|
FWriteV32(dst, Reduce4(vec, FloatMin<float32_t, int32_t>));
|
|
return memory;
|
|
}
|
|
|
|
DEF_SEM(FMAXV_32_Reduce, V128W dst, V128 src) {
|
|
auto vec = FReadV32(src);
|
|
FWriteV32(dst, Reduce4(vec, FloatMax<float32_t, int32_t>));
|
|
return memory;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
DEF_ISEL(FMINV_ASIMDALL_ONLY_SD_4S) = FMINV_32_Reduce;
|
|
DEF_ISEL(FMAXV_ASIMDALL_ONLY_SD_4S) = FMAXV_32_Reduce;
|
|
|
|
namespace {
|
|
|
|
template <typename S>
|
|
DEF_SEM(NOT_8, V128W dst, S src) {
|
|
auto vec = UReadV8(src);
|
|
auto res = UNotV8(vec);
|
|
UWriteV8(dst, res);
|
|
return memory;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
DEF_ISEL(NOT_ASIMDMISC_R_8B) = NOT_8<V64>;
|
|
DEF_ISEL(NOT_ASIMDMISC_R_16B) = NOT_8<V128>;
|
|
|
|
namespace {
|
|
|
|
template <typename T, size_t count>
|
|
DEF_SEM(EXT, V128W dst, T src1, T src2, I32 src3) {
|
|
auto lsb = Read(src3);
|
|
auto vn = UReadV8(src1);
|
|
auto vm = UReadV8(src2);
|
|
uint8v16_t result = {};
|
|
_Pragma("unroll") for (size_t i = 0, max_i = count; i + lsb < max_i; ++i) {
|
|
result.elems[count - 1 - i] = UExtractV8(vm, i + lsb);
|
|
}
|
|
_Pragma("unroll") for (size_t i = lsb; i < count; ++i) {
|
|
result.elems[count - 1 - i] = UExtractV8(vn, i - lsb);
|
|
}
|
|
UWriteV8(dst, result);
|
|
return memory;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
DEF_ISEL(EXT_ASIMDEXT_ONLY_8B) = EXT<V64, 8>;
|
|
DEF_ISEL(EXT_ASIMDEXT_ONLY_16B) = EXT<V128, 16>;
|
|
|
|
|
|
// TODO(pag):
|
|
// FMINV_ASIMDALL_ONLY_H
|
|
// FMAXV_ASIMDALL_ONLY_H
|
|
// FMINNMV_ASIMDALL_ONLY_H
|
|
// FMINNMV_ASIMDALL_ONLY_SD
|
|
// FMAXNMV_ASIMDALL_ONLY_H
|
|
// FMAXNMV_ASIMDALL_ONLY_SD
|
|
|
|
DEF_SEM(USHR_64B, V128W dst, V128W src, I64 shift) {
|
|
auto vec = UExtractV64(UReadV64(src), 0);
|
|
auto sft = Read(shift);
|
|
auto shifted = UShr128(vec, sft);
|
|
uint64v2_t temp_vec = {};
|
|
temp_vec = UInsertV64(temp_vec, 1, 0);
|
|
temp_vec = UInsertV64(temp_vec, 0, (uint64_t) shifted);
|
|
UWriteV64(dst, temp_vec);
|
|
return memory;
|
|
}
|
|
|
|
DEF_ISEL(USHR_ASISDSHF_R) = USHR_64B;
|