mirror of
https://github.com/asmjit/asmjit
synced 2026-06-08 13:13:30 +00:00
kobalicek
b56f4176cb
* Denested src folder to root, renamed testing to asmjit-testing
* Refactored how headers are included into <asmjit/...> form. This
is necessary as compilers would never simplify a path once a ..
appears in include directory - then paths such as ../core/../core
appeared in asserts, which was ugly
* Moved support utilities into asmjit/support/... (still included
by asmjit/core.h for convenience and compatibility)
* Added CMakePresets.json for making it easy to develop AsmJit
* Reworked CMakeLists to be shorter and use CMake option(),
etc... This simplifies it and makes it using more standard
features
* ASMJIT_EMBED now creates asmjit_embed INTERFACE library,
which is accessible via asmjit::asmjit target - this simplifies
embedding and makes it the same as library targets from a CMake
perspective
* Removed ASMJIT_DEPS - this is now provided by cmake target
aliases - 'asmjit::asmjit' so users should not need this variable
* Changed meaning of ASMJIT_LIBS - this now contains only AsmJit
dependencies without asmjit::asmjit target alias. Don't rely on
ASMJIT_LIBS anymore as it's only used internally
* Removed ASMJIT_NO_DEPRECATED option - AsmJit is not going
to provide controllable deprecations in the future
* Removed ASMJIT_NO_VALIDATION in favor of ASMJIT_NO_INTROSPECTION,
which now controls query, features, and validation API presence
* Removed ASMJIT_DIR option - it was never really needed
* Removed AMX_TRANSPOSE feature from instruction database (X86).
Intel has removed it as well, so it's a feature that won't
be siliconized
4392 lines
205 KiB
C++
4392 lines
205 KiB
C++
// This file is part of AsmJit project <https://asmjit.com>
|
|
//
|
|
// See <asmjit/core.h> or LICENSE.md for license and copyright information
|
|
// SPDX-License-Identifier: Zlib
|
|
|
|
#include <asmjit/core/api-build_p.h>
|
|
#include <asmjit/ujit/ujitbase.h>
|
|
|
|
#if defined(ASMJIT_UJIT_AARCH64)
|
|
|
|
#include <asmjit/ujit/unicompiler.h>
|
|
#include <asmjit/ujit/unicompiler_utils_p.h>
|
|
#include <asmjit/ujit/unicondition.h>
|
|
|
|
ASMJIT_BEGIN_SUB_NAMESPACE(ujit)
|
|
|
|
using GPExt = UniCompiler::GPExt;
|
|
using ASIMDExt = UniCompiler::ASIMDExt;
|
|
namespace Inst { using namespace a64::Inst; }
|
|
|
|
// ujit::UniCompiler - Construction & Destruction
|
|
// ==============================================
|
|
|
|
UniCompiler::UniCompiler(BackendCompiler* cc, const CpuFeatures& features, CpuHints cpu_hints, VecConstTableRef ct_ref) noexcept
|
|
: cc(cc),
|
|
_ct_ref(ct_ref),
|
|
_features(features),
|
|
_cpu_hints(cpu_hints),
|
|
_vec_reg_count(32),
|
|
_common_table_offset(0) {
|
|
|
|
_scalar_op_behavior = ScalarOpBehavior::kZeroing;
|
|
_fmin_fmax_op_behavior = FMinFMaxOpBehavior::kFiniteValue;
|
|
_fmadd_op_behavior = FMAddOpBehavior::kFMAStoreToAccumulator;
|
|
_float_to_int_outside_range_behavior = FloatToIntOutsideRangeBehavior::kSaturatedValue;
|
|
|
|
_init_extensions(features);
|
|
}
|
|
|
|
UniCompiler::~UniCompiler() noexcept {}
|
|
|
|
// ujit::UniCompiler - CPU Architecture, Features and Optimization Options
|
|
// =======================================================================
|
|
|
|
void UniCompiler::_init_extensions(const CpuFeatures& features) noexcept {
|
|
uint64_t gp_ext_mask = 0;
|
|
uint64_t asimd_ext_mask = 0;
|
|
|
|
if (features.arm().has_cssc() ) gp_ext_mask |= uint64_t(1) << uint32_t(GPExt::kCSSC);
|
|
if (features.arm().has_flagm() ) gp_ext_mask |= uint64_t(1) << uint32_t(GPExt::kFLAGM);
|
|
if (features.arm().has_flagm2() ) gp_ext_mask |= uint64_t(1) << uint32_t(GPExt::kFLAGM2);
|
|
if (features.arm().has_ls64() ) gp_ext_mask |= uint64_t(1) << uint32_t(GPExt::kLS64);
|
|
if (features.arm().has_ls64_v() ) gp_ext_mask |= uint64_t(1) << uint32_t(GPExt::kLS64_V);
|
|
if (features.arm().has_lse() ) gp_ext_mask |= uint64_t(1) << uint32_t(GPExt::kLSE);
|
|
if (features.arm().has_lse128() ) gp_ext_mask |= uint64_t(1) << uint32_t(GPExt::kLSE128);
|
|
if (features.arm().has_lse2() ) gp_ext_mask |= uint64_t(1) << uint32_t(GPExt::kLSE2);
|
|
|
|
if (features.arm().has_asimd() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kASIMD);
|
|
if (features.arm().has_bf16() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kBF16);
|
|
if (features.arm().has_dotprod() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kDOTPROD);
|
|
if (features.arm().has_fcma() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kFCMA);
|
|
if (features.arm().has_fhm() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kFHM);
|
|
if (features.arm().has_fp16() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kFP16);
|
|
if (features.arm().has_fp16conv()) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kFP16CONV);
|
|
if (features.arm().has_fp8() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kFP8);
|
|
if (features.arm().has_frintts() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kFRINTTS);
|
|
if (features.arm().has_i8mm() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kI8MM);
|
|
if (features.arm().has_jscvt() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kJSCVT);
|
|
if (features.arm().has_pmull() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kPMULL);
|
|
if (features.arm().has_rdm() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kRDM);
|
|
if (features.arm().has_sha1() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kSHA1);
|
|
if (features.arm().has_sha256() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kSHA256);
|
|
if (features.arm().has_sha3() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kSHA3);
|
|
if (features.arm().has_sha512() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kSHA512);
|
|
if (features.arm().has_sm3() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kSM3);
|
|
if (features.arm().has_sm4() ) asimd_ext_mask |= uint64_t(1) << uint32_t(ASIMDExt::kSM4);
|
|
|
|
_gp_ext_mask = gp_ext_mask;
|
|
_asimd_ext_mask = asimd_ext_mask;
|
|
}
|
|
|
|
VecWidth UniCompiler::max_vec_width_from_cpu_features() noexcept {
|
|
return VecWidth::k128;
|
|
}
|
|
|
|
void UniCompiler::init_vec_width(VecWidth vw) noexcept {
|
|
ASMJIT_ASSERT(vw == VecWidth::k128);
|
|
Support::maybe_unused(vw);
|
|
|
|
_vec_width = VecWidth::k128;
|
|
_vec_reg_type = RegType::kVec128;
|
|
_vec_type_id = TypeId::kInt32x4;
|
|
_vec_multiplier = 1u;
|
|
}
|
|
|
|
bool UniCompiler::has_masked_access_of(uint32_t data_size) const noexcept {
|
|
switch (data_size) {
|
|
case 1: return has_cpu_hint(CpuHints::kVecMaskedOps8);
|
|
case 2: return has_cpu_hint(CpuHints::kVecMaskedOps16);
|
|
case 4: return has_cpu_hint(CpuHints::kVecMaskedOps32);
|
|
case 8: return has_cpu_hint(CpuHints::kVecMaskedOps64);
|
|
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// ujit::UniCompiler - Embed
|
|
// =========================
|
|
|
|
void UniCompiler::embed_jump_table(Span<const Label> jump_table, const Label& jump_table_base, uint32_t entry_size) {
|
|
static const uint8_t zeros[8] {};
|
|
|
|
for (const Label& label : jump_table) {
|
|
if (label.is_valid()) {
|
|
cc->embed_label_delta(label, jump_table_base, entry_size);
|
|
}
|
|
else {
|
|
cc->embed(zeros, entry_size);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ujit::UniCompiler - Function
|
|
// ============================
|
|
|
|
void UniCompiler::hook_func() noexcept {
|
|
FuncNode* func = cc->func();
|
|
_func_init_hook = func;
|
|
}
|
|
|
|
void UniCompiler::unhook_func() noexcept {
|
|
_func_init_hook = nullptr;
|
|
}
|
|
|
|
// ujit::UniCompiler - Constants
|
|
// =============================
|
|
|
|
void UniCompiler::_init_vec_const_table_ptr() {
|
|
const void* global = ct_ptr<void>();
|
|
|
|
if (!_common_table_ptr.is_valid()) {
|
|
ScopedInjector injector(cc, &_func_init_hook);
|
|
_common_table_ptr = new_gpz("common_table_ptr");
|
|
cc->mov(_common_table_ptr, (int64_t)global + _common_table_offset);
|
|
}
|
|
}
|
|
|
|
Operand UniCompiler::simd_const(const void* c, Bcst bcst_width, VecWidth const_width) {
|
|
return simd_vec_const(c, bcst_width, const_width);
|
|
}
|
|
|
|
Operand UniCompiler::simd_const(const void* c, Bcst bcst_width, const Vec& similar_to) {
|
|
Support::maybe_unused(similar_to);
|
|
return simd_vec_const(c, bcst_width, VecWidth::k128);
|
|
}
|
|
|
|
Operand UniCompiler::simd_const(const void* c, Bcst bcst_width, const VecArray& similar_to) {
|
|
ASMJIT_ASSERT(!similar_to.is_empty());
|
|
Support::maybe_unused(bcst_width, similar_to);
|
|
|
|
return simd_vec_const(c, bcst_width, VecWidth::k128);
|
|
}
|
|
|
|
Vec UniCompiler::simd_vec_const(const void* c, Bcst bcst_width, VecWidth const_width) {
|
|
Support::maybe_unused(bcst_width);
|
|
Support::maybe_unused(const_width);
|
|
|
|
size_t n = _vec_consts.size();
|
|
for (size_t i = 0; i < n; i++) {
|
|
if (_vec_consts[i].ptr == c) {
|
|
return Vec(OperandSignature{RegTraits<RegType::kVec128>::kSignature}, _vec_consts[i].virt_reg_id);
|
|
}
|
|
}
|
|
|
|
return Vec(OperandSignature{RegTraits<RegType::kVec128>::kSignature}, _new_vec_const(c, true).id());
|
|
}
|
|
|
|
Vec UniCompiler::simd_vec_const(const void* c, Bcst bcst_width, const Vec& similar_to) {
|
|
Support::maybe_unused(similar_to);
|
|
return simd_vec_const(c, bcst_width, VecWidth::k128);
|
|
}
|
|
|
|
Vec UniCompiler::simd_vec_const(const void* c, Bcst bcst_width, const VecArray& similar_to) {
|
|
Support::maybe_unused(similar_to);
|
|
return simd_vec_const(c, bcst_width, VecWidth::k128);
|
|
}
|
|
|
|
Mem UniCompiler::simd_mem_const(const void* c, Bcst bcst_width, VecWidth const_width) {
|
|
Support::maybe_unused(bcst_width, const_width);
|
|
return _get_mem_const(c);
|
|
}
|
|
|
|
Mem UniCompiler::simd_mem_const(const void* c, Bcst bcst_width, const Vec& similar_to) {
|
|
Support::maybe_unused(bcst_width, similar_to);
|
|
return _get_mem_const(c);
|
|
}
|
|
|
|
Mem UniCompiler::simd_mem_const(const void* c, Bcst bcst_width, const VecArray& similar_to) {
|
|
Support::maybe_unused(bcst_width, similar_to);
|
|
return _get_mem_const(c);
|
|
}
|
|
|
|
Mem UniCompiler::_get_mem_const(const void* c) {
|
|
// Make sure we are addressing a constant from the `ct` constant pool.
|
|
const void* ct_addr = ct_ptr<void>();
|
|
ASMJIT_ASSERT((uintptr_t)c >= (uintptr_t)ct_addr &&
|
|
(uintptr_t)c < (uintptr_t)ct_addr + _ct_ref.size);
|
|
|
|
// One GP register is sacrificed to hold the pointer to the `ct`.
|
|
_init_vec_const_table_ptr();
|
|
|
|
int32_t disp = int32_t((intptr_t)c - (intptr_t)ct_addr);
|
|
return mem_ptr(_common_table_ptr, disp - _common_table_offset);
|
|
}
|
|
|
|
Vec UniCompiler::_new_vec_const(const void* c, bool is_unique_const) {
|
|
Support::maybe_unused(is_unique_const);
|
|
|
|
Vec vec;
|
|
const char* special_const_name = nullptr;
|
|
|
|
if (special_const_name) {
|
|
vec = new_vec_with_width(vec_width(), special_const_name);
|
|
}
|
|
else {
|
|
uint64_t u0 = static_cast<const uint64_t*>(c)[0];
|
|
uint64_t u1 = static_cast<const uint64_t*>(c)[1];
|
|
|
|
if (u0 != u1)
|
|
vec = new_vec_with_width(vec_width(), "c_0x%016llX%016llX", (unsigned long long)u1, (unsigned long long)u0);
|
|
else if ((u0 >> 32) != (u0 & 0xFFFFFFFFu))
|
|
vec = new_vec_with_width(vec_width(), "c_0x%016llX", (unsigned long long)u0);
|
|
else if (((u0 >> 16) & 0xFFFFu) != (u0 & 0xFFFFu))
|
|
vec = new_vec_with_width(vec_width(), "c_0x%08X", (unsigned)(u0 & 0xFFFFFFFFu));
|
|
else
|
|
vec = new_vec_with_width(vec_width(), "c_0x%04X", (unsigned)(u0 & 0xFFFFu));
|
|
}
|
|
|
|
VecConstData const_data;
|
|
const_data.ptr = c;
|
|
const_data.virt_reg_id = vec.id();
|
|
_vec_consts.append(arena(), const_data);
|
|
|
|
if (c == &ct().p_0000000000000000) {
|
|
ScopedInjector inject(cc, &_func_init_hook);
|
|
v_zero_i(vec.v128());
|
|
}
|
|
else {
|
|
// NOTE: _get_mem_const() must be outside of injected code as it uses injection too.
|
|
Mem m = _get_mem_const(c);
|
|
|
|
ScopedInjector inject(cc, &_func_init_hook);
|
|
v_loadavec(vec, m);
|
|
}
|
|
|
|
return vec;
|
|
}
|
|
|
|
Vec UniCompiler::simd_const_16b(const void* data16) {
|
|
size_t n = _vec_consts_ex.size();
|
|
|
|
for (size_t i = 0; i < n; i++) {
|
|
if (memcmp(_vec_consts_ex[i].data, data16, 16) == 0) {
|
|
return Vec(OperandSignature{RegTraits<RegType::kVec128>::kSignature}, _vec_consts_ex[i].virt_reg_id);
|
|
}
|
|
}
|
|
|
|
Vec vec = new_vec128("const");
|
|
VecConstDataEx entry;
|
|
|
|
memcpy(entry.data, data16, 16);
|
|
entry.virt_reg_id = vec.id();
|
|
_vec_consts_ex.append(arena(), entry);
|
|
|
|
Mem mem = cc->new_const(ConstPoolScope::kLocal, data16, 16);
|
|
{
|
|
ScopedInjector inject(cc, &_func_init_hook);
|
|
v_loadavec(vec, mem);
|
|
}
|
|
|
|
return vec;
|
|
}
|
|
|
|
// ujit::UniCompiler - Stack
|
|
// =========================
|
|
|
|
Mem UniCompiler::tmp_stack(StackId id, uint32_t size) {
|
|
ASMJIT_ASSERT(Support::is_power_of_2(size));
|
|
ASMJIT_ASSERT(size <= 32);
|
|
|
|
// Only used by asserts.
|
|
Support::maybe_unused(size);
|
|
|
|
Mem& stack = _tmp_stack[size_t(id)];
|
|
if (!stack.base_id()) {
|
|
stack = cc->new_stack(32, 16, "tmp_stack");
|
|
}
|
|
return stack;
|
|
}
|
|
|
|
// ujit::UniCompiler - General Purpose Instructions - Utilities
|
|
// ============================================================
|
|
|
|
struct MemInst {
|
|
uint16_t inst_id;
|
|
uint16_t mem_size;
|
|
};
|
|
|
|
static ASMJIT_NOINLINE void gp_emit_mem_op(UniCompiler& uc, Gp r, Mem m, MemInst ii) {
|
|
BackendCompiler* cc = uc.cc;
|
|
InstId inst_id = ii.inst_id;
|
|
|
|
if (m.has_index() && m.has_shift()) {
|
|
// AArch64 limitation: shift can be the same size as the size of the read operation - HWord << 1, Word << 2, etc...
|
|
// Other shift operations are not supported at the architectural level, so we have to perform it explicitly.
|
|
uint32_t mem_size = ii.mem_size ? uint32_t(ii.mem_size) : r.size();
|
|
uint32_t shift = m.shift();
|
|
|
|
if (mem_size != (1u << shift)) {
|
|
Gp tmp = uc.new_gpz("@mem_addr");
|
|
cc->add(tmp, m.base_reg().as<Gp>(), m.index_reg().as<Gp>(), a64::Shift(m.shift_op(), shift));
|
|
m = a64::ptr(tmp);
|
|
}
|
|
}
|
|
|
|
cc->emit(inst_id, r, m);
|
|
}
|
|
|
|
static constexpr Gp gp_zero_regs[2] = { a64::wzr, a64::xzr };
|
|
|
|
static ASMJIT_INLINE const Gp& gp_zero_as(const Gp& ref) noexcept {
|
|
return gp_zero_regs[size_t(ref.is_gp64())];
|
|
}
|
|
|
|
static ASMJIT_NOINLINE Gp gp_force_reg(UniCompiler& uc, const Operand_& op, const Gp& ref) {
|
|
ASMJIT_ASSERT(op.is_gp() || op.is_mem() || op.is_imm());
|
|
|
|
Gp reg;
|
|
|
|
if (op.is_gp()) {
|
|
reg = op.as<Gp>();
|
|
reg.set_signature(ref.signature());
|
|
return reg;
|
|
}
|
|
|
|
if (op.is_imm() && op.as<Imm>().value() == 0) {
|
|
return gp_zero_as(ref);
|
|
}
|
|
|
|
BackendCompiler* cc = uc.cc;
|
|
reg = uc.new_similar_reg(ref, "@tmp");
|
|
|
|
if (op.is_mem()) {
|
|
gp_emit_mem_op(uc, reg, op.as<Mem>(), MemInst{uint16_t(Inst::kIdLdr), uint16_t(reg.size())});
|
|
}
|
|
else {
|
|
cc->mov(reg, op.as<Imm>());
|
|
}
|
|
return reg;
|
|
}
|
|
|
|
// ujit::UniCompiler - General Purpose Instructions - Conditions
|
|
// =============================================================
|
|
|
|
struct ConditionOpInfo {
|
|
uint16_t inst_id;
|
|
uint16_t reserved;
|
|
};
|
|
|
|
static constexpr ConditionOpInfo condition_op_info[size_t(UniOpCond::kMaxValue) + 1] = {
|
|
{ Inst::kIdAnds, 0 }, // UniOpCond::kAssignAnd
|
|
{ Inst::kIdOrr , 0 }, // UniOpCond::kAssignOr
|
|
{ Inst::kIdEor , 0 }, // UniOpCond::kAssignXor
|
|
{ Inst::kIdAdds, 0 }, // UniOpCond::kAssignAdd
|
|
{ Inst::kIdSubs, 0 }, // UniOpCond::kAssignSub
|
|
{ Inst::kIdNone, 0 }, // UniOpCond::kAssignShr
|
|
{ Inst::kIdTst , 0 }, // UniOpCond::kTest
|
|
{ Inst::kIdNone, 0 }, // UniOpCond::kBitTest
|
|
{ Inst::kIdCmp , 0 } // UniOpCond::kCompare
|
|
};
|
|
|
|
class ConditionApplier : public UniCondition {
|
|
public:
|
|
ASMJIT_INLINE ConditionApplier(const UniCondition& condition) noexcept : UniCondition(condition) {
|
|
// The first operand must always be a register.
|
|
ASMJIT_ASSERT(a.is_gp());
|
|
}
|
|
|
|
ASMJIT_NOINLINE void optimize(UniCompiler& uc) noexcept {
|
|
Support::maybe_unused(uc);
|
|
|
|
switch (op) {
|
|
case UniOpCond::kCompare:
|
|
if (b.is_imm() && b.as<Imm>().value() == 0 && (cond == CondCode::kEqual || cond == CondCode::kNotEqual)) {
|
|
op = UniOpCond::kTest;
|
|
b = a;
|
|
reverse();
|
|
}
|
|
break;
|
|
|
|
case UniOpCond::kBitTest: {
|
|
if (b.is_imm()) {
|
|
uint64_t bit_index = b.as<Imm>().value_as<uint64_t>();
|
|
op = UniOpCond::kTest;
|
|
b = Imm(1u << bit_index);
|
|
}
|
|
break;
|
|
}
|
|
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
ASMJIT_INLINE void reverse() noexcept {
|
|
cond = a64::reverse_cond(cond);
|
|
}
|
|
|
|
ASMJIT_NOINLINE void emit(UniCompiler& uc) {
|
|
BackendCompiler* cc = uc.cc;
|
|
ConditionOpInfo info = condition_op_info[size_t(op)];
|
|
|
|
Gp aGp = a.as<Gp>();
|
|
|
|
switch (op) {
|
|
case UniOpCond::kAssignAnd: {
|
|
if (b.is_imm() && a64::Utils::is_logical_imm(b.as<Imm>().value_as<uint64_t>(), aGp.size() * 8)) {
|
|
cc->emit(info.inst_id, aGp, aGp, b.as<Imm>());
|
|
}
|
|
else {
|
|
cc->emit(info.inst_id, aGp, aGp, gp_force_reg(uc, b, aGp));
|
|
}
|
|
return;
|
|
}
|
|
|
|
case UniOpCond::kAssignAdd:
|
|
case UniOpCond::kAssignSub: {
|
|
if (b.is_imm() && a64::Utils::is_add_sub_imm(b.as<Imm>().value_as<uint64_t>())) {
|
|
cc->emit(info.inst_id, aGp, aGp, b.as<Imm>());
|
|
}
|
|
else {
|
|
cc->emit(info.inst_id, aGp, aGp, gp_force_reg(uc, b, aGp));
|
|
}
|
|
return;
|
|
}
|
|
|
|
case UniOpCond::kAssignXor:
|
|
if (b.is_imm()) {
|
|
const Imm& bImm = b.as<Imm>();
|
|
if (bImm.value() == -1 || (aGp.size() == 4 && bImm.value_as<uint32_t>() == 0xFFFFFFFFu)) {
|
|
cc->mvn_(aGp, aGp);
|
|
cc->tst(aGp, aGp);
|
|
return;
|
|
}
|
|
}
|
|
[[fallthrough]];
|
|
|
|
case UniOpCond::kAssignOr: {
|
|
if (b.is_imm() && a64::Utils::is_logical_imm(b.as<Imm>().value_as<uint64_t>(), aGp.size() * 8)) {
|
|
cc->emit(info.inst_id, aGp, aGp, b.as<Imm>());
|
|
cc->tst(aGp, aGp);
|
|
}
|
|
else {
|
|
cc->emit(info.inst_id, aGp, aGp, gp_force_reg(uc, b, aGp));
|
|
cc->tst(aGp, aGp);
|
|
}
|
|
return;
|
|
}
|
|
|
|
case UniOpCond::kAssignShr: {
|
|
if (b.is_imm()) {
|
|
cc->adds(aGp, gp_zero_as(aGp), aGp, a64::lsr(b.as<Imm>().value_as<uint32_t>()));
|
|
}
|
|
else {
|
|
cc->lsr(aGp, aGp, gp_force_reg(uc, b, aGp));
|
|
cc->tst(aGp, aGp);
|
|
}
|
|
return;
|
|
}
|
|
|
|
case UniOpCond::kTest: {
|
|
if (b.is_imm() && a64::Utils::is_logical_imm(b.as<Imm>().value_as<uint64_t>(), aGp.size() * 8)) {
|
|
cc->emit(info.inst_id, aGp, b.as<Imm>());
|
|
}
|
|
else {
|
|
cc->emit(info.inst_id, aGp, gp_force_reg(uc, b, aGp));
|
|
}
|
|
return;
|
|
}
|
|
|
|
case UniOpCond::kCompare: {
|
|
if (b.is_imm() && a64::Utils::is_add_sub_imm(b.as<Imm>().value_as<uint64_t>())) {
|
|
cc->emit(info.inst_id, aGp, b.as<Imm>());
|
|
}
|
|
else {
|
|
cc->emit(info.inst_id, aGp, gp_force_reg(uc, b, aGp));
|
|
}
|
|
return;
|
|
}
|
|
|
|
case UniOpCond::kBitTest: {
|
|
Gp tmp = uc.new_similar_reg(aGp);
|
|
cc->lsr(tmp, aGp, gp_force_reg(uc, b, aGp));
|
|
cc->tst(tmp, Imm(1));
|
|
return;
|
|
}
|
|
|
|
default:
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
}
|
|
};
|
|
|
|
// ujit::UniCompiler - General Purpose Instructions - Emit
|
|
// =======================================================
|
|
|
|
void UniCompiler::emit_mov(const Gp& dst, const Operand_& src) {
|
|
if (src.is_mem()) {
|
|
gp_emit_mem_op(*this, dst, src.as<Mem>(), MemInst{uint16_t(Inst::kIdLdr), uint16_t(dst.size())});
|
|
}
|
|
else {
|
|
cc->emit(Inst::kIdMov, dst, src);
|
|
}
|
|
}
|
|
|
|
void UniCompiler::emit_m(UniOpM op, const Mem& m_) {
|
|
static constexpr MemInst st_inst[] = {
|
|
{ Inst::kIdNone, 0 }, // kPrefetch
|
|
{ Inst::kIdStr , 0 }, // kStoreZeroReg
|
|
{ Inst::kIdStrb, 1 }, // kStoreZeroU8
|
|
{ Inst::kIdStrh, 2 }, // kStoreZeroU16
|
|
{ Inst::kIdStr , 4 }, // kStoreZeroU32
|
|
{ Inst::kIdStr , 8 } // kStoreZeroU64
|
|
};
|
|
|
|
Gp zero = gp_zero_regs[size_t(op == UniOpM::kStoreZeroReg || op == UniOpM::kStoreZeroU64)];
|
|
MemInst ii = st_inst[size_t(op)];
|
|
|
|
if (!ii.inst_id) {
|
|
return;
|
|
}
|
|
|
|
gp_emit_mem_op(*this, zero, m_, ii);
|
|
}
|
|
|
|
void UniCompiler::emit_rm(UniOpRM op, const Gp& dst, const Mem& src) {
|
|
static constexpr MemInst ld_inst[] = {
|
|
{ Inst::kIdLdr , 0 }, // kLoadReg
|
|
{ Inst::kIdLdrsb, 1 }, // kLoadI8
|
|
{ Inst::kIdLdrb , 1 }, // kLoadU8
|
|
{ Inst::kIdLdrsh, 2 }, // kLoadI16
|
|
{ Inst::kIdLdrh , 2 }, // kLoadU16
|
|
{ Inst::kIdLdr , 4 }, // kLoadI32
|
|
{ Inst::kIdLdr , 4 }, // kLoadU32
|
|
{ Inst::kIdLdr , 8 }, // kLoadI64
|
|
{ Inst::kIdLdr , 8 }, // kLoadU64
|
|
{ Inst::kIdLdrb , 1 }, // kLoadMergeU8
|
|
{ Inst::kIdLdrb , 1 }, // kLoadShiftU8
|
|
{ Inst::kIdLdrh , 2 }, // kLoadMergeU16
|
|
{ Inst::kIdLdrh , 2 } // kLoadShiftU16
|
|
};
|
|
|
|
static constexpr uint32_t ld_32_mask =
|
|
(1u << uint32_t(uint32_t(UniOpRM::kLoadU8 ))) |
|
|
(1u << uint32_t(uint32_t(UniOpRM::kLoadU16))) |
|
|
(1u << uint32_t(uint32_t(UniOpRM::kLoadU32))) ;
|
|
|
|
Gp r(dst);
|
|
Mem m(src);
|
|
MemInst ii = ld_inst[size_t(op)];
|
|
|
|
switch (op) {
|
|
case UniOpRM::kLoadReg:
|
|
case UniOpRM::kLoadI8:
|
|
case UniOpRM::kLoadU8:
|
|
case UniOpRM::kLoadI16:
|
|
case UniOpRM::kLoadU16:
|
|
case UniOpRM::kLoadI32:
|
|
case UniOpRM::kLoadU32:
|
|
case UniOpRM::kLoadI64:
|
|
case UniOpRM::kLoadU64: {
|
|
if (op == UniOpRM::kLoadI32 && dst.is_gp64()) {
|
|
ii.inst_id = uint16_t(Inst::kIdLdrsw);
|
|
}
|
|
|
|
if ((ld_32_mask >> uint32_t(op)) & 1u) {
|
|
r = r.w();
|
|
}
|
|
|
|
gp_emit_mem_op(*this, r, m, ii);
|
|
return;
|
|
}
|
|
|
|
case UniOpRM::kLoadShiftU8:
|
|
case UniOpRM::kLoadShiftU16: {
|
|
Gp tmp = new_similar_reg(r);
|
|
gp_emit_mem_op(*this, tmp.r32(), m, ii);
|
|
cc->orr(r, tmp, r, a64::lsl(ii.mem_size * 8));
|
|
return;
|
|
}
|
|
|
|
case UniOpRM::kLoadMergeU8:
|
|
case UniOpRM::kLoadMergeU16: {
|
|
Gp tmp = new_similar_reg(r);
|
|
gp_emit_mem_op(*this, tmp.r32(), m, ii);
|
|
cc->orr(r, r, tmp);
|
|
return;
|
|
}
|
|
|
|
default: {
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
}
|
|
}
|
|
|
|
struct UniOpMRInfo {
|
|
uint32_t op_inst : 14;
|
|
uint32_t store_inst : 14;
|
|
uint32_t size : 4;
|
|
};
|
|
|
|
void UniCompiler::emit_mr(UniOpMR op, const Mem& dst, const Gp& src) {
|
|
static constexpr UniOpMRInfo op_info_table[] = {
|
|
{ Inst::kIdNone, Inst::kIdStr , 0 }, // kStoreReg
|
|
{ Inst::kIdNone, Inst::kIdStrb, 1 }, // kStoreU8
|
|
{ Inst::kIdNone, Inst::kIdStrh, 2 }, // kStoreU16
|
|
{ Inst::kIdNone, Inst::kIdStr , 4 }, // kStoreU32
|
|
{ Inst::kIdNone, Inst::kIdStr , 8 }, // kStoreU64
|
|
{ Inst::kIdAdd , Inst::kIdStr , 0 }, // kAddReg
|
|
{ Inst::kIdAdd , Inst::kIdStrb, 1 }, // kAddU8
|
|
{ Inst::kIdAdd , Inst::kIdStrh, 2 }, // kAddU16
|
|
{ Inst::kIdAdd , Inst::kIdStr , 4 }, // kAddU32
|
|
{ Inst::kIdAdd , Inst::kIdStr , 8 } // kAddU64
|
|
};
|
|
|
|
Mem m(dst);
|
|
Gp r(src);
|
|
const UniOpMRInfo& op_info = op_info_table[size_t(op)];
|
|
|
|
if (op_info.size >= 1u && op_info.size <= 4) {
|
|
r = r.w();
|
|
}
|
|
|
|
if (op_info.op_inst == Inst::kIdNone) {
|
|
cc->emit(op_info.store_inst, r, m);
|
|
}
|
|
else {
|
|
Gp tmp = new_similar_reg(r, "@tmp");
|
|
switch (op_info.size) {
|
|
case 0: load(tmp, m); break;
|
|
case 1: load_u8(tmp, m); break;
|
|
case 2: load_u16(tmp, m); break;
|
|
case 4: load_u32(tmp, m); break;
|
|
case 8: load_u64(tmp, m); break;
|
|
}
|
|
cc->emit(op_info.op_inst, tmp, tmp, r);
|
|
cc->emit(op_info.store_inst, tmp, m);
|
|
}
|
|
}
|
|
|
|
void UniCompiler::emit_cmov(const Gp& dst, const Operand_& sel, const UniCondition& condition) {
|
|
ConditionApplier ca(condition);
|
|
ca.optimize(*this);
|
|
ca.emit(*this);
|
|
cc->csel(dst, gp_force_reg(*this, sel, dst), dst, condition.cond);
|
|
}
|
|
|
|
void UniCompiler::emit_select(const Gp& dst, const Operand_& sel1_, const Operand_& sel2_, const UniCondition& condition) {
|
|
ConditionApplier ca(condition);
|
|
ca.optimize(*this);
|
|
ca.emit(*this);
|
|
|
|
Gp sel1 = gp_force_reg(*this, sel1_, dst);
|
|
Gp sel2 = gp_force_reg(*this, sel2_, dst);
|
|
cc->csel(dst, sel1, sel2, condition.cond);
|
|
}
|
|
|
|
void UniCompiler::emit_2i(UniOpRR op, const Gp& dst, const Operand_& src_) {
|
|
// ArithOp Reg, Any
|
|
// ----------------
|
|
|
|
if (src_.is_reg_or_mem()) {
|
|
Gp src = gp_force_reg(*this, src_, dst);
|
|
|
|
switch (op) {
|
|
case UniOpRR::kAbs: {
|
|
if (has_cssc()) {
|
|
cc->abs(dst, src);
|
|
}
|
|
else {
|
|
cc->cmp(src, 0);
|
|
cc->cneg(dst, src, CondCode::kMI);
|
|
}
|
|
return;
|
|
}
|
|
|
|
case UniOpRR::kNeg: {
|
|
cc->neg(dst, src);
|
|
return;
|
|
}
|
|
|
|
case UniOpRR::kNot: {
|
|
cc->mvn_(dst, src);
|
|
return;
|
|
}
|
|
|
|
case UniOpRR::kBSwap: {
|
|
cc->rev(dst, src);
|
|
return;
|
|
}
|
|
|
|
case UniOpRR::kCLZ: {
|
|
cc->clz(dst, src);
|
|
return;
|
|
}
|
|
|
|
case UniOpRR::kCTZ: {
|
|
if (has_cssc()) {
|
|
cc->ctz(dst, src);
|
|
}
|
|
else {
|
|
cc->rbit(dst, src);
|
|
cc->clz(dst, dst);
|
|
}
|
|
return;
|
|
}
|
|
|
|
case UniOpRR::kReflect: {
|
|
cc->eor(dst, src, src, a64::asr(dst.size() * 8u - 1));
|
|
return;
|
|
}
|
|
|
|
default:
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
}
|
|
|
|
// Everything should be handled, so this should never be reached!
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
|
|
static constexpr uint64_t kOp3ICommutativeMask =
|
|
(uint64_t(1) << unsigned(UniOpRRR::kAnd )) |
|
|
(uint64_t(1) << unsigned(UniOpRRR::kOr )) |
|
|
(uint64_t(1) << unsigned(UniOpRRR::kXor )) |
|
|
(uint64_t(1) << unsigned(UniOpRRR::kAdd )) |
|
|
(uint64_t(1) << unsigned(UniOpRRR::kMul )) |
|
|
(uint64_t(1) << unsigned(UniOpRRR::kSMin)) |
|
|
(uint64_t(1) << unsigned(UniOpRRR::kSMax)) |
|
|
(uint64_t(1) << unsigned(UniOpRRR::kUMin)) |
|
|
(uint64_t(1) << unsigned(UniOpRRR::kUMax)) ;
|
|
|
|
static ASMJIT_INLINE_NODEBUG bool is_op_3i_commutative(UniOpRRR op) noexcept {
|
|
return (kOp3ICommutativeMask & (uint64_t(1) << unsigned(op))) != 0;
|
|
}
|
|
|
|
void UniCompiler::emit_3i(UniOpRRR op, const Gp& dst, const Operand_& src1_, const Operand_& src2_) {
|
|
Operand src1(src1_);
|
|
Operand src2(src2_);
|
|
|
|
if (!src1.is_reg()) {
|
|
if (src2.is_reg() && is_op_3i_commutative(op)) {
|
|
ASMJIT_ASSERT(src2.is_gp());
|
|
std::swap(src1, src2);
|
|
}
|
|
else {
|
|
src1 = gp_force_reg(*this, src1, dst);
|
|
}
|
|
}
|
|
|
|
static constexpr uint16_t addsub_inst[2] = { Inst::kIdAdd, Inst::kIdSub };
|
|
static constexpr uint16_t sminmax_inst[2] = { Inst::kIdSmin, Inst::kIdSmax };
|
|
static constexpr uint16_t uminmax_inst[2] = { Inst::kIdUmin, Inst::kIdUmax };
|
|
static constexpr uint16_t logical_inst[4] = { Inst::kIdAnd, Inst::kIdOrr, Inst::kIdEor, Inst::kIdBic };
|
|
static constexpr uint16_t shift_inst[3] = { Inst::kIdLsl, Inst::kIdLsr, Inst::kIdAsr };
|
|
|
|
// ArithOp Reg, Reg, Imm
|
|
// ---------------------
|
|
|
|
if (src2.is_imm()) {
|
|
Gp a = src1.as<Gp>().clone_as(dst);
|
|
Imm b = src2.as<Imm>();
|
|
|
|
switch (op) {
|
|
case UniOpRRR::kXor:
|
|
if (b.value() == -1 || (b.value_as<uint32_t>() == 0xFFFFFFFFu && dst.size() == 4)) {
|
|
cc->mvn_(dst, a);
|
|
return;
|
|
}
|
|
[[fallthrough]];
|
|
|
|
case UniOpRRR::kAnd:
|
|
case UniOpRRR::kOr:
|
|
case UniOpRRR::kBic: {
|
|
if (a64::Utils::is_logical_imm(b.value_as<uint64_t>(), a.size() * 8u)) {
|
|
cc->emit(logical_inst[size_t(op) - size_t(UniOpRRR::kAnd)], dst, a, b);
|
|
return;
|
|
}
|
|
|
|
// If the immediate value is not encodable, we have to use a register.
|
|
break;
|
|
}
|
|
|
|
case UniOpRRR::kAdd:
|
|
case UniOpRRR::kSub: {
|
|
uint64_t value = b.value_as<uint64_t>();
|
|
unsigned reverse = int64_t(value) < 0;
|
|
|
|
if (reverse)
|
|
value = uint64_t(0) - value;
|
|
|
|
if (op == UniOpRRR::kSub)
|
|
reverse = reverse ^ 1u;
|
|
|
|
// TODO: [JIT] Just testing the idea of patching the previous instruction to have a post-index addressing.
|
|
if (!reverse && uint64_t(value) < 256 && dst.id() == a.id()) {
|
|
if (cc->cursor()->type() == NodeType::kInst) {
|
|
InstNode* prev_inst = cc->cursor()->as<InstNode>();
|
|
if (prev_inst->inst_id() == Inst::kIdLdr || prev_inst->inst_id() == Inst::kIdStr ||
|
|
prev_inst->inst_id() == Inst::kIdLdr_v || prev_inst->inst_id() == Inst::kIdStr_v) {
|
|
Mem& mem_op = prev_inst->op(prev_inst->op_count() - 1).as<Mem>();
|
|
if (mem_op.base_reg() == a && !mem_op.has_index() && !mem_op.has_offset()) {
|
|
mem_op.set_offset_mode(arm::OffsetMode::kPostIndex);
|
|
mem_op.add_offset(int64_t(value));
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (Support::is_uint_n<12>(value)) {
|
|
cc->emit(addsub_inst[reverse], dst, a, Imm(value));
|
|
return;
|
|
}
|
|
|
|
// If the immediate value is not encodable, we have to use a register.
|
|
break;
|
|
}
|
|
|
|
case UniOpRRR::kMul: {
|
|
uint64_t value = b.value_as<uint64_t>();
|
|
if (value > 0u) {
|
|
if (Support::is_power_of_2(value)) {
|
|
uint32_t shift = Support::ctz(value);
|
|
cc->lsl(dst, a, Imm(shift));
|
|
return;
|
|
}
|
|
|
|
// We can still support multiplication with `power_of_2 + 1`
|
|
if (Support::is_power_of_2(--value)) {
|
|
uint32_t shift = Support::ctz(value);
|
|
cc->add(dst, a, a, a64::lsl(shift));
|
|
return;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
|
|
case UniOpRRR::kSMin:
|
|
case UniOpRRR::kSMax: {
|
|
int64_t value = b.value_as<int64_t>();
|
|
|
|
if (value == 0 || value == -1) {
|
|
uint32_t shift = dst.size() * 8u - 1u;
|
|
|
|
// Signed min/max against -1 and 0 can be implemented by using a variation of bitwise instructions
|
|
// with the input value combined with its signs (that's why arithmetic shift right is used).
|
|
static constexpr uint16_t inst_table[4] = {
|
|
Inst::kIdAnd, // smin(a, 0) == and(a, expand_msb(a))
|
|
Inst::kIdOrr, // smin(a, -1) == orr(a, expand_msb(a))
|
|
Inst::kIdBic, // smax(a, 0) == bic(a, expand_msb(a))
|
|
Inst::kIdOrn // smax(a, -1) == orn(a, expand_msb(a))
|
|
};
|
|
|
|
InstId inst_id = inst_table[(size_t(op) - size_t(UniOpRRR::kSMin)) * 2u + size_t(value == -1)];
|
|
cc->emit(inst_id, dst, a, a, Imm(a64::asr(shift)));
|
|
return;
|
|
}
|
|
|
|
if (has_cssc() && Support::is_int_n<8>(value)) {
|
|
cc->emit(sminmax_inst[size_t(op) - size_t(UniOpRRR::kSMin)], dst, a, b);
|
|
return;
|
|
}
|
|
break;
|
|
}
|
|
|
|
case UniOpRRR::kUMin:
|
|
case UniOpRRR::kUMax: {
|
|
uint64_t value = b.value_as<uint64_t>();
|
|
if (has_cssc() && Support::is_uint_n<8>(value)) {
|
|
cc->emit(uminmax_inst[size_t(op) - size_t(UniOpRRR::kUMin)], dst, a, b);
|
|
return;
|
|
}
|
|
break;
|
|
}
|
|
|
|
case UniOpRRR::kSll:
|
|
case UniOpRRR::kSrl:
|
|
case UniOpRRR::kSra: {
|
|
cc->emit(shift_inst[size_t(op) - size_t(UniOpRRR::kSll)], dst, a, b);
|
|
return;
|
|
}
|
|
|
|
case UniOpRRR::kRol:
|
|
case UniOpRRR::kRor: {
|
|
uint32_t width = dst.size() * 8u;
|
|
uint32_t value = b.value_as<uint32_t>() & (width - 1);
|
|
|
|
if (op == UniOpRRR::kRol)
|
|
value = width - value;
|
|
|
|
cc->ror(dst, a, Imm(value));
|
|
return;
|
|
}
|
|
|
|
case UniOpRRR::kSBound: {
|
|
// TODO: CSSC
|
|
// if (has_cssc() && Support::is_uint8(value)) {
|
|
// }
|
|
break;
|
|
}
|
|
|
|
default:
|
|
// Unhandled instruction means to use a register instead of immediate.
|
|
break;
|
|
}
|
|
}
|
|
|
|
// ArithOp Reg, Reg, Reg
|
|
// ---------------------
|
|
|
|
{
|
|
src2 = gp_force_reg(*this, src2, dst);
|
|
|
|
Gp a = src1.as<Gp>();
|
|
Gp b = src2.as<Gp>();
|
|
|
|
switch (op) {
|
|
case UniOpRRR::kAnd:
|
|
case UniOpRRR::kOr:
|
|
case UniOpRRR::kXor:
|
|
case UniOpRRR::kBic: {
|
|
cc->emit(logical_inst[size_t(op) - size_t(UniOpRRR::kAnd)], dst, a, b);
|
|
return;
|
|
}
|
|
|
|
case UniOpRRR::kAdd:
|
|
case UniOpRRR::kSub: {
|
|
cc->emit(addsub_inst[size_t(op) - size_t(UniOpRRR::kAdd)], dst, a, b);
|
|
return;
|
|
}
|
|
|
|
case UniOpRRR::kMul: {
|
|
cc->mul(dst, a, b);
|
|
return;
|
|
}
|
|
|
|
case UniOpRRR::kUDiv: {
|
|
cc->udiv(dst, a, b);
|
|
return;
|
|
}
|
|
|
|
case UniOpRRR::kUMod: {
|
|
Gp tmp = new_similar_reg(dst, "@tmp");
|
|
cc->udiv(tmp, a, b);
|
|
cc->mul(tmp, tmp, b);
|
|
cc->sub(dst, a, tmp);
|
|
return;
|
|
}
|
|
|
|
case UniOpRRR::kSMin:
|
|
case UniOpRRR::kSMax: {
|
|
if (has_cssc()) {
|
|
cc->emit(sminmax_inst[size_t(op) - size_t(UniOpRRR::kSMin)], dst, a, b);
|
|
}
|
|
else {
|
|
cc->cmp(a, b);
|
|
cc->csel(dst, a, b, op == UniOpRRR::kSMin ? CondCode::kLT : CondCode::kGT);
|
|
}
|
|
return;
|
|
}
|
|
|
|
case UniOpRRR::kUMin:
|
|
case UniOpRRR::kUMax: {
|
|
if (has_cssc()) {
|
|
cc->emit(uminmax_inst[size_t(op) - size_t(UniOpRRR::kUMin)], dst, a, b);
|
|
}
|
|
else {
|
|
cc->cmp(a, b);
|
|
cc->csel(dst, a, b, op == UniOpRRR::kUMin ? CondCode::kLO : CondCode::kHI);
|
|
}
|
|
return;
|
|
}
|
|
|
|
case UniOpRRR::kSll:
|
|
case UniOpRRR::kSrl:
|
|
case UniOpRRR::kSra: {
|
|
cc->emit(shift_inst[size_t(op) - size_t(UniOpRRR::kSll)], dst, a, b);
|
|
return;
|
|
}
|
|
|
|
case UniOpRRR::kRol: {
|
|
Gp tmp = new_similar_reg(dst, "@rol_rev");
|
|
cc->neg(tmp, b);
|
|
cc->ror(dst, a, tmp);
|
|
return;
|
|
}
|
|
|
|
case UniOpRRR::kRor: {
|
|
cc->ror(dst, a, b);
|
|
return;
|
|
}
|
|
|
|
case UniOpRRR::kSBound: {
|
|
cc->cmp(a, b);
|
|
cc->bic(dst, a, a, a64::asr(dst.size() * 8u - 1));
|
|
cc->csel(dst, dst, b, CondCode::kSignedLT);
|
|
return;
|
|
}
|
|
|
|
default:
|
|
// Everything should be handled, so this should never be reached!
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
}
|
|
}
|
|
|
|
void UniCompiler::emit_j(const Operand_& target) {
|
|
cc->emit(Inst::kIdB, target);
|
|
}
|
|
|
|
void UniCompiler::emit_j_if(const Label& target, const UniCondition& condition) {
|
|
ConditionApplier ca(condition);
|
|
ca.optimize(*this);
|
|
ca.emit(*this);
|
|
cc->b(ca.cond, target);
|
|
}
|
|
|
|
void UniCompiler::adds_u8(const Gp& dst, const Gp& src1, const Gp& src2) {
|
|
ASMJIT_ASSERT(dst.size() == src1.size());
|
|
ASMJIT_ASSERT(dst.size() == src2.size());
|
|
|
|
cc->add(dst, src1, src2);
|
|
if (has_cssc()) {
|
|
cc->umin(dst, dst, 255);
|
|
return;
|
|
}
|
|
|
|
Gp tmp = new_similar_reg(dst, "@tmp");
|
|
cc->cmp(dst, 255);
|
|
cc->mov(tmp, 255);
|
|
cc->csel(dst, dst, tmp, CondCode::kLO);
|
|
}
|
|
|
|
void UniCompiler::inv_u8(const Gp& dst, const Gp& src) {
|
|
cc->eor(dst, src, 0xFF);
|
|
}
|
|
|
|
void UniCompiler::div_255_u32(const Gp& dst, const Gp& src) {
|
|
ASMJIT_ASSERT(dst.size() == src.size());
|
|
|
|
// dst = src + 128;
|
|
// dst = (dst + (dst >> 8)) >> 8
|
|
cc->add(dst, src, 128);
|
|
cc->add(dst, dst, dst, a64::lsr(8));
|
|
cc->lsr(dst, dst, 8);
|
|
}
|
|
|
|
void UniCompiler::mul_257_hu16(const Gp& dst, const Gp& src) {
|
|
ASMJIT_ASSERT(dst.size() == src.size());
|
|
cc->add(dst, src, src, a64::lsl(8));
|
|
cc->lsr(dst, dst, 16);
|
|
}
|
|
|
|
void UniCompiler::add_scaled(const Gp& dst, const Gp& a_, int b) {
|
|
Gp a = a_.clone_as(dst);
|
|
|
|
if (Support::is_power_of_2(b)) {
|
|
uint32_t shift = Support::ctz(b);
|
|
cc->add(dst, dst, a, a64::lsl(shift));
|
|
}
|
|
else if (b == 3 && dst.id() == a.id()) {
|
|
cc->add(dst, dst, dst, a64::lsl(1));
|
|
}
|
|
else {
|
|
Gp tmp = new_similar_reg(dst, "@tmp");
|
|
cc->mov(tmp, b);
|
|
cc->madd(dst, a, tmp, dst);
|
|
}
|
|
}
|
|
|
|
void UniCompiler::add_ext(const Gp& dst, const Gp& src_, const Gp& idx_, uint32_t scale, int32_t disp) {
|
|
ASMJIT_ASSERT(scale != 0);
|
|
|
|
Gp src = src_.clone_as(dst);
|
|
Gp idx = idx_.clone_as(dst);
|
|
|
|
if (Support::is_power_of_2(scale)) {
|
|
cc->add(dst, src, idx, a64::lsl(Support::ctz(scale)));
|
|
}
|
|
else {
|
|
Gp tmp = new_similar_reg(dst, "@tmp");
|
|
|
|
if (scale == 3) {
|
|
cc->add(tmp, src, idx, a64::lsl(1));
|
|
cc->add(dst, tmp, idx);
|
|
}
|
|
else {
|
|
cc->mov(tmp, scale);
|
|
cc->mul(tmp, tmp, idx);
|
|
cc->add(dst, src, tmp);
|
|
}
|
|
}
|
|
|
|
if (disp) {
|
|
if (disp > 0)
|
|
cc->add(dst, dst, disp);
|
|
else
|
|
cc->sub(dst, dst, -disp);
|
|
}
|
|
}
|
|
|
|
void UniCompiler::lea(const Gp& dst, const Mem& src) {
|
|
Gp base = src.base_reg().as<Gp>();
|
|
|
|
if (src.has_index()) {
|
|
add_ext(dst, base, src.index_reg().as<Gp>(), 1u << src.shift(), src.offset_lo32());
|
|
}
|
|
else if (src.offset_lo32()) {
|
|
cc->add(dst, base, src.offset_lo32());
|
|
}
|
|
else {
|
|
cc->mov(dst, base);
|
|
}
|
|
}
|
|
|
|
// ujit::UniCompiler - Vector Instructions - Constants
|
|
// ===================================================
|
|
|
|
//! Floating point mode is used in places that are generic and implement various functionality that needs more
|
|
//! than a single instruction. Typically implementing either higher level concepts or missing functionality.
|
|
enum FloatMode : uint32_t {
|
|
//! Used by non-floating point instructions.
|
|
kNone = 0,
|
|
|
|
kF32S = 1,
|
|
kF64S = 2,
|
|
kF32V = 3,
|
|
kF64V = 4
|
|
};
|
|
|
|
enum class ElementSize : uint32_t {
|
|
kNA = 0,
|
|
k8 = 0,
|
|
k16 = 1,
|
|
k32 = 2,
|
|
k64 = 3
|
|
};
|
|
|
|
enum class SameVecOp : uint32_t {
|
|
kNone = 0,
|
|
kZero = 1,
|
|
kOnes = 2,
|
|
kSrc = 3
|
|
};
|
|
|
|
enum class VecPart : uint32_t {
|
|
kNA = 0,
|
|
kLo = 1,
|
|
kHi = 2
|
|
};
|
|
|
|
enum class NarrowingOp : uint32_t {
|
|
kNone,
|
|
kI16ToI8,
|
|
kI16ToU8,
|
|
kU16ToU8,
|
|
kI32ToI16,
|
|
kI32ToU16,
|
|
kU32ToU16,
|
|
kI64ToI32,
|
|
kI64ToU32,
|
|
kU64ToU32
|
|
};
|
|
|
|
enum class NarrowingMode : uint32_t {
|
|
kTruncate,
|
|
kSaturateSToU,
|
|
kSaturateSToS,
|
|
kSaturateUToU
|
|
};
|
|
|
|
// ujit::UniCompiler - Vector Instructions - UniOp Information
|
|
// ============================================================
|
|
|
|
struct UniOpVInfo {
|
|
//! \name Members
|
|
//! \{
|
|
|
|
uint32_t inst_id : 13;
|
|
ASIMDExt asimd_ext : 6;
|
|
uint32_t commutative : 1;
|
|
uint32_t comparison : 1;
|
|
uint32_t reverse : 1;
|
|
SameVecOp same_vec_op : 2;
|
|
FloatMode float_mode : 3;
|
|
ElementSize dst_element : 3;
|
|
VecPart dst_part : 2;
|
|
ElementSize src_element : 3;
|
|
VecPart src_part : 2;
|
|
uint32_t imm : 8;
|
|
uint32_t reserved2 : 19;
|
|
|
|
//! \}
|
|
};
|
|
|
|
#define DEFINE_OP(inst_id, ext, commutative, comparison, reverse, same_vec_op, float_mode, dst_element, dst_part, src_element, src_part, imm) \
|
|
UniOpVInfo { \
|
|
inst_id, \
|
|
ASIMDExt::ext, \
|
|
commutative, \
|
|
comparison, \
|
|
reverse, \
|
|
SameVecOp::same_vec_op, \
|
|
FloatMode::float_mode, \
|
|
ElementSize::dst_element, \
|
|
VecPart::dst_part, \
|
|
ElementSize::src_element, \
|
|
VecPart::src_part, \
|
|
imm, \
|
|
0 \
|
|
}
|
|
|
|
static constexpr UniOpVInfo opcode_info_2v[size_t(UniOpVV::kMaxValue) + 1] = {
|
|
DEFINE_OP(Inst::kIdMov_v , kIntrin, 0, 0, 0, kNone, kNone, kNA, kNA, kNA, kNA, 0x00u), // kMov.
|
|
DEFINE_OP(Inst::kIdMov_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kLo, k64, kLo, 0x00u), // kMovU64.
|
|
DEFINE_OP(Inst::kIdDup_v , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kBroadcastU8Z.
|
|
DEFINE_OP(Inst::kIdDup_v , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kBroadcastU16Z.
|
|
DEFINE_OP(Inst::kIdDup_v , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kBroadcastU8.
|
|
DEFINE_OP(Inst::kIdDup_v , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kBroadcastU16.
|
|
DEFINE_OP(Inst::kIdDup_v , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kBroadcastU32.
|
|
DEFINE_OP(Inst::kIdDup_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kBroadcastU64.
|
|
DEFINE_OP(Inst::kIdDup_v , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kBroadcastF32.
|
|
DEFINE_OP(Inst::kIdDup_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kBroadcastF64.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kBroadcastV128_U32.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kBroadcastV128_U64.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kBroadcastV128_F32.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kBroadcastV128_F64.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kBroadcastV256_U32.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kBroadcastV256_U64.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kBroadcastV256_F32.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kBroadcastV256_F64.
|
|
DEFINE_OP(Inst::kIdAbs_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kAbsI8.
|
|
DEFINE_OP(Inst::kIdAbs_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kAbsI16.
|
|
DEFINE_OP(Inst::kIdAbs_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kAbsI32.
|
|
DEFINE_OP(Inst::kIdAbs_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kAbsI64.
|
|
DEFINE_OP(Inst::kIdMvn_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kNotU32.
|
|
DEFINE_OP(Inst::kIdMvn_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kNotU64.
|
|
DEFINE_OP(Inst::kIdSshll_v , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k8 , kLo, 0x00u), // kCvtI8LoToI16
|
|
DEFINE_OP(Inst::kIdSshll2_v , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k8 , kHi, 0x00u), // kCvtI8HiToI16
|
|
DEFINE_OP(Inst::kIdUshll_v , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k8 , kLo, 0x00u), // kCvtU8LoToU16
|
|
DEFINE_OP(Inst::kIdUshll2_v , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k8 , kHi, 0x00u), // kCvtU8HiToU16
|
|
DEFINE_OP(Inst::kIdSshll_v , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k8 , kLo, 0x00u), // kCvtI8ToI32
|
|
DEFINE_OP(Inst::kIdUshll_v , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k8 , kLo, 0x00u), // kCvtU8ToU32
|
|
DEFINE_OP(Inst::kIdSshll_v , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k16, kLo, 0x00u), // kCvtI16LoToI32
|
|
DEFINE_OP(Inst::kIdSshll2_v , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k16, kHi, 0x00u), // kCvtI16HiToI32
|
|
DEFINE_OP(Inst::kIdUshll_v , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k16, kLo, 0x00u), // kCvtU16LoToU32
|
|
DEFINE_OP(Inst::kIdUshll2_v , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k16, kHi, 0x00u), // kCvtU16HiToU32
|
|
DEFINE_OP(Inst::kIdSshll_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k32, kLo, 0x00u), // kCvtI32LoToI64
|
|
DEFINE_OP(Inst::kIdSshll2_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k32, kHi, 0x00u), // kCvtI32HiToI64
|
|
DEFINE_OP(Inst::kIdUshll_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k32, kLo, 0x00u), // kCvtU32LoToU64
|
|
DEFINE_OP(Inst::kIdUshll2_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k32, kHi, 0x00u), // kCvtU32HiToU64
|
|
DEFINE_OP(Inst::kIdFabs_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kAbsF32S.
|
|
DEFINE_OP(Inst::kIdFabs_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kAbsF64S.
|
|
DEFINE_OP(Inst::kIdFabs_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kAbsF32.
|
|
DEFINE_OP(Inst::kIdFabs_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kAbsF64.
|
|
DEFINE_OP(Inst::kIdFneg_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kNegF32S.
|
|
DEFINE_OP(Inst::kIdFneg_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kNegF64S.
|
|
DEFINE_OP(Inst::kIdFneg_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kNegF32.
|
|
DEFINE_OP(Inst::kIdFneg_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kNegF64.
|
|
DEFINE_OP(Inst::kIdMvn_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kNotF32.
|
|
DEFINE_OP(Inst::kIdMvn_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kNotF64.
|
|
DEFINE_OP(Inst::kIdFrintz_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kTruncF32S.
|
|
DEFINE_OP(Inst::kIdFrintz_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kTruncF64S.
|
|
DEFINE_OP(Inst::kIdFrintz_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kTruncF32.
|
|
DEFINE_OP(Inst::kIdFrintz_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kTruncF64.
|
|
DEFINE_OP(Inst::kIdFrintm_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kFloorF32S.
|
|
DEFINE_OP(Inst::kIdFrintm_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kFloorF64S.
|
|
DEFINE_OP(Inst::kIdFrintm_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kFloorF32.
|
|
DEFINE_OP(Inst::kIdFrintm_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kFloorF64.
|
|
DEFINE_OP(Inst::kIdFrintp_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kCeilF32S.
|
|
DEFINE_OP(Inst::kIdFrintp_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kCeilF64S.
|
|
DEFINE_OP(Inst::kIdFrintp_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCeilF32.
|
|
DEFINE_OP(Inst::kIdFrintp_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kCeilF64.
|
|
DEFINE_OP(Inst::kIdFrintn_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kRoundEvenF32S.
|
|
DEFINE_OP(Inst::kIdFrintn_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kRoundEvenF64S.
|
|
DEFINE_OP(Inst::kIdFrintn_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kRoundEvenF32.
|
|
DEFINE_OP(Inst::kIdFrintn_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kRoundEvenF64.
|
|
DEFINE_OP(Inst::kIdFrinta_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kRoundHalfAwayF32S.
|
|
DEFINE_OP(Inst::kIdFrinta_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kRoundHalfAwayF64S.
|
|
DEFINE_OP(Inst::kIdFrinta_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kRoundHalfAwayF32.
|
|
DEFINE_OP(Inst::kIdFrinta_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kRoundHalfAwayF64.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kRoundHalfUpF32S.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kRoundHalfUpF64S.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kRoundHalfUpF32.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kRoundHalfUpF64.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kRcpF32.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kRcpF64.
|
|
DEFINE_OP(Inst::kIdFsqrt_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kSqrtF32S.
|
|
DEFINE_OP(Inst::kIdFsqrt_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kSqrtF64S.
|
|
DEFINE_OP(Inst::kIdFsqrt_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kSqrtF32.
|
|
DEFINE_OP(Inst::kIdFsqrt_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kSqrtF64.
|
|
DEFINE_OP(Inst::kIdFcvt_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k32, kNA, 0x00u), // kCvtF32ToF64S.
|
|
DEFINE_OP(Inst::kIdFcvt_v , kASIMD , 0, 0, 0, kNone, kF64S, k32, kNA, k64, kNA, 0x00u), // kCvtF64ToF32S.
|
|
DEFINE_OP(Inst::kIdScvtf_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCvtI32ToF32.
|
|
DEFINE_OP(Inst::kIdFcvtl_v , kASIMD , 0, 0, 0, kNone, kF32V, k64, kNA, k32, kLo, 0x00u), // kCvtF32LoToF64.
|
|
DEFINE_OP(Inst::kIdFcvtl2_v , kASIMD , 0, 0, 0, kNone, kF32V, k64, kNA, k32, kHi, 0x00u), // kCvtF32HiToF64.
|
|
DEFINE_OP(Inst::kIdFcvtn_v , kASIMD , 0, 0, 0, kNone, kF64V, k32, kLo, k64, kNA, 0x00u), // kCvtF64ToF32Lo.
|
|
DEFINE_OP(Inst::kIdFcvtn2_v , kASIMD , 0, 0, 0, kNone, kF64V, k32, kHi, k64, kNA, 0x00u), // kCvtF64ToF32Hi.
|
|
DEFINE_OP(Inst::kIdSshll_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k32, kLo, 0x00u), // kCvtI32LoToF64.
|
|
DEFINE_OP(Inst::kIdSshll2_v , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k32, kHi, 0x00u), // kCvtI32HiToF64.
|
|
DEFINE_OP(Inst::kIdFcvtzs_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCvtTruncF32ToI32.
|
|
DEFINE_OP(Inst::kIdFcvtzs_v , kIntrin, 0, 0, 0, kNone, kF64V, k32, kLo, k64, kLo, 0x00u), // kCvtTruncF64ToI32Lo.
|
|
DEFINE_OP(Inst::kIdFcvtzs_v , kIntrin, 0, 0, 0, kNone, kF64V, k32, kHi, k64, kHi, 0x00u), // kCvtTruncF64ToI32Hi.
|
|
DEFINE_OP(Inst::kIdFcvtns_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCvtRoundF32ToI32.
|
|
DEFINE_OP(Inst::kIdFcvtns_v , kIntrin, 0, 0, 0, kNone, kF64V, k32, kLo, k64, kLo, 0x00u), // kCvtRoundF64ToI32Lo.
|
|
DEFINE_OP(Inst::kIdFcvtns_v , kIntrin, 0, 0, 0, kNone, kF64V, k32, kHi, k64, kHi, 0x00u) // kCvtRoundF64ToI32Hi.
|
|
};
|
|
|
|
static constexpr UniOpVInfo opcode_info_2vs[size_t(UniOpVR::kMaxValue) + 1] = {
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kMov.
|
|
DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kMovU32.
|
|
DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kMovU64.
|
|
DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kInsertU8.
|
|
DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kInsertU16.
|
|
DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInsertU32.
|
|
DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInsertU64.
|
|
DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kExtractU8.
|
|
DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kExtractU16.
|
|
DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kExtractU32.
|
|
DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kExtractU64.
|
|
DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, kNA, kNA, 0x00u), // kCvtIntToF32.
|
|
DEFINE_OP(Inst::kIdNone , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, kNA, kNA, 0x00u), // kCvtIntToF64.
|
|
DEFINE_OP(Inst::kIdFcvtzs_v , kASIMD , 0, 0, 0, kNone, kF32S, kNA, kNA, k32, kNA, 0x00u), // kCvtTruncF32ToInt.
|
|
DEFINE_OP(Inst::kIdFcvtns_v , kASIMD , 0, 0, 0, kNone, kF32S, kNA, kNA, k32, kNA, 0x00u), // kCvtRoundF32ToInt.
|
|
DEFINE_OP(Inst::kIdFcvtzs_v , kASIMD , 0, 0, 0, kNone, kF64S, kNA, kNA, k64, kNA, 0x00u), // kCvtTruncF64ToInt.
|
|
DEFINE_OP(Inst::kIdFcvtns_v , kASIMD , 0, 0, 0, kNone, kF64S, kNA, kNA, k64, kNA, 0x00u) // kCvtRoundF64ToInt.
|
|
};
|
|
|
|
static constexpr UniOpVInfo opcode_info_2vi[size_t(UniOpVVI::kMaxValue) + 1] = {
|
|
DEFINE_OP(Inst::kIdShl_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSllU16.
|
|
DEFINE_OP(Inst::kIdShl_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kSllU32.
|
|
DEFINE_OP(Inst::kIdShl_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kSllU64.
|
|
DEFINE_OP(Inst::kIdUshr_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSrlU16.
|
|
DEFINE_OP(Inst::kIdUshr_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kSrlU32.
|
|
DEFINE_OP(Inst::kIdUshr_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kSrlU64.
|
|
DEFINE_OP(Inst::kIdSshr_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSraI16.
|
|
DEFINE_OP(Inst::kIdSshr_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kSraI32.
|
|
DEFINE_OP(Inst::kIdSshr_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kSraI64.
|
|
DEFINE_OP(Inst::kIdExt_v , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kSllbU128.
|
|
DEFINE_OP(Inst::kIdExt_v , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kSrlbU128.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSwizzleU16x4.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSwizzleLoU16x4.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSwizzleHiU16x4.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kSwizzleU32x4.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kSwizzleU64x2.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kSwizzleF32x4.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kSwizzleF64x2.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kSwizzleU64x4.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kSwizzleF64x4.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kExtractV128_I32.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kExtractV128_I64.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kExtractV128_F32.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kExtractV128_F64.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kExtractV256_I32.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kExtractV256_I64.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kExtractV256_F32.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kExtractV256_F64.
|
|
DEFINE_OP(Inst::kIdUrshr_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSrlRndU16.
|
|
DEFINE_OP(Inst::kIdUrshr_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kSrlRndU32.
|
|
DEFINE_OP(Inst::kIdUrshr_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kSrlRndU64.
|
|
DEFINE_OP(Inst::kIdUsra_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSrlAccU16.
|
|
DEFINE_OP(Inst::kIdUsra_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kSrlAccU32.
|
|
DEFINE_OP(Inst::kIdUsra_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kSrlAccU64.
|
|
DEFINE_OP(Inst::kIdUrsra_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSrlRndAccU16.
|
|
DEFINE_OP(Inst::kIdUrsra_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kSrlRndAccU32.
|
|
DEFINE_OP(Inst::kIdUrsra_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kSrlRndAccU64.
|
|
DEFINE_OP(Inst::kIdShrn_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k16, kLo, 0x00u), // kSrlnLoU16.
|
|
DEFINE_OP(Inst::kIdShrn2_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k16, kHi, 0x00u), // kSrlnHiU16.
|
|
DEFINE_OP(Inst::kIdShrn_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k32, kLo, 0x00u), // kSrlnLoU32.
|
|
DEFINE_OP(Inst::kIdShrn2_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k32, kHi, 0x00u), // kSrlnHiU32.
|
|
DEFINE_OP(Inst::kIdShrn_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k64, kLo, 0x00u), // kSrlnLoU64.
|
|
DEFINE_OP(Inst::kIdShrn2_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k64, kHi, 0x00u), // kSrlnHiU64.
|
|
DEFINE_OP(Inst::kIdRshrn_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kLo, k16, kNA, 0x00u), // kSrlnRndLoU16.
|
|
DEFINE_OP(Inst::kIdRshrn2_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kHi, k16, kNA, 0x00u), // kSrlnRndHiU16.
|
|
DEFINE_OP(Inst::kIdRshrn_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kLo, k32, kNA, 0x00u), // kSrlnRndLoU32.
|
|
DEFINE_OP(Inst::kIdRshrn2_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kHi, k32, kNA, 0x00u), // kSrlnRndHiU32.
|
|
DEFINE_OP(Inst::kIdRshrn_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kLo, k64, kNA, 0x00u), // kSrlnRndLoU64.
|
|
DEFINE_OP(Inst::kIdRshrn2_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kHi, k64, kNA, 0x00u), // kSrlnRndHiU64.
|
|
};
|
|
|
|
static constexpr UniOpVInfo opcode_info_3v[size_t(UniOpVVV::kMaxValue) + 1] = {
|
|
DEFINE_OP(Inst::kIdAnd_v , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kAndU32.
|
|
DEFINE_OP(Inst::kIdAnd_v , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kAndU64.
|
|
DEFINE_OP(Inst::kIdOrr_v , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kOrU32.
|
|
DEFINE_OP(Inst::kIdOrr_v , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kOrU64.
|
|
DEFINE_OP(Inst::kIdEor_v , kASIMD , 1, 0, 0, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kXorU32.
|
|
DEFINE_OP(Inst::kIdEor_v , kASIMD , 1, 0, 0, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kXorU64.
|
|
DEFINE_OP(Inst::kIdBic_v , kASIMD , 0, 0, 1, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kAndnU32.
|
|
DEFINE_OP(Inst::kIdBic_v , kASIMD , 0, 0, 1, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kAndnU64.
|
|
DEFINE_OP(Inst::kIdBic_v , kASIMD , 0, 0, 0, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kBicU32.
|
|
DEFINE_OP(Inst::kIdBic_v , kASIMD , 0, 0, 0, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kBicU64.
|
|
DEFINE_OP(Inst::kIdNone , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kAvgrU8.
|
|
DEFINE_OP(Inst::kIdNone , kASIMD , 1, 0, 0, kSrc , kNone, k16, kNA, k16, kNA, 0x00u), // kAvgrU16.
|
|
DEFINE_OP(Inst::kIdAdd_v , kASIMD , 1, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kAddU8.
|
|
DEFINE_OP(Inst::kIdAdd_v , kASIMD , 1, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kAddU16.
|
|
DEFINE_OP(Inst::kIdAdd_v , kASIMD , 1, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kAddU32.
|
|
DEFINE_OP(Inst::kIdAdd_v , kASIMD , 1, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kAddU64.
|
|
DEFINE_OP(Inst::kIdSub_v , kASIMD , 0, 0, 0, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kSubU8.
|
|
DEFINE_OP(Inst::kIdSub_v , kASIMD , 0, 0, 0, kZero, kNone, k16, kNA, k16, kNA, 0x00u), // kSubU16.
|
|
DEFINE_OP(Inst::kIdSub_v , kASIMD , 0, 0, 0, kZero, kNone, k32, kNA, k32, kNA, 0x00u), // kSubU32.
|
|
DEFINE_OP(Inst::kIdSub_v , kASIMD , 0, 0, 0, kZero, kNone, k64, kNA, k64, kNA, 0x00u), // kSubU64.
|
|
DEFINE_OP(Inst::kIdSqadd_v , kASIMD , 1, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kAddsI8.
|
|
DEFINE_OP(Inst::kIdUqadd_v , kASIMD , 1, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kAddsU8.
|
|
DEFINE_OP(Inst::kIdSqadd_v , kASIMD , 1, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kAddsI16.
|
|
DEFINE_OP(Inst::kIdUqadd_v , kASIMD , 1, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kAddsU16.
|
|
DEFINE_OP(Inst::kIdSqsub_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kSubsI8.
|
|
DEFINE_OP(Inst::kIdUqsub_v , kASIMD , 0, 0, 0, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kSubsU8.
|
|
DEFINE_OP(Inst::kIdSqsub_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kSubsI16.
|
|
DEFINE_OP(Inst::kIdUqsub_v , kASIMD , 0, 0, 0, kZero, kNone, k16, kNA, k16, kNA, 0x00u), // kSubsU16.
|
|
DEFINE_OP(Inst::kIdMul_v , kASIMD , 1, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kMulU16.
|
|
DEFINE_OP(Inst::kIdMul_v , kASIMD , 1, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kMulU32.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 1, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kMulU64.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 1, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kMulhI16.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 1, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kMulhU16.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 1, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kMulU64_LoU32.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 1, 0, 0, kNone, kNone, k32, kNA, k16, kNA, 0x00u), // kMHAddI16_I32.
|
|
DEFINE_OP(Inst::kIdSmin_v , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kMinI8.
|
|
DEFINE_OP(Inst::kIdUmin_v , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kMinU8.
|
|
DEFINE_OP(Inst::kIdSmin_v , kASIMD , 1, 0, 0, kSrc , kNone, k16, kNA, k16, kNA, 0x00u), // kMinI16.
|
|
DEFINE_OP(Inst::kIdUmin_v , kASIMD , 1, 0, 0, kSrc , kNone, k16, kNA, k16, kNA, 0x00u), // kMinU16.
|
|
DEFINE_OP(Inst::kIdSmin_v , kASIMD , 1, 0, 0, kSrc , kNone, k32, kNA, k32, kNA, 0x00u), // kMinI32.
|
|
DEFINE_OP(Inst::kIdUmin_v , kASIMD , 1, 0, 0, kSrc , kNone, k32, kNA, k32, kNA, 0x00u), // kMinU32.
|
|
DEFINE_OP(Inst::kIdCmgt_v , kIntrin, 1, 0, 0, kSrc , kNone, k64, kNA, k64, kNA, 0x00u), // kMinI64.
|
|
DEFINE_OP(Inst::kIdCmhi_v , kIntrin, 1, 0, 0, kSrc , kNone, k64, kNA, k64, kNA, 0x00u), // kMinU64.
|
|
DEFINE_OP(Inst::kIdSmax_v , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kMaxI8.
|
|
DEFINE_OP(Inst::kIdUmax_v , kASIMD , 1, 0, 0, kSrc , kNone, k8 , kNA, k8 , kNA, 0x00u), // kMaxU8.
|
|
DEFINE_OP(Inst::kIdSmax_v , kASIMD , 1, 0, 0, kSrc , kNone, k16, kNA, k16, kNA, 0x00u), // kMaxI16.
|
|
DEFINE_OP(Inst::kIdUmax_v , kASIMD , 1, 0, 0, kSrc , kNone, k16, kNA, k16, kNA, 0x00u), // kMaxU16.
|
|
DEFINE_OP(Inst::kIdSmax_v , kASIMD , 1, 0, 0, kSrc , kNone, k32, kNA, k32, kNA, 0x00u), // kMaxI32.
|
|
DEFINE_OP(Inst::kIdUmax_v , kASIMD , 1, 0, 0, kSrc , kNone, k32, kNA, k32, kNA, 0x00u), // kMaxU32.
|
|
DEFINE_OP(Inst::kIdCmgt_v , kIntrin, 1, 0, 0, kSrc , kNone, k64, kNA, k64, kNA, 0x01u), // kMaxI64.
|
|
DEFINE_OP(Inst::kIdCmhi_v , kIntrin, 1, 0, 0, kSrc , kNone, k64, kNA, k64, kNA, 0x01u), // kMaxU64.
|
|
DEFINE_OP(Inst::kIdCmeq_v , kASIMD , 1, 1, 0, kOnes, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpEqU8.
|
|
DEFINE_OP(Inst::kIdCmeq_v , kASIMD , 1, 1, 0, kOnes, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpEqU16.
|
|
DEFINE_OP(Inst::kIdCmeq_v , kASIMD , 1, 1, 0, kOnes, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpEqU32.
|
|
DEFINE_OP(Inst::kIdCmeq_v , kASIMD , 1, 1, 0, kOnes, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpEqU64.
|
|
DEFINE_OP(Inst::kIdCmgt_v , kASIMD , 0, 1, 0, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpGtI8.
|
|
DEFINE_OP(Inst::kIdCmhi_v , kASIMD , 0, 1, 0, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpGtU8.
|
|
DEFINE_OP(Inst::kIdCmgt_v , kASIMD , 0, 1, 0, kZero, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpGtI16.
|
|
DEFINE_OP(Inst::kIdCmhi_v , kASIMD , 0, 1, 0, kZero, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpGtU16.
|
|
DEFINE_OP(Inst::kIdCmgt_v , kASIMD , 0, 1, 0, kZero, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpGtI32.
|
|
DEFINE_OP(Inst::kIdCmhi_v , kASIMD , 0, 1, 0, kZero, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpGtU32.
|
|
DEFINE_OP(Inst::kIdCmgt_v , kASIMD , 0, 1, 0, kZero, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpGtI64.
|
|
DEFINE_OP(Inst::kIdCmhi_v , kASIMD , 0, 1, 0, kZero, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpGtU64.
|
|
DEFINE_OP(Inst::kIdCmge_v , kASIMD , 0, 1, 0, kOnes, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpGeI8.
|
|
DEFINE_OP(Inst::kIdCmhs_v , kASIMD , 0, 1, 0, kOnes, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpGeU8.
|
|
DEFINE_OP(Inst::kIdCmge_v , kASIMD , 0, 1, 0, kOnes, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpGeI16.
|
|
DEFINE_OP(Inst::kIdCmhs_v , kASIMD , 0, 1, 0, kOnes, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpGeU16.
|
|
DEFINE_OP(Inst::kIdCmge_v , kASIMD , 0, 1, 0, kOnes, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpGeI32.
|
|
DEFINE_OP(Inst::kIdCmhs_v , kASIMD , 0, 1, 0, kOnes, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpGeU32.
|
|
DEFINE_OP(Inst::kIdCmge_v , kASIMD , 0, 1, 0, kOnes, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpGeI64.
|
|
DEFINE_OP(Inst::kIdCmhs_v , kASIMD , 0, 1, 0, kOnes, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpGeU64.
|
|
DEFINE_OP(Inst::kIdCmgt_v , kASIMD , 0, 1, 1, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpLtI8.
|
|
DEFINE_OP(Inst::kIdCmhi_v , kASIMD , 0, 1, 1, kZero, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpLtU8.
|
|
DEFINE_OP(Inst::kIdCmgt_v , kASIMD , 0, 1, 1, kZero, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpLtI16.
|
|
DEFINE_OP(Inst::kIdCmhi_v , kASIMD , 0, 1, 1, kZero, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpLtU16.
|
|
DEFINE_OP(Inst::kIdCmgt_v , kASIMD , 0, 1, 1, kZero, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpLtI32.
|
|
DEFINE_OP(Inst::kIdCmhi_v , kASIMD , 0, 1, 1, kZero, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpLtU32.
|
|
DEFINE_OP(Inst::kIdCmgt_v , kASIMD , 0, 1, 1, kZero, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpLtI64.
|
|
DEFINE_OP(Inst::kIdCmhi_v , kASIMD , 0, 1, 1, kZero, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpLtU64.
|
|
DEFINE_OP(Inst::kIdCmge_v , kASIMD , 0, 1, 1, kOnes, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpLeI8.
|
|
DEFINE_OP(Inst::kIdCmhs_v , kASIMD , 0, 1, 1, kOnes, kNone, k8 , kNA, k8 , kNA, 0x00u), // kCmpLeU8.
|
|
DEFINE_OP(Inst::kIdCmge_v , kASIMD , 0, 1, 1, kOnes, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpLeI16.
|
|
DEFINE_OP(Inst::kIdCmhs_v , kASIMD , 0, 1, 1, kOnes, kNone, k16, kNA, k16, kNA, 0x00u), // kCmpLeU16.
|
|
DEFINE_OP(Inst::kIdCmge_v , kASIMD , 0, 1, 1, kOnes, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpLeI32.
|
|
DEFINE_OP(Inst::kIdCmhs_v , kASIMD , 0, 1, 1, kOnes, kNone, k32, kNA, k32, kNA, 0x00u), // kCmpLeU32.
|
|
DEFINE_OP(Inst::kIdCmge_v , kASIMD , 0, 1, 1, kOnes, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpLeI64.
|
|
DEFINE_OP(Inst::kIdCmhs_v , kASIMD , 0, 1, 1, kOnes, kNone, k64, kNA, k64, kNA, 0x00u), // kCmpLeU64.
|
|
DEFINE_OP(Inst::kIdAnd_v , kASIMD , 1, 0, 0, kSrc , kF32V, k8 , kNA, k8 , kNA, 0x00u), // kAndF32.
|
|
DEFINE_OP(Inst::kIdAnd_v , kASIMD , 1, 0, 0, kSrc , kF64V, k8 , kNA, k8 , kNA, 0x00u), // kAndF64.
|
|
DEFINE_OP(Inst::kIdOrr_v , kASIMD , 1, 0, 0, kSrc , kF32V, k8 , kNA, k8 , kNA, 0x00u), // kOrF32.
|
|
DEFINE_OP(Inst::kIdOrr_v , kASIMD , 1, 0, 0, kSrc , kF64V, k8 , kNA, k8 , kNA, 0x00u), // kOrF64.
|
|
DEFINE_OP(Inst::kIdEor_v , kASIMD , 1, 0, 0, kZero, kF32V, k8 , kNA, k8 , kNA, 0x00u), // kXorF32.
|
|
DEFINE_OP(Inst::kIdEor_v , kASIMD , 1, 0, 0, kZero, kF64V, k8 , kNA, k8 , kNA, 0x00u), // kXorF64.
|
|
DEFINE_OP(Inst::kIdBic_v , kASIMD , 0, 0, 1, kZero, kF32V, k8 , kNA, k8 , kNA, 0x00u), // kAndnF32.
|
|
DEFINE_OP(Inst::kIdBic_v , kASIMD , 0, 0, 1, kZero, kF64V, k8 , kNA, k8 , kNA, 0x00u), // kAndnF64.
|
|
DEFINE_OP(Inst::kIdBic_v , kASIMD , 0, 0, 0, kZero, kF32V, k8 , kNA, k8 , kNA, 0x00u), // kBicF32.
|
|
DEFINE_OP(Inst::kIdBic_v , kASIMD , 0, 0, 0, kZero, kF64V, k8 , kNA, k8 , kNA, 0x00u), // kBicF64.
|
|
DEFINE_OP(Inst::kIdFadd_v , kASIMD , 1, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kAddF32S.
|
|
DEFINE_OP(Inst::kIdFadd_v , kASIMD , 1, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kAddF64S.
|
|
DEFINE_OP(Inst::kIdFadd_v , kASIMD , 1, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kAddF32.
|
|
DEFINE_OP(Inst::kIdFadd_v , kASIMD , 1, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kAddF64.
|
|
DEFINE_OP(Inst::kIdFsub_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kSubF32S.
|
|
DEFINE_OP(Inst::kIdFsub_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kSubF64S.
|
|
DEFINE_OP(Inst::kIdFsub_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kSubF32.
|
|
DEFINE_OP(Inst::kIdFsub_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kSubF64.
|
|
DEFINE_OP(Inst::kIdFmul_v , kASIMD , 1, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kMulF32S.
|
|
DEFINE_OP(Inst::kIdFmul_v , kASIMD , 1, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kMulF64S.
|
|
DEFINE_OP(Inst::kIdFmul_v , kASIMD , 1, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kMulF32.
|
|
DEFINE_OP(Inst::kIdFmul_v , kASIMD , 1, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kMulF64.
|
|
DEFINE_OP(Inst::kIdFdiv_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kDivF32S.
|
|
DEFINE_OP(Inst::kIdFdiv_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kDivF64S.
|
|
DEFINE_OP(Inst::kIdFdiv_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kDivF32.
|
|
DEFINE_OP(Inst::kIdFdiv_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kDivF64.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kModF32S.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kModF64S.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kModF32.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kModF64.
|
|
DEFINE_OP(Inst::kIdFminnm_v , kASIMD , 1, 0, 0, kSrc , kF32S, k32, kNA, k32, kNA, 0x00u), // kMinF32S.
|
|
DEFINE_OP(Inst::kIdFminnm_v , kASIMD , 1, 0, 0, kSrc , kF64S, k64, kNA, k64, kNA, 0x00u), // kMinF64S.
|
|
DEFINE_OP(Inst::kIdFminnm_v , kASIMD , 1, 0, 0, kSrc , kF32V, k32, kNA, k32, kNA, 0x00u), // kMinF32.
|
|
DEFINE_OP(Inst::kIdFminnm_v , kASIMD , 1, 0, 0, kSrc , kF64V, k64, kNA, k64, kNA, 0x00u), // kMinF64.
|
|
DEFINE_OP(Inst::kIdFmaxnm_v , kASIMD , 1, 0, 0, kSrc , kF32S, k32, kNA, k32, kNA, 0x00u), // kMaxF32S.
|
|
DEFINE_OP(Inst::kIdFmaxnm_v , kASIMD , 1, 0, 0, kSrc , kF64S, k64, kNA, k64, kNA, 0x00u), // kMaxF64S.
|
|
DEFINE_OP(Inst::kIdFmaxnm_v , kASIMD , 1, 0, 0, kSrc , kF32V, k32, kNA, k32, kNA, 0x00u), // kMaxF32.
|
|
DEFINE_OP(Inst::kIdFmaxnm_v , kASIMD , 1, 0, 0, kSrc , kF64V, k64, kNA, k64, kNA, 0x00u), // kMaxF64.
|
|
DEFINE_OP(Inst::kIdFcmeq_v , kASIMD , 1, 1, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kCmpEqF32S (eq ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmeq_v , kASIMD , 1, 1, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kCmpEqF64S (eq ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmeq_v , kASIMD , 1, 1, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCmpEqF32 (eq ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmeq_v , kASIMD , 1, 1, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kCmpEqF64 (eq ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kCmpNeF32S (ne ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kCmpNeF64S (ne ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCmpNeF32 (ne ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kCmpNeF64 (ne ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmgt_v , kASIMD , 0, 1, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kCmpGtF32S (gt ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmgt_v , kASIMD , 0, 1, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kCmpGtF64S (gt ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmgt_v , kASIMD , 0, 1, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCmpGtF32 (gt ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmgt_v , kASIMD , 0, 1, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kCmpGtF64 (gt ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmge_v , kASIMD , 0, 1, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kCmpGeF32S (ge ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmge_v , kASIMD , 0, 1, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kCmpGeF64S (ge ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmge_v , kASIMD , 0, 1, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCmpGeF32 (ge ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmge_v , kASIMD , 0, 1, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kCmpGeF64 (ge ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmgt_v , kASIMD , 0, 1, 1, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kCmpLtF32S (lt ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmgt_v , kASIMD , 0, 1, 1, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kCmpLtF64S (lt ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmgt_v , kASIMD , 0, 1, 1, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCmpLtF32 (lt ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmgt_v , kASIMD , 0, 1, 1, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kCmpLtF64 (lt ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmge_v , kASIMD , 0, 1, 1, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kCmpLeF32S (le ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmge_v , kASIMD , 0, 1, 1, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kCmpLeF64S (le ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmge_v , kASIMD , 0, 1, 1, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCmpLeF32 (le ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmge_v , kASIMD , 0, 1, 1, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kCmpLeF64 (le ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kCmpOrdF32S (ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kCmpOrdF64S (ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kCmpOrdF32 (ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kCmpOrdF64 (ordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x01u), // kCmpUnordF32S (unordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x01u), // kCmpUnordF64S (unordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x01u), // kCmpUnordF32 (unordered quiet).
|
|
DEFINE_OP(Inst::kIdFcmeq_v , kIntrin, 1, 1, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x01u), // kCmpUnordF64 (unordered quiet).
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kHAddF64.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kCombineLoHiU64.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kCombineLoHiF64.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kSrc , kNone, k64, kNA, k64, kNA, 0x00u), // kCombineHiLoU64.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kSrc , kNone, k64, kNA, k64, kNA, 0x00u), // kCombineHiLoF64.
|
|
DEFINE_OP(Inst::kIdZip1_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kInterleaveLoU8.
|
|
DEFINE_OP(Inst::kIdZip2_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kInterleaveHiU8.
|
|
DEFINE_OP(Inst::kIdZip1_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kInterleaveLoU16.
|
|
DEFINE_OP(Inst::kIdZip2_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kInterleaveHiU16.
|
|
DEFINE_OP(Inst::kIdZip1_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInterleaveLoU32.
|
|
DEFINE_OP(Inst::kIdZip2_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInterleaveHiU32.
|
|
DEFINE_OP(Inst::kIdZip1_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInterleaveLoU64.
|
|
DEFINE_OP(Inst::kIdZip2_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInterleaveHiU64.
|
|
DEFINE_OP(Inst::kIdZip1_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInterleaveLoF32.
|
|
DEFINE_OP(Inst::kIdZip2_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInterleaveHiF32.
|
|
DEFINE_OP(Inst::kIdZip1_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInterleaveLoF64.
|
|
DEFINE_OP(Inst::kIdZip2_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInterleaveHiF64.
|
|
DEFINE_OP(Inst::kIdSqxtn_v , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k16, kNA, 0x00u), // kPacksI16_I8.
|
|
DEFINE_OP(Inst::kIdSqxtun_v , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k16, kNA, 0x00u), // kPacksI16_U8.
|
|
DEFINE_OP(Inst::kIdSqxtn_v , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k32, kNA, 0x00u), // kPacksI32_I16.
|
|
DEFINE_OP(Inst::kIdSqxtun_v , kIntrin, 0, 0, 0, kNone, kNone, k16, kNA, k32, kNA, 0x00u), // kPacksI32_U16.
|
|
DEFINE_OP(Inst::kIdTbl_v , kASIMD , 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kSwizzlev_U8.
|
|
DEFINE_OP(Inst::kIdSmull_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k8 , kLo, 0x00u), // kMulwLoI8.
|
|
DEFINE_OP(Inst::kIdUmull_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k8 , kLo, 0x00u), // kMulwLoU8.
|
|
DEFINE_OP(Inst::kIdSmull2_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k8 , kHi, 0x00u), // kMulwHiI8.
|
|
DEFINE_OP(Inst::kIdUmull2_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k8 , kHi, 0x00u), // kMulwHiU8.
|
|
DEFINE_OP(Inst::kIdSmull_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k16, kLo, 0x00u), // kMulwLoI16.
|
|
DEFINE_OP(Inst::kIdUmull_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k16, kLo, 0x00u), // kMulwLoU16.
|
|
DEFINE_OP(Inst::kIdSmull2_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k16, kHi, 0x00u), // kMulwHiI16.
|
|
DEFINE_OP(Inst::kIdUmull2_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k16, kHi, 0x00u), // kMulwHiU16.
|
|
DEFINE_OP(Inst::kIdSmull_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k32, kLo, 0x00u), // kMulwLoI32.
|
|
DEFINE_OP(Inst::kIdUmull_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k32, kLo, 0x00u), // kMulwLoU32.
|
|
DEFINE_OP(Inst::kIdSmull2_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k32, kHi, 0x00u), // kMulwHiI32.
|
|
DEFINE_OP(Inst::kIdUmull2_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k32, kHi, 0x00u), // kMulwHiU32.
|
|
DEFINE_OP(Inst::kIdSmlal_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k8 , kLo, 0x00u), // kMAddwLoI8.
|
|
DEFINE_OP(Inst::kIdUmlal_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k8 , kLo, 0x00u), // kMAddwLoU8.
|
|
DEFINE_OP(Inst::kIdSmlal2_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k8 , kHi, 0x00u), // kMAddwHiI8.
|
|
DEFINE_OP(Inst::kIdUmlal2_v , kASIMD , 0, 0, 0, kNone, kNone, k16, kNA, k8 , kHi, 0x00u), // kMAddwHiU8.
|
|
DEFINE_OP(Inst::kIdSmlal_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k16, kLo, 0x00u), // kMAddwLoI16.
|
|
DEFINE_OP(Inst::kIdUmlal_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k16, kLo, 0x00u), // kMAddwLoU16.
|
|
DEFINE_OP(Inst::kIdSmlal2_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k16, kHi, 0x00u), // kMAddwHiI16.
|
|
DEFINE_OP(Inst::kIdUmlal2_v , kASIMD , 0, 0, 0, kNone, kNone, k32, kNA, k16, kHi, 0x00u), // kMAddwHiU16.
|
|
DEFINE_OP(Inst::kIdSmlal_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k32, kLo, 0x00u), // kMAddwLoI32.
|
|
DEFINE_OP(Inst::kIdUmlal_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k32, kLo, 0x00u), // kMAddwLoU32.
|
|
DEFINE_OP(Inst::kIdSmlal2_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k32, kHi, 0x00u), // kMAddwHiI32.
|
|
DEFINE_OP(Inst::kIdUmlal2_v , kASIMD , 0, 0, 0, kNone, kNone, k64, kNA, k32, kHi, 0x00u) // kMAddwHiU32.
|
|
};
|
|
|
|
static constexpr UniOpVInfo opcode_info_3vi[size_t(UniOpVVVI::kMaxValue) + 1] = {
|
|
DEFINE_OP(Inst::kIdExt_v , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kAlignr_U128.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInterleaveShuffleU32x4.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInterleaveShuffleU64x2.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInterleaveShuffleF32x4.
|
|
DEFINE_OP(Inst::kIdNone , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInterleaveShuffleF64x2.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInsertV128_U32.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInsertV128_F32.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInsertV128_U64.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInsertV128_F64.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInsertV256_U32.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kInsertV256_F32.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u), // kInsertV256_U64.
|
|
DEFINE_OP(0 , kIntrin, 0, 0, 0, kNone, kNone, k64, kNA, k64, kNA, 0x00u) // kInsertV256_F64.
|
|
};
|
|
|
|
static constexpr UniOpVInfo opcode_info_4v[size_t(UniOpVVV::kMaxValue) + 1] = {
|
|
DEFINE_OP(Inst::kIdBsl_v , kIntrin, 0, 0, 0, kNone, kNone, k8 , kNA, k8 , kNA, 0x00u), // kBlendV_U8.
|
|
DEFINE_OP(Inst::kIdMla_v , kIntrin, 1, 0, 0, kNone, kNone, k16, kNA, k16, kNA, 0x00u), // kMAddU16.
|
|
DEFINE_OP(Inst::kIdMla_v , kIntrin, 1, 0, 0, kNone, kNone, k32, kNA, k32, kNA, 0x00u), // kMAddU32.
|
|
DEFINE_OP(Inst::kIdFmadd_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kMAddF32S.
|
|
DEFINE_OP(Inst::kIdFmadd_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kMAddF64S.
|
|
DEFINE_OP(Inst::kIdFmla_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kMAddF32.
|
|
DEFINE_OP(Inst::kIdFmla_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kMAddF64.
|
|
DEFINE_OP(Inst::kIdFnmsub_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kMSubF32S.
|
|
DEFINE_OP(Inst::kIdFnmsub_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kMSubF64S.
|
|
DEFINE_OP(Inst::kIdFmla_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x01u), // kMSubF32.
|
|
DEFINE_OP(Inst::kIdFmla_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x01u), // kMSubF64.
|
|
DEFINE_OP(Inst::kIdFmsub_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kNMAddF32S.
|
|
DEFINE_OP(Inst::kIdFmsub_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kNMAddF64S.
|
|
DEFINE_OP(Inst::kIdFmls_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x00u), // kNMAddF32.
|
|
DEFINE_OP(Inst::kIdFmls_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x00u), // kNMAddF64.
|
|
DEFINE_OP(Inst::kIdFnmadd_v , kASIMD , 0, 0, 0, kNone, kF32S, k32, kNA, k32, kNA, 0x00u), // kNMSubF32S.
|
|
DEFINE_OP(Inst::kIdFnmadd_v , kASIMD , 0, 0, 0, kNone, kF64S, k64, kNA, k64, kNA, 0x00u), // kNMSubF64S.
|
|
DEFINE_OP(Inst::kIdFmls_v , kASIMD , 0, 0, 0, kNone, kF32V, k32, kNA, k32, kNA, 0x01u), // kNMSubF32.
|
|
DEFINE_OP(Inst::kIdFmls_v , kASIMD , 0, 0, 0, kNone, kF64V, k64, kNA, k64, kNA, 0x01u) // kNMSubF64.
|
|
};
|
|
|
|
#undef DEFINE_OP
|
|
|
|
struct UniOpVMInfo {
|
|
//! \name Members
|
|
//! \{
|
|
|
|
uint32_t cvt_op : 16;
|
|
uint32_t mem_size : 8;
|
|
ElementSize element : 3;
|
|
|
|
//! \}
|
|
};
|
|
|
|
#define DEFINE_OP(cvt_op, size, element) UniOpVMInfo { uint32_t(cvt_op), size, ElementSize::element }
|
|
|
|
static constexpr UniOpVMInfo opcode_info_2vm[size_t(UniOpVM::kMaxValue) + 1] = {
|
|
DEFINE_OP(0 , 1, k8 ), // kLoad8.
|
|
DEFINE_OP(0 , 2, k16), // kLoad16_U16.
|
|
DEFINE_OP(0 , 4, k32), // kLoad32_U32.
|
|
DEFINE_OP(0 , 4, k32), // kLoad32_F32.
|
|
DEFINE_OP(0 , 8, k32), // kLoad64_U32.
|
|
DEFINE_OP(0 , 8, k64), // kLoad64_U64.
|
|
DEFINE_OP(0 , 8, k32), // kLoad64_F32.
|
|
DEFINE_OP(0 , 8, k64), // kLoad64_F64.
|
|
DEFINE_OP(0 , 16, k32), // kLoad128_U32.
|
|
DEFINE_OP(0 , 16, k64), // kLoad128_U64.
|
|
DEFINE_OP(0 , 16, k32), // kLoad128_F32.
|
|
DEFINE_OP(0 , 16, k64), // kLoad128_F64.
|
|
DEFINE_OP(0 , 32, k32), // kLoad256_U32.
|
|
DEFINE_OP(0 , 32, k64), // kLoad256_U64.
|
|
DEFINE_OP(0 , 32, k32), // kLoad256_F32.
|
|
DEFINE_OP(0 , 32, k64), // kLoad256_F64.
|
|
DEFINE_OP(0 , 64, k32), // kLoad512_U32.
|
|
DEFINE_OP(0 , 64, k64), // kLoad512_U64.
|
|
DEFINE_OP(0 , 64, k32), // kLoad512_F32.
|
|
DEFINE_OP(0 , 64, k64), // kLoad512_F64.
|
|
DEFINE_OP(0 , 0, k32), // kLoadN_U32.
|
|
DEFINE_OP(0 , 0, k64), // kLoadN_U64.
|
|
DEFINE_OP(0 , 0, k32), // kLoadN_F32.
|
|
DEFINE_OP(0 , 0, k64), // kLoadN_F64.
|
|
DEFINE_OP(1 , 2, kNA), // kLoadCvt16_U8ToU64.
|
|
DEFINE_OP(1 , 4, kNA), // kLoadCvt32_U8ToU64.
|
|
DEFINE_OP(1 , 8, kNA), // kLoadCvt64_U8ToU64.
|
|
DEFINE_OP(UniOpVV::kCvtI8LoToI16 , 4, kNA), // kLoadCvt32_I8ToI16.
|
|
DEFINE_OP(UniOpVV::kCvtU8LoToU16 , 4, kNA), // kLoadCvt32_U8ToU16.
|
|
DEFINE_OP(UniOpVV::kCvtI8ToI32 , 4, kNA), // kLoadCvt32_I8ToI32.
|
|
DEFINE_OP(UniOpVV::kCvtU8ToU32 , 4, kNA), // kLoadCvt32_U8ToU32.
|
|
DEFINE_OP(UniOpVV::kCvtI16LoToI32, 4, kNA), // kLoadCvt32_I16ToI32.
|
|
DEFINE_OP(UniOpVV::kCvtU16LoToU32, 4, kNA), // kLoadCvt32_U16ToU32.
|
|
DEFINE_OP(UniOpVV::kCvtI32LoToI64, 4, kNA), // kLoadCvt32_I32ToI64.
|
|
DEFINE_OP(UniOpVV::kCvtU32LoToU64, 4, kNA), // kLoadCvt32_U32ToU64.
|
|
DEFINE_OP(UniOpVV::kCvtI8LoToI16 , 8, kNA), // kLoadCvt64_I8ToI16.
|
|
DEFINE_OP(UniOpVV::kCvtU8LoToU16 , 8, kNA), // kLoadCvt64_U8ToU16.
|
|
DEFINE_OP(UniOpVV::kCvtI8ToI32 , 8, kNA), // kLoadCvt64_I8ToI32.
|
|
DEFINE_OP(UniOpVV::kCvtU8ToU32 , 8, kNA), // kLoadCvt64_U8ToU32.
|
|
DEFINE_OP(UniOpVV::kCvtI16LoToI32, 8, kNA), // kLoadCvt64_I16ToI32.
|
|
DEFINE_OP(UniOpVV::kCvtU16LoToU32, 8, kNA), // kLoadCvt64_U16ToU32.
|
|
DEFINE_OP(UniOpVV::kCvtI32LoToI64, 8, kNA), // kLoadCvt64_I32ToI64.
|
|
DEFINE_OP(UniOpVV::kCvtU32LoToU64, 8, kNA), // kLoadCvt64_U32ToU64.
|
|
DEFINE_OP(UniOpVV::kCvtI8LoToI16 , 16, kNA), // kLoadCvt128_I8ToI16.
|
|
DEFINE_OP(UniOpVV::kCvtU8LoToU16 , 16, kNA), // kLoadCvt128_U8ToU16.
|
|
DEFINE_OP(UniOpVV::kCvtI8ToI32 , 16, kNA), // kLoadCvt128_I8ToI32.
|
|
DEFINE_OP(UniOpVV::kCvtU8ToU32 , 16, kNA), // kLoadCvt128_U8ToU32.
|
|
DEFINE_OP(UniOpVV::kCvtI16LoToI32, 16, kNA), // kLoadCvt128_I16ToI32.
|
|
DEFINE_OP(UniOpVV::kCvtU16LoToU32, 16, kNA), // kLoadCvt128_U16ToU32.
|
|
DEFINE_OP(UniOpVV::kCvtI32LoToI64, 16, kNA), // kLoadCvt128_I32ToI64.
|
|
DEFINE_OP(UniOpVV::kCvtU32LoToU64, 16, kNA), // kLoadCvt128_U32ToU64.
|
|
DEFINE_OP(UniOpVV::kCvtI8LoToI16 , 32, kNA), // kLoadCvt256_I8ToI16.
|
|
DEFINE_OP(UniOpVV::kCvtU8LoToU16 , 32, kNA), // kLoadCvt256_U8ToU16.
|
|
DEFINE_OP(UniOpVV::kCvtI16LoToI32, 32, kNA), // kLoadCvt256_I16ToI32.
|
|
DEFINE_OP(UniOpVV::kCvtU16LoToU32, 32, kNA), // kLoadCvt256_U16ToU32.
|
|
DEFINE_OP(UniOpVV::kCvtI32LoToI64, 32, kNA), // kLoadCvt256_I32ToI64.
|
|
DEFINE_OP(UniOpVV::kCvtU32LoToU64, 32, kNA), // kLoadCvt256_U32ToU64.
|
|
DEFINE_OP(1 , 0, kNA), // kLoadCvtN_U8ToU64.
|
|
DEFINE_OP(UniOpVV::kCvtI8LoToI16 , 0, kNA), // kLoadCvtN_I8ToI16.
|
|
DEFINE_OP(UniOpVV::kCvtU8LoToU16 , 0, kNA), // kLoadCvtN_U8ToU16.
|
|
DEFINE_OP(UniOpVV::kCvtI8ToI32 , 0, kNA), // kLoadCvtN_I8ToI32.
|
|
DEFINE_OP(UniOpVV::kCvtU8ToU32 , 0, kNA), // kLoadCvtN_U8ToU32.
|
|
DEFINE_OP(UniOpVV::kCvtI16LoToI32, 0, kNA), // kLoadCvtN_I16ToI32.
|
|
DEFINE_OP(UniOpVV::kCvtU16LoToU32, 0, kNA), // kLoadCvtN_U16ToU32.
|
|
DEFINE_OP(UniOpVV::kCvtI32LoToI64, 0, kNA), // kLoadCvtN_I32ToI64.
|
|
DEFINE_OP(UniOpVV::kCvtU32LoToU64, 0, kNA), // kLoadCvtN_U32ToU64.
|
|
DEFINE_OP(0 , 1, k8 ), // kLoadInsertU8.
|
|
DEFINE_OP(0 , 2, k16), // kLoadInsertU16.
|
|
DEFINE_OP(0 , 4, k32), // kLoadInsertU32.
|
|
DEFINE_OP(0 , 8, k64), // kLoadInsertU64.
|
|
DEFINE_OP(0 , 4, k32), // kLoadInsertF32.
|
|
DEFINE_OP(0 , 8, k64), // kLoadInsertF32x2.
|
|
DEFINE_OP(0 , 8, k64) // kLoadInsertF64.
|
|
};
|
|
|
|
static constexpr UniOpVMInfo opcode_info_2mv[size_t(UniOpMV::kMaxValue) + 1] = {
|
|
DEFINE_OP(NarrowingOp::kNone , 1, k8 ), // kStore8.
|
|
DEFINE_OP(NarrowingOp::kNone , 2, k16), // kStore16_U16.
|
|
DEFINE_OP(NarrowingOp::kNone , 4, k32), // kStore32_U32.
|
|
DEFINE_OP(NarrowingOp::kNone , 4, k32), // kStore32_F32.
|
|
DEFINE_OP(NarrowingOp::kNone , 8, k32), // kStore64_U32.
|
|
DEFINE_OP(NarrowingOp::kNone , 8, k64), // kStore64_U64.
|
|
DEFINE_OP(NarrowingOp::kNone , 8, k32), // kStore64_F32.
|
|
DEFINE_OP(NarrowingOp::kNone , 8, k64), // kStore64_F64.
|
|
DEFINE_OP(NarrowingOp::kNone , 16, k32), // kStore128_U32.
|
|
DEFINE_OP(NarrowingOp::kNone , 16, k64), // kStore128_U64.
|
|
DEFINE_OP(NarrowingOp::kNone , 16, k32), // kStore128_F32.
|
|
DEFINE_OP(NarrowingOp::kNone , 16, k64), // kStore128_F64.
|
|
DEFINE_OP(NarrowingOp::kNone , 32, k32), // kStore256_U32.
|
|
DEFINE_OP(NarrowingOp::kNone , 32, k64), // kStore256_U64.
|
|
DEFINE_OP(NarrowingOp::kNone , 32, k32), // kStore256_F32.
|
|
DEFINE_OP(NarrowingOp::kNone , 32, k64), // kStore256_F64.
|
|
DEFINE_OP(NarrowingOp::kNone , 64, k32), // kStore512_U32.
|
|
DEFINE_OP(NarrowingOp::kNone , 64, k64), // kStore512_U64.
|
|
DEFINE_OP(NarrowingOp::kNone , 64, k32), // kStore512_F32.
|
|
DEFINE_OP(NarrowingOp::kNone , 64, k64), // kStore512_F64.
|
|
DEFINE_OP(NarrowingOp::kNone , 0, k32), // kStoreN_U32.
|
|
DEFINE_OP(NarrowingOp::kNone , 0, k64), // kStoreN_U64.
|
|
DEFINE_OP(NarrowingOp::kNone , 0, k32), // kStoreN_F32.
|
|
DEFINE_OP(NarrowingOp::kNone , 0, k64) // kStoreN_F64.
|
|
/*
|
|
DEFINE_OP(NarrowingOp::kU16ToU8 , 8, kNA), // kStoreCvtz64_U16ToU8.
|
|
DEFINE_OP(NarrowingOp::kU32ToU16 , 8, kNA), // kStoreCvtz64_U32ToU16.
|
|
DEFINE_OP(NarrowingOp::kU64ToU32 , 8, kNA), // kStoreCvtz64_U64ToU32.
|
|
DEFINE_OP(NarrowingOp::kI16ToI8 , 8, kNA), // kStoreCvts64_I16ToI8.
|
|
DEFINE_OP(NarrowingOp::kI16ToU8 , 8, kNA), // kStoreCvts64_I16ToU8.
|
|
DEFINE_OP(NarrowingOp::kU16ToU8 , 8, kNA), // kStoreCvts64_U16ToU8.
|
|
DEFINE_OP(NarrowingOp::kI32ToI16 , 8, kNA), // kStoreCvts64_I32ToI16.
|
|
DEFINE_OP(NarrowingOp::kU32ToU16 , 8, kNA), // kStoreCvts64_U32ToU16.
|
|
DEFINE_OP(NarrowingOp::kI64ToI32 , 8, kNA), // kStoreCvts64_I64ToI32.
|
|
DEFINE_OP(NarrowingOp::kU64ToU32 , 8, kNA), // kStoreCvts64_U64ToU32.
|
|
DEFINE_OP(NarrowingOp::kU16ToU8 , 16, kNA), // kStoreCvtz128_U16ToU8.
|
|
DEFINE_OP(NarrowingOp::kU32ToU16 , 16, kNA), // kStoreCvtz128_U32ToU16.
|
|
DEFINE_OP(NarrowingOp::kU64ToU32 , 16, kNA), // kStoreCvtz128_U64ToU32.
|
|
DEFINE_OP(NarrowingOp::kI16ToI8 , 16, kNA), // kStoreCvts128_I16ToI8.
|
|
DEFINE_OP(NarrowingOp::kI16ToU8 , 16, kNA), // kStoreCvts128_I16ToU8.
|
|
DEFINE_OP(NarrowingOp::kU16ToU8 , 16, kNA), // kStoreCvts128_U16ToU8.
|
|
DEFINE_OP(NarrowingOp::kI32ToI16 , 16, kNA), // kStoreCvts128_I32ToI16.
|
|
DEFINE_OP(NarrowingOp::kU32ToU16 , 16, kNA), // kStoreCvts128_U32ToU16.
|
|
DEFINE_OP(NarrowingOp::kI64ToI32 , 16, kNA), // kStoreCvts128_I64ToI32.
|
|
DEFINE_OP(NarrowingOp::kU64ToU32 , 16, kNA), // kStoreCvts128_U64ToU32.
|
|
DEFINE_OP(NarrowingOp::kU16ToU8 , 32, kNA), // kStoreCvtz256_U16ToU8.
|
|
DEFINE_OP(NarrowingOp::kU32ToU16 , 32, kNA), // kStoreCvtz256_U32ToU16.
|
|
DEFINE_OP(NarrowingOp::kU64ToU32 , 32, kNA), // kStoreCvtz256_U64ToU32.
|
|
DEFINE_OP(NarrowingOp::kI16ToI8 , 32, kNA), // kStoreCvts256_I16ToI8.
|
|
DEFINE_OP(NarrowingOp::kI16ToU8 , 32, kNA), // kStoreCvts256_I16ToU8.
|
|
DEFINE_OP(NarrowingOp::kU16ToU8 , 32, kNA), // kStoreCvts256_U16ToU8.
|
|
DEFINE_OP(NarrowingOp::kI32ToI16 , 32, kNA), // kStoreCvts256_I32ToI16.
|
|
DEFINE_OP(NarrowingOp::kU32ToU16 , 32, kNA), // kStoreCvts256_U32ToU16.
|
|
DEFINE_OP(NarrowingOp::kI64ToI32 , 32, kNA), // kStoreCvts256_I64ToI32.
|
|
DEFINE_OP(NarrowingOp::kU64ToU32 , 32, kNA), // kStoreCvts256_U64ToU32.
|
|
DEFINE_OP(NarrowingOp::kU16ToU8 , 0, kNA), // kStoreCvtzN_U16ToU8.
|
|
DEFINE_OP(NarrowingOp::kU32ToU16 , 0, kNA), // kStoreCvtzN_U32ToU16.
|
|
DEFINE_OP(NarrowingOp::kU64ToU32 , 0, kNA), // kStoreCvtzN_U64ToU32.
|
|
DEFINE_OP(NarrowingOp::kI16ToI8 , 0, kNA), // kStoreCvtsN_I16ToI8.
|
|
DEFINE_OP(NarrowingOp::kI16ToU8 , 0, kNA), // kStoreCvtsN_I16ToU8.
|
|
DEFINE_OP(NarrowingOp::kU16ToU8 , 0, kNA), // kStoreCvtsN_U16ToU8.
|
|
DEFINE_OP(NarrowingOp::kI32ToI16 , 0, kNA), // kStoreCvtsN_I32ToI16.
|
|
DEFINE_OP(NarrowingOp::kU32ToU16 , 0, kNA), // kStoreCvtsN_U32ToU16.
|
|
DEFINE_OP(NarrowingOp::kI64ToI32 , 0, kNA), // kStoreCvtsN_I64ToI32.
|
|
DEFINE_OP(NarrowingOp::kU64ToU32 , 0, kNA) // kStoreCvtsN_U64ToU32.
|
|
*/
|
|
};
|
|
|
|
#undef DEFINE_OP
|
|
|
|
// ujit::UniCompiler - Vector Instructions - Utility Functions
|
|
// ===========================================================
|
|
|
|
static constexpr uint32_t float_mode_mem_size_table[5] = { 0, 4, 8, 0, 0 };
|
|
|
|
static ASMJIT_INLINE bool is_same_vec(const Vec& a, const Operand_& b) noexcept {
|
|
return b.is_vec() && a.id() == b.id();
|
|
}
|
|
|
|
static ASMJIT_INLINE void vec_set_vec_type(Vec& vec, ElementSize sz) noexcept {
|
|
static constexpr uint32_t signatures[5] = {
|
|
RegTraits<RegType::kVec8>::kSignature,
|
|
RegTraits<RegType::kVec16>::kSignature,
|
|
RegTraits<RegType::kVec32>::kSignature,
|
|
RegTraits<RegType::kVec64>::kSignature,
|
|
RegTraits<RegType::kVec128>::kSignature
|
|
};
|
|
vec.set_signature(OperandSignature{signatures[size_t(sz)]});
|
|
}
|
|
|
|
static ASMJIT_INLINE void vec_set_type(Vec& vec, ElementSize sz) noexcept {
|
|
vec.set_element_type(a64::VecElementType(uint32_t(sz) + 1));
|
|
}
|
|
|
|
static ASMJIT_INLINE void vec_set_type_and_index(Vec& vec, ElementSize sz, uint32_t idx) noexcept {
|
|
vec.set_element_type(a64::VecElementType(uint32_t(sz) + 1));
|
|
vec.set_element_index(idx);
|
|
}
|
|
|
|
static ASMJIT_NOINLINE void vec_load_mem(UniCompiler& uc, const Vec& dst, Mem src, uint32_t mem_size) {
|
|
BackendCompiler* cc = uc.cc;
|
|
|
|
if (src.has_index() && src.has_shift()) {
|
|
// AArch64 limitation: index shift can be the same size as the size of the read operation, so H << 1, S << 2,
|
|
// etc... Other shift values are not supported at the architectural level, so we have to precalculate the address.
|
|
uint32_t shift = src.shift();
|
|
if (mem_size != (1u << shift) || src.has_offset()) {
|
|
Gp base = src.base_reg().as<Gp>();
|
|
Gp index = src.index_reg().as<Gp>();
|
|
|
|
if (src.is_pre_index()) {
|
|
cc->add(base, base, index, a64::Shift(src.shift_op(), shift));
|
|
src = a64::ptr(base, src.offset_lo32());
|
|
}
|
|
else {
|
|
Gp tmp = uc.new_gpz("@mem_addr");
|
|
cc->add(tmp, base, index, a64::Shift(src.shift_op(), shift));
|
|
src = a64::ptr(tmp, src.offset_lo32());
|
|
}
|
|
}
|
|
}
|
|
|
|
switch (mem_size) {
|
|
case 1: cc->ldr(dst.b(), src); break;
|
|
case 2: cc->ldr(dst.h(), src); break;
|
|
case 4: cc->ldr(dst.s(), src); break;
|
|
case 8: cc->ldr(dst.d(), src); break;
|
|
case 16: cc->ldr(dst.q(), src); break;
|
|
default:
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
}
|
|
|
|
static ASMJIT_NOINLINE Vec vec_from_mem(UniCompiler& uc, const Mem& op, const Vec& ref, uint32_t mem_size = 0) {
|
|
Vec vec = uc.new_vec128("@tmp");
|
|
if (mem_size == 0)
|
|
mem_size = ref.size();
|
|
vec_load_mem(uc, vec, op, mem_size);
|
|
return vec.clone_as(ref);
|
|
}
|
|
|
|
static ASMJIT_INLINE Vec as_vec(UniCompiler& uc, const Operand_& op, const Vec& ref, uint32_t mem_size = 0) {
|
|
if (op.is_vec())
|
|
return op.as<Vec>().clone_as(ref);
|
|
else
|
|
return vec_from_mem(uc, op.as<Mem>(), ref, mem_size);
|
|
}
|
|
|
|
static ASMJIT_INLINE Vec as_vec(UniCompiler& uc, const Operand_& op, const Vec& ref, FloatMode fm) {
|
|
if (op.is_vec())
|
|
return op.as<Vec>().clone_as(ref);
|
|
else
|
|
return vec_from_mem(uc, op.as<Mem>(), ref, float_mode_mem_size_table[size_t(fm)]);
|
|
}
|
|
|
|
static ASMJIT_NOINLINE Vec vec_mov(UniCompiler& uc, const Vec& dst_, const Operand_& src_) {
|
|
BackendCompiler* cc = uc.cc;
|
|
|
|
Vec dst(dst_);
|
|
vec_set_type(dst, ElementSize::k8);
|
|
|
|
if (src_.is_vec()) {
|
|
if (dst.id() != src_.id()) {
|
|
Vec src = src_.as<Vec>();
|
|
vec_set_type(src, ElementSize::k8);
|
|
cc->mov(dst, src.as<Vec>());
|
|
}
|
|
return dst;
|
|
}
|
|
|
|
if (src_.is_mem()) {
|
|
vec_load_mem(uc, dst, src_.as<Mem>(), dst.size());
|
|
return dst;
|
|
}
|
|
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
|
|
static ASMJIT_NOINLINE void vec_neg(UniCompiler& uc, const Vec& dst, const Vec& src, FloatMode fm) {
|
|
BackendCompiler* cc = uc.cc;
|
|
|
|
if (fm == FloatMode::kF32S)
|
|
cc->mvn_(dst.s(), src.s());
|
|
else if (fm == FloatMode::kF64S)
|
|
cc->mvn_(dst.d(), src.d());
|
|
else
|
|
cc->mvn_(dst.q(), src.q());
|
|
}
|
|
|
|
// ujit::UniCompiler - Vector Instructions - Swizzle 32 Impl
|
|
// =========================================================
|
|
|
|
// [DCBA] <- Mov (DCBA, dcba)
|
|
// [AAAA] <- Dup0 (DCBA, dcba)
|
|
// [BBBB] <- Dup1 (DCBA, dcba)
|
|
// [CCCC] <- Dup2 (DCBA, dcba)
|
|
// [DDDD] <- Dup3 (DCBA, dcba)
|
|
// [CDAB] <- Rev64 (DCBA, dcba)
|
|
// [aDCB] <- Ext4 (DCBA, dcba) [dcb|aDCB|A ]
|
|
// [baDC] <- Ext8 (DCBA, dcba) [ dc|baDC|BA ]
|
|
// [cbaD] <- Ext12 (DCBA, dcba) [ d|cbaD|CBA]
|
|
// [bBaA] <- Zip1_4S(DCBA, dcba)
|
|
// [baBA] <- Zip1_2D(DCBA, dcba)
|
|
// [dDcC] <- Zip2_4S(DCBA, dcba)
|
|
// [dcDC] <- Zip2_2D(DCBA, dcba)
|
|
// [caCA] <- Uzp1_4S(DCBA, dcba)
|
|
// [baBA] <- Uzp1_2D(DCBA, dcba)
|
|
// [dbDB] <- Uzp2_4S(DCBA, dcba)
|
|
// [dcDC] <- Uzp2_2D(DCBA, dcba)
|
|
// [cCaA] <- Trn1_4S(DCBA, dcba)
|
|
// [dDbB] <- Trn2_4S(DCBA, dcba)
|
|
struct Swizzle32Data {
|
|
enum class OpTarget : uint8_t {
|
|
kDst = 0,
|
|
k1 = 1,
|
|
k2 = 2,
|
|
kA = 3,
|
|
|
|
k_ = kDst
|
|
};
|
|
|
|
struct Op {
|
|
//! Swizzle operation does nothing - 'mov' (this can be only the first operation).
|
|
static constexpr uint8_t kMov = 1;
|
|
//! Swizzle operation performs an insert - moves a value from one lane to another.
|
|
static constexpr uint8_t kIns = 2;
|
|
//! Swizzle operation duplicates a lane across all others - 'dup'.
|
|
static constexpr uint8_t kDup = 3;
|
|
//! Swizzle operation rotates a vector - 'ext'.
|
|
static constexpr uint8_t kExt = 4;
|
|
//! Swizzle operation swaps lo/hi elements of 64-bit lanes - 'rev64'.
|
|
static constexpr uint8_t kRev64 = 5;
|
|
//! Swizzle operation can be implemented as a single zip[1|2], uzp[1|2], or trn[1|2] instruction with 32-bit or 64-bit elements.
|
|
static constexpr uint8_t kZipUnzip = 7;
|
|
|
|
static constexpr uint8_t kIns0To1 = uint8_t(kIns ) | (0 << 4) | (1 << 6);
|
|
static constexpr uint8_t kIns0To2 = uint8_t(kIns ) | (0 << 4) | (2 << 6);
|
|
static constexpr uint8_t kIns0To3 = uint8_t(kIns ) | (0 << 4) | (3 << 6);
|
|
static constexpr uint8_t kIns1To0 = uint8_t(kIns ) | (1 << 4) | (0 << 6);
|
|
static constexpr uint8_t kIns1To2 = uint8_t(kIns ) | (1 << 4) | (2 << 6);
|
|
static constexpr uint8_t kIns1To3 = uint8_t(kIns ) | (1 << 4) | (3 << 6);
|
|
static constexpr uint8_t kIns2To0 = uint8_t(kIns ) | (2 << 4) | (0 << 6);
|
|
static constexpr uint8_t kIns2To1 = uint8_t(kIns ) | (2 << 4) | (1 << 6);
|
|
static constexpr uint8_t kIns2To3 = uint8_t(kIns ) | (2 << 4) | (3 << 6);
|
|
static constexpr uint8_t kIns3To0 = uint8_t(kIns ) | (3 << 4) | (0 << 6);
|
|
static constexpr uint8_t kIns3To1 = uint8_t(kIns ) | (3 << 4) | (1 << 6);
|
|
static constexpr uint8_t kIns3To2 = uint8_t(kIns ) | (3 << 4) | (2 << 6);
|
|
static constexpr uint8_t kDup0 = uint8_t(kDup ) | (0 << 4);
|
|
static constexpr uint8_t kDup1 = uint8_t(kDup ) | (1 << 4);
|
|
static constexpr uint8_t kDup2 = uint8_t(kDup ) | (2 << 4);
|
|
static constexpr uint8_t kDup3 = uint8_t(kDup ) | (3 << 4);
|
|
static constexpr uint8_t kExt4 = uint8_t(kExt ) | (1 << 4);
|
|
static constexpr uint8_t kExt8 = uint8_t(kExt ) | (2 << 4);
|
|
static constexpr uint8_t kExt12 = uint8_t(kExt ) | (3 << 4);
|
|
static constexpr uint8_t kZip1_4S = uint8_t(kZipUnzip) | (0 << 4) | (0 << 7);
|
|
static constexpr uint8_t kZip1_2D = uint8_t(kZipUnzip) | (0 << 4) | (1 << 7);
|
|
static constexpr uint8_t kZip2_4S = uint8_t(kZipUnzip) | (1 << 4) | (0 << 7);
|
|
static constexpr uint8_t kZip2_2D = uint8_t(kZipUnzip) | (1 << 4) | (1 << 7);
|
|
static constexpr uint8_t kUzp1_4S = uint8_t(kZipUnzip) | (2 << 4) | (0 << 7);
|
|
static constexpr uint8_t kUzp1_2D = uint8_t(kZipUnzip) | (2 << 4) | (1 << 7);
|
|
static constexpr uint8_t kUzp2_4S = uint8_t(kZipUnzip) | (3 << 4) | (0 << 7);
|
|
static constexpr uint8_t kUzp2_2D = uint8_t(kZipUnzip) | (3 << 4) | (1 << 7);
|
|
static constexpr uint8_t kTrn1_4S = uint8_t(kZipUnzip) | (4 << 4);
|
|
static constexpr uint8_t kTrn2_4S = uint8_t(kZipUnzip) | (5 << 4);
|
|
|
|
// Alias to nothing to make the table easier to read.
|
|
static constexpr uint8_t k_ = 0;
|
|
|
|
uint8_t data;
|
|
|
|
ASMJIT_INLINE_NODEBUG bool is_valid() const noexcept { return data != 0; }
|
|
ASMJIT_INLINE_NODEBUG uint32_t type() const noexcept { return data & 0xF; }
|
|
|
|
ASMJIT_INLINE_NODEBUG uint32_t dup_idx() const noexcept { return (data >> 4) & 0x3; }
|
|
ASMJIT_INLINE_NODEBUG uint32_t ext_imm() const noexcept { return (data >> 2) & (0x3 << 2); }
|
|
|
|
ASMJIT_INLINE_NODEBUG uint32_t zip_op() const noexcept { return (data >> 4) & 0x7; }
|
|
ASMJIT_INLINE_NODEBUG bool zip_s4() const noexcept { return (data & (1 << 7)) == 0; }
|
|
|
|
ASMJIT_INLINE_NODEBUG uint32_t ins_src() const noexcept { return (data >> 4) & 0x3; }
|
|
ASMJIT_INLINE_NODEBUG uint32_t ins_dst() const noexcept { return (data >> 6) & 0x3; }
|
|
};
|
|
|
|
//! \name Members
|
|
//! \{
|
|
|
|
Op ops[3];
|
|
uint8_t flags;
|
|
|
|
//! \}
|
|
|
|
//! \name Accessors
|
|
//! \{
|
|
|
|
ASMJIT_INLINE_NODEBUG bool is_defined() const noexcept { return ops[0].data != 0; }
|
|
ASMJIT_INLINE_NODEBUG Op op(uint32_t index) const noexcept { return ops[index]; }
|
|
ASMJIT_INLINE_NODEBUG OpTarget op_target(uint32_t index) const noexcept { return OpTarget((flags >> (index * 2)) & 0x3); }
|
|
|
|
//! \}
|
|
};
|
|
|
|
// This table provides all combinations for all possible 32-bit swizzles (there is 256 combinations in total).
|
|
// It prioritizes lane moves, and then operations that can have either one or two inputs. Each operation has
|
|
// a target, which specifies whether it replaces the destination or one or both sources that are then passed
|
|
// to a next operation. The last operation must always be `OpTarget::kDst` so the result ends up in the right
|
|
// register.
|
|
//
|
|
// In general the decomposition of operations needed for all swizzles is as follows:
|
|
//
|
|
// - 1 Op Swizzles: 17
|
|
// - 2 Op Swizzles: 156
|
|
// - 3 Op Swizzles: 83
|
|
//
|
|
// Which means that luckily most used swizzles would fall into 1 or 2 operations.
|
|
//
|
|
// NOTE: Moves (InsXToY) operations only happen on the destination as they are destructive, which is perfectly
|
|
// okay as moving them into earlier steps didn't really improve anything.
|
|
#define OP(swiz, op0, target0, op1, target1, op2, target2) { \
|
|
{ \
|
|
{Swizzle32Data::Op::k##op0}, \
|
|
{Swizzle32Data::Op::k##op1}, \
|
|
{Swizzle32Data::Op::k##op2} \
|
|
}, \
|
|
( \
|
|
(uint16_t(Swizzle32Data::OpTarget::k##target0) << 0) | \
|
|
(uint16_t(Swizzle32Data::OpTarget::k##target1) << 2) | \
|
|
(uint16_t(Swizzle32Data::OpTarget::k##target2) << 4) \
|
|
) \
|
|
}
|
|
|
|
static constexpr Swizzle32Data swizzle_32_data[256] = {
|
|
OP(0000, Dup0 , _, _ , _, _ , _), OP(0001, Rev64 , _, Ins1To2, _, Ins1To3, _), OP(0002, Ext8 , _, Ins2To1, _, Ins2To3, _), OP(0003, Dup0 , 2, Ext12 , _, _ , _),
|
|
OP(0010, Zip1_2D, _, Ins0To3, _, _ , _), OP(0011, Rev64 , A, Zip1_4S, _, _ , _), OP(0012, Ext8 , _, Ins3To1, _, Ins2To3, _), OP(0013, Rev64 , 2, Ext12 , _, Ins2To3, _),
|
|
OP(0020, Uzp1_4S, _, Ins0To3, _, _ , _), OP(0021, Ext4 , _, Ins3To2, _, _ , _), OP(0022, Ext4 , A, Trn2_4S, _, _ , _), OP(0023, Ext4 , _, Ins2To0, _, Ins3To2, _),
|
|
OP(0030, Ext8 , _, Ins2To0, _, Ins2To3, _), OP(0031, Dup0 , 2, Uzp2_4S, _, _ , _), OP(0032, Ext8 , _, Ins2To3, _, _ , _), OP(0033, Ext4 , A, Zip2_4S, _, _ , _),
|
|
OP(0100, Zip1_4S, _, Ins0To3, _, _ , _), OP(0101, Rev64 , A, Zip1_2D, _, _ , _), OP(0102, Ext12 , _, Ins3To0, _, Ins1To3, _), OP(0103, Ext12 , _, Ins1To3, _, _ , _),
|
|
OP(0110, Rev64 , 2, Zip1_4S, _, _ , _), OP(0111, Dup1 , 1, Ext4 , _, _ , _), OP(0112, Rev64 , 2, Ext8 , _, Ins2To1, _), OP(0113, Ext12 , _, Ins1To3, _, Ins2To1, _),
|
|
OP(0120, Ext4 , _, Ins0To2, _, Ins3To0, _), OP(0121, Ext4 , _, Ins0To2, _, _ , _), OP(0122, Ext4 , _, Ins0To2, _, Ins1To0, _), OP(0123, Rev64 , A, Ext8 , _, _ , _),
|
|
OP(0130, Ext12 , 2, Zip1_4S, _, _ , _), OP(0131, Ext4 , _, Ins2To1, _, Ins0To2, _), OP(0132, Rev64 , 2, Ext8 , _, _ , _), OP(0133, Ext12 , _, Ins1To3, _, Ins0To1, _),
|
|
OP(0200, Trn1_4S, _, Ins0To3, _, _ , _), OP(0201, Rev64 , _, Ins3To2, _, Ins1To3, _), OP(0202, Ext4 , A, Uzp2_4S, _, _ , _), OP(0203, Uzp1_4S, 2, Ext12 , _, _ , _),
|
|
OP(0210, Mov , _, Ins0To3, _, _ , _), OP(0211, Mov , _, Ins0To3, _, Ins1To0, _), OP(0212, Mov , _, Ins0To3, _, Ins2To0, _), OP(0213, Rev64 , A, Ext8 , 1, Zip1_4S, _),
|
|
OP(0220, Ext8 , 2, Uzp1_4S, _, _ , _), OP(0221, Ext4 , _, Ins1To2, _, _ , _), OP(0222, Dup2 , 1, Ext4 , _, _ , _), OP(0223, Ext4 , _, Ins2To0, _, Ins1To2, _),
|
|
OP(0230, Rev64 , 1, Ext4 , _, _ , _), OP(0231, Ext4 , 2, Uzp2_4S, _, _ , _), OP(0232, Ext8 , _, Ins2To3, _, Ins0To2, _), OP(0233, Rev64 , 1, Ext4 , _, Ins1To0, _),
|
|
OP(0300, Rev64 , _, Ins1To0, _, Ins1To3, _), OP(0301, Rev64 , _, Ins1To3, _, _ , _), OP(0302, Dup0 , 2, Zip2_4S, _, _ , _), OP(0303, Ext4 , A, Zip2_2D, _, _ , _),
|
|
OP(0310, Ext12 , 2, Zip1_2D, _, _ , _), OP(0311, Ext4 , _, Ins0To1, _, _ , _), OP(0312, Dup0 , 2, Zip1_4S, 2, Zip2_4S, _), OP(0313, Uzp2_4S, 1, Ext4 , _, _ , _),
|
|
OP(0320, Ext4 , _, Ins3To0, _, _ , _), OP(0321, Ext4 , _, _ , _, _ , _), OP(0322, Ext4 , _, Ins1To0, _, _ , _), OP(0323, Ext4 , _, Ins2To0, _, _ , _),
|
|
OP(0330, Ext4 , _, Ins2To1, _, Ins3To0, _), OP(0331, Ext4 , _, Ins2To1, _, _ , _), OP(0332, Ext4 , 2, Zip2_4S, _, _ , _), OP(0333, Dup3 , 1, Ext4 , _, _ , _),
|
|
OP(1000, Zip1_4S, _, Ins0To2, _, _ , _), OP(1001, Rev64 , 1, Zip1_4S, _, _ , _), OP(1002, Ext8 , _, Ins2To1, _, _ , _), OP(1003, Ext4 , 1, Ext8 , _, _ , _),
|
|
OP(1010, Zip1_2D, _, _ , _, _ , _), OP(1011, Zip1_2D, _, Ins1To0, _, _ , _), OP(1012, Ext8 , _, Ins3To1, _, _ , _), OP(1013, Ext8 , _, Ins1To0, _, Ins3To1, _),
|
|
OP(1020, Zip1_4S, 2, Uzp1_4S, _, _ , _), OP(1021, Ext4 , 1, Zip1_2D, _, _ , _), OP(1022, Ext8 , _, Ins0To1, _, _ , _), OP(1023, Rev64 , 1, Ext8 , _, _ , _),
|
|
OP(1030, Ext8 , _, Ins2To0, _, _ , _), OP(1031, Ext8 , _, Ins3To0, _, _ , _), OP(1032, Ext8 , _, _ , _, _ , _), OP(1033, Ext8 , _, Ins1To0, _, _ , _),
|
|
OP(1100, Zip1_4S, _, _ , _, _ , _), OP(1101, Zip1_4S, _, Ins2To0, _, _ , _), OP(1102, Ext8 , _, Ins2To1, _, Ins3To2, _), OP(1103, Ext12 , _, Ins2To3, _, _ , _),
|
|
OP(1110, Zip1_4S, _, Ins2To1, _, _ , _), OP(1111, Dup1 , _, _ , _, _ , _), OP(1112, Ext8 , _, Ins3To1, _, Ins3To2, _), OP(1113, Dup1 , 2, Ext12 , _, _ , _),
|
|
OP(1120, Dup1 , 2, Uzp1_4S, _, _ , _), OP(1121, Ext4 , _, Ins0To2, _, Ins0To3, _), OP(1122, Ext8 , _, Ins0To1, _, Ins3To2, _), OP(1123, Ext12 , _, Ins3To1, _, Ins2To3, _),
|
|
OP(1130, Ext8 , _, Ins2To0, _, Ins3To2, _), OP(1131, Uzp2_4S, _, Ins0To3, _, _ , _), OP(1132, Ext8 , _, Ins3To2, _, _ , _), OP(1133, Ext8 , A, Trn2_4S, _, _ , _),
|
|
OP(1200, Zip1_4S, 2, Trn1_4S, _, _ , _), OP(1201, Ext4 , 1, Zip1_4S, _, _ , _), OP(1202, Dup2 , 1, Zip1_4S, _, _ , _), OP(1203, Dup1 , 2, Uzp1_4S, 2, Ext12 , _),
|
|
OP(1210, Mov , _, Ins1To3, _, _ , _), OP(1211, Mov , _, Ins1To0, _, Ins1To3, _), OP(1212, Mov , _, Ins1To3, _, Ins2To0, _), OP(1213, Mov , _, Ins3To0, _, Ins1To3, _),
|
|
OP(1220, Mov , _, Ins1To3, _, Ins2To1, _), OP(1221, Ext4 , _, Ins0To3, _, Ins1To2, _), OP(1222, Ext8 , _, Ins0To1, _, Ins0To2, _), OP(1223, Rev64 , 1, Ext8 , _, Ins1To2, _),
|
|
OP(1230, Rev64 , A, Ext4 , _, _ , _), OP(1231, Ext8 , _, Ins0To2, _, Ins3To0, _), OP(1232, Ext8 , _, Ins0To2, _, _ , _), OP(1233, Ext8 , _, Ins0To2, _, Ins1To0, _),
|
|
OP(1300, Rev64 , _, Ins0To3, _, Ins1To0, _), OP(1301, Rev64 , _, Ins0To3, _, _ , _), OP(1302, Ext8 , 1, Zip1_4S, _, _ , _), OP(1303, Dup3 , 1, Zip1_4S, _, _ , _),
|
|
OP(1310, Mov , _, Ins3To2, _, Ins1To3, _), OP(1311, Trn2_4S, _, Ins0To3, _, _ , _), OP(1312, Dup1 , 2, Zip2_4S, _, _ , _), OP(1313, Ext8 , A, Uzp2_4S, _, _ , _),
|
|
OP(1320, Ext12 , 2, Uzp1_4S, _, _ , _), OP(1321, Ext4 , _, Ins0To3, _, _ , _), OP(1322, Ext4 , _, Ins0To3, _, Ins1To0, _), OP(1323, Ext4 , _, Ins0To3, _, Ins2To0, _),
|
|
OP(1330, Ext8 , _, Ins2To0, _, Ins1To2, _), OP(1331, Ext8 , 2, Uzp2_4S, _, _ , _), OP(1332, Ext8 , _, Ins1To2, _, _ , _), OP(1333, Ext8 , _, Ins1To0, _, Ins1To2, _),
|
|
OP(2000, Uzp1_4S, _, Ins0To1, _, _ , _), OP(2001, Rev64 , _, Ins1To2, _, _ , _), OP(2002, Ext8 , 1, Uzp1_4S, _, _ , _), OP(2003, Ext12 , _, Ins1To2, _, _ , _),
|
|
OP(2010, Zip1_4S, 1, Uzp1_4S, _, _ , _), OP(2011, Dup1 , 1, Uzp1_4S, _, _ , _), OP(2012, Ext8 , _, Ins3To1, _, Ins0To3, _), OP(2013, Ext12 , 1, Uzp1_4S, _, _ , _),
|
|
OP(2020, Uzp1_4S, _, _ , _, _ , _), OP(2021, Rev64 , _, Ins1To2, _, Ins3To1, _), OP(2022, Uzp1_4S, _, Ins1To0, _, _ , _), OP(2023, Ext12 , _, Ins1To2, _, Ins3To1, _),
|
|
OP(2030, Ext8 , _, Ins0To3, _, Ins2To0, _), OP(2031, Rev64 , 1, Uzp1_4S, _, _ , _), OP(2032, Ext8 , _, Ins0To3, _, _ , _), OP(2033, Dup3 , 1, Uzp1_4S, _, _ , _),
|
|
OP(2100, Ext12 , _, Ins1To0, _, _ , _), OP(2101, Rev64 , _, Ins0To2, _, _ , _), OP(2102, Ext12 , _, Ins3To0, _, _ , _), OP(2103, Ext12 , _, _ , _, _ , _),
|
|
OP(2110, Ext4 , 2, Zip1_4S, _, _ , _), OP(2111, Rev64 , _, Ins0To1, _, Ins0To2, _), OP(2112, Ext12 , _, Ins2To1, _, Ins3To0, _), OP(2113, Ext12 , _, Ins2To1, _, _ , _),
|
|
OP(2120, Dup2 , 2, Zip1_4S, _, _ , _), OP(2121, Ext4 , A, Zip1_2D, _, _ , _), OP(2122, Ext12 , _, Ins3To0, _, Ins3To1, _), OP(2123, Ext12 , _, Ins3To1, _, _ , _),
|
|
OP(2130, Dup2 , 2, Ext12 , 2, Zip1_4S, _), OP(2131, Rev64 , _, Ins2To1, _, Ins0To2, _), OP(2132, Ext4 , 2, Ext8 , _, _ , _), OP(2133, Ext12 , _, Ins0To1, _, _ , _),
|
|
OP(2200, Trn1_4S, _, _ , _, _ , _), OP(2201, Rev64 , _, Ins3To2, _, _ , _), OP(2202, Trn1_4S, _, Ins2To0, _, _ , _), OP(2203, Ext12 , _, Ins3To2, _, _ , _),
|
|
OP(2210, Mov , _, Ins2To3, _, _ , _), OP(2211, Ext4 , A, Zip1_4S, _, _ , _), OP(2212, Mov , _, Ins2To0, _, Ins2To3, _), OP(2213, Mov , _, Ins3To0, _, Ins2To3, _),
|
|
OP(2220, Uzp1_4S, _, Ins1To2, _, _ , _), OP(2221, Rev64 , _, Ins3To1, _, Ins3To2, _), OP(2222, Dup2 , _, _ , _, _ , _), OP(2223, Dup2 , 2, Ext12 , _, _ , _),
|
|
OP(2230, Mov , _, Ins3To1, _, Ins2To3, _), OP(2231, Dup2 , 2, Uzp2_4S, _, _ , _), OP(2232, Zip2_2D, _, Ins0To3, _, _ , _), OP(2233, Rev64 , A, Zip2_4S, _, _ , _),
|
|
OP(2300, Rev64 , _, Ins1To0, _, _ , _), OP(2301, Rev64 , _, _ , _, _ , _), OP(2302, Rev64 , _, Ins3To0, _, _ , _), OP(2303, Rev64 , _, Ins2To0, _, _ , _),
|
|
OP(2310, Dup2 , 2, Ext12 , 2, Zip1_2D, _), OP(2311, Rev64 , _, Ins0To1, _, _ , _), OP(2312, Ext12 , 2, Zip2_4S, _, _ , _), OP(2313, Rev64 , _, Ins0To1, _, Ins2To0, _),
|
|
OP(2320, Rev64 , _, Ins1To0, _, Ins3To1, _), OP(2321, Rev64 , _, Ins3To1, _, _ , _), OP(2322, Zip2_4S, _, Ins0To3, _, _ , _), OP(2323, Rev64 , A, Zip2_2D, _, _ , _),
|
|
OP(2330, Rev64 , _, Ins1To0, _, Ins2To1, _), OP(2331, Rev64 , _, Ins2To1, _, _ , _), OP(2332, Rev64 , 2, Zip2_4S, _, _ , _), OP(2333, Rev64 , _, Ins2To0, _, Ins2To1, _),
|
|
OP(3000, Mov , _, Ins0To1, _, Ins0To2, _), OP(3001, Rev64 , _, Ins2To3, _, Ins1To2, _), OP(3002, Ext8 , _, Ins1To3, _, Ins2To1, _), OP(3003, Ext12 , _, Ins0To3, _, Ins1To2, _),
|
|
OP(3010, Mov , _, Ins0To2, _, _ , _), OP(3011, Mov , _, Ins0To2, _, Ins1To0, _), OP(3012, Rev64 , A, Ext12 , _, _ , _), OP(3013, Rev64 , 2, Ext12 , _, _ , _),
|
|
OP(3020, Dup0 , 1, Zip2_4S, _, _ , _), OP(3021, Dup1 , 1, Ext4 , 1, Zip2_4S, _), OP(3022, Ext8 , _, Ins1To3, _, Ins0To1, _), OP(3023, Ext4 , 1, Zip2_4S, _, _ , _),
|
|
OP(3030, Mov , _, Ins0To2, _, Ins3To1, _), OP(3031, Ext8 , _, Ins3To0, _, Ins1To3, _), OP(3032, Ext8 , _, Ins1To3, _, _ , _), OP(3033, Ext8 , _, Ins1To0, _, Ins1To3, _),
|
|
OP(3100, Dup0 , 1, Uzp2_4S, _, _ , _), OP(3101, Rev64 , _, Ins2To3, _, Ins0To2, _), OP(3102, Ext4 , 1, Uzp2_4S, _, _ , _), OP(3103, Ext12 , _, Ins0To3, _, _ , _),
|
|
OP(3110, Mov , _, Ins1To2, _, _ , _), OP(3111, Uzp2_4S, _, Ins0To1, _, _ , _), OP(3112, Mov , _, Ins2To0, _, Ins1To2, _), OP(3113, Ext8 , 1, Uzp2_4S, _, _ , _),
|
|
OP(3120, Rev64 , 2, Uzp1_4S, _, _ , _), OP(3121, Dup1 , 1, Zip2_4S, _, _ , _), OP(3122, Dup2 , 1, Uzp2_4S, _, _ , _), OP(3123, Ext12 , _, Ins3To1, _, Ins0To3, _),
|
|
OP(3130, Dup3 , 2, Zip1_4S, _, _ , _), OP(3131, Uzp2_4S, _, _ , _, _ , _), OP(3132, Zip2_4S, 1, Uzp2_4S, _, _ , _), OP(3133, Uzp2_4S, _, Ins1To0, _, _ , _),
|
|
OP(3200, Mov , _, Ins0To1, _, _ , _), OP(3201, Dup1 , 1, Ext4 , 1, Zip2_2D, _), OP(3202, Mov , _, Ins0To1, _, Ins2To0, _), OP(3203, Ext4 , 1, Zip2_2D, _, _ , _),
|
|
OP(3210, Mov , _, _ , _, _ , _), OP(3211, Mov , _, Ins1To0, _, _ , _), OP(3212, Mov , _, Ins2To0, _, _ , _), OP(3213, Mov , _, Ins3To0, _, _ , _),
|
|
OP(3220, Mov , _, Ins2To1, _, _ , _), OP(3221, Ext12 , 1, Zip2_4S, _, _ , _), OP(3222, Zip2_4S, _, Ins0To2, _, _ , _), OP(3223, Rev64 , 1, Zip2_4S, _, _ , _),
|
|
OP(3230, Mov , _, Ins3To1, _, _ , _), OP(3231, Zip2_4S, 2, Uzp2_4S, _, _ , _), OP(3232, Zip2_2D, _, _ , _, _ , _), OP(3233, Zip2_2D, _, Ins1To0, _, _ , _),
|
|
OP(3300, Mov , _, Ins0To1, _, Ins3To2, _), OP(3301, Rev64 , _, Ins2To3, _, _ , _), OP(3302, Rev64 , _, Ins3To0, _, Ins2To3, _), OP(3303, Rev64 , _, Ins2To0, _, Ins2To3, _),
|
|
OP(3310, Mov , _, Ins3To2, _, _ , _), OP(3311, Trn2_4S, _, _ , _, _ , _), OP(3312, Zip2_4S, 1, Trn2_4S, _, _ , _), OP(3313, Trn2_4S, _, Ins2To0, _, _ , _),
|
|
OP(3320, Dup3 , 2, Uzp1_4S, _, _ , _), OP(3321, Ext4 , _, Ins2To3, _, _ , _), OP(3322, Zip2_4S, _, _ , _, _ , _), OP(3323, Zip2_4S, _, Ins2To0, _, _ , _),
|
|
OP(3330, Mov , _, Ins3To1, _, Ins3To2, _), OP(3331, Uzp2_4S, _, Ins1To2, _, _ , _), OP(3332, Zip2_4S, _, Ins2To1, _, _ , _), OP(3333, Dup3 , _, _ , _, _ , _)
|
|
};
|
|
|
|
#undef OP
|
|
|
|
static void emit_swizzle32_impl(UniCompiler& uc, const Vec& dst, const Vec& src, uint32_t imm) {
|
|
ASMJIT_ASSERT((imm & 0xFCFCFCFC) == 0);
|
|
|
|
BackendCompiler* cc = uc.cc;
|
|
|
|
uint32_t table_index = ((imm & 0x03000000) >> (24 - 6)) |
|
|
((imm & 0x00030000) >> (16 - 4)) |
|
|
((imm & 0x00000300) >> (8 - 2)) | (imm & 0x00000003);
|
|
Swizzle32Data swiz = swizzle_32_data[table_index];
|
|
|
|
if (swiz.is_defined()) {
|
|
Vec op_src[2] = { src, src };
|
|
Vec op_dst;
|
|
|
|
for (uint32_t i = 0; i < 3; i++) {
|
|
Swizzle32Data::Op op = swiz.op(i);
|
|
Swizzle32Data::OpTarget target = swiz.op_target(i);
|
|
|
|
if (!op.is_valid())
|
|
break;
|
|
|
|
if (target == Swizzle32Data::OpTarget::kDst) {
|
|
op_dst = dst;
|
|
}
|
|
else {
|
|
op_dst = uc.new_similar_reg(dst, "@tmp");
|
|
}
|
|
|
|
switch (op.type()) {
|
|
case Swizzle32Data::Op::kMov: {
|
|
vec_mov(uc, op_dst, op_src[0]);
|
|
break;
|
|
}
|
|
|
|
case Swizzle32Data::Op::kIns: {
|
|
uint32_t src_lane = op.ins_src();
|
|
uint32_t dst_lane = op.ins_dst();
|
|
// Insert is always the last operation that only uses the destination register.
|
|
cc->mov(op_dst.s(dst_lane), op_dst.s(src_lane));
|
|
break;
|
|
}
|
|
|
|
case Swizzle32Data::Op::kDup: {
|
|
// Use `dup` if the swizzle is actually a broadcast of a single element.
|
|
uint32_t idx = op.dup_idx();
|
|
cc->dup(op_dst.s4(), op_src[0].s(idx));
|
|
break;
|
|
}
|
|
|
|
case Swizzle32Data::Op::kExt: {
|
|
// Use `ext` if the swizzle is rotating the vector.
|
|
uint32_t n = op.ext_imm();
|
|
cc->ext(op_dst.b16(), op_src[0].b16(), op_src[1].b16(), n);
|
|
break;
|
|
}
|
|
|
|
case Swizzle32Data::Op::kRev64: {
|
|
// Use `rev64` to swap lo/hi elements of 64-bit lanes.
|
|
cc->rev64(op_dst.s4(), op_src[0].s4());
|
|
break;
|
|
}
|
|
|
|
case Swizzle32Data::Op::kZipUnzip: {
|
|
// Use `zip[1|2]`, 'uzp[1|2]', or 'trn[1|2]` if the swizzle can be implemented this way.
|
|
static constexpr uint16_t zip_unzip_inst[8] = {
|
|
Inst::kIdZip1_v,
|
|
Inst::kIdZip2_v,
|
|
Inst::kIdUzp1_v,
|
|
Inst::kIdUzp2_v,
|
|
Inst::kIdTrn1_v,
|
|
Inst::kIdTrn2_v
|
|
};
|
|
|
|
InstId inst_id = zip_unzip_inst[op.zip_op()];
|
|
if (op.zip_s4())
|
|
cc->emit(inst_id, op_dst.s4(), op_src[0].s4(), op_src[1].s4());
|
|
else
|
|
cc->emit(inst_id, op_dst.d2(), op_src[0].d2(), op_src[1].d2());
|
|
break;
|
|
}
|
|
|
|
default:
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
|
|
if (uint32_t(target) & uint32_t(Swizzle32Data::OpTarget::k1)) op_src[0] = op_dst;
|
|
if (uint32_t(target) & uint32_t(Swizzle32Data::OpTarget::k2)) op_src[1] = op_dst;
|
|
}
|
|
}
|
|
else {
|
|
// NOTE: This code is never used at the moment. It's kept if for some reason we would want to avoid using
|
|
// more than 1 or 2 instructions to perform the swizzle. For example on hardware where TBL is faster than
|
|
// other operations combined.
|
|
uint8_t pred_data[16] = { 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xA, 0xB, 0xC, 0xD, 0xE, 0xF };
|
|
|
|
uint32_t d = (imm >> 22) & (0x3 << 2);
|
|
uint32_t c = (imm >> 14) & (0x3 << 2);
|
|
uint32_t b = (imm >> 6) & (0x3 << 2);
|
|
uint32_t a = (imm << 2) & (0x3 << 2);
|
|
|
|
pred_data[ 0] = uint8_t(a);
|
|
pred_data[ 1] = uint8_t(a + 1u);
|
|
pred_data[ 2] = uint8_t(a + 2u);
|
|
pred_data[ 3] = uint8_t(a + 3u);
|
|
pred_data[ 4] = uint8_t(b);
|
|
pred_data[ 5] = uint8_t(b + 1u);
|
|
pred_data[ 6] = uint8_t(b + 2u);
|
|
pred_data[ 7] = uint8_t(b + 3u);
|
|
pred_data[ 8] = uint8_t(c);
|
|
pred_data[ 9] = uint8_t(c + 1u);
|
|
pred_data[10] = uint8_t(c + 2u);
|
|
pred_data[11] = uint8_t(c + 3u);
|
|
pred_data[12] = uint8_t(d);
|
|
pred_data[13] = uint8_t(d + 1u);
|
|
pred_data[14] = uint8_t(d + 2u);
|
|
pred_data[15] = uint8_t(d + 3u);
|
|
|
|
Vec pred = uc.simd_const_16b(pred_data);
|
|
cc->tbl(dst.b16(), src.b16(), pred.b16());
|
|
}
|
|
}
|
|
|
|
// ujit::UniCompiler - Vector Instructions - Interleaved Shuffle 32 Impl
|
|
// =====================================================================
|
|
|
|
struct InterleavedShuffle32Ops {
|
|
struct Op {
|
|
//! Operation swaps lo/hi elements of 64-bit lanes - 'rev64'.
|
|
static constexpr uint32_t kOpRev64 = 0;
|
|
//! Operation performs a 32-bit insert - moves a value from a source lane to a destination lane.
|
|
static constexpr uint32_t kOpInsS = 1;
|
|
//! Operation performs a 64-bit insert - moves a value from a source lane to a destination lane.
|
|
static constexpr uint32_t kOpInsD = 2;
|
|
//! Operation can be implemented as a single zip[1|2], uzp[1|2], trn[1|2], or ext instruction with 32-bit or 64-bit elements.
|
|
static constexpr uint32_t kOpPerm = 3;
|
|
//! Operation duplicates a lane across all others - 'dup'.
|
|
static constexpr uint32_t kOpDup = 4;
|
|
|
|
static constexpr uint32_t kInsS_0To0 = (kOpInsS << 8) | (0 << 0) | (0 << 2);
|
|
static constexpr uint32_t kInsS_0To1 = (kOpInsS << 8) | (0 << 0) | (1 << 2);
|
|
static constexpr uint32_t kInsS_0To2 = (kOpInsS << 8) | (0 << 0) | (2 << 2);
|
|
static constexpr uint32_t kInsS_0To3 = (kOpInsS << 8) | (0 << 0) | (3 << 2);
|
|
static constexpr uint32_t kInsS_1To0 = (kOpInsS << 8) | (1 << 0) | (0 << 2);
|
|
static constexpr uint32_t kInsS_1To1 = (kOpInsS << 8) | (1 << 0) | (1 << 2);
|
|
static constexpr uint32_t kInsS_1To2 = (kOpInsS << 8) | (1 << 0) | (2 << 2);
|
|
static constexpr uint32_t kInsS_1To3 = (kOpInsS << 8) | (1 << 0) | (3 << 2);
|
|
static constexpr uint32_t kInsS_2To0 = (kOpInsS << 8) | (2 << 0) | (0 << 2);
|
|
static constexpr uint32_t kInsS_2To1 = (kOpInsS << 8) | (2 << 0) | (1 << 2);
|
|
static constexpr uint32_t kInsS_2To2 = (kOpInsS << 8) | (2 << 0) | (2 << 2);
|
|
static constexpr uint32_t kInsS_2To3 = (kOpInsS << 8) | (2 << 0) | (3 << 2);
|
|
static constexpr uint32_t kInsS_3To0 = (kOpInsS << 8) | (3 << 0) | (0 << 2);
|
|
static constexpr uint32_t kInsS_3To1 = (kOpInsS << 8) | (3 << 0) | (1 << 2);
|
|
static constexpr uint32_t kInsS_3To2 = (kOpInsS << 8) | (3 << 0) | (2 << 2);
|
|
static constexpr uint32_t kInsS_3To3 = (kOpInsS << 8) | (3 << 0) | (3 << 2);
|
|
static constexpr uint32_t kInsD_0To0 = (kOpInsD << 8) | (0 << 0) | (0 << 2);
|
|
static constexpr uint32_t kInsD_0To1 = (kOpInsD << 8) | (0 << 0) | (1 << 2);
|
|
static constexpr uint32_t kInsD_1To0 = (kOpInsD << 8) | (1 << 0) | (0 << 2);
|
|
static constexpr uint32_t kInsD_1To1 = (kOpInsD << 8) | (1 << 0) | (1 << 2);
|
|
|
|
static constexpr uint32_t kZip1_4S = (kOpPerm << 8) | (0 << 0) | (0 << 3);
|
|
static constexpr uint32_t kZip1_2D = (kOpPerm << 8) | (0 << 0) | (1 << 3);
|
|
static constexpr uint32_t kZip2_4S = (kOpPerm << 8) | (1 << 0) | (0 << 3);
|
|
static constexpr uint32_t kZip2_2D = (kOpPerm << 8) | (1 << 0) | (1 << 3);
|
|
static constexpr uint32_t kUzp1_4S = (kOpPerm << 8) | (2 << 0) | (0 << 3);
|
|
static constexpr uint32_t kUzp1_2D = (kOpPerm << 8) | (2 << 0) | (1 << 3);
|
|
static constexpr uint32_t kUzp2_4S = (kOpPerm << 8) | (3 << 0) | (0 << 3);
|
|
static constexpr uint32_t kUzp2_2D = (kOpPerm << 8) | (3 << 0) | (1 << 3);
|
|
static constexpr uint32_t kTrn1_4S = (kOpPerm << 8) | (4 << 0);
|
|
static constexpr uint32_t kTrn2_4S = (kOpPerm << 8) | (5 << 0);
|
|
static constexpr uint32_t kExt4 = (kOpPerm << 8) | (7 << 0) | (0 << 3);
|
|
static constexpr uint32_t kExt8 = (kOpPerm << 8) | (6 << 0) | (1 << 3);
|
|
static constexpr uint32_t kExt12 = (kOpPerm << 8) | (7 << 0) | (1 << 3);
|
|
|
|
static constexpr uint32_t kRev64 = (kOpRev64 << 8) | (1 << 0); // Dummy bit so we don't end up having all zeros.
|
|
|
|
static constexpr uint32_t kDup0 = (kOpDup << 8) | (0 << 0);
|
|
static constexpr uint32_t kDup1 = (kOpDup << 8) | (1 << 0);
|
|
static constexpr uint32_t kDup2 = (kOpDup << 8) | (2 << 0);
|
|
static constexpr uint32_t kDup3 = (kOpDup << 8) | (3 << 0);
|
|
|
|
static constexpr uint32_t kSrcAA = (0 << 4) | (0 << 6);
|
|
static constexpr uint32_t kSrcAB = (0 << 4) | (1 << 6);
|
|
static constexpr uint32_t kSrcAC = (0 << 4) | (2 << 6);
|
|
static constexpr uint32_t kSrcAD = (0 << 4) | (3 << 6);
|
|
static constexpr uint32_t kSrcBA = (1 << 4) | (0 << 6);
|
|
static constexpr uint32_t kSrcBB = (1 << 4) | (1 << 6);
|
|
static constexpr uint32_t kSrcBC = (1 << 4) | (2 << 6);
|
|
static constexpr uint32_t kSrcBD = (1 << 4) | (3 << 6);
|
|
static constexpr uint32_t kSrcCA = (2 << 4) | (0 << 6);
|
|
static constexpr uint32_t kSrcCB = (2 << 4) | (1 << 6);
|
|
static constexpr uint32_t kSrcCC = (2 << 4) | (2 << 6);
|
|
static constexpr uint32_t kSrcCD = (2 << 4) | (3 << 6);
|
|
static constexpr uint32_t kSrcDA = (3 << 4) | (0 << 6);
|
|
static constexpr uint32_t kSrcDB = (3 << 4) | (1 << 6);
|
|
static constexpr uint32_t kSrcDC = (3 << 4) | (2 << 6);
|
|
static constexpr uint32_t kSrcDD = (3 << 4) | (3 << 6);
|
|
|
|
static constexpr uint32_t kSrcA = kSrcAA;
|
|
static constexpr uint32_t kSrcB = kSrcBB;
|
|
static constexpr uint32_t kSrcC = kSrcCC;
|
|
static constexpr uint32_t kSrcD = kSrcDD;
|
|
|
|
// Alias to nothing to make the table easier to read.
|
|
static constexpr uint32_t k_ = 0;
|
|
static constexpr uint32_t kSrc_ = 0;
|
|
|
|
uint32_t data;
|
|
|
|
ASMJIT_INLINE_NODEBUG bool is_valid() const noexcept { return data != 0u; }
|
|
ASMJIT_INLINE_NODEBUG uint32_t op() const noexcept { return data >> 8; }
|
|
ASMJIT_INLINE_NODEBUG uint32_t is_ins_op() const noexcept { return op() == kOpInsS || op() == kOpInsD; }
|
|
|
|
ASMJIT_INLINE_NODEBUG uint32_t dup_idx() const noexcept { return data & 0x3u; }
|
|
|
|
ASMJIT_INLINE_NODEBUG uint32_t perm_op() const noexcept { return data & 0x7u; }
|
|
ASMJIT_INLINE_NODEBUG bool perm_s4() const noexcept { return (data & (1u << 3)) == 0u; }
|
|
ASMJIT_INLINE_NODEBUG uint32_t perm_ext_imm() const noexcept { return ((data & 0x1) << 2) + (data & 0x8); }
|
|
|
|
ASMJIT_INLINE_NODEBUG uint32_t ins_src() const noexcept { return (data >> 0) & 0x3u; }
|
|
ASMJIT_INLINE_NODEBUG uint32_t ins_dst() const noexcept { return (data >> 2) & 0x3u; }
|
|
|
|
ASMJIT_INLINE_NODEBUG uint32_t src_a() const noexcept { return (data >> 4) & 0x3u; }
|
|
ASMJIT_INLINE_NODEBUG uint32_t src_b() const noexcept { return (data >> 6) & 0x3u; }
|
|
};
|
|
|
|
//! \name Members
|
|
//! \{
|
|
|
|
uint32_t combined;
|
|
|
|
//! \}
|
|
|
|
//! \name Accessors
|
|
//! \{
|
|
|
|
ASMJIT_INLINE_NODEBUG uint32_t count() const noexcept {
|
|
return 1u + uint32_t(((combined >> 11) & 0x7FFu) != 0u) + uint32_t((combined >> 22) != 0u);
|
|
}
|
|
|
|
ASMJIT_INLINE_NODEBUG Op op(uint32_t index) const noexcept {
|
|
return Op{(combined >> (index * 11u)) & 0x7FFu};
|
|
}
|
|
|
|
//! \}
|
|
};
|
|
|
|
// These tables provide all combinations for all possible 32-bit interleaved shuffles (there is 256 combinations
|
|
// in total). It prioritizes lane moves, and then operations that can have either one or two inputs. Initially,
|
|
// there are two sources (A, B), which can be used by any shuffle operation, which result is then referred as C.
|
|
// Then, all consecutive shuffles can use any of A, B, and C as their operands. The last operation is the result.
|
|
//
|
|
// Statistics:
|
|
//
|
|
// - 1 Instruction: 5
|
|
// - 2 Instructions: 113
|
|
// - 3 Instructions: 138
|
|
|
|
#define OP(swiz, op0, src0, op1, src1, op2, src2) { \
|
|
((InterleavedShuffle32Ops::Op::k##op0) << 0) | ((InterleavedShuffle32Ops::Op::kSrc##src0) << 0) | \
|
|
((InterleavedShuffle32Ops::Op::k##op1) << 11) | ((InterleavedShuffle32Ops::Op::kSrc##src1) << 11) | \
|
|
((InterleavedShuffle32Ops::Op::k##op2) << 22) | ((InterleavedShuffle32Ops::Op::kSrc##src2) << 22) \
|
|
}
|
|
|
|
static constexpr InterleavedShuffle32Ops interleaved_shuffle32_ops[256] = {
|
|
OP(4400, Trn1_4S , AB, Zip1_4S , CC, _ , _ ), OP(4401, Rev64 , A , Dup0 , B , Zip1_2D , CD), OP(4402, Ext8 , AA, Dup0 , B , Uzp1_4S , CD), OP(4403, Ext12 , AA, Dup0 , B , Zip1_2D , CD),
|
|
OP(4410, Dup0 , B , Zip1_2D , AC, _ , _ ), OP(4411, Dup1 , A , Dup0 , B , Uzp2_4S , CD), OP(4412, Ext4 , AB, InsS_3To2, C , Rev64 , C ), OP(4413, Ext12 , AA, Dup0 , B , Uzp1_4S , CD),
|
|
OP(4420, Dup0 , B , Uzp1_4S , AC, _ , _ ), OP(4421, Ext4 , AB, InsS_3To2, C , _ , _ ), OP(4422, Ext8 , AB, Trn1_4S , CC, _ , _ ), OP(4423, Rev64 , A , Dup0 , B , Zip2_2D , CD),
|
|
OP(4430, Trn1_4S , AB, Ext12 , AB, Zip1_4S , CD), OP(4431, Dup0 , B , Uzp2_4S , AC, _ , _ ), OP(4432, Dup0 , B , Zip2_2D , AC, _ , _ ), OP(4433, Ext12 , AB, Zip1_4S , CC, _ , _ ),
|
|
OP(4500, Rev64 , B , Dup0 , A , Zip1_2D , DC), OP(4501, Zip1_2D , AB, Rev64 , C , _ , _ ), OP(4502, Uzp1_4S , BA, Ext8 , CB, Rev64 , D ), OP(4503, Ext12 , AA, Rev64 , B , Zip1_2D , CD),
|
|
OP(4510, Rev64 , B , Zip1_2D , AC, _ , _ ), OP(4511, Rev64 , B , Dup1 , A , Zip1_2D , DC), OP(4512, Uzp2_4S , AB, Uzp1_4S , AC, Ext4 , DB), OP(4513, Trn2_4S , AB, Zip1_2D , CB, Ext12 , AD),
|
|
OP(4520, Dup1 , B , Uzp1_4S , CB, Uzp1_4S , AD), OP(4521, Ext4 , AB, InsS_1To2, B , _ , _ ), OP(4522, Rev64 , B , Dup2 , A , Zip1_2D , DC), OP(4523, Ext8 , AB, Rev64 , C , _ , _ ),
|
|
OP(4530, Ext12 , AA, Zip1_2D , CB, Rev64 , D ), OP(4531, Uzp2_4S , AB, InsS_0To3, B , _ , _ ), OP(4532, Rev64 , B , Ext8 , AC, _ , _ ), OP(4533, Rev64 , B , Dup3 , A , Zip1_2D , DC),
|
|
OP(4600, Ext8 , BB, Dup0 , A , Uzp1_4S , DC), OP(4601, Uzp1_4S , BB, Zip1_2D , AC, Rev64 , D ), OP(4602, Uzp1_4S , AB, Rev64 , C , _ , _ ), OP(4603, Uzp1_4S , AB, InsS_3To1, C , Ext12 , AC),
|
|
OP(4610, Dup2 , B , Trn1_4S , CB, Zip1_2D , AD), OP(4611, Ext8 , BB, Dup1 , A , Uzp1_4S , DC), OP(4612, Trn1_4S , AB, Zip1_4S , AC, Zip2_4S , CD), OP(4613, Rev64 , B , Uzp2_4S , AC, Rev64 , D ),
|
|
OP(4620, Ext8 , BB, Uzp1_4S , AC, _ , _ ), OP(4621, Ext4 , AB, InsS_2To2, B , _ , _ ), OP(4622, Ext8 , BB, Dup2 , A , Uzp1_4S , DC), OP(4623, Trn1_4S , AB, Zip2_2D , AC, Ext4 , DB),
|
|
OP(4630, Ext12 , AB, Zip1_4S , AC, InsS_3To2, C ), OP(4631, Ext4 , BB, Uzp2_4S , AC, _ , _ ), OP(4632, Zip2_2D , AB, InsS_0To3, B , _ , _ ), OP(4633, Ext8 , BB, Dup3 , A , Uzp1_4S , DC),
|
|
OP(4700, Ext12 , BB, Dup0 , A , Zip1_2D , DC), OP(4701, Ext12 , BB, Rev64 , A , Zip1_2D , DC), OP(4702, Uzp1_4S , AB, InsS_3To3, B , Rev64 , C ), OP(4703, Ext12 , BB, Ext12 , AA, Zip1_2D , DC),
|
|
OP(4710, Ext12 , BB, Zip1_2D , AC, _ , _ ), OP(4711, Ext12 , BB, Dup1 , A , Zip1_2D , DC), OP(4712, Trn2_4S , BA, InsS_2To0, A , InsS_0To3, B ), OP(4713, Uzp2_4S , AB, InsS_0To2, C , Ext4 , CB),
|
|
OP(4720, Dup3 , B , Uzp1_4S , CB, Uzp1_4S , AD), OP(4721, Ext4 , AB, InsS_3To2, B , _ , _ ), OP(4722, Ext12 , BB, Dup2 , A , Zip1_2D , DC), OP(4723, Ext12 , BB, Rev64 , A , Ext8 , DC),
|
|
OP(4730, Rev64 , A , InsS_3To3, B , Ext4 , CB), OP(4731, Dup3 , B , Ext4 , CB, Uzp2_4S , AD), OP(4732, Ext12 , BB, Ext8 , AC, _ , _ ), OP(4733, Ext12 , BB, Dup3 , A , Zip1_2D , DC),
|
|
OP(5400, Dup0 , A , Zip1_2D , CB, _ , _ ), OP(5401, Rev64 , A , Zip1_2D , CB, _ , _ ), OP(5402, Ext8 , AB, InsS_0To1, A , _ , _ ), OP(5403, Ext12 , AA, Zip1_2D , CB, _ , _ ),
|
|
OP(5410, Zip1_2D , AB, _ , _ , _ , _ ), OP(5411, Dup1 , A , Zip1_2D , CB, _ , _ ), OP(5412, Zip1_2D , AB, InsS_2To0, A , _ , _ ), OP(5413, Zip1_2D , AB, InsS_3To0, A , _ , _ ),
|
|
OP(5420, Uzp1_4S , BA, Ext8 , CB, _ , _ ), OP(5421, Ext12 , BA, Ext8 , CB, _ , _ ), OP(5422, Dup2 , A , Zip1_2D , CB, _ , _ ), OP(5423, Rev64 , A , Ext8 , CB, _ , _ ),
|
|
OP(5430, Zip1_2D , AB, InsS_3To1, A , _ , _ ), OP(5431, Uzp2_4S , BA, Ext8 , CB, _ , _ ), OP(5432, Ext8 , AB, _ , _ , _ , _ ), OP(5433, Dup3 , A , Zip1_2D , CB, _ , _ ),
|
|
OP(5500, Dup1 , B , Dup0 , A , Uzp2_4S , DC), OP(5501, Rev64 , A , Dup1 , B , Zip1_2D , CD), OP(5502, Ext8 , AA, Dup1 , B , Uzp1_4S , CD), OP(5503, Ext12 , AA, Dup1 , B , Zip1_2D , CD),
|
|
OP(5510, Dup1 , B , Zip1_2D , AC, _ , _ ), OP(5511, Trn2_4S , AB, Zip1_4S , CC, _ , _ ), OP(5512, Uzp2_4S , AB, Zip2_2D , AC, Trn1_4S , DC), OP(5513, Ext12 , AA, Dup1 , B , Uzp1_4S , CD),
|
|
OP(5520, Dup1 , B , Uzp1_4S , AC, _ , _ ), OP(5521, Ext12 , BA, Dup1 , B , Zip2_2D , CD), OP(5522, Dup2 , A , Dup1 , B , Uzp2_4S , CD), OP(5523, Rev64 , A , Dup1 , B , Zip2_2D , CD),
|
|
OP(5530, Ext12 , AB, Trn1_4S , AC, InsS_3To2, D ), OP(5531, Dup1 , B , Uzp2_4S , AC, _ , _ ), OP(5532, Dup1 , B , Zip2_2D , AC, _ , _ ), OP(5533, Ext12 , AB, Trn1_4S , CC, _ , _ ),
|
|
OP(5600, Trn1_4S , BA, InsS_1To3, B , InsS_1To0, C ), OP(5601, Ext4 , BB, Zip1_2D , AC, Rev64 , D ), OP(5602, Uzp1_4S , AB, InsS_1To2, B , Rev64 , C ), OP(5603, Ext12 , AA, Zip1_2D , CB, InsS_2To2, B ),
|
|
OP(5610, Zip1_2D , AB, InsS_2To2, B , _ , _ ), OP(5611, Zip1_4S , AB, Trn1_4S , CB, Zip2_4S , DC), OP(5612, Trn2_4S , AB, Zip2_4S , AB, Zip1_4S , DC), OP(5613, Trn2_4S , AB, Zip2_4S , CB, Zip1_4S , DC),
|
|
OP(5620, Dup2 , B , Uzp2_4S , CB, Uzp1_4S , AD), OP(5621, Rev64 , B , Ext4 , AC, InsS_3To2, C ), OP(5622, Uzp1_4S , AB, Zip1_2D , CB, Trn2_4S , CD), OP(5623, Uzp1_4S , AB, Uzp2_4S , CB, Ext12 , AD),
|
|
OP(5630, Ext12 , AB, Trn1_4S , AC, InsS_3To2, C ), OP(5631, Dup2 , B , Zip1_2D , CB, Uzp2_4S , AD), OP(5632, Zip2_2D , AB, InsS_1To3, B , _ , _ ), OP(5633, Ext12 , AB, InsS_3To1, C , Ext12 , AC),
|
|
OP(5700, Ext12 , BB, Dup0 , A , Uzp1_4S , DC), OP(5701, Uzp2_4S , BB, Zip1_2D , AC, Rev64 , D ), OP(5702, Rev64 , B , Uzp1_4S , AC, Rev64 , D ), OP(5703, Uzp2_4S , BB, InsS_0To0, A , Ext12 , AC),
|
|
OP(5710, Zip1_2D , AB, InsS_3To2, B , _ , _ ), OP(5711, Ext12 , BB, Dup1 , A , Uzp1_4S , DC), OP(5712, Uzp2_4S , AB, InsS_2To1, A , Rev64 , C ), OP(5713, Uzp2_4S , AB, Rev64 , C , _ , _ ),
|
|
OP(5720, Ext12 , BB, Uzp1_4S , AC, _ , _ ), OP(5721, Rev64 , B , Ext4 , AC, InsS_2To2, C ), OP(5722, Ext12 , BB, Dup2 , A , Uzp1_4S , DC), OP(5723, Uzp2_4S , BB, Zip2_2D , AC, Rev64 , D ),
|
|
OP(5730, Ext12 , AB, Trn1_4S , AC, InsS_3To2, B ), OP(5731, Ext8 , BB, Uzp2_4S , AC, _ , _ ), OP(5732, Ext8 , AB, InsS_3To2, B , _ , _ ), OP(5733, Ext12 , BB, Dup3 , A , Uzp1_4S , DC),
|
|
OP(6400, Dup0 , A , Uzp1_4S , CB, _ , _ ), OP(6401, Dup1 , A , Uzp1_4S , CA, Uzp1_4S , DB), OP(6402, Ext8 , AA, Uzp1_4S , CB, _ , _ ), OP(6403, Dup3 , A , Uzp1_4S , CA, Uzp1_4S , DB),
|
|
OP(6410, Uzp1_4S , BB, Zip1_2D , AC, _ , _ ), OP(6411, Dup1 , A , Uzp1_4S , CB, _ , _ ), OP(6412, Dup2 , A , Uzp2_4S , CA, Uzp1_4S , DB), OP(6413, Ext12 , AA, Uzp1_4S , CB, _ , _ ),
|
|
OP(6420, Uzp1_4S , AB, _ , _ , _ , _ ), OP(6421, Uzp1_4S , AB, InsS_1To0, A , _ , _ ), OP(6422, Dup2 , A , Uzp1_4S , CB, _ , _ ), OP(6423, Uzp1_4S , AB, InsS_3To0, A , _ , _ ),
|
|
OP(6430, Uzp1_4S , AB, InsS_3To1, A , _ , _ ), OP(6431, Rev64 , B , Uzp2_4S , AC, _ , _ ), OP(6432, Uzp1_4S , BB, Zip2_2D , AC, _ , _ ), OP(6433, Dup3 , A , Uzp1_4S , CB, _ , _ ),
|
|
OP(6500, Ext12 , BB, Dup0 , A , Zip2_2D , DC), OP(6501, Ext4 , BB, Rev64 , A , Zip1_2D , DC), OP(6502, Rev64 , A , Ext12 , CB, InsS_1To1, C ), OP(6503, Ext12 , AB, InsS_0To1, A , _ , _ ),
|
|
OP(6510, Ext4 , BB, Zip1_2D , AC, _ , _ ), OP(6511, Ext12 , BB, Dup1 , A , Zip2_2D , DC), OP(6512, Rev64 , A , Ext12 , CB, InsS_0To1, C ), OP(6513, Ext12 , AB, InsS_1To1, A , _ , _ ),
|
|
OP(6520, Uzp1_4S , AB, InsS_1To2, B , _ , _ ), OP(6521, Trn2_4S , AB, Zip2_4S , AB, Zip1_4S , CD), OP(6522, Ext12 , BB, Dup2 , A , Zip2_2D , DC), OP(6523, Ext12 , AB, InsS_2To1, A , _ , _ ),
|
|
OP(6530, Ext4 , BA, Trn2_4S , CA, Ext8 , DC), OP(6531, Uzp2_4S , AB, InsS_2To3, B , _ , _ ), OP(6532, Ext12 , BB, Zip2_2D , AC, _ , _ ), OP(6533, Ext12 , AB, InsS_0To1, C , _ , _ ),
|
|
OP(6600, Dup2 , B , Dup0 , A , Uzp2_4S , DC), OP(6601, Rev64 , A , Dup2 , B , Zip1_2D , CD), OP(6602, Ext8 , AA, Dup2 , B , Uzp1_4S , CD), OP(6603, Ext12 , AA, Dup2 , B , Zip1_2D , CD),
|
|
OP(6610, Dup2 , B , Zip1_2D , AC, _ , _ ), OP(6611, Dup2 , B , Dup1 , A , Uzp2_4S , DC), OP(6612, Zip2_4S , AB, Trn2_4S , AC, Zip1_4S , CD), OP(6613, Ext12 , AA, Dup2 , B , Uzp1_4S , CD),
|
|
OP(6620, Dup2 , B , Uzp1_4S , AC, _ , _ ), OP(6621, Ext12 , BA, Dup2 , B , Zip2_2D , CD), OP(6622, Trn1_4S , AB, Zip2_4S , CC, _ , _ ), OP(6623, Rev64 , A , Dup2 , B , Zip2_2D , CD),
|
|
OP(6630, Trn1_4S , AB, InsS_3To1, A , InsS_3To2, C ), OP(6631, Dup2 , B , Uzp2_4S , AC, _ , _ ), OP(6632, Dup2 , B , Zip2_2D , AC, _ , _ ), OP(6633, Dup3 , A , Dup2 , B , Uzp2_4S , CD),
|
|
OP(6700, Rev64 , B , Dup0 , A , Zip2_2D , DC), OP(6701, Ext8 , BA, Rev64 , C , Ext8 , DD), OP(6702, Uzp1_4S , BA, Zip2_2D , CB, Rev64 , D ), OP(6703, Ext4 , AA, Rev64 , B , Zip2_2D , CD),
|
|
OP(6710, Rev64 , B , InsD_0To0, A , _ , _ ), OP(6711, Rev64 , B , Dup1 , A , Zip2_2D , DC), OP(6712, Ext12 , BA, Zip2_2D , CB, Rev64 , D ), OP(6713, Uzp2_4S , BA, Zip2_2D , CB, Rev64 , D ),
|
|
OP(6720, Uzp1_4S , AB, InsS_3To2, B , _ , _ ), OP(6721, Ext12 , BA, Rev64 , B , Zip2_2D , CD), OP(6722, Rev64 , B , Dup2 , A , Zip2_2D , DC), OP(6723, Zip2_2D , AB, Rev64 , C , _ , _ ),
|
|
OP(6730, Ext4 , AA, Zip2_2D , CB, Rev64 , D ), OP(6731, Dup3 , B , Uzp1_4S , CB, Uzp2_4S , AD), OP(6732, Rev64 , B , Zip2_2D , AC, _ , _ ), OP(6733, Rev64 , B , Dup3 , A , Zip2_2D , DC),
|
|
OP(7400, Uzp1_4S , AB, InsS_3To3, B , InsS_0To1, C ), OP(7401, Ext12 , BB, Zip1_2D , AC, Rev64 , D ), OP(7402, Ext8 , AB, Trn1_4S , CA, InsS_3To3, B ), OP(7403, Rev64 , B , InsS_0To0, A , Ext12 , AC),
|
|
OP(7410, Zip1_2D , AB, InsS_3To3, B , _ , _ ), OP(7411, Trn2_4S , AB, InsS_0To2, B , InsS_0To1, C ), OP(7412, Ext4 , AB, Trn2_4S , CB, InsS_0To1, C ), OP(7413, Ext4 , AB, Zip2_4S , CB, InsS_0To1, C ),
|
|
OP(7420, Uzp1_4S , AB, InsS_3To3, B , _ , _ ), OP(7421, Ext12 , BA, Trn1_4S , BC, Ext8 , CD), OP(7422, Ext4 , AB, Trn2_4S , CB, InsS_0To1, D ), OP(7423, Ext12 , BB, Ext8 , AC, Rev64 , D ),
|
|
OP(7430, Uzp2_4S , AB, InsS_0To0, A , InsS_0To2, B ), OP(7431, Uzp2_4S , AB, InsS_0To2, B , _ , _ ), OP(7432, Zip2_2D , AB, InsS_0To2, B , _ , _ ), OP(7433, Trn2_4S , AB, Ext4 , AB, Zip2_4S , DC),
|
|
OP(7500, Dup0 , A , Uzp2_4S , CB, _ , _ ), OP(7501, Uzp2_4S , AB, InsS_0To1, A , _ , _ ), OP(7502, Ext4 , AA, Uzp2_4S , CB, _ , _ ), OP(7503, Dup3 , A , Ext4 , CA, Uzp2_4S , DB),
|
|
OP(7510, Uzp2_4S , BB, Zip1_2D , AC, _ , _ ), OP(7511, Dup1 , A , Uzp2_4S , CB, _ , _ ), OP(7512, Dup2 , A , Zip1_2D , CA, Uzp2_4S , DB), OP(7513, Ext8 , AA, Uzp2_4S , CB, _ , _ ),
|
|
OP(7520, Rev64 , B , Uzp1_4S , AC, _ , _ ), OP(7521, Uzp2_4S , AB, InsS_2To1, A , _ , _ ), OP(7522, Dup2 , A , Uzp2_4S , CB, _ , _ ), OP(7523, Dup3 , A , Uzp1_4S , CA, Uzp2_4S , DB),
|
|
OP(7530, Uzp2_4S , AB, InsS_0To0, A , _ , _ ), OP(7531, Uzp2_4S , AB, _ , _ , _ , _ ), OP(7532, Uzp2_4S , BB, Zip2_2D , AC, _ , _ ), OP(7533, Dup3 , A , Uzp2_4S , CB, _ , _ ),
|
|
OP(7600, Dup0 , A , Zip2_2D , CB, _ , _ ), OP(7601, Rev64 , A , InsD_1To1, B , _ , _ ), OP(7602, Zip2_2D , AB, InsS_0To1, A , _ , _ ), OP(7603, Ext4 , AA, Zip2_2D , CB, _ , _ ),
|
|
OP(7610, Zip2_2D , BB, Zip1_2D , AC, _ , _ ), OP(7611, Dup1 , A , Zip2_2D , CB, _ , _ ), OP(7612, Zip2_2D , AB, InsS_1To1, A , _ , _ ), OP(7613, Dup3 , A , Zip1_4S , CA, Zip2_2D , DB),
|
|
OP(7620, Uzp1_4S , BA, Zip2_2D , CB, _ , _ ), OP(7621, Ext12 , BA, Zip2_2D , CB, _ , _ ), OP(7622, Dup2 , A , Zip2_2D , CB, _ , _ ), OP(7623, Rev64 , A , Zip2_2D , CB, _ , _ ),
|
|
OP(7630, Zip2_2D , AB, InsS_0To0, A , _ , _ ), OP(7631, Uzp2_4S , BA, Zip2_2D , CB, _ , _ ), OP(7632, Zip2_2D , AB, _ , _ , _ , _ ), OP(7633, Dup3 , A , Zip2_2D , CB, _ , _ ),
|
|
OP(7700, Dup3 , B , Dup0 , A , Uzp2_4S , DC), OP(7701, Rev64 , A , Dup3 , B , Zip1_2D , CD), OP(7702, Ext8 , AA, Dup3 , B , Uzp1_4S , CD), OP(7703, Ext12 , AA, Dup3 , B , Zip1_2D , CD),
|
|
OP(7710, Dup3 , B , Zip1_2D , AC, _ , _ ), OP(7711, Dup3 , B , Dup1 , A , Uzp2_4S , DC), OP(7712, Trn2_4S , BA, InsS_2To0, A , InsS_2To3, C ), OP(7713, Ext12 , AA, Dup3 , B , Uzp1_4S , CD),
|
|
OP(7720, Dup3 , B , Uzp1_4S , AC, _ , _ ), OP(7721, Ext12 , BA, Dup3 , B , Zip2_2D , CD), OP(7722, Dup3 , B , Dup2 , A , Uzp2_4S , DC), OP(7723, Rev64 , A , Dup3 , B , Zip2_2D , CD),
|
|
OP(7730, Uzp2_4S , AB, InsS_0To0, A , InsS_3To2, C ), OP(7731, Dup3 , B , Uzp2_4S , AC, _ , _ ), OP(7732, Dup3 , B , Zip2_2D , AC, _ , _ ), OP(7733, Trn2_4S , AB, Zip2_4S , CC, _ , _ )
|
|
};
|
|
|
|
static constexpr InterleavedShuffle32Ops interleaved_shuffle32_ops_dst_same_as_b[256] = {
|
|
OP(4400, Trn1_4S , AB, Zip1_4S , CC, _ , _ ), OP(4401, Rev64 , A , Dup0 , B , Zip1_2D , CD), OP(4402, Ext8 , AA, Dup0 , B , Uzp1_4S , CD), OP(4403, Ext12 , AA, Dup0 , B , Zip1_2D , CD),
|
|
OP(4410, Dup0 , B , Zip1_2D , AC, _ , _ ), OP(4411, Dup1 , A , Dup0 , B , Uzp2_4S , CD), OP(4412, Ext4 , AB, InsS_3To2, C , Rev64 , C ), OP(4413, Ext12 , AA, Dup0 , B , Uzp1_4S , CD),
|
|
OP(4420, Dup0 , B , Uzp1_4S , AC, _ , _ ), OP(4421, Ext4 , AB, InsS_3To2, C , _ , _ ), OP(4422, Ext8 , AB, Trn1_4S , CC, _ , _ ), OP(4423, Rev64 , A , Dup0 , B , Zip2_2D , CD),
|
|
OP(4430, Trn1_4S , AB, Ext12 , AB, Zip1_4S , CD), OP(4431, Dup0 , B , Uzp2_4S , AC, _ , _ ), OP(4432, Dup0 , B , Zip2_2D , AC, _ , _ ), OP(4433, Ext12 , AB, Zip1_4S , CC, _ , _ ),
|
|
OP(4500, Rev64 , B , Dup0 , A , Zip1_2D , DC), OP(4501, Zip1_2D , AB, Rev64 , C , _ , _ ), OP(4502, Uzp1_4S , BA, Ext8 , CB, Rev64 , D ), OP(4503, Ext12 , AA, Rev64 , B , Zip1_2D , CD),
|
|
OP(4510, Rev64 , B , Zip1_2D , AC, _ , _ ), OP(4511, Rev64 , B , Dup1 , A , Zip1_2D , DC), OP(4512, Uzp2_4S , AB, Uzp1_4S , AC, Ext4 , DB), OP(4513, Trn2_4S , AB, Zip1_2D , CB, Ext12 , AD),
|
|
OP(4520, Dup1 , B , Uzp1_4S , CB, Uzp1_4S , AD), OP(4521, Ext4 , AB, InsS_1To2, B , _ , _ ), OP(4522, Rev64 , B , Dup2 , A , Zip1_2D , DC), OP(4523, Ext8 , AB, Rev64 , C , _ , _ ),
|
|
OP(4530, Ext12 , AA, Zip1_2D , CB, Rev64 , D ), OP(4531, Uzp2_4S , AB, InsS_0To3, B , _ , _ ), OP(4532, Rev64 , B , Ext8 , AC, _ , _ ), OP(4533, Rev64 , B , Dup3 , A , Zip1_2D , DC),
|
|
OP(4600, Ext8 , BB, Dup0 , A , Uzp1_4S , DC), OP(4601, Uzp1_4S , BB, Zip1_2D , AC, Rev64 , D ), OP(4602, Uzp1_4S , AB, Rev64 , C , _ , _ ), OP(4603, Uzp1_4S , AB, InsS_3To1, C , Ext12 , AC),
|
|
OP(4610, Dup2 , B , Trn1_4S , CB, Zip1_2D , AD), OP(4611, Ext8 , BB, Dup1 , A , Uzp1_4S , DC), OP(4612, Trn1_4S , AB, Zip1_4S , AC, Zip2_4S , CD), OP(4613, Rev64 , B , Uzp2_4S , AC, Rev64 , D ),
|
|
OP(4620, Ext8 , BB, Uzp1_4S , AC, _ , _ ), OP(4621, Ext4 , AB, InsS_2To2, B , _ , _ ), OP(4622, Ext8 , BB, Dup2 , A , Uzp1_4S , DC), OP(4623, Trn1_4S , AB, Zip2_2D , AC, Ext4 , DB),
|
|
OP(4630, Ext12 , AB, Zip1_4S , AC, InsS_3To2, C ), OP(4631, Ext4 , BB, Uzp2_4S , AC, _ , _ ), OP(4632, Zip2_2D , AB, InsS_0To3, B , _ , _ ), OP(4633, Ext8 , BB, Dup3 , A , Uzp1_4S , DC),
|
|
OP(4700, Ext12 , BB, Dup0 , A , Zip1_2D , DC), OP(4701, Ext12 , BB, Rev64 , A , Zip1_2D , DC), OP(4702, Uzp1_4S , AB, InsS_3To3, B , Rev64 , C ), OP(4703, Ext12 , BB, Ext12 , AA, Zip1_2D , DC),
|
|
OP(4710, Ext12 , BB, Zip1_2D , AC, _ , _ ), OP(4711, Ext12 , BB, Dup1 , A , Zip1_2D , DC), OP(4712, Ext12 , BB, Zip1_2D , AC, InsS_2To0, A ), OP(4713, Uzp2_4S , AB, InsS_0To2, C , Ext4 , CB),
|
|
OP(4720, Dup3 , B , Uzp1_4S , CB, Uzp1_4S , AD), OP(4721, Ext4 , AB, InsS_3To2, B , _ , _ ), OP(4722, Ext12 , BB, Dup2 , A , Zip1_2D , DC), OP(4723, Ext12 , BB, Rev64 , A , Ext8 , DC),
|
|
OP(4730, Rev64 , A , InsS_3To3, B , Ext4 , CB), OP(4731, Dup3 , B , Ext4 , CB, Uzp2_4S , AD), OP(4732, Ext12 , BB, Ext8 , AC, _ , _ ), OP(4733, Ext12 , BB, Dup3 , A , Zip1_2D , DC),
|
|
OP(5400, Dup0 , A , Zip1_2D , CB, _ , _ ), OP(5401, Rev64 , A , Zip1_2D , CB, _ , _ ), OP(5402, Ext8 , AB, InsS_0To1, A , _ , _ ), OP(5403, Ext12 , AA, Zip1_2D , CB, _ , _ ),
|
|
OP(5410, Zip1_2D , AB, _ , _ , _ , _ ), OP(5411, Dup1 , A , Zip1_2D , CB, _ , _ ), OP(5412, Zip1_2D , AB, InsS_2To0, A , _ , _ ), OP(5413, Zip1_2D , AB, InsS_3To0, A , _ , _ ),
|
|
OP(5420, Uzp1_4S , BA, Ext8 , CB, _ , _ ), OP(5421, Ext12 , BA, Ext8 , CB, _ , _ ), OP(5422, Dup2 , A , Zip1_2D , CB, _ , _ ), OP(5423, Rev64 , A , Ext8 , CB, _ , _ ),
|
|
OP(5430, Zip1_2D , AB, InsS_3To1, A , _ , _ ), OP(5431, Uzp2_4S , BA, Ext8 , CB, _ , _ ), OP(5432, Ext8 , AB, _ , _ , _ , _ ), OP(5433, Dup3 , A , Zip1_2D , CB, _ , _ ),
|
|
OP(5500, Dup1 , B , Dup0 , A , Uzp2_4S , DC), OP(5501, Rev64 , A , Dup1 , B , Zip1_2D , CD), OP(5502, Ext8 , AA, Dup1 , B , Uzp1_4S , CD), OP(5503, Ext12 , AA, Dup1 , B , Zip1_2D , CD),
|
|
OP(5510, Dup1 , B , Zip1_2D , AC, _ , _ ), OP(5511, Trn2_4S , AB, Zip1_4S , CC, _ , _ ), OP(5512, Uzp2_4S , AB, Zip2_2D , AC, Trn1_4S , DC), OP(5513, Ext12 , AA, Dup1 , B , Uzp1_4S , CD),
|
|
OP(5520, Dup1 , B , Uzp1_4S , AC, _ , _ ), OP(5521, Ext12 , BA, Dup1 , B , Zip2_2D , CD), OP(5522, Dup2 , A , Dup1 , B , Uzp2_4S , CD), OP(5523, Rev64 , A , Dup1 , B , Zip2_2D , CD),
|
|
OP(5530, Ext12 , AB, Trn1_4S , AC, InsS_3To2, D ), OP(5531, Dup1 , B , Uzp2_4S , AC, _ , _ ), OP(5532, Dup1 , B , Zip2_2D , AC, _ , _ ), OP(5533, Ext12 , AB, Trn1_4S , CC, _ , _ ),
|
|
OP(5600, Trn1_4S , BA, InsS_1To3, B , InsS_1To0, C ), OP(5601, Ext4 , BB, Zip1_2D , AC, Rev64 , D ), OP(5602, Uzp1_4S , AB, InsS_1To2, B , Rev64 , C ), OP(5603, Trn1_4S , BA, InsS_1To3, B , InsS_3To0, A ),
|
|
OP(5610, Zip1_2D , AB, InsS_2To2, B , _ , _ ), OP(5611, Zip1_4S , AB, Trn1_4S , CB, Zip2_4S , DC), OP(5612, Trn2_4S , AB, Zip2_4S , AB, Zip1_4S , DC), OP(5613, Trn2_4S , AB, Zip2_4S , CB, Zip1_4S , DC),
|
|
OP(5620, Dup2 , B , Uzp2_4S , CB, Uzp1_4S , AD), OP(5621, Rev64 , B , Ext4 , AC, InsS_3To2, C ), OP(5622, Uzp1_4S , AB, Zip1_2D , CB, Trn2_4S , CD), OP(5623, Uzp1_4S , AB, Uzp2_4S , CB, Ext12 , AD),
|
|
OP(5630, Ext12 , AB, Trn1_4S , AC, InsS_3To2, C ), OP(5631, Dup2 , B , Zip1_2D , CB, Uzp2_4S , AD), OP(5632, Zip2_2D , AB, InsS_1To3, B , _ , _ ), OP(5633, Ext12 , AB, InsS_3To1, C , Ext12 , AC),
|
|
OP(5700, Ext12 , BB, Dup0 , A , Uzp1_4S , DC), OP(5701, Uzp2_4S , BB, Zip1_2D , AC, Rev64 , D ), OP(5702, Rev64 , B , Uzp1_4S , AC, Rev64 , D ), OP(5703, Uzp2_4S , BB, InsS_0To0, A , Ext12 , AC),
|
|
OP(5710, Zip1_2D , AB, InsS_3To2, B , _ , _ ), OP(5711, Ext12 , BB, Dup1 , A , Uzp1_4S , DC), OP(5712, Uzp2_4S , AB, InsS_2To1, A , Rev64 , C ), OP(5713, Uzp2_4S , AB, Rev64 , C , _ , _ ),
|
|
OP(5720, Ext12 , BB, Uzp1_4S , AC, _ , _ ), OP(5721, Rev64 , B , Ext4 , AC, InsS_2To2, C ), OP(5722, Ext12 , BB, Dup2 , A , Uzp1_4S , DC), OP(5723, Uzp2_4S , BB, Zip2_2D , AC, Rev64 , D ),
|
|
OP(5730, Ext12 , BB, Uzp1_4S , AC, InsS_3To1, A ), OP(5731, Ext8 , BB, Uzp2_4S , AC, _ , _ ), OP(5732, Ext8 , AB, InsS_3To2, B , _ , _ ), OP(5733, Ext12 , BB, Dup3 , A , Uzp1_4S , DC),
|
|
OP(6400, Dup0 , A , Uzp1_4S , CB, _ , _ ), OP(6401, Dup1 , A , Uzp1_4S , CA, Uzp1_4S , DB), OP(6402, Ext8 , AA, Uzp1_4S , CB, _ , _ ), OP(6403, Dup3 , A , Uzp1_4S , CA, Uzp1_4S , DB),
|
|
OP(6410, Uzp1_4S , BB, Zip1_2D , AC, _ , _ ), OP(6411, Dup1 , A , Uzp1_4S , CB, _ , _ ), OP(6412, Dup2 , A , Uzp2_4S , CA, Uzp1_4S , DB), OP(6413, Ext12 , AA, Uzp1_4S , CB, _ , _ ),
|
|
OP(6420, Uzp1_4S , AB, _ , _ , _ , _ ), OP(6421, Uzp1_4S , AB, InsS_1To0, A , _ , _ ), OP(6422, Dup2 , A , Uzp1_4S , CB, _ , _ ), OP(6423, Uzp1_4S , AB, InsS_3To0, A , _ , _ ),
|
|
OP(6430, Uzp1_4S , AB, InsS_3To1, A , _ , _ ), OP(6431, Rev64 , B , Uzp2_4S , AC, _ , _ ), OP(6432, Uzp1_4S , BB, Zip2_2D , AC, _ , _ ), OP(6433, Dup3 , A , Uzp1_4S , CB, _ , _ ),
|
|
OP(6500, Ext12 , BB, Dup0 , A , Zip2_2D , DC), OP(6501, Ext4 , BB, Rev64 , A , Zip1_2D , DC), OP(6502, Rev64 , A , Ext12 , CB, InsS_1To1, C ), OP(6503, Ext12 , AB, InsS_0To1, A , _ , _ ),
|
|
OP(6510, Ext4 , BB, Zip1_2D , AC, _ , _ ), OP(6511, Ext12 , BB, Dup1 , A , Zip2_2D , DC), OP(6512, Rev64 , A , Ext12 , CB, InsS_0To1, C ), OP(6513, Ext12 , AB, InsS_1To1, A , _ , _ ),
|
|
OP(6520, Uzp1_4S , AB, InsS_1To2, B , _ , _ ), OP(6521, Trn2_4S , AB, Zip2_4S , AB, Zip1_4S , CD), OP(6522, Ext12 , BB, Dup2 , A , Zip2_2D , DC), OP(6523, Ext12 , AB, InsS_2To1, A , _ , _ ),
|
|
OP(6530, Ext4 , BA, Trn2_4S , CA, Ext8 , DC), OP(6531, Uzp2_4S , AB, InsS_2To3, B , _ , _ ), OP(6532, Ext12 , BB, Zip2_2D , AC, _ , _ ), OP(6533, Ext12 , AB, InsS_0To1, C , _ , _ ),
|
|
OP(6600, Dup2 , B , Dup0 , A , Uzp2_4S , DC), OP(6601, Rev64 , A , Dup2 , B , Zip1_2D , CD), OP(6602, Ext8 , AA, Dup2 , B , Uzp1_4S , CD), OP(6603, Ext12 , AA, Dup2 , B , Zip1_2D , CD),
|
|
OP(6610, Dup2 , B , Zip1_2D , AC, _ , _ ), OP(6611, Dup2 , B , Dup1 , A , Uzp2_4S , DC), OP(6612, Zip2_4S , AB, Trn2_4S , AC, Zip1_4S , CD), OP(6613, Ext12 , AA, Dup2 , B , Uzp1_4S , CD),
|
|
OP(6620, Dup2 , B , Uzp1_4S , AC, _ , _ ), OP(6621, Ext12 , BA, Dup2 , B , Zip2_2D , CD), OP(6622, Trn1_4S , AB, Zip2_4S , CC, _ , _ ), OP(6623, Rev64 , A , Dup2 , B , Zip2_2D , CD),
|
|
OP(6630, Trn1_4S , AB, InsS_3To1, A , InsS_3To2, C ), OP(6631, Dup2 , B , Uzp2_4S , AC, _ , _ ), OP(6632, Dup2 , B , Zip2_2D , AC, _ , _ ), OP(6633, Dup3 , A , Dup2 , B , Uzp2_4S , CD),
|
|
OP(6700, Rev64 , B , Dup0 , A , Zip2_2D , DC), OP(6701, Ext8 , BA, Rev64 , C , Ext8 , DD), OP(6702, Uzp1_4S , BA, Zip2_2D , CB, Rev64 , D ), OP(6703, Ext4 , AA, Rev64 , B , Zip2_2D , CD),
|
|
OP(6710, Rev64 , B , InsD_0To0, A , _ , _ ), OP(6711, Rev64 , B , Dup1 , A , Zip2_2D , DC), OP(6712, Ext12 , BA, Zip2_2D , CB, Rev64 , D ), OP(6713, Uzp2_4S , BA, Zip2_2D , CB, Rev64 , D ),
|
|
OP(6720, Uzp1_4S , AB, InsS_3To2, B , _ , _ ), OP(6721, Ext12 , BA, Rev64 , B , Zip2_2D , CD), OP(6722, Rev64 , B , Dup2 , A , Zip2_2D , DC), OP(6723, Zip2_2D , AB, Rev64 , C , _ , _ ),
|
|
OP(6730, Ext4 , AA, Zip2_2D , CB, Rev64 , D ), OP(6731, Dup3 , B , Uzp1_4S , CB, Uzp2_4S , AD), OP(6732, Rev64 , B , Zip2_2D , AC, _ , _ ), OP(6733, Rev64 , B , Dup3 , A , Zip2_2D , DC),
|
|
OP(7400, Uzp1_4S , AB, InsS_3To3, B , InsS_0To1, C ), OP(7401, Ext12 , BB, Zip1_2D , AC, Rev64 , D ), OP(7402, Ext4 , AB, Trn2_4S , CB, InsS_0To1, A ), OP(7403, Rev64 , B , InsS_0To0, A , Ext12 , AC),
|
|
OP(7410, Zip1_2D , AB, InsS_3To3, B , _ , _ ), OP(7411, Trn2_4S , AB, InsS_0To2, B , InsS_0To1, C ), OP(7412, Ext4 , AB, Trn2_4S , CB, InsS_0To1, C ), OP(7413, Ext4 , AB, Zip2_4S , CB, InsS_0To1, C ),
|
|
OP(7420, Uzp1_4S , AB, InsS_3To3, B , _ , _ ), OP(7421, Ext12 , BA, Trn1_4S , BC, Ext8 , CD), OP(7422, Ext4 , AB, Trn2_4S , CB, InsS_0To1, D ), OP(7423, Ext12 , BB, Ext8 , AC, Rev64 , D ),
|
|
OP(7430, Uzp2_4S , AB, InsS_0To2, B , InsS_0To0, A ), OP(7431, Uzp2_4S , AB, InsS_0To2, B , _ , _ ), OP(7432, Zip2_2D , AB, InsS_0To2, B , _ , _ ), OP(7433, Trn2_4S , AB, Ext4 , AB, Zip2_4S , DC),
|
|
OP(7500, Dup0 , A , Uzp2_4S , CB, _ , _ ), OP(7501, Uzp2_4S , AB, InsS_0To1, A , _ , _ ), OP(7502, Ext4 , AA, Uzp2_4S , CB, _ , _ ), OP(7503, Dup3 , A , Ext4 , CA, Uzp2_4S , DB),
|
|
OP(7510, Uzp2_4S , BB, Zip1_2D , AC, _ , _ ), OP(7511, Dup1 , A , Uzp2_4S , CB, _ , _ ), OP(7512, Dup2 , A , Zip1_2D , CA, Uzp2_4S , DB), OP(7513, Ext8 , AA, Uzp2_4S , CB, _ , _ ),
|
|
OP(7520, Rev64 , B , Uzp1_4S , AC, _ , _ ), OP(7521, Uzp2_4S , AB, InsS_2To1, A , _ , _ ), OP(7522, Dup2 , A , Uzp2_4S , CB, _ , _ ), OP(7523, Dup3 , A , Uzp1_4S , CA, Uzp2_4S , DB),
|
|
OP(7530, Uzp2_4S , AB, InsS_0To0, A , _ , _ ), OP(7531, Uzp2_4S , AB, _ , _ , _ , _ ), OP(7532, Uzp2_4S , BB, Zip2_2D , AC, _ , _ ), OP(7533, Dup3 , A , Uzp2_4S , CB, _ , _ ),
|
|
OP(7600, Dup0 , A , Zip2_2D , CB, _ , _ ), OP(7601, Rev64 , A , InsD_1To1, B , _ , _ ), OP(7602, Zip2_2D , AB, InsS_0To1, A , _ , _ ), OP(7603, Ext4 , AA, Zip2_2D , CB, _ , _ ),
|
|
OP(7610, Zip2_2D , BB, Zip1_2D , AC, _ , _ ), OP(7611, Dup1 , A , Zip2_2D , CB, _ , _ ), OP(7612, Zip2_2D , AB, InsS_1To1, A , _ , _ ), OP(7613, Dup3 , A , Zip1_4S , CA, Zip2_2D , DB),
|
|
OP(7620, Uzp1_4S , BA, Zip2_2D , CB, _ , _ ), OP(7621, Ext12 , BA, Zip2_2D , CB, _ , _ ), OP(7622, Dup2 , A , Zip2_2D , CB, _ , _ ), OP(7623, Rev64 , A , Zip2_2D , CB, _ , _ ),
|
|
OP(7630, Zip2_2D , AB, InsS_0To0, A , _ , _ ), OP(7631, Uzp2_4S , BA, Zip2_2D , CB, _ , _ ), OP(7632, Zip2_2D , AB, _ , _ , _ , _ ), OP(7633, Dup3 , A , Zip2_2D , CB, _ , _ ),
|
|
OP(7700, Dup3 , B , Dup0 , A , Uzp2_4S , DC), OP(7701, Rev64 , A , Dup3 , B , Zip1_2D , CD), OP(7702, Ext8 , AA, Dup3 , B , Uzp1_4S , CD), OP(7703, Ext12 , AA, Dup3 , B , Zip1_2D , CD),
|
|
OP(7710, Dup3 , B , Zip1_2D , AC, _ , _ ), OP(7711, Dup3 , B , Dup1 , A , Uzp2_4S , DC), OP(7712, Trn2_4S , BA, InsS_2To0, A , InsS_2To3, C ), OP(7713, Ext12 , AA, Dup3 , B , Uzp1_4S , CD),
|
|
OP(7720, Dup3 , B , Uzp1_4S , AC, _ , _ ), OP(7721, Ext12 , BA, Dup3 , B , Zip2_2D , CD), OP(7722, Dup3 , B , Dup2 , A , Uzp2_4S , DC), OP(7723, Rev64 , A , Dup3 , B , Zip2_2D , CD),
|
|
OP(7730, Uzp2_4S , AB, InsS_0To0, A , InsS_3To2, C ), OP(7731, Dup3 , B , Uzp2_4S , AC, _ , _ ), OP(7732, Dup3 , B , Zip2_2D , AC, _ , _ ), OP(7733, Trn2_4S , AB, Zip2_4S , CC, _ , _ )
|
|
};
|
|
|
|
#undef OP
|
|
|
|
static void emit_interleaved_shuffle32_impl(UniCompiler& uc, const Vec& dst, const Vec& src1, const Vec& src2, uint32_t imm) {
|
|
ASMJIT_ASSERT((imm & 0xFCFCFCFC) == 0);
|
|
|
|
if (src1.id() == src2.id())
|
|
return emit_swizzle32_impl(uc, dst, src1, imm);
|
|
|
|
BackendCompiler* cc = uc.cc;
|
|
|
|
uint32_t table_index = ((imm & 0x03000000) >> (24 - 6)) |
|
|
((imm & 0x00030000) >> (16 - 4)) |
|
|
((imm & 0x00000300) >> (8 - 2)) | (imm & 0x00000003);
|
|
|
|
// By default the cost matrix tries to avoid inserting A to DST to minimize the number of instructions
|
|
// in case that `dst == a`, which is common. However, we have a also a table that penalizes inserting B
|
|
// to DST, so select the best version.
|
|
InterleavedShuffle32Ops ops;
|
|
if (dst.id() != src2.id())
|
|
ops = interleaved_shuffle32_ops[table_index];
|
|
else
|
|
ops = interleaved_shuffle32_ops_dst_same_as_b[table_index];
|
|
|
|
Vec regs[5] = { src1, src2, Vec(), Vec(), Vec() };
|
|
|
|
uint32_t count = ops.count();
|
|
uint32_t src_use_mask[4] {};
|
|
uint32_t dst_index = count - 1;
|
|
|
|
{
|
|
uint32_t i = count;
|
|
uint32_t all_use_mask = 0;
|
|
|
|
while (i != 0) {
|
|
i--;
|
|
InterleavedShuffle32Ops::Op op = ops.op(i);
|
|
|
|
// Calculate masks that follow.
|
|
all_use_mask |= 1u << op.src_a();
|
|
all_use_mask |= 1u << op.src_b();
|
|
src_use_mask[i] = all_use_mask;
|
|
|
|
// Calculate the last operation where we need a new destination (inserts insert to the last).
|
|
if (op.is_ins_op()) {
|
|
// The last destination would be the previous operation (if not another insert).
|
|
dst_index = i - 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
uint32_t op_index = 2;
|
|
Vec final_dst;
|
|
|
|
for (uint32_t i = 0; i < count; i++) {
|
|
InterleavedShuffle32Ops::Op op = ops.op(i);
|
|
|
|
if (i > 0 && !op.is_ins_op()) {
|
|
op_index++;
|
|
}
|
|
|
|
// Should not be counted if it's invalid - so let's verify it here...
|
|
ASMJIT_ASSERT(op.is_valid());
|
|
|
|
Vec op_dst;
|
|
if (i < dst_index) {
|
|
if (regs[op_index].is_valid()) {
|
|
op_dst = regs[op_index];
|
|
}
|
|
else {
|
|
op_dst = uc.new_similar_reg(dst, "@shuf_tmp_%u", op_index - 2);
|
|
}
|
|
}
|
|
else {
|
|
if (!final_dst.is_valid()) {
|
|
uint32_t msk = 0;
|
|
if (dst.id() == src1.id()) {
|
|
msk = 0x1u;
|
|
}
|
|
else if (dst.id() == src2.id()) {
|
|
msk = 0x2u;
|
|
}
|
|
|
|
if (src_use_mask[i + 1] & msk) {
|
|
// In this case the destination is in conflict with one of the source registers. We have to
|
|
// create a new virtual register and then move it to the real `dst` to not mess up the shuffle.
|
|
ASMJIT_ASSERT(!regs[op_index].is_valid());
|
|
final_dst = uc.new_similar_reg(dst, "@shuf_dst");
|
|
}
|
|
else {
|
|
// Perfect - the destination is not in conflict with any source register.
|
|
final_dst = dst;
|
|
}
|
|
}
|
|
op_dst = final_dst;
|
|
}
|
|
|
|
uint32_t a_index = op.src_a();
|
|
uint32_t b_index = op.src_b();
|
|
|
|
switch (op.op()) {
|
|
case InterleavedShuffle32Ops::Op::kOpInsS: {
|
|
uint32_t src_lane = op.ins_src();
|
|
uint32_t dst_lane = op.ins_dst();
|
|
cc->mov(op_dst.s(dst_lane), regs[a_index].s(src_lane));
|
|
break;
|
|
}
|
|
|
|
case InterleavedShuffle32Ops::Op::kOpInsD: {
|
|
uint32_t src_lane = op.ins_src();
|
|
uint32_t dst_lane = op.ins_dst();
|
|
cc->mov(op_dst.d(dst_lane), regs[a_index].d(src_lane));
|
|
break;
|
|
}
|
|
|
|
case InterleavedShuffle32Ops::Op::kOpPerm: {
|
|
static constexpr uint16_t perm_inst[8] = {
|
|
Inst::kIdZip1_v,
|
|
Inst::kIdZip2_v,
|
|
Inst::kIdUzp1_v,
|
|
Inst::kIdUzp2_v,
|
|
Inst::kIdTrn1_v,
|
|
Inst::kIdTrn2_v,
|
|
Inst::kIdExt_v,
|
|
Inst::kIdExt_v
|
|
};
|
|
|
|
InstId inst_id = perm_inst[op.perm_op()];
|
|
|
|
if (inst_id == Inst::kIdExt_v)
|
|
cc->ext(op_dst.b16(), regs[a_index].b16(), regs[b_index].b16(), op.perm_ext_imm());
|
|
else if (op.perm_s4())
|
|
cc->emit(inst_id, op_dst.s4(), regs[a_index].s4(), regs[b_index].s4());
|
|
else
|
|
cc->emit(inst_id, op_dst.d2(), regs[a_index].d2(), regs[b_index].d2());
|
|
break;
|
|
}
|
|
|
|
case InterleavedShuffle32Ops::Op::kOpRev64: {
|
|
cc->rev64(op_dst.s4(), regs[a_index].s4());
|
|
break;
|
|
}
|
|
|
|
case InterleavedShuffle32Ops::Op::kOpDup: {
|
|
uint32_t idx = op.dup_idx();
|
|
cc->dup(op_dst.s4(), regs[a_index].s(idx));
|
|
break;
|
|
}
|
|
|
|
default:
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
|
|
regs[op_index] = op_dst;
|
|
}
|
|
|
|
vec_mov(uc, dst, final_dst);
|
|
}
|
|
|
|
// ujit::UniCompiler - Vector Instructions - OpArray Iterator
|
|
// ==========================================================
|
|
|
|
template<typename T>
|
|
class OpArrayIter {
|
|
public:
|
|
const T& _op;
|
|
|
|
ASMJIT_INLINE_NODEBUG OpArrayIter(const T& op) noexcept : _op(op) {}
|
|
ASMJIT_INLINE_NODEBUG const T& op() const noexcept { return _op; }
|
|
ASMJIT_INLINE_NODEBUG void next() noexcept {}
|
|
};
|
|
|
|
template<>
|
|
class OpArrayIter<OpArray> {
|
|
public:
|
|
const OpArray& _opArray;
|
|
size_t _i {};
|
|
size_t _n {};
|
|
|
|
ASMJIT_INLINE_NODEBUG OpArrayIter(const OpArray& op_array) noexcept : _opArray(op_array), _i(0), _n(op_array.size()) {}
|
|
ASMJIT_INLINE_NODEBUG const Operand_& op() const noexcept { return _opArray[_i]; }
|
|
ASMJIT_INLINE_NODEBUG void next() noexcept { if (++_i >= _n) _i = 0; }
|
|
};
|
|
|
|
template<typename Src>
|
|
static ASMJIT_INLINE void emit_2v_t(UniCompiler& uc, UniOpVV op, const OpArray& dst_, const Src& src_) {
|
|
size_t n = dst_.size();
|
|
OpArrayIter<Src> src(src_);
|
|
|
|
for (size_t i = 0; i < n; i++) {
|
|
uc.emit_2v(op, dst_[i], src.op());
|
|
src.next();
|
|
}
|
|
}
|
|
|
|
template<typename Src>
|
|
static ASMJIT_INLINE void emit_2vi_t(UniCompiler& uc, UniOpVVI op, const OpArray& dst_, const Src& src_, uint32_t imm) {
|
|
size_t n = dst_.size();
|
|
OpArrayIter<Src> src(src_);
|
|
|
|
for (size_t i = 0; i < n; i++) {
|
|
uc.emit_2vi(op, dst_[i], src.op(), imm);
|
|
src.next();
|
|
}
|
|
}
|
|
|
|
template<typename Src1, typename Src2>
|
|
static ASMJIT_INLINE void emit_3v_t(UniCompiler& uc, UniOpVVV op, const OpArray& dst_, const Src1& src1_, const Src2& src2_) {
|
|
size_t n = dst_.size();
|
|
OpArrayIter<Src1> src1(src1_);
|
|
OpArrayIter<Src2> src2(src2_);
|
|
|
|
for (size_t i = 0; i < n; i++) {
|
|
uc.emit_3v(op, dst_[i], src1.op(), src2.op());
|
|
src1.next();
|
|
src2.next();
|
|
}
|
|
}
|
|
|
|
template<typename Src1, typename Src2>
|
|
static ASMJIT_INLINE void emit_3vi_t(UniCompiler& uc, UniOpVVVI op, const OpArray& dst_, const Src1& src1_, const Src2& src2_, uint32_t imm) {
|
|
size_t n = dst_.size();
|
|
OpArrayIter<Src1> src1(src1_);
|
|
OpArrayIter<Src2> src2(src2_);
|
|
|
|
for (size_t i = 0; i < n; i++) {
|
|
uc.emit_3vi(op, dst_[i], src1.op(), src2.op(), imm);
|
|
src1.next();
|
|
src2.next();
|
|
}
|
|
}
|
|
|
|
template<typename Src1, typename Src2, typename Src3>
|
|
static ASMJIT_INLINE void emit_4v_t(UniCompiler& uc, UniOpVVVV op, const OpArray& dst_, const Src1& src1_, const Src2& src2_, const Src3& src3_) {
|
|
size_t n = dst_.size();
|
|
OpArrayIter<Src1> src1(src1_);
|
|
OpArrayIter<Src2> src2(src2_);
|
|
OpArrayIter<Src3> src3(src3_);
|
|
|
|
for (size_t i = 0; i < n; i++) {
|
|
uc.emit_4v(op, dst_[i], src1.op(), src2.op(), src3.op());
|
|
src1.next();
|
|
src2.next();
|
|
src3.next();
|
|
}
|
|
}
|
|
|
|
// ujit::UniCompiler - Vector Instructions - Emit 2V
|
|
// =================================================
|
|
|
|
void UniCompiler::emit_2v(UniOpVV op, const Operand_& dst_, const Operand_& src_) {
|
|
ASMJIT_ASSERT(dst_.is_vec());
|
|
|
|
Vec dst(dst_.as<Vec>());
|
|
|
|
UniOpVInfo op_info = opcode_info_2v[size_t(op)];
|
|
InstId inst_id = op_info.inst_id;
|
|
|
|
switch (op) {
|
|
case UniOpVV::kMov: {
|
|
if (src_.is_vec()) {
|
|
Vec src = src_.as<Vec>();
|
|
if (dst.size() < 16 || src.size() < 16)
|
|
cc->mov(dst.b8(), src.b8());
|
|
else
|
|
cc->mov(dst.b16(), src.b16());
|
|
return;
|
|
}
|
|
|
|
vec_mov(*this, dst, src_);
|
|
return;
|
|
}
|
|
|
|
case UniOpVV::kMovU64: {
|
|
dst = dst.d();
|
|
Vec src = as_vec(*this, src_, dst);
|
|
|
|
cc->mov(dst.b8(), src.b8());
|
|
return;
|
|
}
|
|
|
|
case UniOpVV::kBroadcastU8Z:
|
|
case UniOpVV::kBroadcastU16Z:
|
|
case UniOpVV::kBroadcastU8:
|
|
case UniOpVV::kBroadcastU16:
|
|
case UniOpVV::kBroadcastU32:
|
|
case UniOpVV::kBroadcastF32:
|
|
case UniOpVV::kBroadcastU64:
|
|
case UniOpVV::kBroadcastF64: {
|
|
ElementSize element_size = ElementSize(op_info.dst_element);
|
|
vec_set_type(dst, element_size);
|
|
|
|
Operand src(src_);
|
|
|
|
if (src.is_mem()) {
|
|
// NOTE: ld1r instruction is pretty limited - it offers only `[base]` or `[base + offset|index]@` (post-index)
|
|
// addressing. This means that it's sometimes just better to emit a regular load folowed by dup than to actually
|
|
// use ld1r.
|
|
Mem& m = src.as<Mem>();
|
|
if (!m.has_index() && !m.has_offset()) {
|
|
cc->ld1r(dst, m);
|
|
return;
|
|
}
|
|
|
|
v_load_iany(dst, src.as<Mem>(), size_t(1) << uint32_t(op_info.src_element), Alignment(1));
|
|
src = dst;
|
|
}
|
|
|
|
if (src.is_vec()) {
|
|
Vec& src_vec = src.as<Vec>();
|
|
vec_set_type_and_index(src_vec, element_size, 0);
|
|
cc->dup(dst, src_vec);
|
|
return;
|
|
}
|
|
else if (src.is_gp()) {
|
|
Gp& src_gp = src.as<Gp>();
|
|
if (element_size <= ElementSize::k32)
|
|
src_gp = src_gp.r32();
|
|
else
|
|
src_gp = src_gp.r64();
|
|
cc->dup(dst, src_gp);
|
|
return;
|
|
}
|
|
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
|
|
case UniOpVV::kBroadcastV128_U32:
|
|
case UniOpVV::kBroadcastV128_U64:
|
|
case UniOpVV::kBroadcastV128_F32:
|
|
case UniOpVV::kBroadcastV128_F64:
|
|
// 128-bit broadcast is an alias of mov as we have only 128-bit vectors...
|
|
v_mov(dst, src_);
|
|
return;
|
|
|
|
case UniOpVV::kBroadcastV256_U32:
|
|
case UniOpVV::kBroadcastV256_U64:
|
|
case UniOpVV::kBroadcastV256_F32:
|
|
case UniOpVV::kBroadcastV256_F64:
|
|
// Unsupported as NEON has only 128-bit vectors.
|
|
ASMJIT_NOT_REACHED();
|
|
|
|
case UniOpVV::kAbsI8:
|
|
case UniOpVV::kAbsI16:
|
|
case UniOpVV::kAbsI32:
|
|
case UniOpVV::kAbsI64:
|
|
case UniOpVV::kNotU32:
|
|
case UniOpVV::kNotU64:
|
|
case UniOpVV::kNotF32:
|
|
case UniOpVV::kNotF64: {
|
|
Vec src = as_vec(*this, src_, dst);
|
|
|
|
vec_set_type(dst, op_info.dst_element);
|
|
vec_set_type(src, op_info.src_element);
|
|
|
|
cc->emit(inst_id, dst, src);
|
|
return;
|
|
}
|
|
|
|
case UniOpVV::kCvtI8ToI32:
|
|
case UniOpVV::kCvtU8ToU32: {
|
|
Vec src = as_vec(*this, src_, dst);
|
|
cc->emit(inst_id, dst.h8(), src.b8(), 0);
|
|
cc->emit(inst_id, dst.s4(), dst.h4(), 0);
|
|
return;
|
|
}
|
|
|
|
case UniOpVV::kCvtI8LoToI16:
|
|
case UniOpVV::kCvtI8HiToI16:
|
|
case UniOpVV::kCvtU8LoToU16:
|
|
case UniOpVV::kCvtU8HiToU16:
|
|
case UniOpVV::kCvtI16LoToI32:
|
|
case UniOpVV::kCvtI16HiToI32:
|
|
case UniOpVV::kCvtU16LoToU32:
|
|
case UniOpVV::kCvtU16HiToU32:
|
|
case UniOpVV::kCvtI32LoToI64:
|
|
case UniOpVV::kCvtI32HiToI64:
|
|
case UniOpVV::kCvtU32LoToU64:
|
|
case UniOpVV::kCvtU32HiToU64: {
|
|
vec_set_type(dst, op_info.dst_element);
|
|
|
|
Vec src;
|
|
|
|
if (op_info.src_part == VecPart::kLo) {
|
|
src = as_vec(*this, src_, dst, 8);
|
|
src = src.v64();
|
|
}
|
|
else if (src_.is_vec()) {
|
|
src = src_.as<Vec>();
|
|
}
|
|
else {
|
|
Mem m(src_.as<Mem>());
|
|
m.add_offset(8);
|
|
src = vec_from_mem(*this, m, dst, 8);
|
|
src = src.v64();
|
|
|
|
// Since we have loaded from memory, we want to use the low-part variant of the instruction.
|
|
inst_id = opcode_info_2v[size_t(op) - 1u].inst_id;
|
|
}
|
|
|
|
vec_set_type(src, op_info.src_element);
|
|
cc->emit(inst_id, dst, src, 0);
|
|
return;
|
|
}
|
|
|
|
case UniOpVV::kRcpF32:
|
|
case UniOpVV::kRcpF64: {
|
|
// Intrinsic.
|
|
const void* one_ptr = op_info.src_element == ElementSize::k32 ? static_cast<const void*>(&ct().f32_1) : static_cast<const void*>(&ct().f64_1);
|
|
|
|
Vec one = simd_vec_const(one_ptr, Bcst::kNA, dst);
|
|
Vec src = as_vec(*this, src_, dst);
|
|
|
|
vec_set_type(dst, op_info.dst_element);
|
|
vec_set_type(one, op_info.dst_element);
|
|
vec_set_type(src, op_info.dst_element);
|
|
|
|
cc->fdiv(dst, one, src);
|
|
return;
|
|
}
|
|
|
|
case UniOpVV::kAbsF32S:
|
|
case UniOpVV::kNegF32S:
|
|
case UniOpVV::kTruncF32S:
|
|
case UniOpVV::kFloorF32S:
|
|
case UniOpVV::kCeilF32S:
|
|
case UniOpVV::kRoundEvenF32S:
|
|
case UniOpVV::kRoundHalfAwayF32S:
|
|
case UniOpVV::kSqrtF32S: {
|
|
dst.set_signature(RegTraits<RegType::kVec32>::kSignature);
|
|
Vec src = as_vec(*this, src_, dst);
|
|
|
|
cc->emit(inst_id, dst, src);
|
|
return;
|
|
}
|
|
|
|
case UniOpVV::kAbsF64S:
|
|
case UniOpVV::kNegF64S:
|
|
case UniOpVV::kTruncF64S:
|
|
case UniOpVV::kFloorF64S:
|
|
case UniOpVV::kCeilF64S:
|
|
case UniOpVV::kRoundEvenF64S:
|
|
case UniOpVV::kRoundHalfAwayF64S:
|
|
case UniOpVV::kSqrtF64S: {
|
|
dst.set_signature(RegTraits<RegType::kVec64>::kSignature);
|
|
Vec src = as_vec(*this, src_, dst);
|
|
|
|
cc->emit(inst_id, dst, src);
|
|
return;
|
|
}
|
|
|
|
case UniOpVV::kRoundHalfUpF32S:
|
|
case UniOpVV::kRoundHalfUpF64S:
|
|
case UniOpVV::kRoundHalfUpF32:
|
|
case UniOpVV::kRoundHalfUpF64: {
|
|
// Intrinsic.
|
|
const void* one_ptr =
|
|
op_info.src_element == ElementSize::k32
|
|
? static_cast<const void*>(&ct().f32_0_5_minus_1ulp)
|
|
: static_cast<const void*>(&ct().f64_0_5_minus_1ulp);
|
|
|
|
Vec one = simd_vec_const(one_ptr, Bcst::kNA, dst);
|
|
Vec src = as_vec(*this, src_, dst);
|
|
|
|
if (op == UniOpVV::kRoundHalfUpF32S) {
|
|
dst.set_signature(RegTraits<RegType::kVec32>::kSignature);
|
|
src.set_signature(RegTraits<RegType::kVec32>::kSignature);
|
|
one.set_signature(RegTraits<RegType::kVec32>::kSignature);
|
|
}
|
|
else if (op == UniOpVV::kRoundHalfUpF64S) {
|
|
dst.set_signature(RegTraits<RegType::kVec64>::kSignature);
|
|
src.set_signature(RegTraits<RegType::kVec64>::kSignature);
|
|
one.set_signature(RegTraits<RegType::kVec64>::kSignature);
|
|
}
|
|
else {
|
|
vec_set_type(dst, op_info.dst_element);
|
|
vec_set_type(one, op_info.src_element);
|
|
vec_set_type(src, op_info.src_element);
|
|
}
|
|
|
|
cc->fadd(dst, src, one);
|
|
cc->frintm(dst, dst);
|
|
|
|
return;
|
|
}
|
|
|
|
case UniOpVV::kAbsF32:
|
|
case UniOpVV::kAbsF64:
|
|
case UniOpVV::kNegF32:
|
|
case UniOpVV::kNegF64:
|
|
case UniOpVV::kTruncF32:
|
|
case UniOpVV::kTruncF64:
|
|
case UniOpVV::kFloorF32:
|
|
case UniOpVV::kFloorF64:
|
|
case UniOpVV::kCeilF32:
|
|
case UniOpVV::kCeilF64:
|
|
case UniOpVV::kRoundEvenF32:
|
|
case UniOpVV::kRoundEvenF64:
|
|
case UniOpVV::kRoundHalfAwayF32:
|
|
case UniOpVV::kRoundHalfAwayF64:
|
|
case UniOpVV::kSqrtF32:
|
|
case UniOpVV::kSqrtF64:
|
|
case UniOpVV::kCvtI32ToF32:
|
|
case UniOpVV::kCvtRoundF32ToI32:
|
|
case UniOpVV::kCvtTruncF32ToI32: {
|
|
Vec src = as_vec(*this, src_, dst);
|
|
|
|
vec_set_type(dst, op_info.dst_element);
|
|
vec_set_type(src, op_info.src_element);
|
|
|
|
cc->emit(inst_id, dst, src);
|
|
return;
|
|
}
|
|
|
|
case UniOpVV::kCvtF32ToF64S:
|
|
case UniOpVV::kCvtF64ToF32S: {
|
|
Vec src = as_vec(*this, src_, dst);
|
|
|
|
vec_set_vec_type(dst, op_info.dst_element);
|
|
vec_set_vec_type(src, op_info.src_element);
|
|
|
|
cc->fcvt(dst, src);
|
|
return;
|
|
}
|
|
|
|
case UniOpVV::kCvtF32HiToF64:
|
|
if (src_.is_mem()) {
|
|
Vec src = as_vec(*this, src_.as<Mem>().clone_adjusted(8), dst, 8).v64();
|
|
|
|
vec_set_type(dst, op_info.dst_element);
|
|
vec_set_type(src, op_info.src_element);
|
|
|
|
cc->emit(opcode_info_2v[size_t(op) - 1u].inst_id, dst, src);
|
|
return;
|
|
}
|
|
[[fallthrough]];
|
|
|
|
case UniOpVV::kCvtF32LoToF64: {
|
|
Vec src = as_vec(*this, src_, dst, 8);
|
|
|
|
if (op_info.src_part == VecPart::kLo) {
|
|
src = src.v64();
|
|
}
|
|
|
|
vec_set_type(dst, op_info.dst_element);
|
|
vec_set_type(src, op_info.src_element);
|
|
|
|
cc->emit(inst_id, dst, src);
|
|
return;
|
|
}
|
|
|
|
case UniOpVV::kCvtI32HiToF64:
|
|
if (src_.is_mem()) {
|
|
Vec src = as_vec(*this, src_.as<Mem>().clone_adjusted(8), dst, 8).v64();
|
|
|
|
vec_set_type(dst, op_info.dst_element);
|
|
vec_set_type(src, op_info.src_element);
|
|
|
|
cc->emit(opcode_info_2v[size_t(op) - 1u].inst_id, dst, src, 0);
|
|
cc->scvtf(dst.d2(), dst.d2());
|
|
return;
|
|
}
|
|
[[fallthrough]];
|
|
|
|
case UniOpVV::kCvtI32LoToF64: {
|
|
Vec src = as_vec(*this, src_, dst, 8);
|
|
|
|
if (op_info.src_part == VecPart::kLo) {
|
|
src = src.v64();
|
|
}
|
|
|
|
vec_set_type(dst, op_info.dst_element);
|
|
vec_set_type(src, op_info.src_element);
|
|
|
|
cc->emit(inst_id, dst, src, 0);
|
|
cc->scvtf(dst.d2(), dst.d2());
|
|
return;
|
|
}
|
|
|
|
case UniOpVV::kCvtF64ToF32Lo:
|
|
case UniOpVV::kCvtF64ToF32Hi: {
|
|
dst = dst.q();
|
|
Vec src = as_vec(*this, src_, dst);
|
|
|
|
if (op_info.dst_part == VecPart::kLo) {
|
|
dst = dst.d();
|
|
}
|
|
|
|
vec_set_type(dst, op_info.dst_element);
|
|
vec_set_type(src, op_info.src_element);
|
|
|
|
cc->emit(inst_id, dst, src);
|
|
return;
|
|
}
|
|
|
|
case UniOpVV::kCvtTruncF64ToI32Lo:
|
|
case UniOpVV::kCvtRoundF64ToI32Lo:
|
|
case UniOpVV::kCvtTruncF64ToI32Hi:
|
|
case UniOpVV::kCvtRoundF64ToI32Hi: {
|
|
dst = dst.q();
|
|
|
|
Vec src = as_vec(*this, src_, dst);
|
|
Vec tmp = new_similar_reg(dst, "@tmp");
|
|
|
|
cc->emit(inst_id, tmp.d2(), src.d2());
|
|
|
|
if (op_info.dst_part == VecPart::kLo) {
|
|
cc->sqxtn(dst.s2(), tmp.d2());
|
|
}
|
|
else {
|
|
cc->sqxtn2(dst.s4(), tmp.d2());
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
default:
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
}
|
|
|
|
void UniCompiler::emit_2v(UniOpVV op, const OpArray& dst_, const Operand_& src_) { emit_2v_t(*this, op, dst_, src_); }
|
|
void UniCompiler::emit_2v(UniOpVV op, const OpArray& dst_, const OpArray& src_) { emit_2v_t(*this, op, dst_, src_); }
|
|
|
|
// ujit::UniCompiler - Vector Instructions - Emit 2VI
|
|
// ==================================================
|
|
|
|
void UniCompiler::emit_2vi(UniOpVVI op, const Operand_& dst_, const Operand_& src_, uint32_t imm) {
|
|
ASMJIT_ASSERT(dst_.is_vec());
|
|
|
|
Vec dst(dst_.as<Vec>());
|
|
|
|
UniOpVInfo op_info = opcode_info_2vi[size_t(op)];
|
|
InstId inst_id = op_info.inst_id;
|
|
|
|
switch (op) {
|
|
case UniOpVVI::kSllbU128:
|
|
case UniOpVVI::kSrlbU128: {
|
|
ASMJIT_ASSERT(imm < 16);
|
|
|
|
Vec src = as_vec(*this, src_, dst);
|
|
|
|
// If the shift is used to extract a high 64-bit element and zero the rest of the register.
|
|
if (op == UniOpVVI::kSrlbU128 && imm == 8) {
|
|
cc->dup(dst.d(), src.d(1));
|
|
return;
|
|
}
|
|
|
|
// If the shift is used to extract the last 32-bit element and zero the rest of the register.
|
|
if (op == UniOpVVI::kSrlbU128 && imm == 12) {
|
|
cc->dup(dst.s(), src.s(3));
|
|
return;
|
|
}
|
|
|
|
Vec zero = simd_vec_zero(dst);
|
|
vec_set_type(dst, ElementSize::k8);
|
|
vec_set_type(src, ElementSize::k8);
|
|
vec_set_type(zero, ElementSize::k8);
|
|
|
|
if (op == UniOpVVI::kSllbU128)
|
|
cc->ext(dst, zero, src, 16u - imm);
|
|
else
|
|
cc->ext(dst, src, zero, imm);
|
|
return;
|
|
}
|
|
|
|
case UniOpVVI::kSwizzleU16x4:
|
|
case UniOpVVI::kSwizzleLoU16x4:
|
|
case UniOpVVI::kSwizzleHiU16x4: {
|
|
Vec src = as_vec(*this, src_, dst);
|
|
|
|
uint8_t pred_data[16] = { 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xA, 0xB, 0xC, 0xD, 0xE, 0xF };
|
|
|
|
uint32_t d = (imm >> 23) & (0x3 << 1);
|
|
uint32_t c = (imm >> 15) & (0x3 << 1);
|
|
uint32_t b = (imm >> 7) & (0x3 << 1);
|
|
uint32_t a = (imm << 1) & (0x3 << 1);
|
|
|
|
if (op != UniOpVVI::kSwizzleHiU16x4) {
|
|
pred_data[ 0] = uint8_t(a);
|
|
pred_data[ 1] = uint8_t(a + 1u);
|
|
pred_data[ 2] = uint8_t(b);
|
|
pred_data[ 3] = uint8_t(b + 1u);
|
|
pred_data[ 4] = uint8_t(c);
|
|
pred_data[ 5] = uint8_t(c + 1u);
|
|
pred_data[ 6] = uint8_t(d);
|
|
pred_data[ 7] = uint8_t(d + 1u);
|
|
}
|
|
|
|
if (op != UniOpVVI::kSwizzleLoU16x4) {
|
|
pred_data[ 8] = uint8_t(a + 8u);
|
|
pred_data[ 9] = uint8_t(a + 9u);
|
|
pred_data[10] = uint8_t(b + 8u);
|
|
pred_data[11] = uint8_t(b + 9u);
|
|
pred_data[12] = uint8_t(c + 8u);
|
|
pred_data[13] = uint8_t(c + 9u);
|
|
pred_data[14] = uint8_t(d + 8u);
|
|
pred_data[15] = uint8_t(d + 9u);
|
|
}
|
|
|
|
Vec pred = simd_const_16b(pred_data);
|
|
cc->tbl(dst.b16(), src.b16(), pred.b16());
|
|
return;
|
|
}
|
|
|
|
case UniOpVVI::kSwizzleU32x4:
|
|
case UniOpVVI::kSwizzleF32x4: {
|
|
Vec src = as_vec(*this, src_, dst);
|
|
emit_swizzle32_impl(*this, dst, src, imm);
|
|
return;
|
|
}
|
|
|
|
case UniOpVVI::kSwizzleU64x2:
|
|
case UniOpVVI::kSwizzleF64x2: {
|
|
Vec src = as_vec(*this, src_, dst);
|
|
|
|
// Use `dup` to broadcast one 64-bit elements.
|
|
if (Swizzle2{imm} == swizzle(0, 0) ||
|
|
Swizzle2{imm} == swizzle(1, 1)) {
|
|
uint32_t idx = imm & 0x1;
|
|
cc->dup(dst.d2(), src.d(idx));
|
|
return;
|
|
}
|
|
|
|
// Use `ext` to swap two 64-bit elements.
|
|
if (Swizzle2{imm} == swizzle(0, 1)) {
|
|
cc->ext(dst.b16(), src.b16(), src.b16(), 8);
|
|
return;
|
|
}
|
|
|
|
// NOP...
|
|
if (Swizzle2{imm} == swizzle(1, 0)) {
|
|
cc->mov(dst.b16(), src.b16());
|
|
return;
|
|
}
|
|
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
|
|
case UniOpVVI::kSwizzleF64x4:
|
|
case UniOpVVI::kSwizzleU64x4:
|
|
case UniOpVVI::kExtractV128_I32:
|
|
case UniOpVVI::kExtractV128_I64:
|
|
case UniOpVVI::kExtractV128_F32:
|
|
case UniOpVVI::kExtractV128_F64:
|
|
case UniOpVVI::kExtractV256_I32:
|
|
case UniOpVVI::kExtractV256_I64:
|
|
case UniOpVVI::kExtractV256_F32:
|
|
case UniOpVVI::kExtractV256_F64:
|
|
// Unsupported as NEON has only 128-bit vectors.
|
|
ASMJIT_NOT_REACHED();
|
|
|
|
default: {
|
|
Vec src = as_vec(*this, src_, dst);
|
|
|
|
if (op_info.dst_part == VecPart::kLo) dst = dst.d();
|
|
if (op_info.src_part == VecPart::kLo) src = src.d();
|
|
|
|
vec_set_type(dst, op_info.dst_element);
|
|
vec_set_type(src, op_info.src_element);
|
|
|
|
cc->emit(inst_id, dst, src, imm);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
void UniCompiler::emit_2vi(UniOpVVI op, const OpArray& dst_, const Operand_& src_, uint32_t imm) { emit_2vi_t(*this, op, dst_, src_, imm); }
|
|
void UniCompiler::emit_2vi(UniOpVVI op, const OpArray& dst_, const OpArray& src_, uint32_t imm) { emit_2vi_t(*this, op, dst_, src_, imm); }
|
|
|
|
// ujit::UniCompiler - Vector Instructions - Emit 2VS
|
|
// ==================================================
|
|
|
|
void UniCompiler::emit_2vs(UniOpVR op, const Operand_& dst_, const Operand_& src_, uint32_t idx) {
|
|
UniOpVInfo op_info = opcode_info_2vs[size_t(op)];
|
|
|
|
switch (op) {
|
|
case UniOpVR::kMov: {
|
|
ASMJIT_ASSERT(dst_.is_reg());
|
|
ASMJIT_ASSERT(src_.is_reg());
|
|
|
|
if (dst_.is_gp() && src_.is_vec()) {
|
|
if (dst_.as<Reg>().size() == 4)
|
|
cc->mov(dst_.as<Gp>(), src_.as<Vec>().s(0));
|
|
else
|
|
cc->mov(dst_.as<Gp>(), src_.as<Vec>().d(0));
|
|
return;
|
|
}
|
|
|
|
if (dst_.is_vec() && src_.is_gp()) {
|
|
if (src_.as<Reg>().size() == 4)
|
|
cc->fmov(dst_.as<Vec>().s(), src_.as<Gp>());
|
|
else
|
|
cc->fmov(dst_.as<Vec>().d(), src_.as<Gp>());
|
|
return;
|
|
}
|
|
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
|
|
case UniOpVR::kMovU32:
|
|
case UniOpVR::kMovU64: {
|
|
ASMJIT_ASSERT(dst_.is_reg());
|
|
ASMJIT_ASSERT(src_.is_reg());
|
|
|
|
if (dst_.is_gp() && src_.is_vec()) {
|
|
if (op == UniOpVR::kMovU32)
|
|
cc->mov(dst_.as<Gp>().r32(), src_.as<Vec>().s(0));
|
|
else
|
|
cc->mov(dst_.as<Gp>().r64(), src_.as<Vec>().d(0));
|
|
return;
|
|
}
|
|
|
|
if (dst_.is_vec() && src_.is_gp()) {
|
|
if (op == UniOpVR::kMovU32)
|
|
cc->fmov(dst_.as<Vec>().s(), src_.as<Gp>().r32());
|
|
else
|
|
cc->fmov(dst_.as<Vec>().d(), src_.as<Gp>().r64());
|
|
return;
|
|
}
|
|
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
|
|
case UniOpVR::kInsertU8:
|
|
case UniOpVR::kInsertU16:
|
|
case UniOpVR::kInsertU32:
|
|
case UniOpVR::kInsertU64: {
|
|
ASMJIT_ASSERT(dst_.is_vec());
|
|
ASMJIT_ASSERT(src_.is_gp());
|
|
|
|
Vec dst(dst_.as<Vec>());
|
|
Gp src(src_.as<Gp>());
|
|
|
|
vec_set_type_and_index(dst, op_info.dst_element, idx);
|
|
src.set_signature(op == UniOpVR::kInsertU64 ? RegTraits<RegType::kGp64>::kSignature : RegTraits<RegType::kGp32>::kSignature);
|
|
|
|
cc->mov(dst, src);
|
|
return;
|
|
}
|
|
|
|
case UniOpVR::kExtractU8:
|
|
case UniOpVR::kExtractU16:
|
|
case UniOpVR::kExtractU32:
|
|
case UniOpVR::kExtractU64: {
|
|
ASMJIT_ASSERT(dst_.is_gp());
|
|
ASMJIT_ASSERT(src_.is_vec());
|
|
|
|
Gp dst(dst_.as<Gp>());
|
|
Vec src(src_.as<Vec>());
|
|
|
|
dst.set_signature(op == UniOpVR::kExtractU64 ? RegTraits<RegType::kGp64>::kSignature : RegTraits<RegType::kGp32>::kSignature);
|
|
vec_set_type_and_index(src, op_info.dst_element, idx);
|
|
|
|
cc->mov(dst, src);
|
|
return;
|
|
}
|
|
|
|
case UniOpVR::kCvtIntToF32:
|
|
case UniOpVR::kCvtIntToF64: {
|
|
ASMJIT_ASSERT(dst_.is_vec());
|
|
ASMJIT_ASSERT(src_.is_gp());
|
|
|
|
Vec dst(dst_.as<Vec>());
|
|
vec_set_vec_type(dst, op_info.dst_element);
|
|
cc->scvtf(dst, src_.as<Gp>());
|
|
return;
|
|
}
|
|
|
|
case UniOpVR::kCvtTruncF32ToInt:
|
|
case UniOpVR::kCvtRoundF32ToInt:
|
|
case UniOpVR::kCvtTruncF64ToInt:
|
|
case UniOpVR::kCvtRoundF64ToInt: {
|
|
ASMJIT_ASSERT(dst_.is_gp());
|
|
ASMJIT_ASSERT(src_.is_vec());
|
|
|
|
Vec src(src_.as<Vec>());
|
|
vec_set_vec_type(src, op_info.src_element);
|
|
cc->emit(op_info.inst_id, dst_, src);
|
|
return;
|
|
}
|
|
|
|
default:
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
}
|
|
|
|
// ujit::UniCompiler - Vector Instructions - Emit 2VM
|
|
// ==================================================
|
|
|
|
void UniCompiler::emit_vm(UniOpVM op, const Vec& dst_, const Mem& src_, Alignment alignment, uint32_t idx) {
|
|
ASMJIT_ASSERT(dst_.is_vec());
|
|
ASMJIT_ASSERT(src_.is_mem());
|
|
|
|
Support::maybe_unused(alignment);
|
|
|
|
Vec dst(dst_);
|
|
Mem src(src_);
|
|
UniOpVMInfo op_info = opcode_info_2vm[size_t(op)];
|
|
|
|
switch (op) {
|
|
case UniOpVM::kLoad8:
|
|
case UniOpVM::kLoad16_U16:
|
|
case UniOpVM::kLoad32_U32:
|
|
case UniOpVM::kLoad32_F32:
|
|
case UniOpVM::kLoad64_U32:
|
|
case UniOpVM::kLoad64_U64:
|
|
case UniOpVM::kLoad64_F32:
|
|
case UniOpVM::kLoad64_F64:
|
|
case UniOpVM::kLoad128_U32:
|
|
case UniOpVM::kLoad128_U64:
|
|
case UniOpVM::kLoad128_F32:
|
|
case UniOpVM::kLoad128_F64: {
|
|
vec_load_mem(*this, dst, src, op_info.mem_size);
|
|
return;
|
|
}
|
|
|
|
case UniOpVM::kLoadN_U32:
|
|
case UniOpVM::kLoadN_U64:
|
|
case UniOpVM::kLoadN_F32:
|
|
case UniOpVM::kLoadN_F64: {
|
|
vec_load_mem(*this, dst.q(), src, 16);
|
|
return;
|
|
}
|
|
|
|
case UniOpVM::kLoadCvtN_U8ToU64:
|
|
case UniOpVM::kLoadCvt16_U8ToU64: {
|
|
Gp tmp = new_gp32("@tmp");
|
|
cc->ldrh(tmp, src);
|
|
cc->mov(dst.b(), tmp);
|
|
cc->lsr(tmp, tmp, 8);
|
|
cc->mov(dst.b(8), tmp);
|
|
return;
|
|
}
|
|
|
|
case UniOpVM::kLoadCvt32_I8ToI16:
|
|
case UniOpVM::kLoadCvt32_U8ToU16:
|
|
case UniOpVM::kLoadCvt32_I8ToI32:
|
|
case UniOpVM::kLoadCvt32_U8ToU32:
|
|
case UniOpVM::kLoadCvt32_I16ToI32:
|
|
case UniOpVM::kLoadCvt32_U16ToU32:
|
|
case UniOpVM::kLoadCvt32_I32ToI64:
|
|
case UniOpVM::kLoadCvt32_U32ToU64:
|
|
case UniOpVM::kLoadCvt64_I8ToI16:
|
|
case UniOpVM::kLoadCvt64_U8ToU16:
|
|
case UniOpVM::kLoadCvt64_I16ToI32:
|
|
case UniOpVM::kLoadCvt64_U16ToU32:
|
|
case UniOpVM::kLoadCvt64_I32ToI64:
|
|
case UniOpVM::kLoadCvt64_U32ToU64: {
|
|
vec_load_mem(*this, dst, src, op_info.mem_size);
|
|
emit_2v(UniOpVV(op_info.cvt_op), dst, dst);
|
|
return;
|
|
}
|
|
|
|
case UniOpVM::kLoadCvtN_I8ToI16:
|
|
case UniOpVM::kLoadCvtN_I8ToI32:
|
|
case UniOpVM::kLoadCvtN_U8ToU16:
|
|
case UniOpVM::kLoadCvtN_U8ToU32:
|
|
case UniOpVM::kLoadCvtN_I16ToI32:
|
|
case UniOpVM::kLoadCvtN_U16ToU32:
|
|
case UniOpVM::kLoadCvtN_I32ToI64:
|
|
case UniOpVM::kLoadCvtN_U32ToU64: {
|
|
vec_load_mem(*this, dst, src, dst.size() / 2u);
|
|
emit_2v(UniOpVV(op_info.cvt_op), dst, dst);
|
|
return;
|
|
}
|
|
|
|
case UniOpVM::kLoadInsertU8:
|
|
case UniOpVM::kLoadInsertU16:
|
|
case UniOpVM::kLoadInsertU32:
|
|
case UniOpVM::kLoadInsertF32:
|
|
case UniOpVM::kLoadInsertU64:
|
|
case UniOpVM::kLoadInsertF32x2:
|
|
case UniOpVM::kLoadInsertF64: {
|
|
if (!src.has_index() && !src.has_offset()) {
|
|
vec_set_type_and_index(dst, op_info.element, idx);
|
|
cc->ld1(dst, src);
|
|
}
|
|
else {
|
|
Vec tmp = new_similar_reg(dst, "@tmp");
|
|
v_load_iany(tmp, src, op_info.mem_size, Alignment(1));
|
|
|
|
vec_set_type_and_index(dst, op_info.element, idx);
|
|
vec_set_type_and_index(tmp, op_info.element, 0);
|
|
cc->mov(dst, tmp);
|
|
}
|
|
return;
|
|
}
|
|
|
|
case UniOpVM::kLoad256_U32:
|
|
case UniOpVM::kLoad256_U64:
|
|
case UniOpVM::kLoad256_F32:
|
|
case UniOpVM::kLoad256_F64:
|
|
case UniOpVM::kLoad512_U32:
|
|
case UniOpVM::kLoad512_U64:
|
|
case UniOpVM::kLoad512_F32:
|
|
case UniOpVM::kLoad512_F64:
|
|
case UniOpVM::kLoadCvt32_U8ToU64:
|
|
case UniOpVM::kLoadCvt64_U8ToU64:
|
|
case UniOpVM::kLoadCvt64_I8ToI32:
|
|
case UniOpVM::kLoadCvt64_U8ToU32:
|
|
case UniOpVM::kLoadCvt128_I8ToI32:
|
|
case UniOpVM::kLoadCvt128_U8ToU32:
|
|
case UniOpVM::kLoadCvt128_I8ToI16:
|
|
case UniOpVM::kLoadCvt128_U8ToU16:
|
|
case UniOpVM::kLoadCvt128_I16ToI32:
|
|
case UniOpVM::kLoadCvt128_U16ToU32:
|
|
case UniOpVM::kLoadCvt128_I32ToI64:
|
|
case UniOpVM::kLoadCvt128_U32ToU64:
|
|
case UniOpVM::kLoadCvt256_I8ToI16:
|
|
case UniOpVM::kLoadCvt256_U8ToU16:
|
|
case UniOpVM::kLoadCvt256_I16ToI32:
|
|
case UniOpVM::kLoadCvt256_U16ToU32:
|
|
case UniOpVM::kLoadCvt256_I32ToI64:
|
|
case UniOpVM::kLoadCvt256_U32ToU64:
|
|
// Unsupported as NEON has only 128-bit vectors.
|
|
ASMJIT_NOT_REACHED();
|
|
|
|
default:
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
}
|
|
|
|
void UniCompiler::emit_vm(UniOpVM op, const OpArray& dst_, const Mem& src_, Alignment alignment, uint32_t idx) {
|
|
Support::maybe_unused(alignment);
|
|
|
|
size_t i = 0;
|
|
size_t n = dst_.size();
|
|
|
|
if (!n)
|
|
return;
|
|
|
|
Mem src(src_);
|
|
UniOpVMInfo op_info = opcode_info_2vm[size_t(op)];
|
|
|
|
uint32_t overridden_mem_size = op_info.mem_size;
|
|
uint32_t mem_size = overridden_mem_size ? overridden_mem_size : dst_[0].as<Vec>().size();
|
|
|
|
if (op <= UniOpVM::kLoadN_F64 && !src.has_index() && !src.has_offset() && mem_size >= 4) {
|
|
while (i + 2 <= n) {
|
|
const Vec& dst0 = dst_[i + 0].as<Vec>();
|
|
const Vec& dst1 = dst_[i + 1].as<Vec>();
|
|
|
|
if (mem_size == 4)
|
|
cc->ldp(dst0.s(), dst1.s(), src);
|
|
else if (mem_size == 8)
|
|
cc->ldp(dst0.d(), dst1.d(), src);
|
|
else
|
|
cc->ldp(dst0.q(), dst1.q(), src);
|
|
|
|
src.add_offset_lo32(int32_t(mem_size * 2));
|
|
|
|
i += 2;
|
|
}
|
|
}
|
|
|
|
while (i < n) {
|
|
ASMJIT_ASSERT(dst_[i].is_vec());
|
|
|
|
const Vec& dst = dst_[i].as<Vec>();
|
|
mem_size = dst.size();
|
|
|
|
emit_vm(op, dst, src, Alignment(1), idx);
|
|
|
|
src.add_offset_lo32(int32_t(mem_size));
|
|
i++;
|
|
}
|
|
}
|
|
|
|
void UniCompiler::emit_mv(UniOpMV op, const Mem& dst_, const Vec& src_, Alignment alignment, uint32_t idx) {
|
|
ASMJIT_ASSERT(dst_.is_mem());
|
|
ASMJIT_ASSERT(src_.is_vec());
|
|
|
|
Support::maybe_unused(alignment);
|
|
|
|
Mem dst(dst_);
|
|
Vec src(src_);
|
|
|
|
switch (op) {
|
|
case UniOpMV::kStore8: {
|
|
cc->str(src.b(), dst);
|
|
return;
|
|
}
|
|
|
|
case UniOpMV::kStore16_U16: {
|
|
cc->str(src.h(), dst);
|
|
return;
|
|
}
|
|
|
|
case UniOpMV::kStore32_U32:
|
|
case UniOpMV::kStore32_F32: {
|
|
cc->str(src.s(), dst);
|
|
return;
|
|
}
|
|
|
|
case UniOpMV::kStore64_U32:
|
|
case UniOpMV::kStore64_U64:
|
|
case UniOpMV::kStore64_F32:
|
|
case UniOpMV::kStore64_F64: {
|
|
cc->str(src.d(), dst);
|
|
return;
|
|
}
|
|
|
|
case UniOpMV::kStore128_U32:
|
|
case UniOpMV::kStore128_U64:
|
|
case UniOpMV::kStore128_F32:
|
|
case UniOpMV::kStore128_F64: {
|
|
cc->str(src.q(), dst);
|
|
return;
|
|
}
|
|
|
|
case UniOpMV::kStoreN_U32:
|
|
case UniOpMV::kStoreN_U64:
|
|
case UniOpMV::kStoreN_F32:
|
|
case UniOpMV::kStoreN_F64: {
|
|
cc->str(src, dst);
|
|
return;
|
|
}
|
|
|
|
/*
|
|
case UniOpMV::kStoreCvtz64_U16ToU8:
|
|
case UniOpMV::kStoreCvtz64_U32ToU16:
|
|
case UniOpMV::kStoreCvtz64_U64ToU32:
|
|
case UniOpMV::kStoreCvts64_I16ToI8:
|
|
case UniOpMV::kStoreCvts64_I16ToU8:
|
|
case UniOpMV::kStoreCvts64_U16ToU8:
|
|
case UniOpMV::kStoreCvts64_I32ToI16:
|
|
case UniOpMV::kStoreCvts64_U32ToU16:
|
|
case UniOpMV::kStoreCvts64_I64ToI32:
|
|
case UniOpMV::kStoreCvts64_U64ToU32:
|
|
case UniOpMV::kStoreCvtzN_U16ToU8:
|
|
case UniOpMV::kStoreCvtzN_U32ToU16:
|
|
case UniOpMV::kStoreCvtzN_U64ToU32:
|
|
case UniOpMV::kStoreCvtsN_I16ToI8:
|
|
case UniOpMV::kStoreCvtsN_I16ToU8:
|
|
case UniOpMV::kStoreCvtsN_U16ToU8:
|
|
case UniOpMV::kStoreCvtsN_I32ToI16:
|
|
case UniOpMV::kStoreCvtsN_U32ToU16:
|
|
case UniOpMV::kStoreCvtsN_I64ToI32:
|
|
case UniOpMV::kStoreCvtsN_U64ToU32:
|
|
UNIMPLEMENTED();
|
|
return;
|
|
*/
|
|
|
|
case UniOpMV::kStore256_U32:
|
|
case UniOpMV::kStore256_U64:
|
|
case UniOpMV::kStore256_F32:
|
|
case UniOpMV::kStore256_F64:
|
|
case UniOpMV::kStore512_U32:
|
|
case UniOpMV::kStore512_U64:
|
|
case UniOpMV::kStore512_F32:
|
|
case UniOpMV::kStore512_F64:
|
|
/*
|
|
case UniOpMV::kStoreCvtz128_U16ToU8:
|
|
case UniOpMV::kStoreCvtz128_U32ToU16:
|
|
case UniOpMV::kStoreCvtz128_U64ToU32:
|
|
case UniOpMV::kStoreCvts128_I16ToI8:
|
|
case UniOpMV::kStoreCvts128_I16ToU8:
|
|
case UniOpMV::kStoreCvts128_U16ToU8:
|
|
case UniOpMV::kStoreCvts128_I32ToI16:
|
|
case UniOpMV::kStoreCvts128_U32ToU16:
|
|
case UniOpMV::kStoreCvts128_I64ToI32:
|
|
case UniOpMV::kStoreCvts128_U64ToU32:
|
|
case UniOpMV::kStoreCvtz256_U16ToU8:
|
|
case UniOpMV::kStoreCvtz256_U32ToU16:
|
|
case UniOpMV::kStoreCvtz256_U64ToU32:
|
|
case UniOpMV::kStoreCvts256_I16ToI8:
|
|
case UniOpMV::kStoreCvts256_I16ToU8:
|
|
case UniOpMV::kStoreCvts256_U16ToU8:
|
|
case UniOpMV::kStoreCvts256_I32ToI16:
|
|
case UniOpMV::kStoreCvts256_U32ToU16:
|
|
case UniOpMV::kStoreCvts256_I64ToI32:
|
|
case UniOpMV::kStoreCvts256_U64ToU32:
|
|
*/
|
|
// Unsupported as NEON has only 128-bit vectors.
|
|
ASMJIT_NOT_REACHED();
|
|
|
|
case UniOpMV::kStoreExtractU16: {
|
|
if (idx == 0) {
|
|
cc->str(src.h(), dst);
|
|
}
|
|
else {
|
|
cc->st1(src.h(idx), dst);
|
|
}
|
|
return;
|
|
}
|
|
|
|
case UniOpMV::kStoreExtractU32: {
|
|
if (idx == 0) {
|
|
cc->str(src.s(), dst);
|
|
}
|
|
else {
|
|
cc->st1(src.s(idx), dst);
|
|
}
|
|
return;
|
|
}
|
|
|
|
case UniOpMV::kStoreExtractU64: {
|
|
if (idx == 0) {
|
|
cc->str(src.d(), dst);
|
|
}
|
|
else {
|
|
cc->st1(src.d(idx), dst);
|
|
}
|
|
return;
|
|
}
|
|
|
|
default:
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
}
|
|
|
|
void UniCompiler::emit_mv(UniOpMV op, const Mem& dst_, const OpArray& src_, Alignment alignment, uint32_t idx) {
|
|
Support::maybe_unused(alignment);
|
|
|
|
size_t i = 0;
|
|
size_t n = src_.size();
|
|
|
|
if (!n)
|
|
return;
|
|
|
|
Mem dst(dst_);
|
|
UniOpVMInfo op_info = opcode_info_2mv[size_t(op)];
|
|
|
|
uint32_t overridden_mem_size = op_info.mem_size;
|
|
uint32_t mem_size = overridden_mem_size ? overridden_mem_size : src_[0].as<Vec>().size();
|
|
|
|
if (op <= UniOpMV::kStoreN_F64 && mem_size >= 4) {
|
|
bool good_offset = (uint32_t(dst.offset_lo32()) & (mem_size - 1)) == 0u;
|
|
|
|
if (good_offset) {
|
|
while (i + 2 <= n) {
|
|
const Vec& src0 = src_[i + 0].as<Vec>();
|
|
const Vec& src1 = src_[i + 1].as<Vec>();
|
|
|
|
if (mem_size == 4)
|
|
cc->stp(src0.s(), src1.s(), dst);
|
|
else if (mem_size == 8)
|
|
cc->stp(src0.d(), src1.d(), dst);
|
|
else
|
|
cc->stp(src0.q(), src1.q(), dst);
|
|
|
|
dst.add_offset_lo32(int32_t(mem_size * 2));
|
|
|
|
i += 2;
|
|
}
|
|
}
|
|
}
|
|
|
|
while (i < n) {
|
|
ASMJIT_ASSERT(src_[i].is_vec());
|
|
|
|
const Vec& src = src_[i].as<Vec>();
|
|
emit_mv(op, dst, src, Alignment(1), idx);
|
|
|
|
dst.add_offset_lo32(int32_t(mem_size));
|
|
i++;
|
|
}}
|
|
|
|
// ujit::UniCompiler - Vector Instructions - Emit 3V
|
|
// =================================================
|
|
|
|
static void emit_3v_op(
|
|
UniCompiler& uc,
|
|
InstId inst_id,
|
|
Vec dst, Vec src1, Operand_ src2_,
|
|
FloatMode float_mode,
|
|
ElementSize dst_element, VecPart dst_part,
|
|
ElementSize src_element, VecPart src_part,
|
|
uint32_t reversed) {
|
|
|
|
Vec src2;
|
|
|
|
switch (float_mode) {
|
|
case FloatMode::kF32S: {
|
|
dst = dst.s();
|
|
src1 = src1.s();
|
|
src2 = as_vec(uc, src2_, dst, 4);
|
|
break;
|
|
}
|
|
|
|
case FloatMode::kF64S: {
|
|
dst = dst.d();
|
|
src1 = src1.d();
|
|
src2 = as_vec(uc, src2_, dst, 8);
|
|
break;
|
|
}
|
|
|
|
default: {
|
|
src2 = as_vec(uc, src2_, dst);
|
|
|
|
if (dst_part == VecPart::kLo) {
|
|
dst = dst.d();
|
|
}
|
|
|
|
if (src_part == VecPart::kLo) {
|
|
src1 = src1.d();
|
|
src2 = src2.d();
|
|
}
|
|
|
|
vec_set_type(dst, dst_element);
|
|
vec_set_type(src1, src_element);
|
|
vec_set_type(src2, src_element);
|
|
break;
|
|
}
|
|
}
|
|
|
|
BackendCompiler* cc = uc.cc;
|
|
if (reversed)
|
|
cc->emit(inst_id, dst, src2, src1);
|
|
else
|
|
cc->emit(inst_id, dst, src1, src2);
|
|
}
|
|
|
|
void UniCompiler::emit_3v(UniOpVVV op, const Operand_& dst_, const Operand_& src1_, const Operand_& src2_) {
|
|
ASMJIT_ASSERT(dst_.is_vec());
|
|
ASMJIT_ASSERT(src1_.is_vec());
|
|
|
|
Vec dst(dst_.as<Vec>());
|
|
Vec src1(src1_.as<Vec>().clone_as(dst));
|
|
|
|
UniOpVInfo op_info = opcode_info_3v[size_t(op)];
|
|
InstId inst_id = op_info.inst_id;
|
|
|
|
if (is_same_vec(src1, src2_)) {
|
|
switch (op_info.same_vec_op) {
|
|
case SameVecOp::kZero: {
|
|
cc->movi(dst.b16(), 0);
|
|
return;
|
|
}
|
|
|
|
case SameVecOp::kOnes: {
|
|
cc->movi(dst.b16(), 0xFF);
|
|
return;
|
|
}
|
|
|
|
case SameVecOp::kSrc: {
|
|
vec_mov(*this, dst, src1);
|
|
return;
|
|
}
|
|
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
switch (op) {
|
|
// dst = a - (floor(a / b) * b).
|
|
case UniOpVVV::kModF32S:
|
|
case UniOpVVV::kModF64S:
|
|
case UniOpVVV::kModF32:
|
|
case UniOpVVV::kModF64: {
|
|
Vec src2 = as_vec(*this, src2_, dst, op_info.float_mode);
|
|
Vec tmp = new_similar_reg(dst, "@tmp1");
|
|
|
|
UniOpVVV sub_op = translate_op(op, UniOpVVV::kModF32S, UniOpVVV::kSubF32S);
|
|
UniOpVVV mul_op = translate_op(op, UniOpVVV::kModF32S, UniOpVVV::kMulF32S);
|
|
UniOpVVV div_op = translate_op(op, UniOpVVV::kModF32S, UniOpVVV::kDivF32S);
|
|
UniOpVV trunc_op = translate_op(op, UniOpVVV::kModF32S, UniOpVV::kTruncF32S);
|
|
|
|
emit_3v(div_op, tmp, src1, src2);
|
|
emit_2v(trunc_op, tmp, tmp);
|
|
emit_3v(mul_op, tmp, tmp, src2);
|
|
emit_3v(sub_op, dst, src1, tmp);
|
|
|
|
return;
|
|
}
|
|
|
|
case UniOpVVV::kMulU64: {
|
|
Vec src2 = as_vec(*this, src2_, dst);
|
|
Vec tmp1 = new_similar_reg(dst, "@tmp1");
|
|
Vec tmp2 = new_similar_reg(dst, "@tmp2");
|
|
Vec tmp3 = new_similar_reg(dst, "@tmp3");
|
|
|
|
cc->rev64(tmp1.s4(), src1.s4());
|
|
cc->xtn(tmp2.s2(), src1.d2());
|
|
cc->mul(tmp1.s4(), tmp1.s4(), src2.s4());
|
|
cc->xtn(tmp3.s2(), src2.d2());
|
|
cc->uaddlp(tmp1.d2(), tmp1.s4());
|
|
cc->shl(dst.d2(), tmp1.d2(), 32);
|
|
cc->umlal(dst.d2(), tmp2.s2(), tmp3.s2());
|
|
|
|
return;
|
|
}
|
|
|
|
case UniOpVVV::kMulhI16: {
|
|
Vec src2 = as_vec(*this, src2_, dst);
|
|
Vec tmp = new_similar_reg(dst, "@tmp");
|
|
|
|
cc->smull(tmp.s4(), src1.h4(), src2.h4());
|
|
cc->smull2(dst.s4(), src1.h8(), src2.h8());
|
|
cc->uzp2(dst.h8(), tmp.h8(), dst.h8());
|
|
return;
|
|
}
|
|
|
|
case UniOpVVV::kMulhU16: {
|
|
Vec src2 = as_vec(*this, src2_, dst);
|
|
Vec tmp = new_similar_reg(dst, "@tmp");
|
|
|
|
cc->umull(tmp.s4(), src1.h4(), src2.h4());
|
|
cc->umull2(dst.s4(), src1.h8(), src2.h8());
|
|
cc->uzp2(dst.h8(), tmp.h8(), dst.h8());
|
|
return;
|
|
}
|
|
|
|
case UniOpVVV::kMulU64_LoU32: {
|
|
Vec src2 = as_vec(*this, src2_, dst);
|
|
Vec tmp1 = new_similar_reg(dst, "@tmp1");
|
|
Vec tmp2 = new_similar_reg(dst, "@tmp2");
|
|
Vec tmp3 = dst;
|
|
|
|
if (dst.id() == src1.id() || dst.id() == src2.id())
|
|
tmp3 = new_similar_reg(dst, "@tmp3");
|
|
|
|
cc->xtn(tmp1.s2(), src1.d2());
|
|
cc->shl(tmp3.d2(), src2.d2(), 32);
|
|
cc->xtn(tmp2.s2(), src2.d2());
|
|
cc->mul(dst.s4(), tmp3.s4(), src1.s4());
|
|
cc->umlal(dst.d2(), tmp1.s2(), tmp2.s2());
|
|
|
|
return;
|
|
}
|
|
|
|
case UniOpVVV::kMHAddI16_I32: {
|
|
Vec src2 = as_vec(*this, src2_, dst);
|
|
|
|
Vec al = new_similar_reg(dst, "@al");
|
|
Vec ah = new_similar_reg(dst, "@ah");
|
|
Vec bl = new_similar_reg(dst, "@bl");
|
|
Vec bh = new_similar_reg(dst, "@bh");
|
|
|
|
cc->xtn(al.h4(), src1.s4());
|
|
cc->xtn(bl.h4(), src2.s4());
|
|
|
|
cc->shrn(ah.h4(), src1.s4(), 16);
|
|
cc->shrn(bh.h4(), src2.s4(), 16);
|
|
|
|
cc->smull(dst.s4(), al.h4(), bl.h4());
|
|
cc->smlal(dst.s4(), ah.h4(), bh.h4());
|
|
|
|
return;
|
|
}
|
|
|
|
case UniOpVVV::kMinI64:
|
|
case UniOpVVV::kMinU64:
|
|
case UniOpVVV::kMaxI64:
|
|
case UniOpVVV::kMaxU64: {
|
|
Vec src2 = as_vec(*this, src2_, dst);
|
|
|
|
// Min/Max is commutative, so let's make dst only overlap src1.
|
|
if (dst.id() == src2.id()) {
|
|
std::swap(src1, src2);
|
|
}
|
|
|
|
bool dst_overlaps_src = dst.id() == src1.id();
|
|
|
|
Vec tmp = dst;
|
|
if (dst_overlaps_src) {
|
|
tmp = new_similar_reg(dst, "@tmp");
|
|
}
|
|
|
|
// Let's emit a nicer sequence for u64 maximum.
|
|
if (op == UniOpVVV::kMaxU64) {
|
|
cc->uqsub(tmp.d2(), src1.d2(), src2.d2());
|
|
cc->add(dst.d2(), tmp.d2(), src2.d2());
|
|
return;
|
|
}
|
|
|
|
cc->emit(inst_id, tmp.d2(), src1.d2(), src2.d2());
|
|
|
|
if (op_info.imm)
|
|
v_blendv_u8(dst, src2, src1, tmp);
|
|
else
|
|
v_blendv_u8(dst, src1, src2, tmp);
|
|
|
|
return;
|
|
}
|
|
|
|
case UniOpVVV::kCmpNeF32S:
|
|
case UniOpVVV::kCmpNeF64S:
|
|
case UniOpVVV::kCmpNeF32:
|
|
case UniOpVVV::kCmpNeF64: {
|
|
emit_3v_op(*this, inst_id, dst, src1, src2_, op_info.float_mode, op_info.dst_element, op_info.dst_part, op_info.src_element, op_info.src_part, 0);
|
|
vec_neg(*this, dst, dst, op_info.float_mode);
|
|
return;
|
|
}
|
|
|
|
case UniOpVVV::kCmpOrdF32S:
|
|
case UniOpVVV::kCmpOrdF64S:
|
|
case UniOpVVV::kCmpOrdF32:
|
|
case UniOpVVV::kCmpOrdF64:
|
|
case UniOpVVV::kCmpUnordF32S:
|
|
case UniOpVVV::kCmpUnordF64S:
|
|
case UniOpVVV::kCmpUnordF32:
|
|
case UniOpVVV::kCmpUnordF64: {
|
|
if (is_same_vec(src1, src2_)) {
|
|
emit_3v_op(*this, Inst::kIdFcmeq_v, dst, src1, src1, op_info.float_mode, op_info.dst_element, op_info.dst_part, op_info.src_element, op_info.src_part, 0);
|
|
}
|
|
else {
|
|
// This takes advantage of the following:
|
|
//
|
|
// When FPCR.AH is 0, the behavior is as follows (ASSUMED):
|
|
//
|
|
// - Negative zero compares less than positive zero.
|
|
// - When FPCR.DN is 0, if either element is a NaN, the result is a quiet NaN.
|
|
// - When FPCR.DN is 1, if either element is a NaN, the result is Default NaN.
|
|
//
|
|
// When FPCR.AH is 1, the behavior is as follows (USED FOR X86 EMULATION - NOT ASSUMED):
|
|
//
|
|
// - If both elements are zeros, regardless of the sign of either zero, the result is the second element.
|
|
// - If either element is a NaN, regardless of the value of FPCR.DN, the result is the second element.
|
|
Vec src2 = as_vec(*this, src2_, dst, op_info.float_mode);
|
|
emit_3v_op(*this, Inst::kIdFmin_v, dst, src1, src2, op_info.float_mode, op_info.dst_element, op_info.dst_part, op_info.src_element, op_info.src_part, 0);
|
|
emit_3v_op(*this, Inst::kIdFcmeq_v, dst, dst, dst, op_info.float_mode, op_info.dst_element, op_info.dst_part, op_info.src_element, op_info.src_part, 0);
|
|
}
|
|
|
|
if (op_info.imm)
|
|
vec_neg(*this, dst, dst, op_info.float_mode);
|
|
|
|
return;
|
|
}
|
|
|
|
case UniOpVVV::kHAddF64: {
|
|
Vec tmp = new_similar_reg(dst, "@tmp");
|
|
Vec src2 = as_vec(*this, src2_, dst);
|
|
|
|
if (src1.id() == src2.id()) {
|
|
cc->ext(tmp.b16(), src1.b16(), src1.b16(), 8);
|
|
cc->fadd(dst.d2(), src1.d2(), tmp.d2());
|
|
}
|
|
else {
|
|
cc->zip1(tmp.d2(), src1.d2(), src2.d2());
|
|
cc->zip2(dst.d2(), src1.d2(), src2.d2());
|
|
cc->fadd(dst.d2(), dst.d2(), tmp.d2());
|
|
}
|
|
return;
|
|
}
|
|
|
|
case UniOpVVV::kCombineLoHiU64:
|
|
case UniOpVVV::kCombineLoHiF64: {
|
|
// Intrinsic - dst = {src1.u64[0], src2.64[1]} - combining low part of src1 and high part of src1.
|
|
Vec src2 = as_vec(*this, src2_, dst);
|
|
|
|
vec_set_type(dst, ElementSize::k8);
|
|
vec_set_type(src1, ElementSize::k8);
|
|
vec_set_type(src2, ElementSize::k8);
|
|
|
|
// `EXT dst, a, b, #n` -> `dst = b:a >> #n*8`
|
|
cc->ext(dst, src2, src1, 8);
|
|
return;
|
|
}
|
|
|
|
case UniOpVVV::kCombineHiLoU64:
|
|
case UniOpVVV::kCombineHiLoF64: {
|
|
// Intrinsic - dst = {src1.u64[1], src2.64[0]} - combining high part of src1 and low part of src2.
|
|
Vec src2 = as_vec(*this, src2_, dst);
|
|
|
|
if (is_same_vec(dst, src1)) {
|
|
if (is_same_vec(dst, src2))
|
|
return;
|
|
cc->mov(dst.d(0), src2.d(0));
|
|
}
|
|
else if (is_same_vec(dst, src2)) {
|
|
cc->mov(dst.d(1), src1.d(1));
|
|
}
|
|
else {
|
|
cc->mov(dst.b16(), src1.b16());
|
|
cc->mov(dst.d(0), src2.d(0));
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
case UniOpVVV::kPacksI16_I8:
|
|
case UniOpVVV::kPacksI16_U8:
|
|
case UniOpVVV::kPacksI32_I16:
|
|
case UniOpVVV::kPacksI32_U16: {
|
|
static constexpr uint16_t pack_lo_inst[4] = { Inst::kIdSqxtn_v , Inst::kIdSqxtun_v , Inst::kIdSqxtn_v , Inst::kIdSqxtun_v };
|
|
static constexpr uint16_t pack_hi_inst[4] = { Inst::kIdSqxtn2_v, Inst::kIdSqxtun2_v, Inst::kIdSqxtn2_v, Inst::kIdSqxtun2_v };
|
|
|
|
size_t id = size_t(op) - size_t(UniOpVVV::kPacksI16_I8);
|
|
|
|
Vec src2 = as_vec(*this, src2_, dst);
|
|
vec_set_type(src1, op_info.src_element);
|
|
vec_set_type(src2, op_info.src_element);
|
|
|
|
if (src1.id() == src2.id()) {
|
|
Vec dst_d = dst.d();
|
|
|
|
vec_set_type(dst, op_info.dst_element);
|
|
vec_set_type(dst_d, op_info.dst_element);
|
|
|
|
cc->emit(pack_lo_inst[id], dst_d, src1);
|
|
cc->mov(dst.d(1), dst.d(0));
|
|
}
|
|
else {
|
|
Vec tmp = dst;
|
|
if (dst.id() == src1.id() || dst.id() == src2.id()) {
|
|
tmp = new_similar_reg(dst, "@tmp");
|
|
}
|
|
|
|
a64::Vec tmp_d = tmp.d();
|
|
|
|
vec_set_type(tmp, op_info.dst_element);
|
|
vec_set_type(tmp_d, op_info.dst_element);
|
|
|
|
cc->emit(pack_lo_inst[id], tmp_d, src1);
|
|
cc->emit(pack_hi_inst[id], tmp, src2);
|
|
|
|
if (dst.id() != tmp.id()) {
|
|
cc->mov(dst.b16(), tmp.b16());
|
|
}
|
|
}
|
|
return;
|
|
}
|
|
|
|
default: {
|
|
emit_3v_op(*this, inst_id, dst, src1, src2_, op_info.float_mode, op_info.dst_element, op_info.dst_part, op_info.src_element, op_info.src_part, op_info.reverse);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
void UniCompiler::emit_3v(UniOpVVV op, const OpArray& dst_, const Operand_& src1_, const OpArray& src2_) { emit_3v_t(*this, op, dst_, src1_, src2_); }
|
|
void UniCompiler::emit_3v(UniOpVVV op, const OpArray& dst_, const OpArray& src1_, const Operand_& src2_) { emit_3v_t(*this, op, dst_, src1_, src2_); }
|
|
void UniCompiler::emit_3v(UniOpVVV op, const OpArray& dst_, const OpArray& src1_, const OpArray& src2_) { emit_3v_t(*this, op, dst_, src1_, src2_); }
|
|
|
|
// ujit::UniCompiler - Vector Instructions - Emit 3VI
|
|
// ==================================================
|
|
|
|
void UniCompiler::emit_3vi(UniOpVVVI op, const Operand_& dst_, const Operand_& src1_, const Operand_& src2_, uint32_t imm) {
|
|
ASMJIT_ASSERT(dst_.is_vec());
|
|
ASMJIT_ASSERT(src1_.is_vec());
|
|
|
|
Vec dst(dst_.as<Vec>());
|
|
Vec src1(src1_.as<Vec>().clone_as(dst));
|
|
|
|
// Not used at the moment - maybe the info is not needed in this case.
|
|
UniOpVInfo op_info = opcode_info_3vi[size_t(op)];
|
|
Support::maybe_unused(op_info);
|
|
|
|
switch (op) {
|
|
case UniOpVVVI::kAlignr_U128: {
|
|
ASMJIT_ASSERT(imm < 16);
|
|
|
|
if (imm == 0) {
|
|
vec_mov(*this, dst, src2_);
|
|
return;
|
|
}
|
|
|
|
Vec src2 = as_vec(*this, src2_, dst);
|
|
vec_set_type(dst, ElementSize::k8);
|
|
vec_set_type(src1, ElementSize::k8);
|
|
vec_set_type(src2, ElementSize::k8);
|
|
cc->ext(dst, src2, src1, imm);
|
|
return;
|
|
}
|
|
|
|
case UniOpVVVI::kInterleaveShuffleU32x4:
|
|
case UniOpVVVI::kInterleaveShuffleF32x4: {
|
|
ASMJIT_ASSERT((imm & 0xFCFCFCFC) == 0);
|
|
|
|
Vec src2 = as_vec(*this, src2_, dst);
|
|
emit_interleaved_shuffle32_impl(*this, dst, src1, src2, imm);
|
|
return;
|
|
}
|
|
|
|
case UniOpVVVI::kInterleaveShuffleU64x2:
|
|
case UniOpVVVI::kInterleaveShuffleF64x2: {
|
|
ASMJIT_ASSERT((imm & 0xFFFCFEFE) == 0);
|
|
|
|
Vec src2 = as_vec(*this, src2_, dst);
|
|
|
|
if (src1.id() == src2.id()) {
|
|
v_swizzle_u64x2(dst, src1, Swizzle2{imm});
|
|
return;
|
|
}
|
|
|
|
if (Swizzle2{imm} == swizzle(0, 0))
|
|
cc->zip1(dst.d2(), src1.d2(), src2.d2());
|
|
else if (Swizzle2{imm} == swizzle(1, 1))
|
|
cc->zip2(dst.d2(), src1.d2(), src2.d2());
|
|
else if (Swizzle2{imm} == swizzle(1, 0))
|
|
v_combine_hi_lo_u64(dst, src2, src1);
|
|
else
|
|
v_combine_lo_hi_u64(dst, src2, src1);
|
|
|
|
return;
|
|
}
|
|
|
|
case UniOpVVVI::kInsertV128_U32:
|
|
case UniOpVVVI::kInsertV128_F32:
|
|
case UniOpVVVI::kInsertV128_U64:
|
|
case UniOpVVVI::kInsertV128_F64:
|
|
case UniOpVVVI::kInsertV256_U32:
|
|
case UniOpVVVI::kInsertV256_F32:
|
|
case UniOpVVVI::kInsertV256_U64:
|
|
case UniOpVVVI::kInsertV256_F64:
|
|
// Unsupported as NEON has only 128-bit vectors.
|
|
ASMJIT_NOT_REACHED();
|
|
|
|
default:
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
}
|
|
|
|
void UniCompiler::emit_3vi(UniOpVVVI op, const OpArray& dst_, const Operand_& src1_, const OpArray& src2_, uint32_t imm) { emit_3vi_t(*this, op, dst_, src1_, src2_, imm); }
|
|
void UniCompiler::emit_3vi(UniOpVVVI op, const OpArray& dst_, const OpArray& src1_, const Operand_& src2_, uint32_t imm) { emit_3vi_t(*this, op, dst_, src1_, src2_, imm); }
|
|
void UniCompiler::emit_3vi(UniOpVVVI op, const OpArray& dst_, const OpArray& src1_, const OpArray& src2_, uint32_t imm) { emit_3vi_t(*this, op, dst_, src1_, src2_, imm); }
|
|
|
|
// ujit::UniCompiler - Vector Instructions - Emit 4V
|
|
// =================================================
|
|
|
|
void UniCompiler::emit_4v(UniOpVVVV op, const Operand_& dst_, const Operand_& src1_, const Operand_& src2_, const Operand_& src3_) {
|
|
ASMJIT_ASSERT(dst_.is_vec());
|
|
ASMJIT_ASSERT(src1_.is_vec());
|
|
|
|
Vec dst(dst_.as<Vec>());
|
|
Vec src1(src1_.as<Vec>().clone_as(dst));
|
|
|
|
UniOpVInfo op_info = opcode_info_4v[size_t(op)];
|
|
InstId inst_id = op_info.inst_id;
|
|
|
|
switch (op) {
|
|
case UniOpVVVV::kBlendV_U8: {
|
|
Vec src2 = as_vec(*this, src2_, dst);
|
|
Vec src3 = as_vec(*this, src3_, dst);
|
|
|
|
vec_set_type(dst, op_info.dst_element);
|
|
vec_set_type(src1, op_info.src_element);
|
|
vec_set_type(src2, op_info.src_element);
|
|
vec_set_type(src3, op_info.src_element);
|
|
|
|
// We can pick between these depending on register arrangement:
|
|
// - BSL (bitwise select)
|
|
// - BIT (bitwise insert if true)
|
|
// - BIF (bitwise insert if false)
|
|
|
|
if (dst.id() == src1.id()) {
|
|
cc->bit(dst, src2, src3);
|
|
return;
|
|
}
|
|
|
|
if (dst.id() == src2.id()) {
|
|
cc->bif(dst, src1, src3);
|
|
return;
|
|
}
|
|
|
|
vec_mov(*this, dst, src3);
|
|
cc->bsl(dst, src2, src1);
|
|
return;
|
|
}
|
|
|
|
case UniOpVVVV::kMAddF32S:
|
|
case UniOpVVVV::kMAddF64S:
|
|
case UniOpVVVV::kMSubF32S:
|
|
case UniOpVVVV::kMSubF64S:
|
|
case UniOpVVVV::kNMAddF32S:
|
|
case UniOpVVVV::kNMAddF64S:
|
|
case UniOpVVVV::kNMSubF32S:
|
|
case UniOpVVVV::kNMSubF64S: {
|
|
Vec src2;
|
|
Vec src3;
|
|
|
|
if (op_info.float_mode == FloatMode::kF32S) {
|
|
dst = dst.s();
|
|
src1 = src1.s();
|
|
src2 = as_vec(*this, src2_, dst, 4);
|
|
src3 = as_vec(*this, src3_, dst, 4);
|
|
}
|
|
else {
|
|
dst = dst.d();
|
|
src1 = src1.d();
|
|
src2 = as_vec(*this, src2_, dst, 8);
|
|
src3 = as_vec(*this, src3_, dst, 8);
|
|
}
|
|
|
|
cc->emit(inst_id, dst, src1, src2, src3);
|
|
return;
|
|
}
|
|
|
|
case UniOpVVVV::kMAddU16:
|
|
case UniOpVVVV::kMAddU32:
|
|
case UniOpVVVV::kMAddF32:
|
|
case UniOpVVVV::kMAddF64:
|
|
case UniOpVVVV::kMSubF32:
|
|
case UniOpVVVV::kMSubF64:
|
|
case UniOpVVVV::kNMAddF32:
|
|
case UniOpVVVV::kNMAddF64:
|
|
case UniOpVVVV::kNMSubF32:
|
|
case UniOpVVVV::kNMSubF64: {
|
|
Vec src2 = as_vec(*this, src2_, dst);
|
|
Vec src3;
|
|
|
|
bool negate_acc = op_info.imm != 0;
|
|
bool dst_overlaps = dst.id() == src1.id() || dst.id() == src2.id();
|
|
bool destructible = is_same_vec(dst, src3_) || !src3_.is_reg();
|
|
|
|
if (!dst_overlaps && src3_.is_mem()) {
|
|
vec_load_mem(*this, dst, src3_.as<Mem>(), dst.size());
|
|
src3 = dst;
|
|
}
|
|
else {
|
|
src3 = as_vec(*this, src3_, dst);
|
|
}
|
|
|
|
vec_set_type(dst, op_info.dst_element);
|
|
vec_set_type(src1, op_info.src_element);
|
|
vec_set_type(src2, op_info.src_element);
|
|
vec_set_type(src3, op_info.src_element);
|
|
|
|
if (destructible) {
|
|
if (negate_acc)
|
|
cc->fneg(src3, src3);
|
|
|
|
cc->emit(inst_id, src3, src1, src2);
|
|
|
|
if (dst.id() != src3.id())
|
|
cc->mov(dst, src3);
|
|
return;
|
|
}
|
|
|
|
Vec tmp = dst;
|
|
|
|
if (dst_overlaps) {
|
|
tmp = new_similar_reg(dst, "@tmp");
|
|
vec_set_type(tmp, op_info.dst_element);
|
|
}
|
|
|
|
if (negate_acc)
|
|
cc->fneg(tmp, src3);
|
|
else
|
|
cc->mov(tmp, src3);
|
|
|
|
cc->emit(inst_id, tmp, src1, src2);
|
|
|
|
if (dst.id() != tmp.id())
|
|
cc->mov(dst, tmp);
|
|
return;
|
|
}
|
|
|
|
default: {
|
|
ASMJIT_NOT_REACHED();
|
|
}
|
|
}
|
|
}
|
|
|
|
void UniCompiler::emit_4v(UniOpVVVV op, const OpArray& dst_, const Operand_& src1_, const Operand_& src2_, const OpArray& src3_) { emit_4v_t(*this, op, dst_, src1_, src2_, src3_); }
|
|
void UniCompiler::emit_4v(UniOpVVVV op, const OpArray& dst_, const Operand_& src1_, const OpArray& src2_, const Operand& src3_) { emit_4v_t(*this, op, dst_, src1_, src2_, src3_); }
|
|
void UniCompiler::emit_4v(UniOpVVVV op, const OpArray& dst_, const Operand_& src1_, const OpArray& src2_, const OpArray& src3_) { emit_4v_t(*this, op, dst_, src1_, src2_, src3_); }
|
|
void UniCompiler::emit_4v(UniOpVVVV op, const OpArray& dst_, const OpArray& src1_, const Operand_& src2_, const Operand& src3_) { emit_4v_t(*this, op, dst_, src1_, src2_, src3_); }
|
|
void UniCompiler::emit_4v(UniOpVVVV op, const OpArray& dst_, const OpArray& src1_, const Operand_& src2_, const OpArray& src3_) { emit_4v_t(*this, op, dst_, src1_, src2_, src3_); }
|
|
void UniCompiler::emit_4v(UniOpVVVV op, const OpArray& dst_, const OpArray& src1_, const OpArray& src2_, const Operand& src3_) { emit_4v_t(*this, op, dst_, src1_, src2_, src3_); }
|
|
void UniCompiler::emit_4v(UniOpVVVV op, const OpArray& dst_, const OpArray& src1_, const OpArray& src2_, const OpArray& src3_) { emit_4v_t(*this, op, dst_, src1_, src2_, src3_); }
|
|
|
|
ASMJIT_END_SUB_NAMESPACE
|
|
|
|
#endif
|