admin/asmjit-asmjit
1
0
Fork 0
mirror of https://github.com/asmjit/asmjit synced 2026-06-08 13:13:30 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
master
asmjit-asmjit/asmjit/x86/x86rapass.cpp
T
kobalicek b56f4176cb Codebase update and improvements, instruction DB update 
* Denested src folder to root, renamed testing to asmjit-testing

  * Refactored how headers are included into <asmjit/...> form. This
    is necessary as compilers would never simplify a path once a ..
    appears in include directory - then paths such as ../core/../core
    appeared in asserts, which was ugly

  * Moved support utilities into asmjit/support/... (still included
    by asmjit/core.h for convenience and compatibility)

  * Added CMakePresets.json for making it easy to develop AsmJit

  * Reworked CMakeLists to be shorter and use CMake option(),
    etc... This simplifies it and makes it using more standard
    features

  * ASMJIT_EMBED now creates asmjit_embed INTERFACE library,
    which is accessible via asmjit::asmjit target - this simplifies
    embedding and makes it the same as library targets from a CMake
    perspective

  * Removed ASMJIT_DEPS - this is now provided by cmake target
    aliases - 'asmjit::asmjit' so users should not need this variable

  * Changed meaning of ASMJIT_LIBS - this now contains only AsmJit
    dependencies without asmjit::asmjit target alias. Don't rely on
    ASMJIT_LIBS anymore as it's only used internally

  * Removed ASMJIT_NO_DEPRECATED option - AsmJit is not going
    to provide controllable deprecations in the future

  * Removed ASMJIT_NO_VALIDATION in favor of ASMJIT_NO_INTROSPECTION,
    which now controls query, features, and validation API presence

  * Removed ASMJIT_DIR option - it was never really needed

  * Removed AMX_TRANSPOSE feature from instruction database (X86).
    Intel has removed it as well, so it's a feature that won't
    be siliconized
2025-11-02 22:31:46 +01:00

1654 lines
58 KiB
C++
Raw Permalink Blame History

// This file is part of AsmJit project <https://asmjit.com>
//
// See <asmjit/core.h> or LICENSE.md for license and copyright information
// SPDX-License-Identifier: Zlib
#include <asmjit/core/api-build_p.h>
#if !defined(ASMJIT_NO_X86) && !defined(ASMJIT_NO_COMPILER)
#include <asmjit/core/cpuinfo.h>
#include <asmjit/core/formatter_p.h>
#include <asmjit/core/type.h>
#include <asmjit/support/support.h>
#include <asmjit/x86/x86assembler.h>
#include <asmjit/x86/x86compiler.h>
#include <asmjit/x86/x86instapi_p.h>
#include <asmjit/x86/x86instdb_p.h>
#include <asmjit/x86/x86emithelper_p.h>
#include <asmjit/x86/x86rapass_p.h>
ASMJIT_BEGIN_SUB_NAMESPACE(x86)
// x86::X86RAPass - Utilities
// ==========================
[[nodiscard]]
static ASMJIT_INLINE uint64_t ra_imm_mask_from_size(uint32_t size) noexcept {
ASMJIT_ASSERT(size > 0 && size < 256);
static constexpr uint64_t masks[] = {
0x00000000000000FFu, // 1
0x000000000000FFFFu, // 2
0x00000000FFFFFFFFu, // 4
0xFFFFFFFFFFFFFFFFu, // 8
0x0000000000000000u, // 16
0x0000000000000000u, // 32
0x0000000000000000u, // 64
0x0000000000000000u, // 128
0x0000000000000000u // 256
};
return masks[Support::ctz(size)];
}
static const RegMask ra_consecutive_lead_count_to_reg_mask_filter[5] = {
0xFFFFFFFFu, // [0] No consecutive.
0x00000000u, // [1] Invalid, never used.
0x55555555u, // [2] Even registers.
0x00000000u, // [3] Invalid, never used.
0x11111111u // [4] Every fourth register.
};
[[nodiscard]]
static ASMJIT_INLINE RATiedFlags ra_use_out_flags_from_rw_flags(OpRWFlags rw_flags) noexcept {
static constexpr RATiedFlags map[] = {
RATiedFlags::kNone,
RATiedFlags::kRead | RATiedFlags::kUse, // kRead
RATiedFlags::kWrite | RATiedFlags::kOut, // kWrite
RATiedFlags::kRW | RATiedFlags::kUse, // kRW
RATiedFlags::kNone,
RATiedFlags::kRead | RATiedFlags::kUse | RATiedFlags::kUseRM, // kRead | kRegMem
RATiedFlags::kWrite | RATiedFlags::kOut | RATiedFlags::kOutRM, // kWrite | kRegMem
RATiedFlags::kRW | RATiedFlags::kUse | RATiedFlags::kUseRM // kRW | kRegMem
};
return map[uint32_t(rw_flags & (OpRWFlags::kRW | OpRWFlags::kRegMem))];
}
[[nodiscard]]
static ASMJIT_INLINE RATiedFlags ra_reg_rw_flags(OpRWFlags flags) noexcept {
return (RATiedFlags)ra_use_out_flags_from_rw_flags(flags);
}
[[nodiscard]]
static ASMJIT_INLINE RATiedFlags ra_mem_base_rw_flags(OpRWFlags flags) noexcept {
constexpr uint32_t kShift = Support::ctz_const<OpRWFlags::kMemBaseRW>;
return (RATiedFlags)ra_use_out_flags_from_rw_flags(OpRWFlags(uint32_t(flags) >> kShift) & OpRWFlags::kRW);
}
[[nodiscard]]
static ASMJIT_INLINE RATiedFlags ra_mem_index_rw_flags(OpRWFlags flags) noexcept {
constexpr uint32_t kShift = Support::ctz_const<OpRWFlags::kMemIndexRW>;
return (RATiedFlags)ra_use_out_flags_from_rw_flags(OpRWFlags(uint32_t(flags) >> kShift) & OpRWFlags::kRW);
}
// x86::RACFGBuilder
// =================
class RACFGBuilder : public RACFGBuilderT<RACFGBuilder> {
public:
Arch _arch;
bool _is_64bit;
const EmitHelperInstructionIds& _ids;
ASMJIT_INLINE_NODEBUG RACFGBuilder(X86RAPass& pass) noexcept
: RACFGBuilderT<RACFGBuilder>(pass),
_arch(pass.cc().arch()),
_is_64bit(pass.register_size() == 8),
_ids(pass._emit_helper.ids()) {
}
[[nodiscard]]
ASMJIT_INLINE_NODEBUG X86RAPass& pass() const noexcept { return static_cast<X86RAPass&>(_pass); }
[[nodiscard]]
ASMJIT_INLINE_NODEBUG Compiler& cc() const noexcept { return static_cast<Compiler&>(_cc); }
[[nodiscard]]
ASMJIT_INLINE_NODEBUG const EmitHelperInstructionIds& ids() const noexcept { return _ids; }
[[nodiscard]]
Error on_instruction(InstNode* inst, InstControlFlow& cf, RAInstBuilder& ib) noexcept;
[[nodiscard]]
Error on_before_invoke(InvokeNode* invoke_node) noexcept;
[[nodiscard]]
Error on_invoke(InvokeNode* invoke_node, RAInstBuilder& ib) noexcept;
[[nodiscard]]
Error move_vec_to_ptr(InvokeNode* invoke_node, const FuncValue& arg, const Vec& src, Out<Reg> out) noexcept;
[[nodiscard]]
Error move_imm_to_reg_arg(InvokeNode* invoke_node, const FuncValue& arg, const Imm& imm_, Out<Reg> out) noexcept;
[[nodiscard]]
Error move_imm_to_stack_arg(InvokeNode* invoke_node, const FuncValue& arg, const Imm& imm_) noexcept;
[[nodiscard]]
Error move_reg_to_stack_arg(InvokeNode* invoke_node, const FuncValue& arg, const Reg& reg) noexcept;
[[nodiscard]]
Error on_before_ret(FuncRetNode* func_ret) noexcept;
[[nodiscard]]
Error on_ret(FuncRetNode* func_ret, RAInstBuilder& ib) noexcept;
};
// x86::RACFGBuilder - OnInst
// ==========================
Error RACFGBuilder::on_instruction(InstNode* inst, InstControlFlow& cf, RAInstBuilder& ib) noexcept {
InstId inst_id = inst->inst_id();
InstRWInfo rw_info;
if (Inst::is_defined_id(inst_id)) {
Span<const Operand> operands = inst->operands();
ASMJIT_PROPAGATE(InstInternal::query_rw_info(_arch, inst->baseInst(), operands.data(), operands.size(), &rw_info));
const InstDB::InstInfo& inst_info = InstDB::inst_info_by_id(inst_id);
bool has_gpb_hi_constraint = false;
size_t single_reg_ops = 0;
// Copy instruction RW flags to instruction builder except kMovOp, which is propagated manually later.
ib.add_inst_rw_flags(rw_info.inst_flags() & ~InstRWFlags::kMovOp);
// Mask of all operand types used by the instruction - can be used as an optimization later.
uint32_t op_types_mask = 0u;
if (!operands.is_empty()) {
// The mask is for all registers, but we are mostly interested in AVX-512 registers at the moment. The mask
// will be combined with all available registers of the Compiler at the end so we it never use more registers
// than available.
RegMask instruction_allowed_regs = 0xFFFFFFFFu;
uint32_t consecutive_offset = 0;
RAWorkId consecutive_lead_id = kBadWorkId;
RAWorkReg* consecutive_parent = nullptr;
if (inst_info.is_evex()) {
// EVEX instruction and VEX instructions that can be encoded with EVEX have the possibility to use 32 SIMD
// registers (XMM/YMM/ZMM).
if (inst_info.is_vex() && !inst_info.is_evex_compatible()) {
if (inst_info.is_evex_kreg_only()) {
// EVEX encodable only if the first operand is K register (compare instructions).
if (!operands[0].is_mask_reg()) {
instruction_allowed_regs = 0xFFFFu;
}
}
else if (inst_info.is_evex_two_op_only()) {
// EVEX encodable only if the instruction has two operands (gather instructions).
if (operands.size() != 2u) {
instruction_allowed_regs = 0xFFFFu;
}
}
else {
instruction_allowed_regs = 0xFFFFu;
}
}
}
else if (inst_info.is_evex_transformable()) {
ib.add_aggregated_flags(RATiedFlags::kInst_IsTransformable);
}
else {
// Not EVEX, restrict everything to [0-15] registers.
instruction_allowed_regs = 0xFFFFu;
}
for (size_t i = 0u; i < operands.size(); i++) {
const Operand& op = operands[i];
const OpRWInfo& op_rw_info = rw_info.operand(i);
op_types_mask |= 1u << uint32_t(op.op_type());
if (op.is_reg()) {
// Register Operand
// ----------------
const Reg& reg = op.as<Reg>();
RATiedFlags flags = ra_reg_rw_flags(op_rw_info.op_flags());
RegMask allowed_regs = instruction_allowed_regs;
if (op_rw_info.is_unique()) {
flags |= RATiedFlags::kUnique;
}
// X86-specific constraints related to LO|HI general purpose registers. This is only required when the
// register is part of the encoding. If the register is fixed we won't restrict anything as it doesn't
// restrict encoding of other registers.
if (reg.is_gp8() && !op_rw_info.has_op_flag(OpRWFlags::kRegPhysId)) {
flags |= RATiedFlags::kX86_Gpb;
if (!_is_64bit) {
// Restrict to first four - AL|AH|BL|BH|CL|CH|DL|DH. In 32-bit mode it's not possible to access
// SIL|DIL, etc, so this is just enough.
allowed_regs = 0x0Fu;
}
else {
// If we encountered GPB-HI register the situation is much more complicated than in 32-bit mode.
// We need to patch all registers to not use ID higher than 7 and all GPB-LO registers to not use
// index higher than 3. Instead of doing the patching here we just set a flag and will do it later,
// to not complicate this loop.
if (reg.is_gp8_hi()) {
has_gpb_hi_constraint = true;
allowed_regs = 0x0Fu;
}
}
}
uint32_t virt_index = Operand::virt_id_to_index(reg.id());
if (virt_index < Operand::kVirtIdCount) {
RAWorkReg* work_reg;
ASMJIT_PROPAGATE(_pass.virt_index_as_work_reg(&work_reg, virt_index));
// Use RW instead of Write in case that not the whole register is overwritten. This is important
// for liveness as we cannot kill a register that will be used. For example `mov al, 0xFF` is not
// a write-only operation if user allocated the whole `rax` register.
if ((flags & RATiedFlags::kRW) == RATiedFlags::kWrite) {
if (work_reg->reg_byte_mask() & ~(op_rw_info.write_byte_mask() | op_rw_info.extend_byte_mask())) {
// Not write-only operation.
flags = (flags & ~RATiedFlags::kOut) | (RATiedFlags::kRead | RATiedFlags::kUse);
}
}
// Do not use RegMem flag if changing Reg to Mem requires a CPU feature that is not available.
if (rw_info.rm_feature() && Support::test(flags, RATiedFlags::kUseRM | RATiedFlags::kOutRM)) {
if (!cc().code()->cpu_features().has(rw_info.rm_feature())) {
flags &= ~(RATiedFlags::kUseRM | RATiedFlags::kOutRM);
}
}
RegGroup group = work_reg->group();
RegMask use_regs = _pass._available_regs[group] & allowed_regs;
RegMask out_regs = use_regs;
uint32_t use_id = Reg::kIdBad;
uint32_t out_id = Reg::kIdBad;
uint32_t use_rewrite_mask = 0;
uint32_t out_rewrite_mask = 0;
if (op_rw_info.consecutive_lead_count()) {
// There must be a single consecutive register lead, otherwise the RW data is invalid.
if (consecutive_lead_id != kBadWorkId) {
return make_error(Error::kInvalidState);
}
// A consecutive lead register cannot be used as a consecutive +1/+2/+3 register, the registers must be distinct.
if (RATiedReg::consecutive_data_from_flags(flags) != 0) {
return make_error(Error::kNotConsecutiveRegs);
}
flags |= RATiedFlags::kLeadConsecutive | RATiedReg::consecutive_data_to_flags(op_rw_info.consecutive_lead_count() - 1);
consecutive_lead_id = work_reg->work_id();
RegMask filter = ra_consecutive_lead_count_to_reg_mask_filter[op_rw_info.consecutive_lead_count()];
if (Support::test(flags, RATiedFlags::kUse)) {
flags |= RATiedFlags::kUseConsecutive;
use_regs &= filter;
}
else {
flags |= RATiedFlags::kOutConsecutive;
out_regs &= filter;
}
}
if (Support::test(flags, RATiedFlags::kUse)) {
use_rewrite_mask = Support::bit_mask<uint32_t>(inst->_get_rewrite_index(&reg._base_id));
if (op_rw_info.has_op_flag(OpRWFlags::kRegPhysId)) {
use_id = op_rw_info.phys_id();
flags |= RATiedFlags::kUseFixed;
}
else if (op_rw_info.has_op_flag(OpRWFlags::kConsecutive)) {
if (consecutive_lead_id == kBadWorkId) {
return make_error(Error::kInvalidState);
}
if (consecutive_lead_id == work_reg->work_id()) {
return make_error(Error::kOverlappedRegs);
}
flags |= RATiedFlags::kUseConsecutive | RATiedReg::consecutive_data_to_flags(++consecutive_offset);
}
}
else {
out_rewrite_mask = Support::bit_mask<uint32_t>(inst->_get_rewrite_index(&reg._base_id));
if (op_rw_info.has_op_flag(OpRWFlags::kRegPhysId)) {
out_id = op_rw_info.phys_id();
flags |= RATiedFlags::kOutFixed;
}
else if (op_rw_info.has_op_flag(OpRWFlags::kConsecutive)) {
if (consecutive_lead_id == kBadWorkId) {
return make_error(Error::kInvalidState);
}
if (consecutive_lead_id == work_reg->work_id()) {
return make_error(Error::kOverlappedRegs);
}
flags |= RATiedFlags::kOutConsecutive | RATiedReg::consecutive_data_to_flags(++consecutive_offset);
}
}
ASMJIT_PROPAGATE(ib.add(work_reg, flags, use_regs, use_id, use_rewrite_mask, out_regs, out_id, out_rewrite_mask, op_rw_info.rm_size(), consecutive_parent));
if (single_reg_ops == i) {
single_reg_ops++;
}
if (Support::test(flags, RATiedFlags::kLeadConsecutive | RATiedFlags::kUseConsecutive | RATiedFlags::kOutConsecutive)) {
consecutive_parent = work_reg;
}
}
}
else if (op.is_mem()) {
// Memory Operand
// --------------
const Mem& mem = op.as<Mem>();
ib.add_forbidden_flags(RATiedFlags::kUseRM | RATiedFlags::kOutRM);
if (mem.is_reg_home()) {
RAWorkReg* work_reg;
ASMJIT_PROPAGATE(_pass.virt_index_as_work_reg(&work_reg, Operand::virt_id_to_index(mem.base_id())));
if (ASMJIT_UNLIKELY(!_pass.get_or_create_stack_slot(work_reg))) {
return make_error(Error::kOutOfMemory);
}
}
else if (mem.has_base_reg()) {
uint32_t virt_index = Operand::virt_id_to_index(mem.base_id());
if (virt_index < Operand::kVirtIdCount) {
RAWorkReg* work_reg;
ASMJIT_PROPAGATE(_pass.virt_index_as_work_reg(&work_reg, virt_index));
RATiedFlags flags = ra_mem_base_rw_flags(op_rw_info.op_flags());
RegGroup group = work_reg->group();
RegMask in_out_regs = _pass._available_regs[group];
uint32_t use_id = Reg::kIdBad;
uint32_t out_id = Reg::kIdBad;
uint32_t use_rewrite_mask = 0;
uint32_t out_rewrite_mask = 0;
if (Support::test(flags, RATiedFlags::kUse)) {
use_rewrite_mask = Support::bit_mask<uint32_t>(inst->_get_rewrite_index(&mem._base_id));
if (op_rw_info.has_op_flag(OpRWFlags::kMemPhysId)) {
use_id = op_rw_info.phys_id();
flags |= RATiedFlags::kUseFixed;
}
}
else {
out_rewrite_mask = Support::bit_mask<uint32_t>(inst->_get_rewrite_index(&mem._base_id));
if (op_rw_info.has_op_flag(OpRWFlags::kMemPhysId)) {
out_id = op_rw_info.phys_id();
flags |= RATiedFlags::kOutFixed;
}
}
ASMJIT_PROPAGATE(ib.add(work_reg, flags, in_out_regs, use_id, use_rewrite_mask, in_out_regs, out_id, out_rewrite_mask));
}
}
if (mem.has_index_reg()) {
uint32_t virt_index = Operand::virt_id_to_index(mem.index_id());
if (virt_index < Operand::kVirtIdCount) {
RAWorkReg* work_reg;
ASMJIT_PROPAGATE(_pass.virt_index_as_work_reg(&work_reg, virt_index));
RATiedFlags flags = ra_mem_index_rw_flags(op_rw_info.op_flags());
RegGroup group = work_reg->group();
RegMask in_out_regs = _pass._available_regs[group] & instruction_allowed_regs;
// Index registers have never fixed id on X86/x64.
const uint32_t use_id = Reg::kIdBad;
const uint32_t out_id = Reg::kIdBad;
uint32_t use_rewrite_mask = 0;
uint32_t out_rewrite_mask = 0;
if (Support::test(flags, RATiedFlags::kUse)) {
use_rewrite_mask = Support::bit_mask<uint32_t>(inst->_get_rewrite_index(&mem._data[Operand::kDataMemIndexId]));
}
else {
out_rewrite_mask = Support::bit_mask<uint32_t>(inst->_get_rewrite_index(&mem._data[Operand::kDataMemIndexId]));
}
ASMJIT_PROPAGATE(ib.add(work_reg, RATiedFlags::kUse | RATiedFlags::kRead, in_out_regs, use_id, use_rewrite_mask, in_out_regs, out_id, out_rewrite_mask));
}
}
}
}
}
// Handle extra operand (either REP {cx|ecx|rcx} or AVX-512 {k} selector).
if (inst->has_extra_reg()) {
uint32_t virt_index = Operand::virt_id_to_index(inst->extra_reg().id());
if (virt_index < Operand::kVirtIdCount) {
RAWorkReg* work_reg;
ASMJIT_PROPAGATE(_pass.virt_index_as_work_reg(&work_reg, virt_index));
RegGroup group = work_reg->group();
RegMask in_out_regs = _pass._available_regs[group];
uint32_t rewrite_mask = Support::bit_mask<uint32_t>(inst->_get_rewrite_index(&inst->extra_reg()._id));
if (group == RegGroup::kMask) {
// AVX-512 mask selector {k} register - read-only, allocable to any register except {k0}.
ASMJIT_PROPAGATE(ib.add(work_reg, RATiedFlags::kUse | RATiedFlags::kRead, in_out_regs, Reg::kIdBad, rewrite_mask, in_out_regs, Reg::kIdBad, 0));
single_reg_ops = 0u;
}
else {
// REP {cx|ecx|rcx} register - read & write, allocable to {cx|ecx|rcx} only.
ASMJIT_PROPAGATE(ib.add(work_reg, RATiedFlags::kUse | RATiedFlags::kRW, in_out_regs, Gp::kIdCx, rewrite_mask, in_out_regs, Gp::kIdBad, 0));
}
}
else {
RegGroup group = inst->extra_reg().group();
if (group == RegGroup::kMask && inst->extra_reg().id() != 0) {
single_reg_ops = 0u;
}
}
}
// If this instruction has move semantics then check whether it could be eliminated if all virtual registers
// are allocated into the same register. Take into account the virtual size of the destination register as that's
// more important than a physical register size in this case.
if (rw_info.has_inst_flag(InstRWFlags::kMovOp) && !inst->has_extra_reg() && Support::bit_test(op_types_mask, uint32_t(OperandType::kReg))) {
// AVX+ move instructions have 3 operand form - the first two operands must be the same to guarantee move semantics.
if (operands.size() == 2 || (operands.size() == 3 && operands[0] == operands[1])) {
uint32_t virt_index = Operand::virt_id_to_index(operands.first().as<Reg>().id());
if (virt_index < Operand::kVirtIdCount) {
const VirtReg* virt_reg = _cc.virt_reg_by_index(virt_index);
const OpRWInfo& op_rw_info = rw_info.operand(0);
uint64_t remaining_byte_mask = virt_reg->work_reg()->reg_byte_mask() & ~op_rw_info.write_byte_mask();
if (remaining_byte_mask == 0u || (remaining_byte_mask & op_rw_info.extend_byte_mask()) == 0) {
ib.add_inst_rw_flags(InstRWFlags::kMovOp);
}
}
}
}
// Handle X86 constraints.
if (has_gpb_hi_constraint) {
for (RATiedReg& tied_reg : ib) {
RegMask filter = tied_reg.has_flag(RATiedFlags::kX86_Gpb) ? 0x0Fu : 0xFFu;
tied_reg._use_reg_mask &= filter;
tied_reg._out_reg_mask &= filter;
}
}
if (ib.tied_reg_count() == 1) {
// Handle special cases of some instructions where all operands share the same
// register. In such case the single operand becomes read-only or write-only.
InstSameRegHint same_reg_hint = InstSameRegHint::kNone;
if (single_reg_ops == operands.size()) {
same_reg_hint = inst_info.same_reg_hint();
}
else if (operands.size() == 2 && operands[1].is_imm()) {
// Handle some tricks used by X86 asm.
const Reg& reg = operands[0].as<Reg>();
const Imm& imm = operands[1].as<Imm>();
const RAWorkReg* work_reg = ib[0]->work_reg();
uint32_t work_reg_size = work_reg->signature().size();
switch (inst->inst_id()) {
case Inst::kIdOr: {
// Sets the value of the destination register to -1, previous content unused.
if (reg.size() >= 4 || reg.size() >= work_reg_size) {
if (imm.value() == -1 || imm.value_as<uint64_t>() == ra_imm_mask_from_size(reg.size())) {
same_reg_hint = InstSameRegHint::kWO;
}
}
[[fallthrough]];
}
case Inst::kIdAdd:
case Inst::kIdAnd:
case Inst::kIdRol:
case Inst::kIdRor:
case Inst::kIdSar:
case Inst::kIdShl:
case Inst::kIdShr:
case Inst::kIdSub:
case Inst::kIdXor: {
// Updates [E|R]FLAGS without changing the content.
if (reg.size() != 4 || reg.size() >= work_reg_size) {
if (imm.value() == 0) {
same_reg_hint = InstSameRegHint::kRO;
}
}
break;
}
}
}
else if (operands.size() == 4u && operands[3].is_imm()) {
const Imm& imm = operands[3].as<Imm>();
switch (inst->inst_id()) {
case Inst::kIdVpternlogd:
case Inst::kIdVpternlogq: {
uint32_t predicate = uint32_t(imm.value() & 0xFFu);
if (predicate == 0x00u || predicate == 0xFFu) {
ib[0]->make_write_only();
}
break;
}
}
}
switch (same_reg_hint) {
case InstSameRegHint::kNone:
break;
case InstSameRegHint::kRO:
ib[0]->make_read_only();
break;
case InstSameRegHint::kWO:
ib[0]->make_write_only();
break;
}
}
cf = inst_info.control_flow();
}
return Error::kOk;
}
// x86::RACFGBuilder - OnInvoke
// ============================
Error RACFGBuilder::on_before_invoke(InvokeNode* invoke_node) noexcept {
const FuncDetail& fd = invoke_node->detail();
uint32_t arg_count = invoke_node->arg_count();
cc().set_cursor(invoke_node->prev());
RegType native_reg_type = cc()._gp_signature.reg_type();
for (uint32_t arg_index = 0; arg_index < arg_count; arg_index++) {
const FuncValuePack& arg_pack = fd.arg_pack(arg_index);
for (uint32_t value_index = 0; value_index < Globals::kMaxValuePack; value_index++) {
if (!arg_pack[value_index]) {
break;
}
const FuncValue& arg = arg_pack[value_index];
const Operand& op = invoke_node->arg(arg_index, value_index);
if (op.is_none()) {
continue;
}
if (op.is_reg()) {
const Reg& reg = op.as<Reg>();
RAWorkReg* work_reg;
ASMJIT_PROPAGATE(_pass.virt_index_as_work_reg(&work_reg, Operand::virt_id_to_index(reg.id())));
if (arg.is_reg()) {
RegGroup reg_group = work_reg->group();
RegGroup arg_group = RegUtils::group_of(arg.reg_type());
if (arg.is_indirect()) {
if (reg.is_gp()) {
if (reg.reg_type() != native_reg_type) {
return make_error(Error::kInvalidAssignment);
}
// It's considered allocated if this is an indirect argument and the user used GP.
continue;
}
Reg indirect_reg;
ASMJIT_PROPAGATE(move_vec_to_ptr(invoke_node, arg, reg.as<Vec>(), Out(indirect_reg)));
invoke_node->_args[arg_index][value_index] = indirect_reg;
}
else {
if (reg_group != arg_group) {
// TODO: Conversion is not supported.
return make_error(Error::kInvalidAssignment);
}
}
}
else {
if (arg.is_indirect()) {
if (reg.is_gp()) {
if (reg.reg_type() != native_reg_type) {
return make_error(Error::kInvalidAssignment);
}
ASMJIT_PROPAGATE(move_reg_to_stack_arg(invoke_node, arg, reg));
continue;
}
Reg indirect_reg;
ASMJIT_PROPAGATE(move_vec_to_ptr(invoke_node, arg, reg.as<Vec>(), Out(indirect_reg)));
ASMJIT_PROPAGATE(move_reg_to_stack_arg(invoke_node, arg, indirect_reg));
}
else {
ASMJIT_PROPAGATE(move_reg_to_stack_arg(invoke_node, arg, reg));
}
}
}
else if (op.is_imm()) {
if (arg.is_reg()) {
Reg reg;
ASMJIT_PROPAGATE(move_imm_to_reg_arg(invoke_node, arg, op.as<Imm>(), Out(reg)));
invoke_node->_args[arg_index][value_index] = reg;
}
else {
ASMJIT_PROPAGATE(move_imm_to_stack_arg(invoke_node, arg, op.as<Imm>()));
}
}
}
}
cc().set_cursor(invoke_node);
if (fd.has_flag(CallConvFlags::kCalleePopsStack) && fd.arg_stack_size() != 0) {
ASMJIT_PROPAGATE(cc().sub(cc().zsp(), fd.arg_stack_size()));
}
if (fd.has_ret()) {
for (uint32_t value_index = 0; value_index < Globals::kMaxValuePack; value_index++) {
const FuncValue& ret = fd.ret(value_index);
if (!ret) {
break;
}
const Operand& op = invoke_node->ret(value_index);
if (op.is_reg()) {
const Reg& reg = op.as<Reg>();
RAWorkReg* work_reg;
ASMJIT_PROPAGATE(_pass.virt_index_as_work_reg(&work_reg, Operand::virt_id_to_index(reg.id())));
if (ret.is_reg()) {
if (ret.reg_type() == RegType::kX86_St) {
if (work_reg->group() != RegGroup::kVec) {
return make_error(Error::kInvalidAssignment);
}
Reg dst(work_reg->signature(), work_reg->virt_id());
Mem mem;
TypeId type_id = TypeUtils::scalar_of(work_reg->type_id());
if (ret.has_type_id()) {
type_id = ret.type_id();
}
switch (type_id) {
case TypeId::kFloat32:
ASMJIT_PROPAGATE(_pass.use_temporary_mem(mem, 4, 4));
mem.set_size(4);
ASMJIT_PROPAGATE(cc().fstp(mem));
ASMJIT_PROPAGATE(cc().emit(ids().movss(), dst.as<Vec>(), mem));
break;
case TypeId::kFloat64:
ASMJIT_PROPAGATE(_pass.use_temporary_mem(mem, 8, 4));
mem.set_size(8);
ASMJIT_PROPAGATE(cc().fstp(mem));
ASMJIT_PROPAGATE(cc().emit(ids().movsd(), dst.as<Vec>(), mem));
break;
default:
return make_error(Error::kInvalidAssignment);
}
}
else {
RegGroup reg_group = work_reg->group();
RegGroup ret_group = RegUtils::group_of(ret.reg_type());
if (reg_group != ret_group) {
// TODO: Conversion is not supported.
return make_error(Error::kInvalidAssignment);
}
}
}
}
}
}
// This block has function call(s).
_cur_block->add_flags(RABlockFlags::kHasFuncCalls);
_pass.func()->frame().add_attributes(FuncAttributes::kHasFuncCalls);
_pass.func()->frame().update_call_stack_size(fd.arg_stack_size());
return Error::kOk;
}
Error RACFGBuilder::on_invoke(InvokeNode* invoke_node, RAInstBuilder& ib) noexcept {
uint32_t arg_count = invoke_node->arg_count();
const FuncDetail& fd = invoke_node->detail();
for (uint32_t arg_index = 0; arg_index < arg_count; arg_index++) {
const FuncValuePack& arg_pack = fd.arg_pack(arg_index);
for (uint32_t value_index = 0; value_index < Globals::kMaxValuePack; value_index++) {
if (!arg_pack[value_index]) {
continue;
}
const FuncValue& arg = arg_pack[value_index];
const Operand& op = invoke_node->arg(arg_index, value_index);
if (op.is_none()) {
continue;
}
if (op.is_reg()) {
const Reg& reg = op.as<Reg>();
RAWorkReg* work_reg;
ASMJIT_PROPAGATE(_pass.virt_index_as_work_reg(&work_reg, Operand::virt_id_to_index(reg.id())));
if (arg.is_indirect()) {
RegGroup reg_group = work_reg->group();
if (reg_group != RegGroup::kGp) {
return make_error(Error::kInvalidState);
}
ASMJIT_PROPAGATE(ib.add_call_arg(work_reg, arg.reg_id()));
}
else if (arg.is_reg()) {
RegGroup reg_group = work_reg->group();
RegGroup arg_group = RegUtils::group_of(arg.reg_type());
if (reg_group == arg_group) {
ASMJIT_PROPAGATE(ib.add_call_arg(work_reg, arg.reg_id()));
}
}
}
}
}
for (uint32_t ret_index = 0; ret_index < Globals::kMaxValuePack; ret_index++) {
const FuncValue& ret = fd.ret(ret_index);
if (!ret) {
break;
}
// Not handled here...
const Operand& op = invoke_node->ret(ret_index);
if (ret.reg_type() == RegType::kX86_St) {
continue;
}
if (op.is_reg()) {
const Reg& reg = op.as<Reg>();
RAWorkReg* work_reg;
ASMJIT_PROPAGATE(_pass.virt_index_as_work_reg(&work_reg, Operand::virt_id_to_index(reg.id())));
if (ret.is_reg()) {
RegGroup reg_group = work_reg->group();
RegGroup ret_group = RegUtils::group_of(ret.reg_type());
if (reg_group == ret_group) {
ASMJIT_PROPAGATE(ib.add_call_ret(work_reg, ret.reg_id()));
}
}
else {
return make_error(Error::kInvalidAssignment);
}
}
}
// Setup clobbered registers.
for (RegGroup group : Support::enumerate(RegGroup::kMaxVirt)) {
ib._clobbered[group] = Support::lsb_mask<RegMask>(_pass._phys_reg_count.get(group)) & ~fd.preserved_regs(group);
}
return Error::kOk;
}
// x86::RACFGBuilder - MoveVecToPtr
// ================================
static inline OperandSignature vec_reg_signature_by_size(uint32_t size) noexcept {
return OperandSignature{
size >= 64 ? RegTraits<RegType::kVec512>::kSignature :
size >= 32 ? RegTraits<RegType::kVec256>::kSignature :
RegTraits<RegType::kVec128>::kSignature
};
}
Error RACFGBuilder::move_vec_to_ptr(InvokeNode* invoke_node, const FuncValue& arg, const Vec& src, Out<Reg> out) noexcept {
Support::maybe_unused(invoke_node);
ASMJIT_ASSERT(arg.is_reg());
uint32_t arg_size = TypeUtils::size_of(arg.type_id());
if (arg_size == 0) {
return make_error(Error::kInvalidState);
}
if (arg_size < 16) {
arg_size = 16;
}
uint32_t arg_stack_offset = Support::align_up(invoke_node->detail()._arg_stack_size, arg_size);
_func_node->frame().update_call_stack_alignment(arg_size);
invoke_node->detail()._arg_stack_size = arg_stack_offset + arg_size;
Vec vec_reg(vec_reg_signature_by_size(arg_size), src.id());
Mem vec_ptr = ptr(_pass._sp.as<Gp>(), int32_t(arg_stack_offset));
uint32_t vec_mov_inst_id = pass()._emit_helper.ids().movaps();
if (arg_size > 16) {
vec_mov_inst_id = Inst::kIdVmovaps;
}
ASMJIT_PROPAGATE(cc()._new_reg(out, RegUtils::type_id_of(cc()._gp_signature.reg_type()), nullptr));
VirtReg* virt_reg = cc().virt_reg_by_id(out->id());
virt_reg->set_weight(BaseRAPass::kCallArgWeight);
ASMJIT_PROPAGATE(cc().lea(out->as<Gp>(), vec_ptr));
ASMJIT_PROPAGATE(cc().emit(vec_mov_inst_id, ptr(out->as<Gp>()), vec_reg));
if (arg.is_stack()) {
Mem stack_ptr = ptr(_pass._sp.as<Gp>(), arg.stack_offset());
ASMJIT_PROPAGATE(cc().mov(stack_ptr, out->as<Gp>()));
}
return Error::kOk;
}
// x86::RACFGBuilder - Move Imm to Reg Arg
// =======================================
Error RACFGBuilder::move_imm_to_reg_arg(InvokeNode* invoke_node, const FuncValue& arg, const Imm& imm_, Out<Reg> out) noexcept {
Support::maybe_unused(invoke_node);
ASMJIT_ASSERT(arg.is_reg());
Imm imm(imm_);
TypeId reg_type_id = TypeId::kUInt32;
switch (arg.type_id()) {
case TypeId::kInt8: imm.sign_extend_int8(); goto MovU32;
case TypeId::kUInt8: imm.zero_extend_uint8(); goto MovU32;
case TypeId::kInt16: imm.sign_extend_int16(); goto MovU32;
case TypeId::kUInt16: imm.zero_extend_uint16(); goto MovU32;
case TypeId::kInt32:
case TypeId::kUInt32:
MovU32:
imm.zero_extend_uint32();
break;
case TypeId::kInt64:
case TypeId::kUInt64:
// Moving to GPD automatically zero extends in 64-bit mode.
if (imm.is_uint32()) {
imm.zero_extend_uint32();
break;
}
reg_type_id = TypeId::kUInt64;
break;
default:
return make_error(Error::kInvalidAssignment);
}
ASMJIT_PROPAGATE(cc()._new_reg(out, reg_type_id, nullptr));
cc().virt_reg_by_id(out->id())->set_weight(BaseRAPass::kCallArgWeight);
return cc().mov(out->as<x86::Gp>(), imm);
}
// x86::RACFGBuilder - Move Imm to Stack Arg
// =========================================
Error RACFGBuilder::move_imm_to_stack_arg(InvokeNode* invoke_node, const FuncValue& arg, const Imm& imm_) noexcept {
Support::maybe_unused(invoke_node);
ASMJIT_ASSERT(arg.is_stack());
Mem stack_ptr = ptr(_pass._sp.as<Gp>(), arg.stack_offset());
Imm imm[2];
stack_ptr.set_size(4);
imm[0] = imm_;
uint32_t mov_count = 0;
// One stack entry has the same size as the native register size. That means that if we want to move a 32-bit
// integer on the stack in 64-bit mode, we need to extend it to a 64-bit integer first. In 32-bit mode, pushing
// a 64-bit on stack is done in two steps by pushing low and high parts separately.
switch (arg.type_id()) {
case TypeId::kInt8: imm[0].sign_extend_int8(); goto MovU32;
case TypeId::kUInt8: imm[0].zero_extend_uint8(); goto MovU32;
case TypeId::kInt16: imm[0].sign_extend_int16(); goto MovU32;
case TypeId::kUInt16: imm[0].zero_extend_uint16(); goto MovU32;
case TypeId::kInt32:
case TypeId::kUInt32:
case TypeId::kFloat32:
MovU32:
imm[0].zero_extend_uint32();
mov_count = 1;
break;
case TypeId::kInt64:
case TypeId::kUInt64:
case TypeId::kFloat64:
case TypeId::kMmx32:
case TypeId::kMmx64:
if (_is_64bit && imm[0].is_int32()) {
stack_ptr.set_size(8);
mov_count = 1;
break;
}
imm[1].set_value(imm[0].uint_hi32());
imm[0].zero_extend_uint32();
mov_count = 2;
break;
default:
return make_error(Error::kInvalidAssignment);
}
for (uint32_t i = 0; i < mov_count; i++) {
ASMJIT_PROPAGATE(cc().mov(stack_ptr, imm[i]));
stack_ptr.add_offset_lo32(int32_t(stack_ptr.size()));
}
return Error::kOk;
}
// x86::RACFGBuilder - MoveRegToStackArg
// =====================================
Error RACFGBuilder::move_reg_to_stack_arg(InvokeNode* invoke_node, const FuncValue& arg, const Reg& reg) noexcept {
Support::maybe_unused(invoke_node);
ASMJIT_ASSERT(arg.is_stack());
Mem stack_ptr = ptr(_pass._sp.as<Gp>(), arg.stack_offset());
Reg r0, r1;
VirtReg* vr = cc().virt_reg_by_id(reg.id());
uint32_t register_size = cc().register_size();
InstId inst_id = 0;
TypeId dst_type_id = arg.type_id();
TypeId src_type_id = vr->type_id();
switch (dst_type_id) {
case TypeId::kInt64:
case TypeId::kUInt64:
// Extend BYTE->QWORD (GP).
if (TypeUtils::is_gp8(src_type_id)) {
r1.set_reg_t<RegType::kGp8Lo>(reg.id());
inst_id = (dst_type_id == TypeId::kInt64 && src_type_id == TypeId::kInt8) ? Inst::kIdMovsx : Inst::kIdMovzx;
goto ExtendMovGpXQ;
}
// Extend WORD->QWORD (GP).
if (TypeUtils::is_gp16(src_type_id)) {
r1.set_reg_t<RegType::kGp16>(reg.id());
inst_id = (dst_type_id == TypeId::kInt64 && src_type_id == TypeId::kInt16) ? Inst::kIdMovsx : Inst::kIdMovzx;
goto ExtendMovGpXQ;
}
// Extend DWORD->QWORD (GP).
if (TypeUtils::is_gp32(src_type_id)) {
r1.set_reg_t<RegType::kGp32>(reg.id());
inst_id = Inst::kIdMovsxd;
if (dst_type_id == TypeId::kInt64 && src_type_id == TypeId::kInt32) {
goto ExtendMovGpXQ;
}
else {
goto ZeroExtendGpDQ;
}
}
// Move QWORD (GP).
if (TypeUtils::is_gp64(src_type_id)) goto MovGpQ;
if (TypeUtils::is_mmx(src_type_id)) goto MovMmQ;
if (TypeUtils::is_vec(src_type_id)) goto MovXmmQ;
break;
case TypeId::kInt32:
case TypeId::kUInt32:
case TypeId::kInt16:
case TypeId::kUInt16:
// DWORD <- WORD (Zero|Sign Extend).
if (TypeUtils::is_gp16(src_type_id)) {
bool is_dst_signed = dst_type_id == TypeId::kInt16 || dst_type_id == TypeId::kInt32;
bool is_src_signed = src_type_id == TypeId::kInt8 || src_type_id == TypeId::kInt16;
r1.set_reg_t<RegType::kGp16>(reg.id());
inst_id = is_dst_signed && is_src_signed ? Inst::kIdMovsx : Inst::kIdMovzx;
goto ExtendMovGpD;
}
// DWORD <- BYTE (Zero|Sign Extend).
if (TypeUtils::is_gp8(src_type_id)) {
bool is_dst_signed = dst_type_id == TypeId::kInt16 || dst_type_id == TypeId::kInt32;
bool is_src_signed = src_type_id == TypeId::kInt8 || src_type_id == TypeId::kInt16;
r1.set_reg_t<RegType::kGp8Lo>(reg.id());
inst_id = is_dst_signed && is_src_signed ? Inst::kIdMovsx : Inst::kIdMovzx;
goto ExtendMovGpD;
}
[[fallthrough]];
case TypeId::kInt8:
case TypeId::kUInt8:
if (TypeUtils::is_int(src_type_id)) goto MovGpD;
if (TypeUtils::is_mmx(src_type_id)) goto MovMmD;
if (TypeUtils::is_vec(src_type_id)) goto MovXmmD;
break;
case TypeId::kMmx32:
case TypeId::kMmx64:
// Extend BYTE->QWORD (GP).
if (TypeUtils::is_gp8(src_type_id)) {
r1.set_reg_t<RegType::kGp8Lo>(reg.id());
inst_id = Inst::kIdMovzx;
goto ExtendMovGpXQ;
}
// Extend WORD->QWORD (GP).
if (TypeUtils::is_gp16(src_type_id)) {
r1.set_reg_t<RegType::kGp16>(reg.id());
inst_id = Inst::kIdMovzx;
goto ExtendMovGpXQ;
}
if (TypeUtils::is_gp32(src_type_id)) goto ExtendMovGpDQ;
if (TypeUtils::is_gp64(src_type_id)) goto MovGpQ;
if (TypeUtils::is_mmx(src_type_id)) goto MovMmQ;
if (TypeUtils::is_vec(src_type_id)) goto MovXmmQ;
break;
case TypeId::kFloat32:
case TypeId::kFloat32x1:
if (TypeUtils::is_vec(src_type_id)) goto MovXmmD;
break;
case TypeId::kFloat64:
case TypeId::kFloat64x1:
if (TypeUtils::is_vec(src_type_id)) goto MovXmmQ;
break;
default:
if (TypeUtils::is_vec(dst_type_id) && reg.as<Reg>().is_vec()) {
stack_ptr.set_size(TypeUtils::size_of(dst_type_id));
uint32_t vec_mov_inst_id = pass()._emit_helper.ids().movaps();
if (TypeUtils::is_vec128(dst_type_id)) {
r0.set_reg_t<RegType::kVec128>(reg.id());
}
else if (TypeUtils::is_vec256(dst_type_id)) {
r0.set_reg_t<RegType::kVec256>(reg.id());
}
else if (TypeUtils::is_vec512(dst_type_id)) {
r0.set_reg_t<RegType::kVec512>(reg.id());
}
else {
break;
}
return cc().emit(vec_mov_inst_id, stack_ptr, r0);
}
break;
}
return make_error(Error::kInvalidAssignment);
// Extend+Move Gp.
ExtendMovGpD:
stack_ptr.set_size(4);
r0.set_reg_t<RegType::kGp32>(reg.id());
ASMJIT_PROPAGATE(cc().emit(inst_id, r0, r1));
ASMJIT_PROPAGATE(cc().emit(Inst::kIdMov, stack_ptr, r0));
return Error::kOk;
ExtendMovGpXQ:
if (register_size == 8) {
stack_ptr.set_size(8);
r0.set_reg_t<RegType::kGp64>(reg.id());
ASMJIT_PROPAGATE(cc().emit(inst_id, r0, r1));
ASMJIT_PROPAGATE(cc().emit(Inst::kIdMov, stack_ptr, r0));
}
else {
stack_ptr.set_size(4);
r0.set_reg_t<RegType::kGp32>(reg.id());
ASMJIT_PROPAGATE(cc().emit(inst_id, r0, r1));
ExtendMovGpDQ:
ASMJIT_PROPAGATE(cc().emit(Inst::kIdMov, stack_ptr, r0));
stack_ptr.add_offset_lo32(4);
ASMJIT_PROPAGATE(cc().emit(Inst::kIdAnd, stack_ptr, 0));
}
return Error::kOk;
ZeroExtendGpDQ:
stack_ptr.set_size(4);
r0.set_reg_t<RegType::kGp32>(reg.id());
goto ExtendMovGpDQ;
MovGpD:
stack_ptr.set_size(4);
r0.set_reg_t<RegType::kGp32>(reg.id());
return cc().emit(Inst::kIdMov, stack_ptr, r0);
MovGpQ:
stack_ptr.set_size(8);
r0.set_reg_t<RegType::kGp64>(reg.id());
return cc().emit(Inst::kIdMov, stack_ptr, r0);
MovMmD:
stack_ptr.set_size(4);
r0.set_reg_t<RegType::kX86_Mm>(reg.id());
return cc().emit(ids().movd(), stack_ptr, r0);
MovMmQ:
stack_ptr.set_size(8);
r0.set_reg_t<RegType::kX86_Mm>(reg.id());
return cc().emit(ids().movq(), stack_ptr, r0);
MovXmmD:
stack_ptr.set_size(4);
r0.set_reg_t<RegType::kVec128>(reg.id());
return cc().emit(ids().movss(), stack_ptr, r0);
MovXmmQ:
stack_ptr.set_size(8);
r0.set_reg_t<RegType::kVec128>(reg.id());
return cc().emit(ids().movlps(), stack_ptr, r0);
}
// x86::RACFGBuilder - OnReg
// =========================
Error RACFGBuilder::on_before_ret(FuncRetNode* func_ret) noexcept {
const FuncDetail& func_detail = _pass.func()->detail();
Span<const Operand> operands = func_ret->operands();
cc().set_cursor(func_ret->prev());
for (size_t i = 0; i < operands.size(); i++) {
const Operand& op = operands[i];
const FuncValue& ret = func_detail.ret(i);
if (!op.is_reg()) {
continue;
}
if (ret.reg_type() == RegType::kX86_St) {
const Reg& reg = op.as<Reg>();
uint32_t virt_index = Operand::virt_id_to_index(reg.id());
if (virt_index < Operand::kVirtIdCount) {
RAWorkReg* work_reg;
ASMJIT_PROPAGATE(_pass.virt_index_as_work_reg(&work_reg, virt_index));
if (work_reg->group() != RegGroup::kVec) {
return make_error(Error::kInvalidAssignment);
}
Reg src(work_reg->signature(), work_reg->virt_id());
Mem mem;
TypeId type_id = TypeUtils::scalar_of(work_reg->type_id());
if (ret.has_type_id()) {
type_id = ret.type_id();
}
switch (type_id) {
case TypeId::kFloat32:
ASMJIT_PROPAGATE(_pass.use_temporary_mem(mem, 4, 4));
mem.set_size(4);
ASMJIT_PROPAGATE(cc().emit(ids().movss(), mem, src.as<Vec>()));
ASMJIT_PROPAGATE(cc().fld(mem));
break;
case TypeId::kFloat64:
ASMJIT_PROPAGATE(_pass.use_temporary_mem(mem, 8, 4));
mem.set_size(8);
ASMJIT_PROPAGATE(cc().emit(ids().movsd(), mem, src.as<Vec>()));
ASMJIT_PROPAGATE(cc().fld(mem));
break;
default:
return make_error(Error::kInvalidAssignment);
}
}
}
}
return Error::kOk;
}
Error RACFGBuilder::on_ret(FuncRetNode* func_ret, RAInstBuilder& ib) noexcept {
const FuncDetail& func_detail = _pass.func()->detail();
Span<const Operand> operands = func_ret->operands();
for (size_t i = 0u; i < operands.size(); i++) {
const Operand& op = operands[i];
if (op.is_none()) {
continue;
}
const FuncValue& ret = func_detail.ret(i);
if (ASMJIT_UNLIKELY(!ret.is_reg())) {
return make_error(Error::kInvalidAssignment);
}
// Not handled here...
if (ret.reg_type() == RegType::kX86_St) {
continue;
}
if (op.is_reg()) {
// Register return value.
const Reg& reg = op.as<Reg>();
uint32_t virt_index = Operand::virt_id_to_index(reg.id());
if (virt_index < Operand::kVirtIdCount) {
RAWorkReg* work_reg;
ASMJIT_PROPAGATE(_pass.virt_index_as_work_reg(&work_reg, virt_index));
RegGroup group = work_reg->group();
RegMask in_out_regs = _pass._available_regs[group];
ASMJIT_PROPAGATE(ib.add(work_reg, RATiedFlags::kUse | RATiedFlags::kRead, in_out_regs, ret.reg_id(), 0, in_out_regs, Reg::kIdBad, 0));
}
}
else {
return make_error(Error::kInvalidAssignment);
}
}
return Error::kOk;
}
// x86::X86RAPass - Construction & Destruction
// ===========================================
X86RAPass::X86RAPass(BaseCompiler& cc) noexcept
: BaseRAPass(cc) { _emit_helper_ptr = &_emit_helper; }
X86RAPass::~X86RAPass() noexcept {}
// x86::X86RAPass - OnInit & OnDone
// ================================
void X86RAPass::on_init() noexcept {
Arch arch = cc().arch();
uint32_t base_reg_count = Environment::is_32bit(arch) ? 8u : 16u;
uint32_t simd_reg_count = base_reg_count;
if (Environment::is_64bit(arch) && _func->frame().is_avx512_enabled()) {
simd_reg_count = 32u;
}
_emit_helper.reset(&_cb, _func->frame().is_avx_enabled(), _func->frame().is_avx512_enabled());
_arch_traits = &ArchTraits::by_arch(arch);
_phys_reg_count.set(RegGroup::kGp, base_reg_count);
_phys_reg_count.set(RegGroup::kVec, simd_reg_count);
_phys_reg_count.set(RegGroup::kMask, 8);
_phys_reg_count.set(RegGroup::kX86_MM, 8);
_build_phys_index();
_available_regs[RegGroup::kGp] = Support::lsb_mask<RegMask>(_phys_reg_count.get(RegGroup::kGp));
_available_regs[RegGroup::kVec] = Support::lsb_mask<RegMask>(_phys_reg_count.get(RegGroup::kVec));
_available_regs[RegGroup::kMask] = Support::lsb_mask<RegMask>(_phys_reg_count.get(RegGroup::kMask)) ^ 1u;
_available_regs[RegGroup::kX86_MM] = Support::lsb_mask<RegMask>(_phys_reg_count.get(RegGroup::kX86_MM));
_scratch_reg_indexes[0] = uint8_t(Gp::kIdCx);
_scratch_reg_indexes[1] = uint8_t(base_reg_count - 1);
const FuncFrame& frame = _func->frame();
// The architecture specific setup makes implicitly all registers available. So
// make unavailable all registers that are special and cannot be used in general.
bool has_fp = frame.has_preserved_fp();
make_unavailable(RegGroup::kGp, Gp::kIdSp); // ESP|RSP used as a stack-pointer (SP).
if (has_fp) {
make_unavailable(RegGroup::kGp, Gp::kIdBp); // EBP|RBP used as a frame-pointer (FP).
}
make_unavailable(frame._unavailable_regs);
_sp = cc().zsp();
_fp = cc().zbp();
}
void X86RAPass::on_done() noexcept {}
// x86::X86RAPass - BuildCFG
// =========================
Error X86RAPass::build_cfg_nodes() noexcept {
return RACFGBuilder(*this).run();
}
// x86::X86RAPass - Rewrite
// ========================
static InstId transform_vex_to_evex(InstId inst_id) {
switch (inst_id) {
case Inst::kIdVbroadcastf128: return Inst::kIdVbroadcastf32x4;
case Inst::kIdVbroadcasti128: return Inst::kIdVbroadcasti32x4;
case Inst::kIdVextractf128: return Inst::kIdVextractf32x4;
case Inst::kIdVextracti128: return Inst::kIdVextracti32x4;
case Inst::kIdVinsertf128: return Inst::kIdVinsertf32x4;
case Inst::kIdVinserti128: return Inst::kIdVinserti32x4;
case Inst::kIdVmovdqa: return Inst::kIdVmovdqa32;
case Inst::kIdVmovdqu: return Inst::kIdVmovdqu32;
case Inst::kIdVpand: return Inst::kIdVpandd;
case Inst::kIdVpandn: return Inst::kIdVpandnd;
case Inst::kIdVpor: return Inst::kIdVpord;
case Inst::kIdVpxor: return Inst::kIdVpxord;
case Inst::kIdVroundpd: return Inst::kIdVrndscalepd;
case Inst::kIdVroundps: return Inst::kIdVrndscaleps;
case Inst::kIdVroundsd: return Inst::kIdVrndscalesd;
case Inst::kIdVroundss: return Inst::kIdVrndscaless;
default:
// This should never happen as only transformable instructions should go this path.
ASMJIT_ASSERT(false);
return 0;
}
}
ASMJIT_FAVOR_SPEED Error X86RAPass::rewrite() noexcept {
const size_t virt_count = cc()._virt_regs.size();
return rewrite_iterate([&](BaseNode* node, BaseNode* stop, RABlock* block) noexcept -> Error {
while (node != stop) {
BaseNode* next = node->next();
if (node->is_inst()) {
InstNode* inst = node->as<InstNode>();
RAInst* ra_inst = node->pass_data<RAInst>();
Span<Operand> operands = inst->operands();
// Rewrite virtual registers into physical registers.
if (ra_inst) {
// This data is allocated by Arena passed to `run_on_function()`, which will be reset after the RA pass finishes.
// So reset this data to prevent having a dead pointer after the RA pass is complete.
node->reset_pass_data();
// If the instruction contains pass data (ra_inst) then it was a subject for register allocation and must be
// rewritten to use physical regs.
const RATiedReg* tied_regs = ra_inst->tied_regs();
uint32_t tied_count = ra_inst->tied_count();
RegMask combined_reg_ids = 0;
for (uint32_t i = 0; i < tied_count; i++) {
const RATiedReg& tied_reg = tied_regs[i];
Support::BitWordIterator<uint32_t> use_it(tied_reg.use_rewrite_mask());
if (use_it.has_next()) {
uint32_t use_id = tied_reg.use_id();
do {
inst->_rewrite_id_at_index(use_it.next(), use_id);
} while (use_it.has_next());
combined_reg_ids |= use_id;
}
Support::BitWordIterator<uint32_t> out_it(tied_reg.out_rewrite_mask());
if (out_it.has_next()) {
uint32_t out_id = tied_reg.out_id();
do {
inst->_rewrite_id_at_index(out_it.next(), out_id);
} while (out_it.has_next());
combined_reg_ids |= out_id;
}
}
// If one operand was rewritten from Reg to Mem, we have to ensure that we are using the correct instruction.
if (ra_inst->is_reg_to_mem_patched()) {
switch (inst->inst_id()) {
case Inst::kIdKmovb: {
if (operands[0].is_gp() && operands[1].is_mem()) {
// Transform from [V]MOVD to MOV.
operands[1].as<Mem>().set_size(1);
inst->set_inst_id(Inst::kIdMovzx);
}
break;
}
case Inst::kIdVmovw: {
if (operands[0].is_gp() && operands[1].is_mem()) {
// Transform from [V]MOVD to MOV.
operands[1].as<Mem>().set_size(2);
inst->set_inst_id(Inst::kIdMovzx);
}
break;
}
case Inst::kIdMovd:
case Inst::kIdVmovd:
case Inst::kIdKmovd: {
if (operands[0].is_gp() && operands[1].is_mem()) {
// Transform from [V]MOVD to MOV.
operands[1].as<Mem>().set_size(4);
inst->set_inst_id(Inst::kIdMov);
}
break;
}
case Inst::kIdMovq:
case Inst::kIdVmovq:
case Inst::kIdKmovq: {
if (operands[0].is_gp() && operands[1].is_mem()) {
// Transform from [V]MOVQ to MOV.
operands[1].as<Mem>().set_size(8);
inst->set_inst_id(Inst::kIdMov);
}
break;
}
default:
break;
}
}
// Transform VEX instruction to EVEX when necessary.
if (ra_inst->is_transformable()) {
if (combined_reg_ids >= 16u) {
inst->set_inst_id(transform_vex_to_evex(inst->inst_id()));
}
}
// Remove moves that do not do anything.
//
// Usually these moves are inserted during code generation and originally they used different registers. If RA
// allocated these into the same register such redundant mov would appear.
if (ra_inst->has_inst_rw_flag(InstRWFlags::kMovOp) && !inst->has_extra_reg()) {
if (operands.size() == 2u) {
if (operands[0] == operands[1]) {
cc().remove_node(node);
node = next;
continue;
}
}
}
if (ASMJIT_UNLIKELY(node->type() != NodeType::kInst)) {
// FuncRet terminates the flow, it must either be removed if the exit label is next to it (optimization) or
// patched to an architecture dependent jump instruction that jumps to the function's exit before the epilog.
if (node->type() == NodeType::kFuncRet) {
if (!is_next_to(node, _func->exit_node())) {
cc().set_cursor(node->prev());
ASMJIT_PROPAGATE(emit_jump(_func->exit_node()->label()));
}
BaseNode* prev = node->prev();
cc().remove_node(node);
if (block) {
block->set_last(prev);
}
}
}
}
// Rewrite stack slot addresses.
for (Operand& op : operands) {
if (op.is_mem()) {
BaseMem& mem = op.as<BaseMem>();
if (mem.is_reg_home()) {
uint32_t virt_index = Operand::virt_id_to_index(mem.base_id());
if (ASMJIT_UNLIKELY(virt_index >= virt_count)) {
return make_error(Error::kInvalidVirtId);
}
VirtReg* virt_reg = cc().virt_reg_by_index(virt_index);
RAWorkReg* work_reg = virt_reg->work_reg();
ASMJIT_ASSERT(work_reg != nullptr);
RAStackSlot* slot = work_reg->stack_slot();
int32_t offset = slot->offset();
mem._set_base(_sp.reg_type(), slot->base_reg_id());
mem.clear_reg_home();
mem.add_offset_lo32(offset);
}
}
}
}
node = next;
}
return Error::kOk;
});
}
// x86::X86RAPass - OnEmit
// =======================
Error X86RAPass::emit_move(RAWorkReg* work_reg, uint32_t dst_phys_id, uint32_t src_phys_id) noexcept {
Reg dst(work_reg->signature(), dst_phys_id);
Reg src(work_reg->signature(), src_phys_id);
const char* comment = nullptr;
#ifndef ASMJIT_NO_LOGGING
if (has_diagnostic_option(DiagnosticOptions::kRAAnnotate)) {
_tmp_string.clear();
Formatter::format_virt_reg_name_with_prefix(_tmp_string, "<MOVE> ", 7u, work_reg->virt_reg());
comment = _tmp_string.data();
}
#endif
return _emit_helper.emit_reg_move(dst, src, work_reg->type_id(), comment);
}
Error X86RAPass::emit_swap(RAWorkReg* a_reg, uint32_t a_phys_id, RAWorkReg* b_reg, uint32_t b_phys_id) noexcept {
bool is_64bit = Support::max(a_reg->type_id(), b_reg->type_id()) >= TypeId::kInt64;
OperandSignature sign = is_64bit ? OperandSignature{RegTraits<RegType::kGp64>::kSignature}
: OperandSignature{RegTraits<RegType::kGp32>::kSignature};
#ifndef ASMJIT_NO_LOGGING
if (has_diagnostic_option(DiagnosticOptions::kRAAnnotate)) {
_tmp_string.clear();
Formatter::format_virt_reg_name_with_prefix(_tmp_string, "<SWAP> ", 7u, a_reg->virt_reg());
Formatter::format_virt_reg_name_with_prefix(_tmp_string, ", " , 2u, b_reg->virt_reg());
cc().set_inline_comment(_tmp_string.data());
}
#endif
return cc().emit(Inst::kIdXchg, Reg(sign, a_phys_id), Reg(sign, b_phys_id));
}
Error X86RAPass::emit_load(RAWorkReg* work_reg, uint32_t dst_phys_id) noexcept {
Reg dst_reg(work_reg->signature(), dst_phys_id);
BaseMem src_mem(work_reg_as_mem(work_reg));
const char* comment = nullptr;
#ifndef ASMJIT_NO_LOGGING
if (has_diagnostic_option(DiagnosticOptions::kRAAnnotate)) {
_tmp_string.clear();
Formatter::format_virt_reg_name_with_prefix(_tmp_string, "<LOAD> ", 7u, work_reg->virt_reg());
comment = _tmp_string.data();
}
#endif
return _emit_helper.emit_reg_move(dst_reg, src_mem, work_reg->type_id(), comment);
}
Error X86RAPass::emit_save(RAWorkReg* work_reg, uint32_t src_phys_id) noexcept {
BaseMem dst_mem(work_reg_as_mem(work_reg));
Reg src_reg(work_reg->signature(), src_phys_id);
const char* comment = nullptr;
#ifndef ASMJIT_NO_LOGGING
if (has_diagnostic_option(DiagnosticOptions::kRAAnnotate)) {
_tmp_string.clear();
Formatter::format_virt_reg_name_with_prefix(_tmp_string, "<SAVE> ", 7u, work_reg->virt_reg());
comment = _tmp_string.data();
}
#endif
return _emit_helper.emit_reg_move(dst_mem, src_reg, work_reg->type_id(), comment);
}
Error X86RAPass::emit_jump(const Label& label) noexcept {
return cc().jmp(label);
}
Error X86RAPass::emit_pre_call(InvokeNode* invoke_node) noexcept {
if (invoke_node->detail().has_var_args() && cc().is_64bit()) {
const FuncDetail& fd = invoke_node->detail();
uint32_t arg_count = invoke_node->arg_count();
switch (invoke_node->detail().call_conv().id()) {
case CallConvId::kX64SystemV: {
// AL register contains the number of arguments passed in XMM register(s).
uint32_t n = 0;
for (uint32_t arg_index = 0; arg_index < arg_count; arg_index++) {
const FuncValuePack& arg_pack = fd.arg_pack(arg_index);
for (uint32_t value_index = 0; value_index < Globals::kMaxValuePack; value_index++) {
const FuncValue& arg = arg_pack[value_index];
if (!arg) {
break;
}
if (arg.is_reg() && RegUtils::group_of(arg.reg_type()) == RegGroup::kVec) {
n++;
}
}
}
if (!n) {
ASMJIT_PROPAGATE(cc().xor_(eax, eax));
}
else {
ASMJIT_PROPAGATE(cc().mov(eax, n));
}
break;
}
case CallConvId::kX64Windows: {
// Each double-precision argument passed in XMM must be also passed in GP.
for (uint32_t arg_index = 0; arg_index < arg_count; arg_index++) {
const FuncValuePack& arg_pack = fd.arg_pack(arg_index);
for (uint32_t value_index = 0; value_index < Globals::kMaxValuePack; value_index++) {
const FuncValue& arg = arg_pack[value_index];
if (!arg) {
break;
}
if (arg.is_reg() && RegUtils::group_of(arg.reg_type()) == RegGroup::kVec) {
Gp dst = gpq(fd.call_conv().passed_order(RegGroup::kGp)[arg_index]);
Vec src = xmm(arg.reg_id());
ASMJIT_PROPAGATE(cc().emit(_emit_helper.ids().movq(), dst, src));
}
}
}
break;
}
default:
return make_error(Error::kInvalidState);
}
}
return Error::kOk;
}
ASMJIT_END_SUB_NAMESPACE
#endif // !ASMJIT_NO_X86 && !ASMJIT_NO_COMPILER
Reference in New Issue View Git Blame Copy Permalink