admin/asmjit-asmjit
1
0
Fork 0
mirror of https://github.com/asmjit/asmjit synced 2026-06-08 13:13:30 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
80098456b6b84650e64a26d87de3f7c0fe4d499d
asmjit-asmjit/src/asmjit/x86/x86internal.cpp
T

1634 lines
56 KiB
C++
Raw Blame History

// AsmJit - Machine code generation for C++
//
// * Official AsmJit Home Page: https://asmjit.com
// * Official Github Repository: https://github.com/asmjit/asmjit
//
// Copyright (c) 2008-2020 The AsmJit Authors
//
// This software is provided 'as-is', without any express or implied
// warranty. In no event will the authors be held liable for any damages
// arising from the use of this software.
//
// Permission is granted to anyone to use this software for any purpose,
// including commercial applications, and to alter it and redistribute it
// freely, subject to the following restrictions:
//
// 1. The origin of this software must not be misrepresented; you must not
// claim that you wrote the original software. If you use this software
// in a product, an acknowledgment in the product documentation would be
// appreciated but is not required.
// 2. Altered source versions must be plainly marked as such, and must not be
// misrepresented as being the original software.
// 3. This notice may not be removed or altered from any source distribution.
#include "../core/api-build_p.h"
#ifdef ASMJIT_BUILD_X86
#include "../core/logging.h"
#include "../core/string.h"
#include "../core/support.h"
#include "../core/type.h"
#include "../x86/x86internal_p.h"
// Can be used for debugging...
// #define ASMJIT_DUMP_ARGS_ASSIGNMENT
ASMJIT_BEGIN_SUB_NAMESPACE(x86)
// ============================================================================
// [asmjit::X86Internal - Helpers]
// ============================================================================
static ASMJIT_INLINE uint32_t x86GetXmmMovInst(const FuncFrame& frame) {
bool avx = frame.isAvxEnabled();
bool aligned = frame.hasAlignedVecSR();
return aligned ? (avx ? Inst::kIdVmovaps : Inst::kIdMovaps)
: (avx ? Inst::kIdVmovups : Inst::kIdMovups);
}
static ASMJIT_INLINE uint32_t x86VecTypeIdToRegType(uint32_t typeId) noexcept {
return typeId <= Type::_kIdVec128End ? Reg::kTypeXmm :
typeId <= Type::_kIdVec256End ? Reg::kTypeYmm : Reg::kTypeZmm;
}
//! Converts `size` to a 'kmov?' instructio.
static inline uint32_t x86KmovFromSize(uint32_t size) noexcept {
switch (size) {
case 1: return Inst::kIdKmovb;
case 2: return Inst::kIdKmovw;
case 4: return Inst::kIdKmovd;
case 8: return Inst::kIdKmovq;
default: return Inst::kIdNone;
}
}
// ============================================================================
// [asmjit::X86Internal - FuncDetail]
// ============================================================================
ASMJIT_FAVOR_SIZE Error X86Internal::initFuncDetail(FuncDetail& func, const FuncSignature& sign, uint32_t gpSize) noexcept {
ASMJIT_UNUSED(sign);
const CallConv& cc = func.callConv();
uint32_t archId = cc.archId();
uint32_t stackOffset = cc._spillZoneSize;
uint32_t i;
uint32_t argCount = func.argCount();
if (func.retCount() != 0) {
uint32_t typeId = func._rets[0].typeId();
switch (typeId) {
case Type::kIdI64:
case Type::kIdU64: {
if (archId == ArchInfo::kIdX86) {
// Convert a 64-bit return value to two 32-bit return values.
func._retCount = 2;
typeId -= 2;
// 64-bit value is returned in EDX:EAX on X86.
func._rets[0].initReg(Reg::kTypeGpd, Gp::kIdAx, typeId);
func._rets[1].initReg(Reg::kTypeGpd, Gp::kIdDx, typeId);
break;
}
else {
func._rets[0].initReg(Reg::kTypeGpq, Gp::kIdAx, typeId);
}
break;
}
case Type::kIdI8:
case Type::kIdI16:
case Type::kIdI32: {
func._rets[0].initReg(Reg::kTypeGpd, Gp::kIdAx, Type::kIdI32);
break;
}
case Type::kIdU8:
case Type::kIdU16:
case Type::kIdU32: {
func._rets[0].initReg(Reg::kTypeGpd, Gp::kIdAx, Type::kIdU32);
break;
}
case Type::kIdF32:
case Type::kIdF64: {
uint32_t regType = (archId == ArchInfo::kIdX86) ? Reg::kTypeSt : Reg::kTypeXmm;
func._rets[0].initReg(regType, 0, typeId);
break;
}
case Type::kIdF80: {
// 80-bit floats are always returned by FP0.
func._rets[0].initReg(Reg::kTypeSt, 0, typeId);
break;
}
case Type::kIdMmx32:
case Type::kIdMmx64: {
// MM registers are returned through XMM or GPQ (Win64).
uint32_t regType = Reg::kTypeMm;
if (archId != ArchInfo::kIdX86)
regType = cc.strategy() == CallConv::kStrategyDefault ? Reg::kTypeXmm : Reg::kTypeGpq;
func._rets[0].initReg(regType, 0, typeId);
break;
}
default: {
func._rets[0].initReg(x86VecTypeIdToRegType(typeId), 0, typeId);
break;
}
}
}
if (cc.strategy() == CallConv::kStrategyDefault) {
uint32_t gpzPos = 0;
uint32_t vecPos = 0;
for (i = 0; i < argCount; i++) {
FuncValue& arg = func._args[i];
uint32_t typeId = arg.typeId();
if (Type::isInt(typeId)) {
uint32_t regId = gpzPos < CallConv::kMaxRegArgsPerGroup ? cc._passedOrder[Reg::kGroupGp].id[gpzPos] : uint8_t(BaseReg::kIdBad);
if (regId != BaseReg::kIdBad) {
uint32_t regType = (typeId <= Type::kIdU32) ? Reg::kTypeGpd : Reg::kTypeGpq;
arg.assignRegData(regType, regId);
func.addUsedRegs(Reg::kGroupGp, Support::bitMask(regId));
gpzPos++;
}
else {
uint32_t size = Support::max<uint32_t>(Type::sizeOf(typeId), gpSize);
arg.assignStackOffset(int32_t(stackOffset));
stackOffset += size;
}
continue;
}
if (Type::isFloat(typeId) || Type::isVec(typeId)) {
uint32_t regId = vecPos < CallConv::kMaxRegArgsPerGroup ? cc._passedOrder[Reg::kGroupVec].id[vecPos] : uint8_t(BaseReg::kIdBad);
// If this is a float, but `floatByVec` is false, we have to pass by stack.
if (Type::isFloat(typeId) && !cc.hasFlag(CallConv::kFlagPassFloatsByVec))
regId = BaseReg::kIdBad;
if (regId != BaseReg::kIdBad) {
arg.initTypeId(typeId);
arg.assignRegData(x86VecTypeIdToRegType(typeId), regId);
func.addUsedRegs(Reg::kGroupVec, Support::bitMask(regId));
vecPos++;
}
else {
uint32_t size = Type::sizeOf(typeId);
arg.assignStackOffset(int32_t(stackOffset));
stackOffset += size;
}
continue;
}
}
}
if (cc.strategy() == CallConv::kStrategyWin64) {
for (i = 0; i < argCount; i++) {
FuncValue& arg = func._args[i];
uint32_t typeId = arg.typeId();
uint32_t size = Type::sizeOf(typeId);
if (Type::isInt(typeId) || Type::isMmx(typeId)) {
uint32_t regId = i < CallConv::kMaxRegArgsPerGroup ? cc._passedOrder[Reg::kGroupGp].id[i] : uint8_t(BaseReg::kIdBad);
if (regId != BaseReg::kIdBad) {
uint32_t regType = (size <= 4 && !Type::isMmx(typeId)) ? Reg::kTypeGpd : Reg::kTypeGpq;
arg.assignRegData(regType, regId);
func.addUsedRegs(Reg::kGroupGp, Support::bitMask(regId));
}
else {
arg.assignStackOffset(int32_t(stackOffset));
stackOffset += gpSize;
}
continue;
}
if (Type::isFloat(typeId) || Type::isVec(typeId)) {
uint32_t regId = BaseReg::kIdBad;
if (i < CallConv::kMaxRegArgsPerGroup)
regId = cc._passedOrder[Reg::kGroupVec].id[i];
if (regId != BaseReg::kIdBad && (Type::isFloat(typeId) || cc.hasFlag(CallConv::kFlagVectorCall))) {
uint32_t regType = x86VecTypeIdToRegType(typeId);
arg.assignRegData(regType, regId);
func.addUsedRegs(Reg::kGroupVec, Support::bitMask(regId));
}
else {
arg.assignStackOffset(int32_t(stackOffset));
stackOffset += 8; // Always 8 bytes (float/double).
}
continue;
}
}
}
func._argStackSize = stackOffset;
return kErrorOk;
}
// ============================================================================
// [asmjit::X86FuncArgsContext]
// ============================================================================
static RegInfo x86GetRegForMemToMemMove(uint32_t archId, uint32_t dstTypeId, uint32_t srcTypeId) noexcept {
uint32_t dstSize = Type::sizeOf(dstTypeId);
uint32_t srcSize = Type::sizeOf(srcTypeId);
uint32_t maxSize = Support::max<uint32_t>(dstSize, srcSize);
uint32_t gpSize = archId == ArchInfo::kIdX86 ? 4 : 8;
uint32_t signature = 0;
if (maxSize <= gpSize || (Type::isInt(dstTypeId) && Type::isInt(srcTypeId)))
signature = maxSize <= 4 ? Gpd::kSignature : Gpq::kSignature;
else if (maxSize <= 16)
signature = Xmm::kSignature;
else if (maxSize <= 32)
signature = Ymm::kSignature;
else if (maxSize <= 64)
signature = Zmm::kSignature;
return RegInfo { signature };
}
// Used by both `argsToFuncFrame()` and `emitArgsAssignment()`.
class X86FuncArgsContext {
public:
enum VarId : uint32_t {
kVarIdNone = 0xFF
};
//! Contains information about a single argument or SA register that may need shuffling.
struct Var {
inline void init(const FuncValue& cur_, const FuncValue& out_) noexcept {
cur = cur_;
out = out_;
}
//! Reset the value to its unassigned state.
inline void reset() noexcept {
cur.reset();
out.reset();
}
inline bool isDone() const noexcept { return cur.isDone(); }
inline void markDone() noexcept { cur.addFlags(FuncValue::kFlagIsDone); }
FuncValue cur;
FuncValue out;
};
struct WorkData {
inline void reset() noexcept {
_archRegs = 0;
_workRegs = 0;
_usedRegs = 0;
_assignedRegs = 0;
_dstRegs = 0;
_dstShuf = 0;
_numSwaps = 0;
_numStackArgs = 0;
memset(_reserved, 0, sizeof(_reserved));
memset(_physToVarId, kVarIdNone, 32);
}
inline bool isAssigned(uint32_t regId) const noexcept {
ASMJIT_ASSERT(regId < 32);
return Support::bitTest(_assignedRegs, regId);
}
inline void assign(uint32_t varId, uint32_t regId) noexcept {
ASMJIT_ASSERT(!isAssigned(regId));
ASMJIT_ASSERT(_physToVarId[regId] == kVarIdNone);
_physToVarId[regId] = uint8_t(varId);
_assignedRegs ^= Support::bitMask(regId);
}
inline void reassign(uint32_t varId, uint32_t newId, uint32_t oldId) noexcept {
ASMJIT_ASSERT( isAssigned(oldId));
ASMJIT_ASSERT(!isAssigned(newId));
ASMJIT_ASSERT(_physToVarId[oldId] == varId);
ASMJIT_ASSERT(_physToVarId[newId] == kVarIdNone);
_physToVarId[oldId] = uint8_t(kVarIdNone);
_physToVarId[newId] = uint8_t(varId);
_assignedRegs ^= Support::bitMask(newId) ^ Support::bitMask(oldId);
}
inline void swap(uint32_t aVarId, uint32_t aRegId, uint32_t bVarId, uint32_t bRegId) noexcept {
ASMJIT_ASSERT(isAssigned(aRegId));
ASMJIT_ASSERT(isAssigned(bRegId));
ASMJIT_ASSERT(_physToVarId[aRegId] == aVarId);
ASMJIT_ASSERT(_physToVarId[bRegId] == bVarId);
_physToVarId[aRegId] = uint8_t(bVarId);
_physToVarId[bRegId] = uint8_t(aVarId);
}
inline void unassign(uint32_t varId, uint32_t regId) noexcept {
ASMJIT_UNUSED(varId);
ASMJIT_ASSERT(isAssigned(regId));
ASMJIT_ASSERT(_physToVarId[regId] == varId);
_physToVarId[regId] = uint8_t(kVarIdNone);
_assignedRegs ^= Support::bitMask(regId);
}
inline uint32_t archRegs() const noexcept { return _archRegs; }
inline uint32_t workRegs() const noexcept { return _workRegs; }
inline uint32_t usedRegs() const noexcept { return _usedRegs; }
inline uint32_t assignedRegs() const noexcept { return _assignedRegs; }
inline uint32_t dstRegs() const noexcept { return _dstRegs; }
inline uint32_t availableRegs() const noexcept { return _workRegs & ~_assignedRegs; }
uint32_t _archRegs; //!< All allocable registers provided by the architecture.
uint32_t _workRegs; //!< All registers that can be used by the shuffler.
uint32_t _usedRegs; //!< Registers used by the shuffler (all).
uint32_t _assignedRegs; //!< Assigned registers.
uint32_t _dstRegs; //!< Destination registers assigned to arguments or SA.
uint32_t _dstShuf; //!< Destination registers that require shuffling.
uint8_t _numSwaps; //!< Number of register swaps.
uint8_t _numStackArgs; //!< Number of stack loads.
uint8_t _reserved[6]; //!< Reserved (only used as padding).
uint8_t _physToVarId[32]; //!< Physical ID to variable ID mapping.
};
uint8_t _archId;
bool _hasStackSrc; //!< Has arguments passed via stack (SRC).
bool _hasPreservedFP; //!< Has preserved frame-pointer (FP).
uint8_t _stackDstMask; //!< Has arguments assigned to stack (DST).
uint8_t _regSwapsMask; //!< Register swap groups (bit-mask).
uint8_t _saVarId;
uint32_t _varCount;
WorkData _workData[BaseReg::kGroupVirt];
Var _vars[kFuncArgCountLoHi + 1];
X86FuncArgsContext() noexcept;
inline uint32_t archId() const noexcept { return _archId; }
inline uint32_t varCount() const noexcept { return _varCount; }
inline Var& var(uint32_t varId) noexcept { return _vars[varId]; }
inline const Var& var(uint32_t varId) const noexcept { return _vars[varId]; }
inline uint32_t indexOf(const Var* var) const noexcept { return uint32_t((size_t)(var - _vars)); }
Error initWorkData(const FuncFrame& frame, const FuncArgsAssignment& args) noexcept;
Error markScratchRegs(FuncFrame& frame) noexcept;
Error markDstRegsDirty(FuncFrame& frame) noexcept;
Error markStackArgsReg(FuncFrame& frame) noexcept;
};
X86FuncArgsContext::X86FuncArgsContext() noexcept {
_archId = ArchInfo::kIdNone;
_varCount = 0;
_hasStackSrc = false;
_hasPreservedFP = false;
_stackDstMask = 0;
_regSwapsMask = 0;
_saVarId = kVarIdNone;
for (uint32_t group = 0; group < BaseReg::kGroupVirt; group++)
_workData[group].reset();
}
ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::initWorkData(const FuncFrame& frame, const FuncArgsAssignment& args) noexcept {
// The code has to be updated if this changes.
ASMJIT_ASSERT(BaseReg::kGroupVirt == 4);
uint32_t i;
const FuncDetail& func = *args.funcDetail();
// Initialize ArchType.
uint32_t archId = func.callConv().archId();
uint32_t archRegCount = (archId == ArchInfo::kIdX86) ? 8 : 16;
_archId = uint8_t(archId);
// Initialize `_archRegs`.
_workData[Reg::kGroupGp ]._archRegs = Support::lsbMask<uint32_t>(archRegCount) & ~Support::bitMask(Gp::kIdSp);
_workData[Reg::kGroupVec ]._archRegs = Support::lsbMask<uint32_t>(archRegCount);
_workData[Reg::kGroupMm ]._archRegs = Support::lsbMask<uint32_t>(8);
_workData[Reg::kGroupKReg]._archRegs = Support::lsbMask<uint32_t>(8);
if (frame.hasPreservedFP())
_workData[Reg::kGroupGp]._archRegs &= ~Support::bitMask(Gp::kIdBp);
// Extract information from all function arguments/assignments and build Var[] array.
uint32_t varId = 0;
for (i = 0; i < kFuncArgCountLoHi; i++) {
const FuncValue& dst_ = args.arg(i);
if (!dst_.isAssigned()) continue;
const FuncValue& src_ = func.arg(i);
if (ASMJIT_UNLIKELY(!src_.isAssigned()))
return DebugUtils::errored(kErrorInvalidState);
Var& var = _vars[varId];
var.init(src_, dst_);
FuncValue& src = var.cur;
FuncValue& dst = var.out;
uint32_t dstGroup = 0xFFFFFFFFu;
uint32_t dstId = BaseReg::kIdBad;
WorkData* dstWd = nullptr;
if (dst.isReg()) {
uint32_t dstType = dst.regType();
if (ASMJIT_UNLIKELY(dstType >= Reg::kTypeCount))
return DebugUtils::errored(kErrorInvalidRegType);
// Copy TypeId from source if the destination doesn't have it. The RA
// used by BaseCompiler would never leave TypeId undefined, but users
// of FuncAPI can just assign phys regs without specifying the type.
if (!dst.hasTypeId())
dst.setTypeId(Reg::typeIdOf(dst.regType()));
dstGroup = Reg::groupOf(dstType);
if (ASMJIT_UNLIKELY(dstGroup >= BaseReg::kGroupVirt))
return DebugUtils::errored(kErrorInvalidRegGroup);
dstWd = &_workData[dstGroup];
dstId = dst.regId();
if (ASMJIT_UNLIKELY(dstId >= 32 || !Support::bitTest(dstWd->archRegs(), dstId)))
return DebugUtils::errored(kErrorInvalidPhysId);
if (ASMJIT_UNLIKELY(Support::bitTest(dstWd->dstRegs(), dstId)))
return DebugUtils::errored(kErrorOverlappedRegs);
dstWd->_dstRegs |= Support::bitMask(dstId);
dstWd->_dstShuf |= Support::bitMask(dstId);
dstWd->_usedRegs |= Support::bitMask(dstId);
}
else {
if (!dst.hasTypeId())
dst.setTypeId(src.typeId());
RegInfo regInfo = x86GetRegForMemToMemMove(archId, dst.typeId(), src.typeId());
if (ASMJIT_UNLIKELY(!regInfo.isValid()))
return DebugUtils::errored(kErrorInvalidState);
_stackDstMask = uint8_t(_stackDstMask | Support::bitMask(regInfo.group()));
}
if (src.isReg()) {
uint32_t srcId = src.regId();
uint32_t srcGroup = Reg::groupOf(src.regType());
if (dstGroup == srcGroup) {
dstWd->assign(varId, srcId);
// The best case, register is allocated where it is expected to be.
if (dstId == srcId)
var.markDone();
}
else {
if (ASMJIT_UNLIKELY(srcGroup >= BaseReg::kGroupVirt))
return DebugUtils::errored(kErrorInvalidState);
WorkData& srcData = _workData[srcGroup];
srcData.assign(varId, srcId);
}
}
else {
if (dstWd)
dstWd->_numStackArgs++;
_hasStackSrc = true;
}
varId++;
}
// Initialize WorkData::workRegs.
for (i = 0; i < BaseReg::kGroupVirt; i++)
_workData[i]._workRegs = (_workData[i].archRegs() & (frame.dirtyRegs(i) | ~frame.preservedRegs(i))) | _workData[i].dstRegs() | _workData[i].assignedRegs();
// Create a variable that represents `SARegId` if necessary.
bool saRegRequired = _hasStackSrc && frame.hasDynamicAlignment() && !frame.hasPreservedFP();
WorkData& gpRegs = _workData[BaseReg::kGroupGp];
uint32_t saCurRegId = frame.saRegId();
uint32_t saOutRegId = args.saRegId();
if (saCurRegId != BaseReg::kIdBad) {
// Check if the provided `SARegId` doesn't collide with input registers.
if (ASMJIT_UNLIKELY(gpRegs.isAssigned(saCurRegId)))
return DebugUtils::errored(kErrorOverlappedRegs);
}
if (saOutRegId != BaseReg::kIdBad) {
// Check if the provided `SARegId` doesn't collide with argument assignments.
if (ASMJIT_UNLIKELY(Support::bitTest(gpRegs.dstRegs(), saOutRegId)))
return DebugUtils::errored(kErrorOverlappedRegs);
saRegRequired = true;
}
if (saRegRequired) {
uint32_t ptrTypeId = (archId == ArchInfo::kIdX86) ? Type::kIdU32 : Type::kIdU64;
uint32_t ptrRegType = (archId == ArchInfo::kIdX86) ? BaseReg::kTypeGp32 : BaseReg::kTypeGp64;
_saVarId = uint8_t(varId);
_hasPreservedFP = frame.hasPreservedFP();
Var& var = _vars[varId];
var.reset();
if (saCurRegId == BaseReg::kIdBad) {
if (saOutRegId != BaseReg::kIdBad && !gpRegs.isAssigned(saOutRegId)) {
saCurRegId = saOutRegId;
}
else {
uint32_t availableRegs = gpRegs.availableRegs();
if (!availableRegs)
availableRegs = gpRegs.archRegs() & ~gpRegs.workRegs();
if (ASMJIT_UNLIKELY(!availableRegs))
return DebugUtils::errored(kErrorNoMorePhysRegs);
saCurRegId = Support::ctz(availableRegs);
}
}
var.cur.initReg(ptrRegType, saCurRegId, ptrTypeId);
gpRegs.assign(varId, saCurRegId);
gpRegs._workRegs |= Support::bitMask(saCurRegId);
if (saOutRegId != BaseReg::kIdBad) {
var.out.initReg(ptrRegType, saOutRegId, ptrTypeId);
gpRegs._dstRegs |= Support::bitMask(saOutRegId);
gpRegs._workRegs |= Support::bitMask(saOutRegId);
}
else {
var.markDone();
}
varId++;
}
_varCount = varId;
// Detect register swaps.
for (varId = 0; varId < _varCount; varId++) {
Var& var = _vars[varId];
if (var.cur.isReg() && var.out.isReg()) {
uint32_t srcId = var.cur.regId();
uint32_t dstId = var.out.regId();
uint32_t group = Reg::groupOf(var.cur.regType());
if (group != Reg::groupOf(var.out.regType()))
continue;
WorkData& wd = _workData[group];
if (wd.isAssigned(dstId)) {
Var& other = _vars[wd._physToVarId[dstId]];
if (Reg::groupOf(other.out.regType()) == group && other.out.regId() == srcId) {
wd._numSwaps++;
_regSwapsMask = uint8_t(_regSwapsMask | Support::bitMask(group));
}
}
}
}
return kErrorOk;
}
ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::markDstRegsDirty(FuncFrame& frame) noexcept {
for (uint32_t i = 0; i < BaseReg::kGroupVirt; i++) {
WorkData& wd = _workData[i];
uint32_t regs = wd.usedRegs() | wd._dstShuf;
wd._workRegs |= regs;
frame.addDirtyRegs(i, regs);
}
return kErrorOk;
}
ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::markScratchRegs(FuncFrame& frame) noexcept {
uint32_t groupMask = 0;
// Handle stack to stack moves.
groupMask |= _stackDstMask;
// Handle register swaps.
groupMask |= _regSwapsMask & ~Support::bitMask(BaseReg::kGroupGp);
if (!groupMask)
return kErrorOk;
// selects one dirty register per affected group that can be used as a scratch register.
for (uint32_t group = 0; group < BaseReg::kGroupVirt; group++) {
if (Support::bitTest(groupMask, group)) {
WorkData& wd = _workData[group];
// Initially, pick some clobbered or dirty register.
uint32_t workRegs = wd.workRegs();
uint32_t regs = workRegs & ~(wd.usedRegs() | wd._dstShuf);
// If that didn't work out pick some register which is not in 'used'.
if (!regs) regs = workRegs & ~wd.usedRegs();
// If that didn't work out pick any other register that is allocable.
// This last resort case will, however, result in marking one more
// register dirty.
if (!regs) regs = wd.archRegs() & ~workRegs;
// If that didn't work out we will have to use XORs instead of MOVs.
if (!regs) continue;
uint32_t regMask = Support::blsi(regs);
wd._workRegs |= regMask;
frame.addDirtyRegs(group, regMask);
}
}
return kErrorOk;
}
ASMJIT_FAVOR_SIZE Error X86FuncArgsContext::markStackArgsReg(FuncFrame& frame) noexcept {
// TODO: Validate, improve...
if (_saVarId != kVarIdNone) {
const Var& var = _vars[_saVarId];
frame.setSARegId(var.cur.regId());
}
else if (frame.hasPreservedFP()) {
// Always EBP|RBP if the frame-pointer isn't omitted.
frame.setSARegId(Gp::kIdBp);
}
return kErrorOk;
}
// ============================================================================
// [asmjit::X86Internal - FrameLayout]
// ============================================================================
ASMJIT_FAVOR_SIZE Error X86Internal::initFuncFrame(FuncFrame& frame, const FuncDetail& func) noexcept {
uint32_t archId = func.callConv().archId();
// Initializing FuncFrame means making a copy of some properties of `func`.
// Properties like `_localStackSize` will be set by the user before the frame
// is finalized.
frame.reset();
frame._archId = uint8_t(archId);
frame._spRegId = Gp::kIdSp;
frame._saRegId = Gp::kIdBad;
uint32_t naturalStackAlignment = func.callConv().naturalStackAlignment();
uint32_t minDynamicAlignment = Support::max<uint32_t>(naturalStackAlignment, 16);
if (minDynamicAlignment == naturalStackAlignment)
minDynamicAlignment <<= 1;
frame._naturalStackAlignment = uint8_t(naturalStackAlignment);
frame._minDynamicAlignment = uint8_t(minDynamicAlignment);
frame._redZoneSize = uint8_t(func.redZoneSize());
frame._spillZoneSize = uint8_t(func.spillZoneSize());
frame._finalStackAlignment = uint8_t(frame._naturalStackAlignment);
if (func.hasFlag(CallConv::kFlagCalleePopsStack)) {
frame._calleeStackCleanup = uint16_t(func.argStackSize());
}
// Initial masks of dirty and preserved registers.
for (uint32_t group = 0; group < BaseReg::kGroupVirt; group++) {
frame._dirtyRegs[group] = func.usedRegs(group);
frame._preservedRegs[group] = func.preservedRegs(group);
}
// Exclude ESP/RSP - this register is never included in saved GP regs.
frame._preservedRegs[BaseReg::kGroupGp] &= ~Support::bitMask(Gp::kIdSp);
return kErrorOk;
}
ASMJIT_FAVOR_SIZE Error X86Internal::finalizeFuncFrame(FuncFrame& frame) noexcept {
uint32_t gpSize = frame.archId() == ArchInfo::kIdX86 ? 4 : 8;
// The final stack alignment must be updated accordingly to call and local stack alignments.
uint32_t stackAlignment = frame._finalStackAlignment;
ASMJIT_ASSERT(stackAlignment == Support::max(frame._naturalStackAlignment,
frame._callStackAlignment,
frame._localStackAlignment));
// TODO: Must be configurable.
uint32_t vecSize = 16;
bool hasFP = frame.hasPreservedFP();
bool hasDA = frame.hasDynamicAlignment();
// Include EBP|RBP if the function preserves the frame-pointer.
if (hasFP)
frame._dirtyRegs[Reg::kGroupGp] |= Support::bitMask(Gp::kIdBp);
// These two are identical if the function doesn't align its stack dynamically.
uint32_t saRegId = frame.saRegId();
if (saRegId == BaseReg::kIdBad)
saRegId = Gp::kIdSp;
// Fix stack arguments base-register from ESP|RSP to EBP|RBP in case it was
// not picked before and the function performs dynamic stack alignment.
if (hasDA && saRegId == Gp::kIdSp)
saRegId = Gp::kIdBp;
// Mark as dirty any register but ESP|RSP if used as SA pointer.
if (saRegId != Gp::kIdSp)
frame._dirtyRegs[Reg::kGroupGp] |= Support::bitMask(saRegId);
frame._spRegId = uint8_t(Gp::kIdSp);
frame._saRegId = uint8_t(saRegId);
// Setup stack size used to save preserved registers.
frame._gpSaveSize = uint16_t(Support::popcnt(frame.savedRegs(Reg::kGroupGp )) * gpSize);
frame._nonGpSaveSize = uint16_t(Support::popcnt(frame.savedRegs(Reg::kGroupVec )) * vecSize +
Support::popcnt(frame.savedRegs(Reg::kGroupMm )) * 8 +
Support::popcnt(frame.savedRegs(Reg::kGroupKReg)) * 8);
uint32_t v = 0; // The beginning of the stack frame relative to SP after prolog.
v += frame.callStackSize(); // Count 'callStackSize' <- This is used to call functions.
v = Support::alignUp(v, stackAlignment); // Align to function's stack alignment.
frame._localStackOffset = v; // Store 'localStackOffset' <- Function's local stack starts here.
v += frame.localStackSize(); // Count 'localStackSize' <- Function's local stack ends here.
// If the function's stack must be aligned, calculate the alignment necessary
// to store vector registers, and set `FuncFrame::kAttrAlignedVecSR` to inform
// PEI that it can use instructions that perform aligned stores/loads.
if (stackAlignment >= vecSize && frame._nonGpSaveSize) {
frame.addAttributes(FuncFrame::kAttrAlignedVecSR);
v = Support::alignUp(v, vecSize); // Align '_nonGpSaveOffset'.
}
frame._nonGpSaveOffset = v; // Store '_nonGpSaveOffset' <- Non-GP Save/Restore starts here.
v += frame._nonGpSaveSize; // Count '_nonGpSaveSize' <- Non-GP Save/Restore ends here.
// Calculate if dynamic alignment (DA) slot (stored as offset relative to SP) is required and its offset.
if (hasDA && !hasFP) {
frame._daOffset = v; // Store 'daOffset' <- DA pointer would be stored here.
v += gpSize; // Count 'daOffset'.
}
else {
frame._daOffset = FuncFrame::kTagInvalidOffset;
}
// The return address should be stored after GP save/restore regs. It has
// the same size as `gpSize` (basically the native register/pointer size).
// We don't adjust it now as `v` now contains the exact size that the
// function requires to adjust (call frame + stack frame, vec stack size).
// The stack (if we consider this size) is misaligned now, as it's always
// aligned before the function call - when `call()` is executed it pushes
// the current EIP|RIP onto the stack, and misaligns it by 12 or 8 bytes
// (depending on the architecture). So count number of bytes needed to align
// it up to the function's CallFrame (the beginning).
if (v || frame.hasFuncCalls())
v += Support::alignUpDiff(v + frame.gpSaveSize() + gpSize, stackAlignment);
frame._gpSaveOffset = v; // Store 'gpSaveOffset' <- Function's GP Save/Restore starts here.
frame._stackAdjustment = v; // Store 'stackAdjustment' <- SA used by 'add zsp, SA' and 'sub zsp, SA'.
v += frame._gpSaveSize; // Count 'gpSaveSize' <- Function's GP Save/Restore ends here.
v += gpSize; // Count 'ReturnAddress' <- As CALL pushes onto stack.
// If the function performs dynamic stack alignment then the stack-adjustment must be aligned.
if (hasDA)
frame._stackAdjustment = Support::alignUp(frame._stackAdjustment, stackAlignment);
uint32_t saInvOff = FuncFrame::kTagInvalidOffset;
uint32_t saTmpOff = gpSize + frame._gpSaveSize;
// Calculate where the function arguments start relative to SP.
frame._saOffsetFromSP = hasDA ? saInvOff : v;
// Calculate where the function arguments start relative to FP or user-provided register.
frame._saOffsetFromSA = hasFP ? gpSize * 2 // Return address + frame pointer.
: saTmpOff; // Return address + all saved GP regs.
return kErrorOk;
}
// ============================================================================
// [asmjit::X86Internal - ArgsToFrameInfo]
// ============================================================================
ASMJIT_FAVOR_SIZE Error X86Internal::argsToFuncFrame(const FuncArgsAssignment& args, FuncFrame& frame) noexcept {
X86FuncArgsContext ctx;
ASMJIT_PROPAGATE(ctx.initWorkData(frame, args));
ASMJIT_PROPAGATE(ctx.markDstRegsDirty(frame));
ASMJIT_PROPAGATE(ctx.markScratchRegs(frame));
ASMJIT_PROPAGATE(ctx.markStackArgsReg(frame));
return kErrorOk;
}
// ============================================================================
// [asmjit::X86Internal - Emit Helpers]
// ============================================================================
ASMJIT_FAVOR_SIZE Error X86Internal::emitRegMove(Emitter* emitter,
const Operand_& dst_,
const Operand_& src_, uint32_t typeId, bool avxEnabled, const char* comment) {
// Invalid or abstract TypeIds are not allowed.
ASMJIT_ASSERT(Type::isValid(typeId) && !Type::isAbstract(typeId));
Operand dst(dst_);
Operand src(src_);
uint32_t instId = Inst::kIdNone;
uint32_t memFlags = 0;
uint32_t overrideMemSize = 0;
enum MemFlags : uint32_t {
kDstMem = 0x1,
kSrcMem = 0x2
};
// Detect memory operands and patch them to have the same size as the register.
// BaseCompiler always sets memory size of allocs and spills, so it shouldn't
// be really necessary, however, after this function was separated from Compiler
// it's better to make sure that the size is always specified, as we can use
// 'movzx' and 'movsx' that rely on it.
if (dst.isMem()) { memFlags |= kDstMem; dst.as<Mem>().setSize(src.size()); }
if (src.isMem()) { memFlags |= kSrcMem; src.as<Mem>().setSize(dst.size()); }
switch (typeId) {
case Type::kIdI8:
case Type::kIdU8:
case Type::kIdI16:
case Type::kIdU16:
// Special case - 'movzx' load.
if (memFlags & kSrcMem) {
instId = Inst::kIdMovzx;
dst.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
}
else if (!memFlags) {
// Change both destination and source registers to GPD (safer, no dependencies).
dst.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
src.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
}
ASMJIT_FALLTHROUGH;
case Type::kIdI32:
case Type::kIdU32:
case Type::kIdI64:
case Type::kIdU64:
instId = Inst::kIdMov;
break;
case Type::kIdMmx32:
instId = Inst::kIdMovd;
if (memFlags) break;
ASMJIT_FALLTHROUGH;
case Type::kIdMmx64 : instId = Inst::kIdMovq ; break;
case Type::kIdMask8 : instId = Inst::kIdKmovb; break;
case Type::kIdMask16: instId = Inst::kIdKmovw; break;
case Type::kIdMask32: instId = Inst::kIdKmovd; break;
case Type::kIdMask64: instId = Inst::kIdKmovq; break;
default: {
uint32_t elementTypeId = Type::baseOf(typeId);
if (Type::isVec32(typeId) && memFlags) {
overrideMemSize = 4;
if (elementTypeId == Type::kIdF32)
instId = avxEnabled ? Inst::kIdVmovss : Inst::kIdMovss;
else
instId = avxEnabled ? Inst::kIdVmovd : Inst::kIdMovd;
break;
}
if (Type::isVec64(typeId) && memFlags) {
overrideMemSize = 8;
if (elementTypeId == Type::kIdF64)
instId = avxEnabled ? Inst::kIdVmovsd : Inst::kIdMovsd;
else
instId = avxEnabled ? Inst::kIdVmovq : Inst::kIdMovq;
break;
}
if (elementTypeId == Type::kIdF32)
instId = avxEnabled ? Inst::kIdVmovaps : Inst::kIdMovaps;
else if (elementTypeId == Type::kIdF64)
instId = avxEnabled ? Inst::kIdVmovapd : Inst::kIdMovapd;
else if (typeId <= Type::_kIdVec256End)
instId = avxEnabled ? Inst::kIdVmovdqa : Inst::kIdMovdqa;
else if (elementTypeId <= Type::kIdU32)
instId = Inst::kIdVmovdqa32;
else
instId = Inst::kIdVmovdqa64;
break;
}
}
if (!instId)
return DebugUtils::errored(kErrorInvalidState);
if (overrideMemSize) {
if (dst.isMem()) dst.as<Mem>().setSize(overrideMemSize);
if (src.isMem()) src.as<Mem>().setSize(overrideMemSize);
}
emitter->setInlineComment(comment);
return emitter->emit(instId, dst, src);
}
ASMJIT_FAVOR_SIZE Error X86Internal::emitArgMove(Emitter* emitter,
const Reg& dst_, uint32_t dstTypeId,
const Operand_& src_, uint32_t srcTypeId, bool avxEnabled, const char* comment) {
// Deduce optional `dstTypeId`, which may be `Type::kIdVoid` in some cases.
if (!dstTypeId) dstTypeId = opData.archRegs.regTypeToTypeId[dst_.type()];
// Invalid or abstract TypeIds are not allowed.
ASMJIT_ASSERT(Type::isValid(dstTypeId) && !Type::isAbstract(dstTypeId));
ASMJIT_ASSERT(Type::isValid(srcTypeId) && !Type::isAbstract(srcTypeId));
Reg dst(dst_);
Operand src(src_);
uint32_t dstSize = Type::sizeOf(dstTypeId);
uint32_t srcSize = Type::sizeOf(srcTypeId);
uint32_t instId = Inst::kIdNone;
// Not a real loop, just 'break' is nicer than 'goto'.
for (;;) {
if (Type::isInt(dstTypeId)) {
if (Type::isInt(srcTypeId)) {
instId = Inst::kIdMovsx;
uint32_t typeOp = (dstTypeId << 8) | srcTypeId;
// Sign extend by using 'movsx'.
if (typeOp == ((Type::kIdI16 << 8) | Type::kIdI8 ) ||
typeOp == ((Type::kIdI32 << 8) | Type::kIdI8 ) ||
typeOp == ((Type::kIdI32 << 8) | Type::kIdI16) ||
typeOp == ((Type::kIdI64 << 8) | Type::kIdI8 ) ||
typeOp == ((Type::kIdI64 << 8) | Type::kIdI16)) break;
// Sign extend by using 'movsxd'.
instId = Inst::kIdMovsxd;
if (typeOp == ((Type::kIdI64 << 8) | Type::kIdI32)) break;
}
if (Type::isInt(srcTypeId) || src_.isMem()) {
// Zero extend by using 'movzx' or 'mov'.
if (dstSize <= 4 && srcSize < 4) {
instId = Inst::kIdMovzx;
dst.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
}
else {
// We should have caught all possibilities where `srcSize` is less
// than 4, so we don't have to worry about 'movzx' anymore. Minimum
// size is enough to determine if we want 32-bit or 64-bit move.
instId = Inst::kIdMov;
srcSize = Support::min(srcSize, dstSize);
dst.setSignature(srcSize == 4 ? Reg::signatureOfT<Reg::kTypeGpd>()
: Reg::signatureOfT<Reg::kTypeGpq>());
if (src.isReg()) src.setSignature(dst.signature());
}
break;
}
// NOTE: The previous branch caught all memory sources, from here it's
// always register to register conversion, so catch the remaining cases.
srcSize = Support::min(srcSize, dstSize);
if (Type::isMmx(srcTypeId)) {
// 64-bit move.
instId = Inst::kIdMovq;
if (srcSize == 8) break;
// 32-bit move.
instId = Inst::kIdMovd;
dst.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
break;
}
if (Type::isMask(srcTypeId)) {
instId = x86KmovFromSize(srcSize);
dst.setSignature(srcSize <= 4 ? Reg::signatureOfT<Reg::kTypeGpd>()
: Reg::signatureOfT<Reg::kTypeGpq>());
break;
}
if (Type::isVec(srcTypeId)) {
// 64-bit move.
instId = avxEnabled ? Inst::kIdVmovq : Inst::kIdMovq;
if (srcSize == 8) break;
// 32-bit move.
instId = avxEnabled ? Inst::kIdVmovd : Inst::kIdMovd;
dst.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
break;
}
}
if (Type::isMmx(dstTypeId)) {
instId = Inst::kIdMovq;
srcSize = Support::min(srcSize, dstSize);
if (Type::isInt(srcTypeId) || src.isMem()) {
// 64-bit move.
if (srcSize == 8) break;
// 32-bit move.
instId = Inst::kIdMovd;
if (src.isReg()) src.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
break;
}
if (Type::isMmx(srcTypeId)) break;
// This will hurt if `avxEnabled`.
instId = Inst::kIdMovdq2q;
if (Type::isVec(srcTypeId)) break;
}
if (Type::isMask(dstTypeId)) {
srcSize = Support::min(srcSize, dstSize);
if (Type::isInt(srcTypeId) || Type::isMask(srcTypeId) || src.isMem()) {
instId = x86KmovFromSize(srcSize);
if (Reg::isGp(src) && srcSize <= 4) src.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
break;
}
}
if (Type::isVec(dstTypeId)) {
// By default set destination to XMM, will be set to YMM|ZMM if needed.
dst.setSignature(Reg::signatureOfT<Reg::kTypeXmm>());
// This will hurt if `avxEnabled`.
if (Reg::isMm(src)) {
// 64-bit move.
instId = Inst::kIdMovq2dq;
break;
}
// Argument conversion.
uint32_t dstElement = Type::baseOf(dstTypeId);
uint32_t srcElement = Type::baseOf(srcTypeId);
if (dstElement == Type::kIdF32 && srcElement == Type::kIdF64) {
srcSize = Support::min(dstSize * 2, srcSize);
dstSize = srcSize / 2;
if (srcSize <= 8)
instId = avxEnabled ? Inst::kIdVcvtss2sd : Inst::kIdCvtss2sd;
else
instId = avxEnabled ? Inst::kIdVcvtps2pd : Inst::kIdCvtps2pd;
if (dstSize == 32)
dst.setSignature(Reg::signatureOfT<Reg::kTypeYmm>());
if (src.isReg())
src.setSignature(Reg::signatureOfVecBySize(srcSize));
break;
}
if (dstElement == Type::kIdF64 && srcElement == Type::kIdF32) {
srcSize = Support::min(dstSize, srcSize * 2) / 2;
dstSize = srcSize * 2;
if (srcSize <= 4)
instId = avxEnabled ? Inst::kIdVcvtsd2ss : Inst::kIdCvtsd2ss;
else
instId = avxEnabled ? Inst::kIdVcvtpd2ps : Inst::kIdCvtpd2ps;
dst.setSignature(Reg::signatureOfVecBySize(dstSize));
if (src.isReg() && srcSize >= 32)
src.setSignature(Reg::signatureOfT<Reg::kTypeYmm>());
break;
}
srcSize = Support::min(srcSize, dstSize);
if (Reg::isGp(src) || src.isMem()) {
// 32-bit move.
if (srcSize <= 4) {
instId = avxEnabled ? Inst::kIdVmovd : Inst::kIdMovd;
if (src.isReg()) src.setSignature(Reg::signatureOfT<Reg::kTypeGpd>());
break;
}
// 64-bit move.
if (srcSize == 8) {
instId = avxEnabled ? Inst::kIdVmovq : Inst::kIdMovq;
break;
}
}
if (Reg::isVec(src) || src.isMem()) {
instId = avxEnabled ? Inst::kIdVmovaps : Inst::kIdMovaps;
uint32_t sign = Reg::signatureOfVecBySize(srcSize);
dst.setSignature(sign);
if (src.isReg()) src.setSignature(sign);
break;
}
}
return DebugUtils::errored(kErrorInvalidState);
}
if (src.isMem())
src.as<Mem>().setSize(srcSize);
emitter->setInlineComment(comment);
return emitter->emit(instId, dst, src);
}
// ============================================================================
// [asmjit::X86Internal - Emit Prolog & Epilog]
// ============================================================================
static ASMJIT_INLINE void X86Internal_setupSaveRestoreInfo(uint32_t group, const FuncFrame& frame, Reg& xReg, uint32_t& xInst, uint32_t& xSize) noexcept {
switch (group) {
case Reg::kGroupVec:
xReg = xmm(0);
xInst = x86GetXmmMovInst(frame);
xSize = xReg.size();
break;
case Reg::kGroupMm:
xReg = mm(0);
xInst = Inst::kIdMovq;
xSize = xReg.size();
break;
case Reg::kGroupKReg:
xReg = k(0);
xInst = Inst::kIdKmovq;
xSize = xReg.size();
break;
}
}
ASMJIT_FAVOR_SIZE Error X86Internal::emitProlog(Emitter* emitter, const FuncFrame& frame) {
uint32_t gpSaved = frame.savedRegs(Reg::kGroupGp);
Gp zsp = emitter->zsp(); // ESP|RSP register.
Gp zbp = emitter->zbp(); // EBP|RBP register.
Gp gpReg = zsp; // General purpose register (temporary).
Gp saReg = zsp; // Stack-arguments base pointer.
// Emit: 'push zbp'
// 'mov zbp, zsp'.
if (frame.hasPreservedFP()) {
gpSaved &= ~Support::bitMask(Gp::kIdBp);
ASMJIT_PROPAGATE(emitter->push(zbp));
ASMJIT_PROPAGATE(emitter->mov(zbp, zsp));
}
// Emit: 'push gp' sequence.
{
Support::BitWordIterator<uint32_t> it(gpSaved);
while (it.hasNext()) {
gpReg.setId(it.next());
ASMJIT_PROPAGATE(emitter->push(gpReg));
}
}
// Emit: 'mov saReg, zsp'.
uint32_t saRegId = frame.saRegId();
if (saRegId != BaseReg::kIdBad && saRegId != Gp::kIdSp) {
saReg.setId(saRegId);
if (frame.hasPreservedFP()) {
if (saRegId != Gp::kIdBp)
ASMJIT_PROPAGATE(emitter->mov(saReg, zbp));
}
else {
ASMJIT_PROPAGATE(emitter->mov(saReg, zsp));
}
}
// Emit: 'and zsp, StackAlignment'.
if (frame.hasDynamicAlignment()) {
ASMJIT_PROPAGATE(emitter->and_(zsp, -int32_t(frame.finalStackAlignment())));
}
// Emit: 'sub zsp, StackAdjustment'.
if (frame.hasStackAdjustment()) {
ASMJIT_PROPAGATE(emitter->sub(zsp, frame.stackAdjustment()));
}
// Emit: 'mov [zsp + DAOffset], saReg'.
if (frame.hasDynamicAlignment() && frame.hasDAOffset()) {
Mem saMem = ptr(zsp, int32_t(frame.daOffset()));
ASMJIT_PROPAGATE(emitter->mov(saMem, saReg));
}
// Emit 'movxxx [zsp + X], {[x|y|z]mm, k}'.
{
Reg xReg;
Mem xBase = ptr(zsp, int32_t(frame.nonGpSaveOffset()));
uint32_t xInst;
uint32_t xSize;
for (uint32_t group = 1; group < BaseReg::kGroupVirt; group++) {
Support::BitWordIterator<uint32_t> it(frame.savedRegs(group));
if (it.hasNext()) {
X86Internal_setupSaveRestoreInfo(group, frame, xReg, xInst, xSize);
do {
xReg.setId(it.next());
ASMJIT_PROPAGATE(emitter->emit(xInst, xBase, xReg));
xBase.addOffsetLo32(int32_t(xSize));
} while (it.hasNext());
}
}
}
return kErrorOk;
}
ASMJIT_FAVOR_SIZE Error X86Internal::emitEpilog(Emitter* emitter, const FuncFrame& frame) {
uint32_t i;
uint32_t regId;
uint32_t gpSize = emitter->gpSize();
uint32_t gpSaved = frame.savedRegs(Reg::kGroupGp);
Gp zsp = emitter->zsp(); // ESP|RSP register.
Gp zbp = emitter->zbp(); // EBP|RBP register.
Gp gpReg = emitter->zsp(); // General purpose register (temporary).
// Don't emit 'pop zbp' in the pop sequence, this case is handled separately.
if (frame.hasPreservedFP())
gpSaved &= ~Support::bitMask(Gp::kIdBp);
// Emit 'movxxx {[x|y|z]mm, k}, [zsp + X]'.
{
Reg xReg;
Mem xBase = ptr(zsp, int32_t(frame.nonGpSaveOffset()));
uint32_t xInst;
uint32_t xSize;
for (uint32_t group = 1; group < BaseReg::kGroupVirt; group++) {
Support::BitWordIterator<uint32_t> it(frame.savedRegs(group));
if (it.hasNext()) {
X86Internal_setupSaveRestoreInfo(group, frame, xReg, xInst, xSize);
do {
xReg.setId(it.next());
ASMJIT_PROPAGATE(emitter->emit(xInst, xReg, xBase));
xBase.addOffsetLo32(int32_t(xSize));
} while (it.hasNext());
}
}
}
// Emit 'emms' and/or 'vzeroupper'.
if (frame.hasMmxCleanup()) ASMJIT_PROPAGATE(emitter->emms());
if (frame.hasAvxCleanup()) ASMJIT_PROPAGATE(emitter->vzeroupper());
if (frame.hasPreservedFP()) {
// Emit 'mov zsp, zbp' or 'lea zsp, [zbp - x]'
int32_t count = int32_t(frame.gpSaveSize() - gpSize);
if (!count)
ASMJIT_PROPAGATE(emitter->mov(zsp, zbp));
else
ASMJIT_PROPAGATE(emitter->lea(zsp, ptr(zbp, -count)));
}
else {
if (frame.hasDynamicAlignment() && frame.hasDAOffset()) {
// Emit 'mov zsp, [zsp + DsaSlot]'.
Mem saMem = ptr(zsp, int32_t(frame.daOffset()));
ASMJIT_PROPAGATE(emitter->mov(zsp, saMem));
}
else if (frame.hasStackAdjustment()) {
// Emit 'add zsp, StackAdjustment'.
ASMJIT_PROPAGATE(emitter->add(zsp, int32_t(frame.stackAdjustment())));
}
}
// Emit 'pop gp' sequence.
if (gpSaved) {
i = gpSaved;
regId = 16;
do {
regId--;
if (i & 0x8000) {
gpReg.setId(regId);
ASMJIT_PROPAGATE(emitter->pop(gpReg));
}
i <<= 1;
} while (regId != 0);
}
// Emit 'pop zbp'.
if (frame.hasPreservedFP())
ASMJIT_PROPAGATE(emitter->pop(zbp));
// Emit 'ret' or 'ret x'.
if (frame.hasCalleeStackCleanup())
ASMJIT_PROPAGATE(emitter->emit(Inst::kIdRet, int(frame.calleeStackCleanup())));
else
ASMJIT_PROPAGATE(emitter->emit(Inst::kIdRet));
return kErrorOk;
}
// ============================================================================
// [asmjit::X86Internal - Emit Arguments Assignment]
// ============================================================================
#ifdef ASMJIT_DUMP_ARGS_ASSIGNMENT
static void dumpFuncValue(String& sb, uint32_t archId, const FuncValue& value) noexcept {
Logging::formatTypeId(sb, value.typeId());
sb.appendChar('@');
if (value.isReg()) {
Logging::formatRegister(sb, 0, nullptr, archId, value.regType(), value.regId());
}
else if (value.isStack()) {
sb.appendFormat("[%d]", value.stackOffset());
}
else {
sb.appendString("<none>");
}
}
static void dumpAssignment(String& sb, const X86FuncArgsContext& ctx) noexcept {
typedef X86FuncArgsContext::Var Var;
uint32_t archId = ctx.archId();
uint32_t varCount = ctx.varCount();
for (uint32_t i = 0; i < varCount; i++) {
const Var& var = ctx.var(i);
const FuncValue& dst = var.out;
const FuncValue& cur = var.cur;
sb.appendFormat("Var%u: ", i);
dumpFuncValue(sb, archId, dst);
sb.appendString(" <- ");
dumpFuncValue(sb, archId, cur);
if (var.isDone())
sb.appendString(" {Done}");
sb.appendChar('\n');
}
}
#endif
ASMJIT_FAVOR_SIZE Error X86Internal::emitArgsAssignment(Emitter* emitter, const FuncFrame& frame, const FuncArgsAssignment& args) {
typedef X86FuncArgsContext::Var Var;
typedef X86FuncArgsContext::WorkData WorkData;
enum WorkFlags : uint32_t {
kWorkNone = 0x00,
kWorkDidSome = 0x01,
kWorkPending = 0x02,
kWorkPostponed = 0x04
};
X86FuncArgsContext ctx;
ASMJIT_PROPAGATE(ctx.initWorkData(frame, args));
#ifdef ASMJIT_DUMP_ARGS_ASSIGNMENT
{
String sb;
dumpAssignment(sb, ctx);
printf("%s\n", sb.data());
}
#endif
uint32_t archId = ctx.archId();
uint32_t varCount = ctx._varCount;
WorkData* workData = ctx._workData;
// Use AVX if it's enabled.
bool avxEnabled = frame.isAvxEnabled();
uint32_t saVarId = ctx._saVarId;
uint32_t saRegId = Gp::kIdSp;
if (frame.hasDynamicAlignment()) {
if (frame.hasPreservedFP())
saRegId = Gp::kIdBp;
else
saRegId = saVarId < varCount ? ctx._vars[saVarId].cur.regId() : frame.saRegId();
}
// --------------------------------------------------------------------------
// Register to stack and stack to stack moves must be first as now we have
// the biggest chance of having as many as possible unassigned registers.
// --------------------------------------------------------------------------
if (ctx._stackDstMask) {
// Base address of all arguments passed by stack.
Mem baseArgPtr = ptr(emitter->gpz(saRegId), int32_t(frame.saOffset(saRegId)));
Mem baseStackPtr = ptr(emitter->gpz(Gp::kIdSp), int32_t(0));
for (uint32_t varId = 0; varId < varCount; varId++) {
Var& var = ctx._vars[varId];
if (!var.out.isStack()) continue;
ASMJIT_ASSERT(var.cur.isReg() || var.cur.isStack());
Reg reg;
if (var.cur.isReg()) {
WorkData& wd = workData[Reg::groupOf(var.cur.regType())];
uint32_t rId = var.cur.regId();
reg.setSignatureAndId(Reg::signatureOf(var.cur.regType()), rId);
wd.unassign(varId, rId);
}
else {
// Stack to reg move - tricky since we move stack to stack we can decide which
// register to use. In general we follow the rule that IntToInt moves will use
// GP regs with possibility to sign or zero extend, and all other moves will
// either use GP or VEC regs depending on the size of the move.
RegInfo rInfo = x86GetRegForMemToMemMove(archId, var.out.typeId(), var.cur.typeId());
if (ASMJIT_UNLIKELY(!rInfo.isValid()))
return DebugUtils::errored(kErrorInvalidState);
WorkData& wd = workData[rInfo.group()];
uint32_t availableRegs = wd.availableRegs();
if (ASMJIT_UNLIKELY(!availableRegs))
return DebugUtils::errored(kErrorInvalidState);
uint32_t rId = Support::ctz(availableRegs);
reg.setSignatureAndId(rInfo.signature(), rId);
ASMJIT_PROPAGATE(
emitArgMove(emitter,
reg,
var.out.typeId(),
baseArgPtr.cloneAdjusted(var.cur.stackOffset()),
var.cur.typeId(),
avxEnabled));
}
// Register to stack move.
ASMJIT_PROPAGATE(
emitRegMove(emitter, baseStackPtr.cloneAdjusted(var.out.stackOffset()), reg, var.cur.typeId(), avxEnabled));
var.markDone();
}
}
// --------------------------------------------------------------------------
// Shuffle all registers that are currently assigned accordingly to the assignment.
// --------------------------------------------------------------------------
uint32_t workFlags = kWorkNone;
for (;;) {
for (uint32_t varId = 0; varId < varCount; varId++) {
Var& var = ctx._vars[varId];
if (var.isDone() || !var.cur.isReg()) continue;
uint32_t curType = var.cur.regType();
uint32_t outType = var.out.regType();
uint32_t curGroup = Reg::groupOf(curType);
uint32_t outGroup = Reg::groupOf(outType);
uint32_t curId = var.cur.regId();
uint32_t outId = var.out.regId();
if (curGroup != outGroup) {
ASMJIT_ASSERT(false);
// Requires a conversion between two register groups.
if (workData[outGroup]._numSwaps) {
// TODO: Postponed
workFlags |= kWorkPending;
}
else {
// TODO:
workFlags |= kWorkPending;
}
}
else {
WorkData& wd = workData[outGroup];
if (!wd.isAssigned(outId)) {
EmitMove:
ASMJIT_PROPAGATE(
emitArgMove(emitter,
Reg::fromTypeAndId(outType, outId), var.out.typeId(),
Reg::fromTypeAndId(curType, curId), var.cur.typeId(), avxEnabled));
wd.reassign(varId, outId, curId);
var.cur.initReg(outType, outId, var.out.typeId());
if (outId == var.out.regId())
var.markDone();
workFlags |= kWorkDidSome | kWorkPending;
}
else {
uint32_t altId = wd._physToVarId[outId];
Var& altVar = ctx._vars[altId];
if (!altVar.out.isInitialized() || (altVar.out.isReg() && altVar.out.regId() == curId)) {
// Swap operation is possible only between two GP registers.
if (curGroup == Reg::kGroupGp) {
uint32_t highestType = Support::max(var.cur.regType(), altVar.cur.regType());
uint32_t signature = highestType == Reg::kTypeGpq ? Reg::signatureOfT<Reg::kTypeGpq>()
: Reg::signatureOfT<Reg::kTypeGpd>();
ASMJIT_PROPAGATE(emitter->emit(Inst::kIdXchg, Reg(signature, outId), Reg(signature, curId)));
wd.swap(varId, curId, altId, outId);
var.cur.setRegId(outId);
var.markDone();
altVar.cur.setRegId(curId);
if (altVar.out.isInitialized())
altVar.markDone();
workFlags |= kWorkDidSome;
}
else {
// If there is a scratch register it can be used to perform the swap.
uint32_t availableRegs = wd.availableRegs();
if (availableRegs) {
uint32_t inOutRegs = wd.dstRegs();
if (availableRegs & ~inOutRegs)
availableRegs &= ~inOutRegs;
outId = Support::ctz(availableRegs);
goto EmitMove;
}
else {
workFlags |= kWorkPending;
}
}
}
else {
workFlags |= kWorkPending;
}
}
}
}
if (!(workFlags & kWorkPending))
break;
// If we did nothing twice it means that something is really broken.
if ((workFlags & (kWorkDidSome | kWorkPostponed)) == kWorkPostponed)
return DebugUtils::errored(kErrorInvalidState);
workFlags = (workFlags & kWorkDidSome) ? kWorkNone : kWorkPostponed;
}
// --------------------------------------------------------------------------
// Load arguments passed by stack into registers. This is pretty simple and
// it never requires multiple iterations like the previous phase.
// --------------------------------------------------------------------------
if (ctx._hasStackSrc) {
uint32_t iterCount = 1;
if (frame.hasDynamicAlignment() && !frame.hasPreservedFP())
saRegId = saVarId < varCount ? ctx._vars[saVarId].cur.regId() : frame.saRegId();
// Base address of all arguments passed by stack.
Mem baseArgPtr = ptr(emitter->gpz(saRegId), int32_t(frame.saOffset(saRegId)));
for (uint32_t iter = 0; iter < iterCount; iter++) {
for (uint32_t varId = 0; varId < varCount; varId++) {
Var& var = ctx._vars[varId];
if (var.isDone()) continue;
if (var.cur.isStack()) {
ASMJIT_ASSERT(var.out.isReg());
uint32_t outId = var.out.regId();
uint32_t outType = var.out.regType();
uint32_t group = Reg::groupOf(outType);
WorkData& wd = ctx._workData[group];
if (outId == saRegId && group == BaseReg::kGroupGp) {
// This register will be processed last as we still need `saRegId`.
if (iterCount == 1) {
iterCount++;
continue;
}
wd.unassign(wd._physToVarId[outId], outId);
}
Reg dstReg = Reg::fromTypeAndId(outType, outId);
Mem srcMem = baseArgPtr.cloneAdjusted(var.cur.stackOffset());
ASMJIT_PROPAGATE(
emitArgMove(emitter,
dstReg, var.out.typeId(),
srcMem, var.cur.typeId(), avxEnabled));
wd.assign(varId, outId);
var.cur.initReg(outType, outId, var.cur.typeId(), FuncValue::kFlagIsDone);
}
}
}
}
return kErrorOk;
}
ASMJIT_END_SUB_NAMESPACE
#endif // ASMJIT_BUILD_X86
Reference in New Issue View Git Blame Copy Permalink