// This file is part of AsmJit project // // See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include #ifndef ASMJIT_NO_COMPILER #include #include #include #include #include #include ASMJIT_BEGIN_NAMESPACE // RABlock - Control Flow // ====================== Error RABlock::append_successor(RABlock* successor) noexcept { RABlock* predecessor = this; if (predecessor->has_successor(successor)) { return Error::kOk; } ASMJIT_PROPAGATE(successor->_predecessors.reserve_additional(arena())); ASMJIT_PROPAGATE(predecessor->_successors.reserve_additional(arena())); predecessor->_successors.append_unchecked(successor); successor->_predecessors.append_unchecked(predecessor); return Error::kOk; } Error RABlock::prepend_successor(RABlock* successor) noexcept { RABlock* predecessor = this; if (predecessor->has_successor(successor)) { return Error::kOk; } ASMJIT_PROPAGATE(successor->_predecessors.reserve_additional(arena())); ASMJIT_PROPAGATE(predecessor->_successors.reserve_additional(arena())); predecessor->_successors.prepend_unchecked(successor); successor->_predecessors.prepend_unchecked(predecessor); return Error::kOk; } // BaseRAPass - Construction & Destruction // ======================================= BaseRAPass::BaseRAPass(BaseCompiler& cc) noexcept : Pass(cc, "RAPass") {} BaseRAPass::~BaseRAPass() noexcept {} static void RAPass_reset_virt_reg_data(BaseRAPass* self) noexcept { for (RAWorkReg* work_reg : self->_work_regs) { VirtReg* virt_reg = work_reg->virt_reg(); // Update the information regarding the stack of the virtual register. if (work_reg->has_stack_slot()) { RAStackSlot* slot = work_reg->stack_slot(); virt_reg->assign_stack_slot(slot->offset()); } // Reset work reg association so it cannot be used by accident (RAWorkReg data will be destroyed). virt_reg->_work_reg = nullptr; } } // BaseRAPass - Run Prepare & Cleanup // ================================== #ifndef ASMJIT_NO_LOGGING static ASMJIT_INLINE void RAPass_prepare_logging(BaseRAPass& pass, Logger* logger) noexcept { DiagnosticOptions diag = pass._cb.diagnostic_options(); pass._logger = logger; if (logger) { pass._format_options = logger->options(); pass._diagnostic_options = diag; } else { pass._format_options.reset(); pass._diagnostic_options = diag & ~(DiagnosticOptions::kRADebugCFG | DiagnosticOptions::kRADebugUnreachable); } } static ASMJIT_INLINE void RAPass_cleanup_logging(BaseRAPass& pass) noexcept { pass._logger = nullptr; pass._format_options.reset(); pass._diagnostic_options = DiagnosticOptions::kNone; } #else static ASMJIT_INLINE void RAPass_prepare_logging(BaseRAPass&, Logger*) noexcept {} static ASMJIT_INLINE void RAPass_cleanup_logging(BaseRAPass&) noexcept {} #endif static void RAPass_prepare_for_function(BaseRAPass* pass, FuncDetail* func_detail) noexcept { pass->_args_assignment.reset(func_detail); pass->_stack_allocator.reset(pass->_arena); } static void RAPass_cleanup_after_function(BaseRAPass* pass) noexcept { pass->_blocks.reset(); pass->_exits.reset(); pass->_pov.reset(); pass->_instruction_count = 0; pass->_created_block_count = 0; pass->_shared_assignments.reset(); pass->_last_timestamp = 0; pass->_arch_traits = nullptr; pass->_phys_reg_index.reset(); pass->_phys_reg_count.reset(); pass->_phys_reg_total = 0; pass->_scratch_reg_indexes.fill(Reg::kIdBad); pass->_available_regs.reset(); pass->_clobbered_regs.reset(); pass->_work_regs.reset(); pass->_work_regs_of_group.for_each([](ArenaVector& regs) { regs.reset(); }); pass->_multi_work_reg_count = 0u; pass->_total_work_reg_count = 0u; pass->_strategy.for_each([](RAStrategy& strategy) { strategy.reset(); }); pass->_global_live_spans.fill(nullptr); pass->_global_live_max_count.reset(); pass->_temporary_mem.reset(); pass->_stack_allocator.reset(nullptr); pass->_args_assignment.reset(nullptr); pass->_num_stack_args_to_stack_slots = 0; pass->_max_work_reg_name_size = 0; } // BaseRAPass - Run & RunOnFunction // ================================ Error BaseRAPass::run(Arena& arena, Logger* logger) { // Find the first function node by skipping all nodes that are not of `NodeType::kFunc` type. // If there is no function in the whole code, we would just return early and not setup anything. BaseNode* node = cc().first_node(); for (;;) { if (!node) { // The code has no function. return Error::kOk; } if (node->type() == NodeType::kFunc) { break; } node = node->next(); } Error err = Error::kOk; FuncNode* func = node->as(); RAPass_prepare_logging(*this, logger); do { // Try to find a second function in the code in order to know whether this function is last. Generally, // there are two use-cases we want to optimize for: The first is generating a function at a time and the // second is generating multiple functions at a time. In the first case we know we can do a little bit // cheaper cleanup at the end as we know we won't be running the register allocator again in this run(). node = func->end_node(); FuncNode* next_func = nullptr; while (node) { if (node->type() == NodeType::kFunc) { next_func = node->as(); break; } node = node->next(); } err = run_on_function(arena, func, next_func != nullptr); if (err != Error::kOk) { break; } func = next_func; } while (func); RAPass_cleanup_logging(*this); if (ASMJIT_UNLIKELY(err != Error::kOk)) { return _cb.report_error(err); } return err; } Error BaseRAPass::run_on_function(Arena& arena, FuncNode* func, [[maybe_unused]] bool last) noexcept { // Initialize all core structures to use `arena` and `func`. BaseNode* end = func->end_node(); _arena = &arena; _func = func; _stop = end->next(); _injection_start = nullptr; _injection_end = end; RAPass_prepare_for_function(this, &_func->_func_detail); // Initialize architecture-specific members. on_init(); // Perform all allocation steps required. Error err = on_perform_all_steps(); // Must be called regardless of the allocation status. on_done(); // Reset possible connections introduced by the register allocator. RAPass_reset_virt_reg_data(this); // Reset all core structures and everything that depends on the passed `Arena`. RAPass_cleanup_after_function(this); _arena = nullptr; _func = nullptr; _stop = nullptr; _injection_start = nullptr; _injection_end = nullptr; // Reset `Arena` as nothing should persist between `run_on_function()` calls. arena.reset(); // We alter the compiler cursor, because it doesn't make sense to reference it after the compilation - some nodes // may disappear and the old cursor could be unreachable, so just set the cursor to the last node for better safety. cc().set_cursor(cc().last_node()); return err; } // BaseRAPass - Perform All Steps // ============================== Error BaseRAPass::on_perform_all_steps() noexcept { ASMJIT_PROPAGATE(build_cfg_nodes()); ASMJIT_PROPAGATE(build_cfg_views()); ASMJIT_PROPAGATE(remove_unreachable_code()); ASMJIT_PROPAGATE(build_cfg_dominators()); ASMJIT_PROPAGATE(build_reg_ids()); ASMJIT_PROPAGATE(build_liveness()); ASMJIT_PROPAGATE(assign_arg_index_to_work_regs()); #ifndef ASMJIT_NO_LOGGING if (has_diagnostic_option(DiagnosticOptions::kRAAnnotate)) { ASMJIT_PROPAGATE(annotate_code()); } #endif ASMJIT_PROPAGATE(run_global_allocator()); ASMJIT_PROPAGATE(run_local_allocator()); ASMJIT_PROPAGATE(update_stack_frame()); ASMJIT_PROPAGATE(insert_prolog_epilog()); ASMJIT_PROPAGATE(rewrite()); return Error::kOk; } // BaseRAPass - Events // =================== void BaseRAPass::on_init() noexcept {} void BaseRAPass::on_done() noexcept {} // BaseRAPass - CFG - Basic Block Management // ========================================= RABlock* BaseRAPass::new_block(BaseNode* initial_node) noexcept { RABlock* block = arena().new_oneshot(this); if (ASMJIT_UNLIKELY(!block)) { return nullptr; } block->set_first(initial_node); block->set_last(initial_node); // Ignore return values here as we don't care if it was successful or not - this is a pre-allocation only // to make the default allocated block close to the block itself. In general, it's very common to have at // least 1 predecessor and successor, and in case of branches it's either 2 successors or predecessors in // case two basic blocks merge. (void)block->_predecessors.reserve_fit(arena(), 2u); (void)block->_successors.reserve_fit(arena(), 2u); _created_block_count++; return block; } RABlock* BaseRAPass::new_block_or_existing_at(LabelNode* label_node, BaseNode** stopped_at) noexcept { if (label_node->has_pass_data()) { return label_node->pass_data(); } FuncNode* func = this->func(); BaseNode* node = label_node->prev(); RABlock* block = nullptr; // Try to find some label, but terminate the loop on any code. We try hard to coalesce code that contains two // consecutive labels or a combination of non-code nodes between 2 or more labels. // // Possible cases that would share the same basic block: // // 1. Two or more consecutive labels: // Label1: // Label2: // // 2. Two or more labels separated by non-code nodes: // Label1: // ; Some comment... // .align 16 // Label2: size_t num_pending_labels = 0; while (node) { if (node->type() == NodeType::kLabel) { // Function has a different NodeType, just make sure this was not messed up as we must never associate // BasicBlock with a `func` itself. ASMJIT_ASSERT(node != func); block = node->pass_data(); if (block) { // Exit node has always a block associated with it. If we went here it means that `label_node` passed // here is after the end of the function and cannot be merged with the function exit block. if (node == func->exit_node()) { block = nullptr; } break; } num_pending_labels++; } else if (node->type() == NodeType::kAlign) { // Align node is fine. } else { break; } node = node->prev(); } if (stopped_at) *stopped_at = node; if (!block) { block = new_block(); if (ASMJIT_UNLIKELY(!block)) { return nullptr; } } label_node->set_pass_data(block); node = label_node; while (num_pending_labels) { node = node->prev(); for (;;) { if (node->type() == NodeType::kLabel) { node->set_pass_data(block); num_pending_labels--; break; } node = node->prev(); ASMJIT_ASSERT(node != nullptr); } } if (!block->first()) { block->set_first(node); block->set_last(label_node); } return block; } Error BaseRAPass::add_block(RABlock* block) noexcept { ASMJIT_PROPAGATE(_blocks.reserve_additional(arena())); block->_block_id = RABlockId(block_count()); _blocks.append_unchecked(block); return Error::kOk; } // BaseRAPass - CFG - Build // ======================== // [[pure virtual]] Error BaseRAPass::build_cfg_nodes() noexcept { return make_error(Error::kInvalidState); } Error BaseRAPass::init_shared_assignments(Span shared_assignments_map) noexcept { if (shared_assignments_map.is_empty()) { return Error::kOk; } uint32_t count = 0; for (RABlock* block : _blocks) { if (block->has_shared_assignment_id()) { uint32_t shared_assignment_id = shared_assignments_map[block->shared_assignment_id()]; block->set_shared_assignment_id(shared_assignment_id); count = Support::max(count, shared_assignment_id + 1); } } ASMJIT_PROPAGATE(_shared_assignments.resize_fit(arena(), count)); // Aggregate all entry scratch GP regs from blocks of the same assignment to the assignment itself. It will then be // used instead of RABlock's own scratch regs mask, as shared assignments have precedence. for (RABlock* block : _blocks) { if (block->has_jump_table()) { Span successors = block->successors(); if (!successors.is_empty()) { RABlock* first_successor = successors[0]; // NOTE: Shared assignments connect all possible successors so we only need the first to propagate exit scratch // GP registers. if (first_successor->has_shared_assignment_id()) { RASharedAssignment& sa = _shared_assignments[first_successor->shared_assignment_id()]; sa.add_entry_scratch_gp_regs(block->exit_scratch_gp_regs()); } else { // This is only allowed if there is a single successor - in that case shared assignment is not necessary. ASMJIT_ASSERT(successors.size() == 1u); } } } if (block->has_shared_assignment_id()) { RASharedAssignment& sa = _shared_assignments[block->shared_assignment_id()]; sa.add_entry_scratch_gp_regs(block->_entry_scratch_gp_regs); } } return Error::kOk; } // BaseRAPass - CFG - Views Order // ============================== // Stack specific to building a post-order-view. It reuses the POV vector in a way that stacked items // are added from the end, which would never collide with items already added to the POV vector as // they are added from the beginning (and the number of stacked items cannot exceed the number of // blocks). Additionally, it needs one vector of uint32_t to store the index of successors where it // ended before it was pushed on the stack. class RAPOVBuilderStack { protected: RABlock** _block_ptr; uint32_t* _index_ptr; uint32_t* _index_begin; public: ASMJIT_INLINE RAPOVBuilderStack(RABlock** pov_stack, uint32_t* index_stack, size_t block_count) noexcept : _block_ptr(pov_stack + block_count), _index_ptr(index_stack), _index_begin(index_stack) {} ASMJIT_INLINE void push(RABlock* block, uint32_t index) noexcept { *--_block_ptr = block; *_index_ptr++ = index; } ASMJIT_INLINE void pop(RABlock*& block, uint32_t& index) noexcept { ASMJIT_ASSERT(_index_ptr != _index_begin); block = *_block_ptr++; index = *--_index_ptr; } ASMJIT_INLINE_NODEBUG bool is_empty() const noexcept { return _index_ptr == _index_begin; } }; Error BaseRAPass::build_cfg_views() noexcept { #ifndef ASMJIT_NO_LOGGING Logger* logger = logger_if(DiagnosticOptions::kRADebugCFG); ASMJIT_RA_LOG_FORMAT("[build_cfg_views]\n"); #endif // !ASMJIT_NO_LOGGING size_t count = block_count(); if (ASMJIT_UNLIKELY(!count)) { return Error::kOk; } ArenaVector indexes; ASMJIT_PROPAGATE(_pov.reserve_fit(arena(), count)); ASMJIT_PROPAGATE(indexes.reserve_fit(arena(), count)); RABlock** pov_data = _pov.data(); size_t pov_index = 0u; RABlock* cur_block = _blocks[0]; uint32_t cur_index = 0u; RAPOVBuilderStack stack(pov_data, indexes.data(), count); // This loop uses reachable bit in a RABlock to mark visited blocks. cur_block->make_reachable(); for (;;) { while (cur_index < cur_block->successors().size()) { RABlock* child = cur_block->successors()[cur_index++]; if (!child->is_reachable()) { // Mark the block as reachable to prevent visiting the same block again. child->make_reachable(); // Add the cur_block block to the stack, we will get back to it later. stack.push(cur_block, cur_index); // Visit the first successor. cur_block = child; cur_index = 0u; } } cur_block->_pov_index = uint32_t(pov_index); pov_data[pov_index++] = cur_block; if (stack.is_empty()) { break; } stack.pop(cur_block, cur_index); } _pov._set_size(pov_index); indexes.release(arena()); ASMJIT_RA_LOG_COMPLEX({ StringTmp<1024> sb; for (RABlock* block : blocks()) { sb.clear(); if (block->has_successors()) { sb.append_format(" #%u -> {", block->block_id()); dump_block_ids(sb, block->successors()); sb.append("}\n"); } else { sb.append_format(" #%u -> {Exit}\n", block->block_id()); } logger->log(sb); } }); return Error::kOk; } // BaseRAPass - CFG - Dominators // ============================= static ASMJIT_INLINE RABlock* intersect_blocks(RABlock* b1, RABlock* b2) noexcept { while (b1 != b2) { while (b2->pov_index() > b1->pov_index()) b1 = b1->idom(); while (b1->pov_index() > b2->pov_index()) b2 = b2->idom(); } return b1; } // Based on "A Simple, Fast Dominance Algorithm". Error BaseRAPass::build_cfg_dominators() noexcept { #ifndef ASMJIT_NO_LOGGING Logger* logger = logger_if(DiagnosticOptions::kRADebugCFG); ASMJIT_RA_LOG_FORMAT("[build_cfg_dominators]\n"); #endif // !ASMJIT_NO_LOGGING if (_blocks.is_empty()) { return Error::kOk; } RABlock* entry_block = this->entry_block(); entry_block->_idom = entry_block; bool changed = true; #ifndef ASMJIT_NO_LOGGING uint32_t iter_count = 0; #endif // !ASMJIT_NO_LOGGING while (changed) { changed = false; #ifndef ASMJIT_NO_LOGGING iter_count++; #endif // !ASMJIT_NO_LOGGING for (RABlock* block : _pov.iterate_reverse()) { if (block == entry_block) { continue; } RABlock* idom = nullptr; Span predecessors = block->predecessors(); for (RABlock* p : predecessors.iterate_reverse()) { if (!p->idom()) { continue; } idom = !idom ? p : intersect_blocks(idom, p); } if (block->idom() != idom) { ASMJIT_ASSUME(idom != nullptr); ASMJIT_RA_LOG_FORMAT(" idom of #%u -> #%u\n", block->block_id(), idom->block_id()); block->_idom = idom; changed = true; } } } ASMJIT_RA_LOG_FORMAT(" done (%u iterations)\n", iter_count); return Error::kOk; } bool BaseRAPass::_strictly_dominates(const RABlock* a, const RABlock* b) const noexcept { ASMJIT_ASSERT(a != nullptr); // There must be at least one block if this function is ASMJIT_ASSERT(b != nullptr); // called, as both `a` and `b` must be valid blocks. ASMJIT_ASSERT(a != b); // Checked by `dominates()` and `strictly_dominates()`. // Nothing strictly dominates the entry block. const RABlock* entry_block = this->entry_block(); if (a == entry_block) { return false; } const RABlock* idom = b->idom(); while (idom != a && idom != entry_block) { idom = idom->idom(); } return idom != entry_block; } const RABlock* BaseRAPass::_nearest_common_dominator(const RABlock* a, const RABlock* b) const noexcept { ASMJIT_ASSERT(a != nullptr); // There must be at least one block if this function is ASMJIT_ASSERT(b != nullptr); // called, as both `a` and `b` must be valid blocks. ASMJIT_ASSERT(a != b); // Checked by `dominates()` and `_strictly_dominates()`. if (a == b) { return a; } // If `a` strictly dominates `b` then `a` is the nearest common dominator. if (_strictly_dominates(a, b)) { return a; } // If `b` strictly dominates `a` then `b` is the nearest common dominator. if (_strictly_dominates(b, a)) { return b; } const RABlock* entry_block = this->entry_block(); RABlockTimestamp timestamp = next_timestamp(); // Mark all A's dominators. const RABlock* block = a->idom(); while (block != entry_block) { block->set_timestamp(timestamp); block = block->idom(); } // Check all B's dominators against marked dominators of A. block = b->idom(); while (block != entry_block) { if (block->has_timestamp(timestamp)) { return block; } block = block->idom(); } return entry_block; } // BaseRAPass - CFG - Utilities // ============================ Error BaseRAPass::remove_unreachable_code() noexcept { size_t num_all_blocks = block_count(); size_t num_reachable_blocks = reachable_block_count(); // All reachable -> nothing to do. if (num_all_blocks == num_reachable_blocks) { return Error::kOk; } #ifndef ASMJIT_NO_LOGGING Logger* logger = logger_if(DiagnosticOptions::kRADebugUnreachable); String& sb = _tmp_string; ASMJIT_RA_LOG_FORMAT("[remove_unreachable_code - detected %zu of %zu unreachable blocks]\n", num_all_blocks - num_reachable_blocks, num_all_blocks); #endif for (RABlock* block : _blocks.iterate()) { if (block->is_reachable()) { continue; } ASMJIT_RA_LOG_FORMAT(" removing code from unreachable block {%u}\n", uint32_t(block->block_id())); BaseNode* first = block->first(); BaseNode* last = block->last(); BaseNode* before_first = first->prev(); BaseNode* after_last = last->next(); BaseNode* node = first; while (node != after_last) { BaseNode* next = node->next(); if (node->is_code() || node->is_removable()) { #ifndef ASMJIT_NO_LOGGING if (logger) { sb.clear(); Formatter::format_node(sb, _format_options, &_cb, node); logger->logf(" %s\n", sb.data()); } #endif cc().remove_node(node); } node = next; } if (before_first->next() == after_last) { block->set_first(nullptr); block->set_last(nullptr); } else { block->set_first(before_first->next()); block->set_last(after_last->prev()); } } return Error::kOk; } BaseNode* BaseRAPass::find_successor_starting_at(BaseNode* node) noexcept { while (node && (node->is_informative() || node->has_no_effect())) { node = node->next(); } return node; } bool BaseRAPass::is_next_to(BaseNode* node, BaseNode* target) noexcept { for (;;) { node = node->next(); if (node == target) { return true; } if (!node) { return false; } if (node->is_code() || node->is_data()) { return false; } } } // BaseRAPass - Registers - VirtReg / WorkReg Mapping // ================================================== Error BaseRAPass::_as_work_reg(RAWorkReg** out, VirtReg* virt_reg) noexcept { // Checked by `as_work_reg()` - must be true. ASMJIT_ASSERT(virt_reg->_work_reg == nullptr); OperandSignature signature = RegUtils::signature_of(virt_reg->reg_type()); RegGroup group = signature.reg_group(); ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); ArenaVector& work_regs_by_group = work_regs(group); ASMJIT_PROPAGATE(work_regs_by_group.reserve_additional(arena())); RAWorkReg* work_reg = arena().new_oneshot(virt_reg, signature, kBadWorkId); if (ASMJIT_UNLIKELY(!work_reg)) { return make_error(Error::kOutOfMemory); } virt_reg->set_work_reg(work_reg); if (!virt_reg->is_stack_area()) { work_reg->set_reg_byte_mask(Support::lsb_mask(virt_reg->virt_size())); } work_regs_by_group.append_unchecked(work_reg); _total_work_reg_count++; // Only used by RA logging. _max_work_reg_name_size = Support::max(_max_work_reg_name_size, virt_reg->name_size()); *out = work_reg; return Error::kOk; } RAStackSlot* BaseRAPass::_create_stack_slot(RAWorkReg* work_reg) noexcept { VirtReg* virt_reg = work_reg->virt_reg(); RAStackSlot* slot = _stack_allocator.new_slot(_sp.id(), virt_reg->virt_size(), virt_reg->alignment(), RAStackSlot::kFlagRegHome); work_reg->_stack_slot = slot; work_reg->mark_stack_used(); return slot; } RAAssignment::WorkToPhysMap* BaseRAPass::new_work_to_phys_map() noexcept { size_t count = work_reg_count(); size_t size = WorkToPhysMap::size_of(count); // If no registers are used it could be zero, in that case return a dummy map instead of NULL. if (ASMJIT_UNLIKELY(!size)) { static const RAAssignment::WorkToPhysMap null_map = {{ 0 }}; return const_cast(&null_map); } WorkToPhysMap* map = arena().alloc_oneshot(size); if (ASMJIT_UNLIKELY(!map)) { return nullptr; } map->reset(count); return map; } RAAssignment::PhysToWorkMap* BaseRAPass::new_phys_to_work_map() noexcept { uint32_t count = phys_reg_total(); size_t size = PhysToWorkMap::size_of(count); PhysToWorkMap* map = arena().alloc_oneshot(size); if (ASMJIT_UNLIKELY(!map)) { return nullptr; } map->reset(count); return map; } // BaseRAPass - Registers - Liveness Analysis and Statistics // ========================================================= ASMJIT_FAVOR_SPEED Error BaseRAPass::build_reg_ids() noexcept { uint32_t count = _total_work_reg_count; ASMJIT_PROPAGATE(_work_regs.reserve_fit(arena(), count)); RAWorkReg** work_regs = _work_regs.data(); _work_regs._set_size(count); uint32_t multi_id = 0u; uint32_t single_id = count; for (uint32_t rg = 0; rg < Globals::kNumVirtGroups; rg++) { for (RAWorkReg* work_reg : _work_regs_of_group[rg]) { if (work_reg->is_within_single_basic_block()) { work_reg->_work_id = RAWorkId(--single_id); work_regs[single_id] = work_reg; } else { work_reg->_work_id = RAWorkId(multi_id); work_reg->_single_basic_block_id = kBadBlockId; work_regs[multi_id++] = work_reg; } } } ASMJIT_ASSERT(single_id == multi_id); _multi_work_reg_count = multi_id; return Error::kOk; } // BaseRAPass - Registers - Liveness Analysis and Statistics // ========================================================= template static ASMJIT_INLINE void BaseRAPass_calculate_initial_in_out(RABlock* block, size_t live_word_count, Support::FixedStack& queue, uint32_t pov_index) noexcept { using BitWord = Support::BitWord; // Calculate LIVE-OUT based on LIVE-IN of all successors and then recalculate LIVE-IN based on LIVE-OUT and KILL bits. Span successors = block->successors(); if (!successors.is_empty()) { BitMutator bm_live_in(block->live_in()); BitMutator bm_live_out(block->live_out()); BitMutator bm_kill(block->kill()); for (RABlock* successor : successors.iterate_reverse()) { if (Support::bool_and(successor->pov_index() > pov_index, !successor->is_enqueued())) { successor->add_flags(RABlockFlags::kIsEnqueued); queue.push(successor); continue; } BitMutator bm_successor_live_in(successor->live_in()); for (uint32_t bw = 0; bw < live_word_count; bw++) { BitWord sc = bm_successor_live_in.bit_word(bw); BitWord in = bm_live_in.bit_word(bw) | sc; BitWord out = bm_live_out.bit_word(bw) | sc; bm_live_in.set_bit_word(bw, in & ~bm_kill.bit_word(bw)); bm_live_out.set_bit_word(bw, out); } bm_live_in.commit(block->live_in()); bm_live_out.commit(block->live_out()); } } } // Calculate LIVE-IN and LIVE-OUT of the given `block` and a single `successor` that has changed its LIVE-IN bits. template static ASMJIT_INLINE Support::BitWord BaseRAPass_recalculateInOut(RABlock* block, size_t live_word_count, RABlock* successor) noexcept { using BitWord = Support::BitWord; BitMutator bm_live_in(block->live_in()); BitMutator bm_live_out(block->live_out()); BitMutator bm_kill(block->kill()); BitMutator bm_successor_live_in(successor->live_in()); BitWord changed = 0u; for (size_t i = 0; i < live_word_count; i++) { BitWord succ_in = bm_successor_live_in.bit_word(i); BitWord in = bm_live_in.bit_word(i); BitWord out = bm_live_out.bit_word(i); BitWord kill = bm_kill.bit_word(i); BitWord new_in = (in | succ_in) & ~kill; BitWord new_out = (out | succ_in); bm_live_in.set_bit_word(i, new_in); bm_live_out.set_bit_word(i, new_out); changed |= in ^ new_in; } bm_live_in.commit(block->live_in()); bm_live_out.commit(block->live_out()); return changed; } template static ASMJIT_INLINE Error BaseRAPass_calculateInOutKill( BaseRAPass* pass, uint32_t* n_uses_per_work_reg, uint32_t* n_outs_per_work_reg, uint32_t* n_insts_per_block, Out num_visits_out ) noexcept { Span pov = pass->_pov.as_span(); size_t multi_work_reg_count = pass->multi_work_reg_count(); size_t multi_work_reg_count_as_bit_words = BitOps::size_in_words(multi_work_reg_count); constexpr RAWorkRegFlags kLiveFlag = RAWorkRegFlags::kSingleBlockLiveFlag; constexpr RAWorkRegFlags kVisitedFlag = RAWorkRegFlags::kSingleBlockVisitedFlag; // Calculate GEN and KILL and then initial LIVE-IN and LIVE-OUT bits. // // GEN is mapped to LIVE-IN, because it's not needed after LIVE-IN is calculated, // which is essentially `LIVE-IN = GEN & ~KILL` - so once we know GEN and KILL for // each block, calculating LIVE-IN is trivial. for (RABlock* block : pov.iterate()) { ASMJIT_PROPAGATE(block->alloc_live_bits(multi_work_reg_count)); BaseNode* node = block->last(); BaseNode* stop = block->first(); BitMutator live_in(block->live_in()); // LIVE-IN which maps to GEN as well. BitMutator kill(block->kill()); // KILL only. RABlockId block_id = block->block_id(); uint32_t inst_count = 0; for (;;) { if (node->is_inst()) { InstNode* inst = node->as(); RAInst* ra_inst = inst->pass_data(); ASMJIT_ASSERT(ra_inst != nullptr); RATiedReg* tied_regs = ra_inst->tied_regs(); uint32_t count = ra_inst->tied_count(); for (uint32_t j = 0; j < count; j++) { RATiedReg* tied_reg = &tied_regs[j]; RAWorkReg* work_reg = tied_reg->work_reg(); RAWorkId work_id = work_reg->work_id(); // Update `n_uses` and `n_outs`. n_uses_per_work_reg[uint32_t(work_id)] += 1u; n_outs_per_work_reg[uint32_t(work_id)] += uint32_t(tied_reg->is_write()); bool is_kill = tied_reg->is_write_only(); RATiedFlags tied_flags = tied_reg->flags(); // Mark as: // KILL - if this VirtReg is killed afterwards. // LAST - if this VirtReg is last in this basic block. if (work_reg->is_within_single_basic_block()) { bool was_kill = !Support::test(work_reg->flags(), kLiveFlag); bool was_last = !Support::test(work_reg->flags(), kVisitedFlag); tied_flags |= Support::bool_as_flag(was_kill); tied_flags |= Support::bool_as_flag(was_last); work_reg->add_flags(kVisitedFlag); work_reg->xor_flags(Support::bool_as_flag(uint32_t(is_kill ^ was_kill))); } else { bool was_kill = kill.bit_at(work_id); bool was_last = !live_in.bit_at(work_id); tied_flags |= Support::bool_as_flag(was_kill); tied_flags |= Support::bool_as_flag(was_last); // KILL if the register is write only, otherwise GEN. live_in.add_bit(work_id, !is_kill); kill.xor_bit(work_id, bool(is_kill ^ was_kill)); } tied_reg->_flags = tied_flags; if (tied_reg->is_lead_consecutive()) { work_reg->mark_lead_consecutive(); } if (tied_reg->has_consecutive_parent()) { RAWorkReg* consecutive_parent_reg = tied_reg->consecutive_parent(); ASMJIT_PROPAGATE(consecutive_parent_reg->add_immediate_consecutive(pass->arena(), work_id)); } } inst_count++; } if (node == stop) { break; } node = node->prev(); ASMJIT_ASSERT(node != nullptr); } n_insts_per_block[uint32_t(block_id)] = inst_count; // Calculate initial LIVE-IN from GEN - LIVE-IN = GEN & ~KILL. live_in.clear_bits(kill); live_in.commit(block->live_in()); kill.commit(block->kill()); } // Calculate initial values of LIVE-OUT and update LIVE-IN accordingly to LIVE-OUT. // // This step requires a queue, however, we only add a node's successors to the queue, which post-order-index // is greater than the post-order-index of the block being processed. This makes the next pass much faster to // converge. uint32_t num_visits = 0u; if (multi_work_reg_count_as_bit_words > 0u) { ArenaVector queue_storage; ASMJIT_PROPAGATE(queue_storage.reserve_fit(pass->arena(), pov.size())); Support::FixedStack queue(queue_storage.data(), pov.size()); for (size_t pov_index = 0u; pov_index < pov.size(); pov_index++) { RABlock* block = pov[pov_index]; BaseRAPass_calculate_initial_in_out(block, multi_work_reg_count_as_bit_words, queue, uint32_t(pov_index)); } // Iteratively keep recalculating LIVE-IN and LIVE-OUT once there are no more changes to the bits. This is // needed as there may be cycles in the CFG, which have to be propagated. This algorithm essentially uses a // work queue where nodes that change are pushed to propagate the changes to all predecessor nodes. while (!queue.is_empty()) { num_visits++; RABlock* block = queue.pop(); block->clear_flags(RABlockFlags::kIsEnqueued); for (RABlock* predecessor : block->predecessors()) { Support::BitWord changed = BaseRAPass_recalculateInOut(predecessor, multi_work_reg_count_as_bit_words, block); if (Support::bool_and(changed, !predecessor->is_enqueued())) { predecessor->add_flags(RABlockFlags::kIsEnqueued); queue.push(predecessor); } } } queue_storage.release(pass->arena()); } num_visits_out = num_visits; return Error::kOk; } ASMJIT_FAVOR_SPEED Error BaseRAPass::build_liveness() noexcept { #ifndef ASMJIT_NO_LOGGING Logger* logger = logger_if(DiagnosticOptions::kRADebugLiveness); #endif ASMJIT_RA_LOG_FORMAT("[build_liveness]\n"); size_t num_all_blocks = block_count(); size_t num_work_regs = work_reg_count(); size_t multi_work_reg_count = _multi_work_reg_count; if (!num_work_regs) { ASMJIT_RA_LOG_FORMAT(" done (no virtual registers)\n"); return Error::kOk; } ArenaVector n_uses_per_work_reg; // Number of USEs of each RAWorkReg. ArenaVector n_outs_per_work_reg; // Number of OUTs of each RAWorkReg. ArenaVector n_insts_per_block; // Number of instructions of each RABlock. ASMJIT_PROPAGATE(n_uses_per_work_reg.resize_fit(arena(), num_work_regs)); ASMJIT_PROPAGATE(n_outs_per_work_reg.resize_fit(arena(), num_work_regs)); ASMJIT_PROPAGATE(n_insts_per_block.resize_fit(arena(), num_all_blocks)); // Calculate GEN/KILL and then IN/OUT of Each Block // ------------------------------------------------ { uint32_t num_visits = 0; if (multi_work_reg_count > 0u && multi_work_reg_count <= Support::bit_size_of) { // If the number of work registers as a mask fits into a single BitWord use a separate code-path that optimizes // for such case. This makes faster generating smaller code that doesn't have many virtual registers in use. ASMJIT_PROPAGATE( BaseRAPass_calculateInOutKill( this, n_uses_per_work_reg.data(), n_outs_per_work_reg.data(), n_insts_per_block.data(), Out(num_visits) ) ); } else { ASMJIT_PROPAGATE( BaseRAPass_calculateInOutKill( this, n_uses_per_work_reg.data(), n_outs_per_work_reg.data(), n_insts_per_block.data(), Out(num_visits) ) ); } ASMJIT_RA_LOG_COMPLEX({ String& sb = _tmp_string; logger->logf(" LiveIn/Out Done (%u visits)\n", num_visits); for (uint32_t i = 0; i < num_all_blocks; i++) { RABlock* block = _blocks[i]; ASMJIT_PROPAGATE(sb.assign_format(" {#%u}\n", block->block_id())); ASMJIT_PROPAGATE(dump_block_liveness(sb, block)); logger->log(sb); } }); } // Reserve the space in each `RAWorkReg` for references // ---------------------------------------------------- for (uint32_t i = 0; i < num_work_regs; i++) { RAWorkReg* work_reg = work_reg_by_id(RAWorkId(i)); ASMJIT_PROPAGATE(work_reg->_refs.reserve_fit(arena(), n_uses_per_work_reg[i])); ASMJIT_PROPAGATE(work_reg->_writes.reserve_fit(arena(), n_outs_per_work_reg[i])); } // These are not needed anymore, so release the memory now so other allocations can reuse it. n_uses_per_work_reg.release(arena()); n_outs_per_work_reg.release(arena()); // Assign block and instruction positions, build LiveCount and LiveSpans // --------------------------------------------------------------------- // This is a starting position, reserving [0, 1] for function arguments. uint32_t position = 2; for (uint32_t i = 0; i < num_all_blocks; i++) { RABlock* block = _blocks[i]; if (!block->is_reachable()) { continue; } BaseNode* node = block->first(); BaseNode* stop = block->last(); Span live_out = block->live_out(); uint32_t end_position = position + n_insts_per_block[i] * 2u; block->set_first_position(NodePosition(position)); block->set_end_position(NodePosition(end_position)); RALiveCount cur_live_count; RALiveCount max_live_count; // Process LIVE-IN. Support::BitVectorIterator it(block->live_in()); while (it.has_next()) { RAWorkReg* work_reg = _work_regs[uint32_t(it.next())]; cur_live_count[work_reg->group()]++; ASMJIT_PROPAGATE(work_reg->live_spans().open_at(arena(), NodePosition(position), NodePosition(end_position))); } for (;;) { if (node->is_inst()) { InstNode* inst = node->as(); RAInst* ra_inst = inst->pass_data(); // Impossible - each processed instruction node must have an associated RAInst. ASMJIT_ASSERT(ra_inst != nullptr); RATiedReg* tied_regs = ra_inst->tied_regs(); uint32_t count = ra_inst->tied_count(); inst->set_position(NodePosition(position)); ra_inst->_live_count = cur_live_count; for (uint32_t j = 0; j < count; j++) { RATiedReg* tied_reg = &tied_regs[j]; RAWorkReg* work_reg = tied_reg->work_reg(); RAWorkId work_id = work_reg->work_id(); // Create refs and writes. work_reg->_refs.append_unchecked(node); if (tied_reg->is_write()) { work_reg->_writes.append_unchecked(node); } // We couldn't calculate this in previous steps, but since we know all LIVE-OUT at this point it becomes // trivial. If this is the last instruction that uses this `work_reg` and it's not LIVE-OUT then it is a // KILL here. if (tied_reg->is_last() && (size_t(work_id) >= multi_work_reg_count || !BitOps::bit_at(live_out, work_id))) { tied_reg->add_flags(RATiedFlags::kKill); } RALiveSpans& live_spans = work_reg->live_spans(); bool was_open; ASMJIT_PROPAGATE(live_spans.open_at( arena(), NodePosition(position + !tied_reg->is_read()), NodePosition(end_position), was_open)); RegGroup group = work_reg->group(); if (!was_open) { cur_live_count[group]++; ra_inst->_live_count[group]++; } if (tied_reg->is_kill()) { live_spans.close_at(NodePosition(position + !tied_reg->is_read() + 1u)); cur_live_count[group]--; } // Update `RAWorkReg::use_id_mask` and `RAWorkReg::hint_reg_id`. if (tied_reg->has_use_id()) { uint32_t use_id = tied_reg->use_id(); work_reg->add_use_id_mask(Support::bit_mask(use_id)); if (!work_reg->has_hint_reg_id() && !Support::bit_test(ra_inst->_clobbered_regs[group], use_id)) { work_reg->set_hint_reg_id(use_id); } } if (tied_reg->use_reg_mask()) { work_reg->restrict_preferred_mask(tied_reg->use_reg_mask()); if (work_reg->is_lead_consecutive()) { work_reg->restrict_consecutive_mask(tied_reg->use_reg_mask()); } } if (tied_reg->out_reg_mask()) { work_reg->restrict_preferred_mask(tied_reg->out_reg_mask()); if (work_reg->is_lead_consecutive()) { work_reg->restrict_consecutive_mask(tied_reg->out_reg_mask()); } } // Update `RAWorkReg::clobber_survival_mask`. if (ra_inst->_clobbered_regs[group] && !tied_reg->is_out_or_kill()) { work_reg->add_clobber_survival_mask(ra_inst->_clobbered_regs[group]); } } if (node->is_invoke()) { func()->frame().update_call_stack_alignment(node->as()->detail().natural_stack_alignment()); } position += 2; max_live_count.op(ra_inst->_live_count); } if (node == stop) { break; } node = node->next(); ASMJIT_ASSERT(node != nullptr); } block->_max_live_count = max_live_count; _global_live_max_count.op(max_live_count); ASMJIT_ASSERT(NodePosition(position) == block->end_position()); } // Calculate WorkReg statistics // ---------------------------- for (uint32_t i = 0; i < num_work_regs; i++) { RAWorkReg* work_reg = _work_regs[i]; RALiveSpans& spans = work_reg->live_spans(); uint32_t width = spans.width(); float freq = width ? float(double(work_reg->_refs.size()) / double(width)) : float(0); RALiveStats& stats = work_reg->live_stats(); stats._width = width; stats._freq = freq; stats._priority = freq + float(int(work_reg->virt_reg()->weight())) * 0.01f; } ASMJIT_RA_LOG_COMPLEX({ String& sb = _tmp_string; sb.clear(); dump_live_spans(sb); logger->log(sb); }); n_insts_per_block.release(arena()); return Error::kOk; } Error BaseRAPass::assign_arg_index_to_work_regs() noexcept { Span live_in = entry_block()->live_in(); uint32_t arg_count = func()->arg_count(); uint32_t multi_work_reg_count = _multi_work_reg_count; for (uint32_t arg_index = 0; arg_index < arg_count; arg_index++) { for (uint32_t value_index = 0; value_index < Globals::kMaxValuePack; value_index++) { // Unassigned argument. const RegOnly& reg_arg = func()->arg_pack(arg_index)[value_index]; if (!reg_arg.is_reg() || !cc().is_virt_id_valid(reg_arg.id())) { continue; } VirtReg* virt_reg = cc().virt_reg_by_id(reg_arg.id()); if (!virt_reg) { continue; } // Unreferenced argument. RAWorkReg* work_reg = virt_reg->work_reg(); if (!work_reg) { continue; } // Overwritten argument. RAWorkId work_id = work_reg->work_id(); if (uint32_t(work_id) >= multi_work_reg_count || !BitOps::bit_at(live_in, work_id)) { continue; } work_reg->set_arg_index(arg_index, value_index); const FuncValue& arg = func()->detail().arg(arg_index, value_index); if (arg.is_reg() && RegUtils::group_of(arg.reg_type()) == work_reg->group()) { work_reg->set_hint_reg_id(arg.reg_id()); } } } return Error::kOk; } // BaseRAPass - Allocation - Global // ================================ #ifndef ASMJIT_NO_LOGGING static void RAPass_dump_spans(String& sb, uint32_t index, const RALiveSpans& live_spans) noexcept { sb.append_format(" %02u: ", index); for (uint32_t i = 0; i < live_spans.size(); i++) { const RALiveSpan& live_span = live_spans[i]; if (i) { sb.append(", "); } sb.append_format("[%u:%u]", live_span.a, live_span.b); } sb.append('\n'); } #endif Error BaseRAPass::run_global_allocator() noexcept { ASMJIT_PROPAGATE(init_global_live_spans()); for (RegGroup group : Support::enumerate(RegGroup::kMaxVirt)) { ASMJIT_PROPAGATE(bin_pack(group)); } return Error::kOk; } ASMJIT_FAVOR_SPEED Error BaseRAPass::init_global_live_spans() noexcept { for (RegGroup group : Support::enumerate(RegGroup::kMaxVirt)) { size_t phys_count = _phys_reg_count.get(group); RALiveSpans* live_spans = nullptr; if (phys_count) { live_spans = arena().alloc_oneshot(phys_count * sizeof(RALiveSpans)); if (ASMJIT_UNLIKELY(!live_spans)) { return make_error(Error::kOutOfMemory); } for (size_t phys_id = 0; phys_id < phys_count; phys_id++) { new(Support::PlacementNew{&live_spans[phys_id]}) RALiveSpans(); } } _global_live_spans[group] = live_spans; } return Error::kOk; } struct RAConsecutiveReg { RAWorkReg* work_reg; RAWorkReg* parent_reg; }; ASMJIT_FAVOR_SPEED Error BaseRAPass::bin_pack(RegGroup group) noexcept { if (work_reg_count(group) == 0) return Error::kOk; #ifndef ASMJIT_NO_LOGGING Logger* logger = logger_if(DiagnosticOptions::kRADebugAssignment); String& sb = _tmp_string; ASMJIT_RA_LOG_FORMAT("[bin_pack] Available=%u (0x%08X) Count=%u RegGroup=%u\n", Support::popcnt(_available_regs[group]), _available_regs[group], work_reg_count(group), uint32_t(group)); #endif uint32_t phys_count = _phys_reg_count.get(group); ArenaVector work_regs; ArenaVector consecutive_regs; RALiveSpans tmp_spans; ArenaVector& group_regs = this->work_regs(group); ASMJIT_PROPAGATE(work_regs.reserve_fit(arena(), group_regs.size())); work_regs.assign_unchecked(group_regs); work_regs.sort([](const RAWorkReg* a, const RAWorkReg* b) noexcept { return b->live_stats().priority() - a->live_stats().priority(); }); size_t num_work_regs = work_regs.size(); RegMask available_regs = _available_regs[group]; RegMask preserved_regs = func()->frame().preserved_regs(group); // First try to pack everything that provides register-id hint as these are most likely function arguments and fixed // (pre-colored) virtual registers. if (!work_regs.is_empty()) { uint32_t dst_index = 0; for (uint32_t index = 0; index < num_work_regs; index++) { RAWorkReg* work_reg = work_regs[index]; if (work_reg->is_lead_consecutive()) { ASMJIT_PROPAGATE(consecutive_regs.append(arena(), RAConsecutiveReg{work_reg, nullptr})); work_reg->mark_processed_consecutive(); } if (work_reg->has_hint_reg_id()) { uint32_t phys_id = work_reg->hint_reg_id(); if (Support::bit_test(available_regs, phys_id)) { RALiveSpans& live = _global_live_spans[group][phys_id]; Error err = tmp_spans.non_overlapping_union_of(arena(), live, work_reg->live_spans()); if (err == Error::kOk) { live.swap(tmp_spans); work_reg->set_home_reg_id(phys_id); work_reg->mark_allocated(); continue; } if (err != Error::kByPass) { return err; } } } work_regs[dst_index++] = work_reg; } work_regs._set_size(dst_index); num_work_regs = dst_index; } // Allocate consecutive registers - both leads and all consecutives. This is important and prioritized over the rest, // because once a lead is allocated we really need to allocate its consecutives, otherwise we may bin pack other // registers into their places, which would result in wrong hints to the local allocator, and then into many moves // or spills. if (!consecutive_regs.is_empty()) { // This loop appends all other consecutive registers into `consecutive_regs` array. Leads are at the beginning, // non-leads follow. for (size_t i = 0;;) { size_t stop = consecutive_regs.size(); if (i == stop) { break; } while (i < stop) { RAWorkReg* work_reg = consecutive_regs[i].work_reg; if (work_reg->has_immediate_consecutives()) { ArenaBitSet::ForEachBitSet it(work_reg->immediate_consecutives()); while (it.has_next()) { RAWorkId consecutive_work_id = RAWorkId(it.next()); RAWorkReg* consecutive_reg = work_reg_by_id(consecutive_work_id); if (!consecutive_reg->is_processed_consecutive()) { ASMJIT_PROPAGATE(consecutive_regs.append(arena(), RAConsecutiveReg{consecutive_reg, work_reg})); consecutive_reg->mark_processed_consecutive(); } } } i++; } } for (RAConsecutiveReg& consecutive_reg : consecutive_regs) { RAWorkReg* work_reg = consecutive_reg.work_reg; if (work_reg->is_allocated()) { continue; } RAWorkReg* parent_reg = consecutive_reg.parent_reg; RegMask phys_regs = 0; if (!parent_reg) { phys_regs = available_regs & work_reg->preferred_mask(); if (!phys_regs) { phys_regs = available_regs & work_reg->consecutive_mask(); // NOTE: This should never be true as it would mean we would never allocate this virtual register // (not here, and not later when local register allocator processes RATiedReg sets). if (ASMJIT_UNLIKELY(!phys_regs)) { return make_error(Error::kConsecutiveRegsAllocation); } } } else if (parent_reg->has_home_reg_id()) { uint32_t consecutive_id = parent_reg->home_reg_id() + 1; // NOTE: We don't support wrapping. If this goes beyond all allocable registers there is something wrong. if (consecutive_id > 31 || !Support::bit_test(available_regs, consecutive_id)) { return make_error(Error::kConsecutiveRegsAllocation); } work_reg->set_hint_reg_id(consecutive_id); phys_regs = Support::bit_mask(consecutive_id); } while (phys_regs) { uint32_t phys_id = Support::bit_size_of - 1 - Support::clz(phys_regs); RALiveSpans& live = _global_live_spans[group][phys_id]; Error err = tmp_spans.non_overlapping_union_of(arena(), live, work_reg->live_spans()); if (err == Error::kOk) { work_reg->set_home_reg_id(phys_id); work_reg->mark_allocated(); live.swap(tmp_spans); break; } if (ASMJIT_UNLIKELY(err != Error::kByPass)) { return err; } phys_regs ^= Support::bit_mask(phys_id); } } } // Try to pack the rest. if (!work_regs.is_empty()) { size_t dst_index = 0; for (size_t index = 0; index < num_work_regs; index++) { RAWorkReg* work_reg = work_regs[index]; if (work_reg->is_allocated()) { continue; } RegMask remaining_phys_regs = available_regs; if (remaining_phys_regs & work_reg->preferred_mask()) { remaining_phys_regs &= work_reg->preferred_mask(); } RegMask phys_regs = remaining_phys_regs & ~preserved_regs; remaining_phys_regs &= preserved_regs; for (;;) { if (!phys_regs) { if (!remaining_phys_regs) { break; } phys_regs = remaining_phys_regs; remaining_phys_regs = 0; } uint32_t phys_id = Support::ctz(phys_regs); if (work_reg->clobber_survival_mask()) { RegMask preferred_mask = (phys_regs | remaining_phys_regs) & work_reg->clobber_survival_mask(); if (preferred_mask) { if (preferred_mask & ~remaining_phys_regs) { preferred_mask &= ~remaining_phys_regs; } phys_id = Support::ctz(preferred_mask); } } RALiveSpans& live = _global_live_spans[group][phys_id]; Error err = tmp_spans.non_overlapping_union_of(arena(), live, work_reg->live_spans()); if (err == Error::kOk) { work_reg->set_home_reg_id(phys_id); work_reg->mark_allocated(); live.swap(tmp_spans); break; } if (ASMJIT_UNLIKELY(err != Error::kByPass)) { return err; } phys_regs &= ~Support::bit_mask(phys_id); remaining_phys_regs &= ~Support::bit_mask(phys_id); } // Keep it in `work_regs` if it was not allocated. if (!phys_regs) { work_regs[dst_index++] = work_reg; } } work_regs._set_size(dst_index); num_work_regs = dst_index; } ASMJIT_RA_LOG_COMPLEX({ for (uint32_t phys_id = 0; phys_id < phys_count; phys_id++) { RALiveSpans& live = _global_live_spans[group][phys_id]; if (live.is_empty()) { continue; } sb.clear(); RAPass_dump_spans(sb, phys_id, live); logger->log(sb); } }); // Maybe unused if logging is disabled. Support::maybe_unused(phys_count); if (work_regs.is_empty()) { ASMJIT_RA_LOG_FORMAT(" completed.\n"); } else { _strategy[group].set_type(RAStrategyType::kComplex); for (RAWorkReg* work_reg : work_regs) { work_reg->mark_stack_preferred(); } ASMJIT_RA_LOG_COMPLEX({ size_t count = work_regs.size(); sb.clear(); sb.append_format(" Unassigned (%zu): ", count); for (uint32_t i = 0; i < num_work_regs; i++) { RAWorkReg* work_reg = work_regs[i]; if (i) { sb.append(", "); } Formatter::format_virt_reg_name(sb, work_reg->virt_reg()); } sb.append('\n'); logger->log(sb); }); } return Error::kOk; } // BaseRAPass - Allocation - Local // =============================== Error BaseRAPass::run_local_allocator() noexcept { RALocalAllocator lra(*this); ASMJIT_PROPAGATE(lra.init()); if (!block_count()) { return Error::kOk; } // The allocation is done when this reaches zero. size_t blocks_remaining = reachable_block_count(); // Current block. uint32_t block_id = 0; RABlock* block = _blocks[block_id]; // The first block (entry) must always be reachable. ASMJIT_ASSERT(block->is_reachable()); // Assign function arguments for the initial block. The `lra` is valid now. ASMJIT_PROPAGATE(lra.make_initial_assignment()); ASMJIT_PROPAGATE(set_block_entry_assignment(block, block, lra._cur_assignment)); // The loop starts from the first block and iterates blocks in order, however, the algorithm also allows to jump to // any other block when finished if it's a jump target. In-order iteration just makes sure that all blocks are visited. for (;;) { BaseNode* first = block->first(); BaseNode* last = block->last(); BaseNode* terminator = block->has_terminator() ? last : nullptr; BaseNode* before_first = first->prev(); BaseNode* after_last = last->next(); bool is_unconditional_jump = false; RABlock* consecutive = block->has_successors() ? block->successors()[0] : nullptr; lra.set_block(block); block->make_allocated(); BaseNode* node = first; while (node != after_last) { BaseNode* next = node->next(); if (node->is_inst()) { InstNode* inst = node->as(); if (ASMJIT_UNLIKELY(inst == terminator)) { Span successors = block->successors(); if (block->has_consecutive()) { ASMJIT_PROPAGATE(lra.alloc_branch(inst, successors.last(), successors.first())); node = next; continue; } else if (successors.size() > 1) { RABlock* cont = block->has_consecutive() ? successors.first() : nullptr; ASMJIT_PROPAGATE(lra.alloc_jump_table(inst, successors, cont)); node = next; continue; } else { // Otherwise this is an unconditional jump, special handling isn't required. is_unconditional_jump = true; } } ASMJIT_PROPAGATE(lra.alloc_instruction(inst)); if (inst->type() == NodeType::kInvoke) { ASMJIT_PROPAGATE(emit_pre_call(inst->as())); } else { ASMJIT_PROPAGATE(lra.spill_after_allocation(inst)); } } node = next; } if (consecutive) { BaseNode* prev = after_last ? after_last->prev() : cc().last_node(); cc().set_cursor(is_unconditional_jump ? prev->prev() : prev); if (consecutive->has_entry_assignment()) { ASMJIT_PROPAGATE(lra.switch_to_assignment(consecutive->entry_phys_to_work_map(), consecutive->live_in(), consecutive->is_allocated(), false)); } else { ASMJIT_PROPAGATE(lra.spill_regs_before_entry(consecutive)); ASMJIT_PROPAGATE(set_block_entry_assignment(consecutive, block, lra._cur_assignment)); lra._cur_assignment.copy_from(consecutive->entry_phys_to_work_map()); } } // Important as the local allocator can insert instructions before // and after any instruction within the basic block. block->set_first(before_first->next()); block->set_last(after_last ? after_last->prev() : cc().last_node()); if (--blocks_remaining == 0) { break; } // Switch to the next consecutive block, if any. if (consecutive) { block = consecutive; if (!block->is_allocated()) { continue; } } // Get the next block. for (;;) { if (++block_id >= block_count()) { block_id = 0; } block = _blocks[block_id]; if (!block->is_reachable() || block->is_allocated() || !block->has_entry_assignment()) { continue; } break; } // If we switched to another block we have to update the local allocator. ASMJIT_PROPAGATE(lra.replace_assignment(block->entry_phys_to_work_map())); } _clobbered_regs.op(lra._clobbered_regs); return Error::kOk; } Error BaseRAPass::set_block_entry_assignment(RABlock* block, const RABlock* from_block, const RAAssignment& from_assignment) noexcept { if (block->has_shared_assignment_id()) { uint32_t shared_assignment_id = block->shared_assignment_id(); // Shouldn't happen. Entry assignment of a block that has a shared-state will assign to all blocks // with the same shared_assignment_id. It's a bug if the shared state has been already assigned. if (!_shared_assignments[shared_assignment_id].is_empty()) { return make_error(Error::kInvalidState); } return set_shared_assignment(shared_assignment_id, from_assignment); } PhysToWorkMap* phys_to_work_map = clone_phys_to_work_map(from_assignment.phys_to_work_map()); if (ASMJIT_UNLIKELY(!phys_to_work_map)) { return make_error(Error::kOutOfMemory); } block->set_entry_assignment(phys_to_work_map); // True if this is the first (entry) block, nothing to do in this case. if (block == from_block) { // Entry block should never have a shared state. if (block->has_shared_assignment_id()) { return make_error(Error::kInvalidState); } return Error::kOk; } Span live_out = from_block->live_out(); Span live_in = block->live_in(); // It's possible that `from_block` has LIVE-OUT regs that `block` doesn't // have in LIVE-IN, these have to be unassigned. { Support::BitVectorOpIterator it(live_out, live_in); while (it.has_next()) { RAWorkId work_id = RAWorkId(it.next()); RAWorkReg* work_reg = work_reg_by_id(work_id); RegGroup group = work_reg->group(); uint32_t phys_id = from_assignment.work_to_phys_id(group, work_id); if (phys_id != RAAssignment::kPhysNone) { phys_to_work_map->unassign(group, phys_id, _phys_reg_index.get(group) + phys_id); } } } return block_entry_assigned(phys_to_work_map); } Error BaseRAPass::set_shared_assignment(uint32_t shared_assignment_id, const RAAssignment& from_assignment) noexcept { ASMJIT_ASSERT(_shared_assignments[shared_assignment_id].is_empty()); PhysToWorkMap* phys_to_work_map = clone_phys_to_work_map(from_assignment.phys_to_work_map()); if (ASMJIT_UNLIKELY(!phys_to_work_map)) { return make_error(Error::kOutOfMemory); } _shared_assignments[shared_assignment_id].assign_phys_to_work_map(phys_to_work_map); ASMJIT_PROPAGATE(_shared_assignments[shared_assignment_id]._live_in.resize(arena(), multi_work_reg_count())); Span shared_live_in = _shared_assignments[shared_assignment_id]._live_in.as_span(); Support::Array shared_assigned {}; for (RABlock* block : blocks()) { if (block->shared_assignment_id() == shared_assignment_id) { ASMJIT_ASSERT(!block->has_entry_assignment()); PhysToWorkMap* entry_phys_to_work_map = clone_phys_to_work_map(from_assignment.phys_to_work_map()); if (ASMJIT_UNLIKELY(!entry_phys_to_work_map)) { return make_error(Error::kOutOfMemory); } block->set_entry_assignment(entry_phys_to_work_map); Span live_in = block->live_in(); BitOps::or_(shared_live_in, shared_live_in, live_in); for (RegGroup group : Support::enumerate(RegGroup::kMaxVirt)) { shared_assigned[group] |= entry_phys_to_work_map->assigned[group]; uint32_t phys_base_index = _phys_reg_index.get(group); Support::BitWordIterator it(entry_phys_to_work_map->assigned[group]); while (it.has_next()) { uint32_t phys_id = it.next(); RAWorkId work_id = entry_phys_to_work_map->work_ids[phys_base_index + phys_id]; // Should not happen as a register that only lives in a single basic block should not appear in the map. ASMJIT_ASSERT(uint32_t(work_id) < _multi_work_reg_count); if (!BitOps::bit_at(live_in, work_id)) { entry_phys_to_work_map->unassign(group, phys_id, phys_base_index + phys_id); } } } } } for (RegGroup group : Support::enumerate(RegGroup::kMaxVirt)) { uint32_t phys_base_index = _phys_reg_index.get(group); Support::BitWordIterator it(_available_regs[group] & ~shared_assigned[group]); while (it.has_next()) { uint32_t phys_id = it.next(); if (Support::bit_test(phys_to_work_map->assigned[group], phys_id)) { phys_to_work_map->unassign(group, phys_id, phys_base_index + phys_id); } } } return block_entry_assigned(phys_to_work_map); } Error BaseRAPass::block_entry_assigned(const PhysToWorkMap* phys_to_work_map) noexcept { // Complex allocation strategy requires to record register assignments upon block entry (or per shared state). for (RegGroup group : Support::enumerate(RegGroup::kMaxVirt)) { if (!_strategy[group].is_complex()) { continue; } uint32_t phys_base_index = _phys_reg_index.get(group); Support::BitWordIterator it(phys_to_work_map->assigned[group]); while (it.has_next()) { uint32_t phys_id = it.next(); RAWorkId work_id = phys_to_work_map->work_ids[phys_base_index + phys_id]; RAWorkReg* work_reg = work_reg_by_id(work_id); work_reg->add_allocated_mask(Support::bit_mask(phys_id)); } } return Error::kOk; } // BaseRAPass - Allocation - Utilities // =================================== Error BaseRAPass::use_temporary_mem(BaseMem& out, uint32_t size, uint32_t alignment) noexcept { ASMJIT_ASSERT(alignment <= 64); if (_temporary_mem.is_none()) { ASMJIT_PROPAGATE(cc()._new_stack(Out(_temporary_mem.as()), size, alignment)); } else { ASMJIT_ASSERT(_temporary_mem.as().is_reg_home()); uint32_t virt_id = _temporary_mem.as().base_id(); VirtReg* virt_reg = cc().virt_reg_by_id(virt_id); cc().set_stack_size(virt_id, Support::max(virt_reg->virt_size(), size), Support::max(virt_reg->alignment(), alignment)); } out = _temporary_mem.as(); return Error::kOk; } // BaseRAPass - Allocation - Prolog & Epilog // ========================================= Error BaseRAPass::update_stack_frame() noexcept { // Update some StackFrame information that we updated during allocation. The only information we don't have at the // moment is final local stack size, which is calculated last. FuncFrame& frame = func()->frame(); for (RegGroup group : Support::enumerate(RegGroup::kMaxVirt)) { frame.add_dirty_regs(group, _clobbered_regs[group]); } frame.set_local_stack_alignment(_stack_allocator.alignment()); // If there are stack arguments that are not assigned to registers upon entry and the function doesn't require // dynamic stack alignment we keep these arguments where they are. This will also mark all stack slots that match // these arguments as allocated. if (_num_stack_args_to_stack_slots) { ASMJIT_PROPAGATE(_mark_stack_args_to_keep()); } // Calculate offsets of all stack slots and update StackSize to reflect the calculated local stack size. ASMJIT_PROPAGATE(_stack_allocator.calculate_stack_frame()); frame.set_local_stack_size(_stack_allocator.stack_size()); // Update the stack frame based on `_args_assignment` and finalize it. Finalization means to apply final calculation // to the stack layout. ASMJIT_PROPAGATE(_args_assignment.update_func_frame(frame)); ASMJIT_PROPAGATE(frame.finalize()); // StackAllocator allocates all stots starting from [0], adjust them when necessary. if (frame.local_stack_offset() != 0) { ASMJIT_PROPAGATE(_stack_allocator.adjust_slot_offsets(int32_t(frame.local_stack_offset()))); } // Again, if there are stack arguments allocated in function's stack we have to handle them. This handles all cases // (either regular or dynamic stack alignment). if (_num_stack_args_to_stack_slots) { ASMJIT_PROPAGATE(_update_stack_args()); } return Error::kOk; } Error BaseRAPass::_mark_stack_args_to_keep() noexcept { FuncFrame& frame = func()->frame(); bool has_sa_reg = frame.has_preserved_fp() || !frame.has_dynamic_alignment(); ArenaVector& work_regs = _work_regs; size_t num_work_regs = work_reg_count(); for (size_t work_id = 0; work_id < num_work_regs; work_id++) { RAWorkReg* work_reg = work_regs[work_id]; if (work_reg->has_flag(RAWorkRegFlags::kStackArgToStack)) { ASMJIT_ASSERT(work_reg->has_arg_index()); const FuncValue& src_arg = _func->detail().arg(work_reg->arg_index()); // If the register doesn't have stack slot then we failed. It doesn't make much sense as it was marked as // `kFlagStackArgToStack`, which requires the WorkReg was live-in upon function entry. RAStackSlot* slot = work_reg->stack_slot(); if (ASMJIT_UNLIKELY(!slot)) { return make_error(Error::kInvalidState); } if (has_sa_reg && src_arg.is_stack() && !src_arg.is_indirect()) { uint32_t type_size = TypeUtils::size_of(src_arg.type_id()); if (type_size == slot->size()) { slot->add_flags(RAStackSlot::kFlagStackArg); continue; } } // NOTE: Update StackOffset here so when `_args_assignment.update_func_frame()` is called it will take into // consideration moving to stack slots. Without this we may miss some scratch registers later. FuncValue& dst_arg = _args_assignment.arg(work_reg->arg_index(), work_reg->arg_value_index()); dst_arg.assign_stack_offset(0); } } return Error::kOk; } Error BaseRAPass::_update_stack_args() noexcept { FuncFrame& frame = func()->frame(); ArenaVector& work_regs = _work_regs; size_t num_work_regs = work_reg_count(); for (size_t work_id = 0; work_id < num_work_regs; work_id++) { RAWorkReg* work_reg = work_regs[work_id]; if (work_reg->has_flag(RAWorkRegFlags::kStackArgToStack)) { ASMJIT_ASSERT(work_reg->has_arg_index()); RAStackSlot* slot = work_reg->stack_slot(); if (ASMJIT_UNLIKELY(!slot)) { return make_error(Error::kInvalidState); } if (slot->is_stack_arg()) { const FuncValue& src_arg = _func->detail().arg(work_reg->arg_index()); if (frame.has_preserved_fp()) { slot->set_base_reg_id(_fp.id()); slot->set_offset(int32_t(frame.sa_offset_from_sa()) + src_arg.stack_offset()); } else { slot->set_offset(int32_t(frame.sa_offset_from_sp()) + src_arg.stack_offset()); } } else { FuncValue& dst_arg = _args_assignment.arg(work_reg->arg_index(), work_reg->arg_value_index()); dst_arg.set_stack_offset(slot->offset()); } } } return Error::kOk; } Error BaseRAPass::insert_prolog_epilog() noexcept { FuncFrame& frame = _func->frame(); cc().set_cursor(func()); ASMJIT_PROPAGATE(cc().emit_prolog(frame)); ASMJIT_PROPAGATE(_emit_helper_ptr->emit_args_assignment(frame, _args_assignment)); cc().set_cursor(func()->exit_node()); ASMJIT_PROPAGATE(cc().emit_epilog(frame)); return Error::kOk; } // BaseRAPass - Rewriter // ===================== // [[pure virtual]] Error BaseRAPass::rewrite() noexcept { return make_error(Error::kInvalidState); } // BaseRAPass - Emit // ================= // [[pure virtual]] Error BaseRAPass::emit_move(RAWorkReg* work_reg, uint32_t dst_phys_id, uint32_t src_phys_id) noexcept { Support::maybe_unused(work_reg, dst_phys_id, src_phys_id); return make_error(Error::kInvalidState); } // [[pure virtual]] Error BaseRAPass::emit_swap(RAWorkReg* a_reg, uint32_t a_phys_id, RAWorkReg* b_reg, uint32_t b_phys_id) noexcept { Support::maybe_unused(a_reg, a_phys_id, b_reg, b_phys_id); return make_error(Error::kInvalidState); } // [[pure virtual]] Error BaseRAPass::emit_load(RAWorkReg* work_reg, uint32_t dst_phys_id) noexcept { Support::maybe_unused(work_reg, dst_phys_id); return make_error(Error::kInvalidState); } // [[pure virtual]] Error BaseRAPass::emit_save(RAWorkReg* work_reg, uint32_t src_phys_id) noexcept { Support::maybe_unused(work_reg, src_phys_id); return make_error(Error::kInvalidState); } // [[pure virtual]] Error BaseRAPass::emit_jump(const Label& label) noexcept { Support::maybe_unused(label); return make_error(Error::kInvalidState); } Error BaseRAPass::emit_pre_call(InvokeNode* invoke_node) noexcept { Support::maybe_unused(invoke_node); return make_error(Error::kOk); } // BaseRAPass - Logging // ==================== #ifndef ASMJIT_NO_LOGGING static void RAPass_formatLiveness(BaseRAPass* pass, String& sb, const RAInst* ra_inst) noexcept { Support::maybe_unused(pass); const RATiedReg* tied_regs = ra_inst->tied_regs(); uint32_t tied_count = ra_inst->tied_count(); for (uint32_t i = 0; i < tied_count; i++) { const RATiedReg& tied_reg = tied_regs[i]; if (i != 0) { sb.append(' '); } Formatter::format_virt_reg_name(sb, tied_reg.work_reg()->virt_reg()); sb.append('{'); sb.append(tied_reg.is_read_write() ? 'X' : tied_reg.is_read() ? 'R' : tied_reg.is_write() ? 'W' : '?'); if (tied_reg.is_lead_consecutive()) { sb.append_format("|Lead[%u]", tied_reg.consecutive_data() + 1u); } if (tied_reg.has_use_id()) { sb.append_format("|Use=%u", tied_reg.use_id()); } else if (tied_reg.is_use()) { sb.append("|Use"); } if (tied_reg.is_use_consecutive() && !tied_reg.is_lead_consecutive()) { sb.append_format("+%u", tied_reg.consecutive_data()); } if (tied_reg.has_out_id()) { sb.append_format("|Out=%u", tied_reg.out_id()); } else if (tied_reg.is_out()) { sb.append("|Out"); } if (tied_reg.is_out_consecutive() && !tied_reg.is_lead_consecutive()) { sb.append_format("+%u", tied_reg.consecutive_data()); } if (tied_reg.is_first()) { sb.append("|First"); } if (tied_reg.is_last()) { sb.append("|Last"); } if (tied_reg.is_kill()) { sb.append("|Kill"); } sb.append("}"); } } ASMJIT_FAVOR_SIZE Error BaseRAPass::annotate_code() noexcept { StringTmp<1024> sb; for (const RABlock* block : _blocks) { BaseNode* node = block->first(); if (!node) { continue; } BaseNode* last = block->last(); for (;;) { sb.clear(); Formatter::format_node(sb, _format_options, &_cb, node); if (has_diagnostic_option(DiagnosticOptions::kRADebugLiveness) && node->is_inst() && node->has_pass_data()) { const RAInst* ra_inst = node->pass_data(); if (ra_inst->tied_count() > 0) { sb.pad_end(40); sb.append(" | "); RAPass_formatLiveness(this, sb, ra_inst); } } node->set_inline_comment(static_cast(cc()._builder_arena.dup(sb.data(), sb.size(), true))); if (node == last) { break; } node = node->next(); } } return Error::kOk; } ASMJIT_FAVOR_SIZE Error BaseRAPass::dump_block_ids(String& sb, Span blocks) noexcept { for (size_t i = 0; i < blocks.size(); i++) { const RABlock* block = blocks[i]; ASMJIT_PROPAGATE(sb.append_format(!i ? "#%u" : ", #%u", uint32_t(block->block_id()))); } return Error::kOk; } ASMJIT_FAVOR_SIZE Error BaseRAPass::dump_block_liveness(String& sb, const RABlock* block) noexcept { for (uint32_t live_type = 0; live_type < RABlock::kLiveCount; live_type++) { const char* info_name = live_type == RABlock::kLiveIn ? "IN " : live_type == RABlock::kLiveOut ? "OUT " : "KILL"; Support::BitVectorIterator it(block->live_bits(live_type)); if (it.has_next()) { bool first = true; sb.append_format(" %s [", info_name); do { const RAWorkReg* work_reg = work_reg_by_id(RAWorkId(it.next())); if (!first) { sb.append(", "); } Formatter::format_virt_reg_name(sb, work_reg->virt_reg()); first = false; } while (it.has_next()); sb.append("]\n"); } } return Error::kOk; } ASMJIT_FAVOR_SIZE Error BaseRAPass::dump_live_spans(String& sb) noexcept { size_t max_size = _max_work_reg_name_size; for (RAWorkReg* work_reg : _work_regs.iterate()) { RALiveStats& stats = work_reg->live_stats(); sb.append(" "); size_t old_size = sb.size(); Formatter::format_virt_reg_name(sb, work_reg->virt_reg()); sb.pad_end(old_size + max_size); sb.append_format(" {ra_id=%-5u virt_id=%-5u width=%-5u freq=%0.5f priority=%0.5f ", uint32_t(work_reg->work_id()), work_reg->virt_id(), stats.width(), double(stats.freq()), double(stats.priority())); if (work_reg->is_within_single_basic_block()) { sb.append_format("bb=#%-4u", uint32_t(work_reg->single_basic_block_id())); } else { sb.append("bb=<...>"); } sb.append_format("}: "); RALiveSpans& live_spans = work_reg->live_spans(); for (uint32_t x = 0; x < live_spans.size(); x++) { const RALiveSpan& live_span = live_spans[x]; if (x) { sb.append(", "); } sb.append_format("[%u:%u]", live_span.a, live_span.b); } sb.append('\n'); } return Error::kOk; } #endif ASMJIT_END_NAMESPACE #endif // !ASMJIT_NO_COMPILER