Function boundary detection: multi-pass prolog, tail-call, .pdata, data pointers

- Extended prolog patterns: MSVC x64 callee-saves, sub rsp >= 0x20, enter
- Tail-call detection: jmp to prolog candidates seeds new functions
- endbr64/endbr32 CET skipping at function starts
- PE .pdata exception table parsing for precise start/end boundaries
- Data-section function pointer scanning (vtables, callbacks)
- PLT stub detection with is_thunk tag
- end_address and is_thunk fields in Function struct and JSON output
- getDataRanges() and getRuntimeFunctions() in Binary interface
- README docs for all new features + research paper reference
This commit is contained in:
x86byte
2026-06-30 10:04:56 +01:00
parent 236e428dce
commit 6c2dca78ab
13 changed files with 322576 additions and 79906 deletions
+150 -22
View File
@@ -20,6 +20,20 @@ static bool regMatch(const string& op, const string& target)
CFGBuilder::CFGBuilder(Binary* binary, Disassembler* disasm, bool subsOnly)
: m_bin(binary), m_dis(disasm), m_subs_only(subsOnly) {}
addr_t CFGBuilder::skipEndbr64(addr_t addr)
{
auto bytes = m_bin->readBytes(addr, 4);
if (bytes.size() >= 4)
{
// endbr64: F3 0F 1E FA
// endbr32: F3 0F 1E FB
if (bytes[0] == 0xF3 && bytes[1] == 0x0F && bytes[2] == 0x1E &&
(bytes[3] == 0xFA || bytes[3] == 0xFB))
return addr + 4;
}
return addr;
}
bool CFGBuilder::isVisited(addr_t addr)
{
return m_blocks.count(addr) > 0;
@@ -239,7 +253,12 @@ BasicBlock CFGBuilder::disassembleBlock(addr_t start, addr_t& next)
{
addr_t t = m_dis->getBranchTarget(inst);
if (t && !m_dis->isIndirectBranch(inst))
{
t = skipEndbr64(t);
block.successors.push_back(t);
if (m_prologs.count(t))
m_funcs.insert(t);
}
else if (m_dis->isIndirectBranch(inst))
{
addr_t resolved = resolveGOT(inst);
@@ -322,6 +341,7 @@ BasicBlock CFGBuilder::disassembleBlock(addr_t start, addr_t& next)
addr_t t = m_dis->getBranchTarget(inst);
if (t && !m_dis->isIndirectBranch(inst))
{
t = skipEndbr64(t);
m_funcs.insert(t);
recordIndirect(inst, t, true, "", "");
}
@@ -342,6 +362,7 @@ BasicBlock CFGBuilder::disassembleBlock(addr_t start, addr_t& next)
((uint64_t)gb[6] << 48) | ((uint64_t)gb[7] << 56);
if (target && target < 0x100000000ULL) resolved = target;
}
resolved = skipEndbr64(resolved);
m_funcs.insert(resolved);
recordIndirect(inst, resolved, true, nm, lb);
}
@@ -352,6 +373,7 @@ BasicBlock CFGBuilder::disassembleBlock(addr_t start, addr_t& next)
resolved = traceReg(reg, block.instructions, block.instructions.size() - 1);
if (resolved)
{
resolved = skipEndbr64(resolved);
m_funcs.insert(resolved);
recordIndirect(inst, resolved, true, "traced", "");
}
@@ -411,8 +433,10 @@ Function CFGBuilder::buildFunction(addr_t start, const string& name)
func.address = start;
func.name = name;
addr_t actual = skipEndbr64(start);
queue<addr_t> q;
q.push(start);
q.push(actual);
while (!q.empty())
{
@@ -428,9 +452,66 @@ Function CFGBuilder::buildFunction(addr_t start, const string& name)
func.blocks.push_back(block);
}
// Compute end_address as max instruction address across all blocks
for (const auto& b : func.blocks)
for (const auto& inst : b.instructions)
if (inst.address + inst.size > func.end_address)
func.end_address = inst.address + inst.size;
// Tag PLT stubs
if (isPLTStub(func)) func.is_thunk = true;
return func;
}
bool CFGBuilder::isPLTStub(const Function& func) const
{
if (func.blocks.size() != 1) return false;
const auto& block = func.blocks[0];
if (block.instructions.empty()) return false;
const auto& first = block.instructions[0];
if (first.mnemonic == "jmp" && first.operands.find('[') != string::npos)
{
int64_t disp = m_dis->getRIPDisp(first);
if (disp != 0)
{
addr_t target = first.address + first.size + disp;
if (m_got.count(target)) return true;
}
}
return false;
}
void CFGBuilder::scanDataPointers()
{
auto ranges = m_bin->getDataRanges();
auto exec = m_bin->getExecutableRanges();
auto isInExec = [&](addr_t addr) -> bool {
for (const auto& r : exec)
if (addr >= r.first && addr < r.second) return true;
return false;
};
for (const auto& r : ranges)
{
addr_t a = r.first;
// Align to 8 bytes
if (a & 7) a = (a + 7) & ~7ULL;
for (; a + 8 <= r.second; a += 8)
{
auto bytes = m_bin->readBytes(a, 8);
if (bytes.size() < 8) continue;
uint64_t val = bytes[0] | ((uint64_t)bytes[1] << 8) |
((uint64_t)bytes[2] << 16) | ((uint64_t)bytes[3] << 24) |
((uint64_t)bytes[4] << 32) | ((uint64_t)bytes[5] << 40) |
((uint64_t)bytes[6] << 48) | ((uint64_t)bytes[7] << 56);
if (val && isInExec(val))
m_funcs.insert(val);
}
}
}
CFG CFGBuilder::build()
{
CFG cfg;
@@ -458,36 +539,67 @@ CFG CFGBuilder::build()
set<addr_t> built;
queue<addr_t> pending;
// Apply .pdata end_address to functions
auto pdata = m_bin->getRuntimeFunctions();
auto setEndAddress = [&](Function& func) {
for (const auto& rf : pdata)
if (rf.first == func.address)
{
func.end_address = rf.second;
break;
}
};
auto isEmpty = [](const Function& f) -> bool {
for (const auto& b : f.blocks)
if (!b.instructions.empty()) return false;
return true;
};
auto scanCallAndJmpTargets = [&](const Function& func) {
for (const auto& b : func.blocks)
for (const auto& inst : b.instructions)
{
if (m_dis->isCall(inst))
{
addr_t t = m_dis->getBranchTarget(inst);
if (t && !m_dis->isIndirectBranch(inst))
{
t = skipEndbr64(t);
m_funcs.insert(t);
}
}
if (m_dis->isUnconditionalBranch(inst))
{
addr_t t = m_dis->getBranchTarget(inst);
if (t && !m_dis->isIndirectBranch(inst))
{
t = skipEndbr64(t);
if (m_prologs.count(t))
m_funcs.insert(t);
}
}
}
};
auto process = [&](addr_t addr, const string& name)
{
addr = skipEndbr64(addr);
if (built.count(addr)) return;
built.insert(addr);
auto func = buildFunction(addr, name);
if (isEmpty(func)) return;
setEndAddress(func);
cfg.functions.push_back(func);
for (const auto& b : func.blocks)
for (const auto& inst : b.instructions)
if (m_dis->isCall(inst))
{
addr_t t = m_dis->getBranchTarget(inst);
if (t && !m_dis->isIndirectBranch(inst))
m_funcs.insert(t);
}
scanCallAndJmpTargets(func);
for (auto d : m_funcs)
if (!built.count(d)) pending.push(d);
};
{
addr_t ep = m_bin->getEntryPoint();
addr_t ep = skipEndbr64(m_bin->getEntryPoint());
string ep_name = "entry";
for (const auto& exp : m_bin->getExportedFunctions())
if (exp.first == ep) { ep_name = exp.second; break; }
@@ -497,7 +609,10 @@ CFG CFGBuilder::build()
if (!m_subs_only)
{
for (const auto& exp : m_bin->getExportedFunctions())
pending.push(exp.first);
pending.push(skipEndbr64(exp.first));
for (const auto& rf : pdata)
if (rf.first != m_bin->getEntryPoint())
pending.push(skipEndbr64(rf.first));
}
map<addr_t, string> imp_map;
@@ -507,6 +622,7 @@ CFG CFGBuilder::build()
while (!pending.empty())
{
addr_t a = pending.front(); pending.pop();
a = skipEndbr64(a);
if (built.count(a)) continue;
string name;
@@ -518,19 +634,11 @@ CFG CFGBuilder::build()
if (it != imp_map.end()) name = it->second;
}
built.insert(a);
auto func = buildFunction(a, name);
if (isEmpty(func)) continue;
setEndAddress(func);
cfg.functions.push_back(func);
for (const auto& b : func.blocks)
for (const auto& inst : b.instructions)
if (m_dis->isCall(inst))
{
addr_t t = m_dis->getBranchTarget(inst);
if (t && !m_dis->isIndirectBranch(inst))
if (!built.count(t)) pending.push(t);
}
scanCallAndJmpTargets(func);
if (!m_subs_only)
{
@@ -539,6 +647,26 @@ CFG CFGBuilder::build()
}
}
// Add data-section function pointers as seeds (if not subs-only)
if (!m_subs_only)
{
scanDataPointers();
for (auto d : m_funcs)
if (!built.count(d)) pending.push(d);
while (!pending.empty())
{
addr_t a = pending.front(); pending.pop();
a = skipEndbr64(a);
if (built.count(a)) continue;
auto func = buildFunction(a, "");
if (isEmpty(func)) continue;
setEndAddress(func);
cfg.functions.push_back(func);
scanCallAndJmpTargets(func);
}
}
cfg.indirect_targets = m_targets;
return cfg;
}
+3
View File
@@ -18,6 +18,9 @@ class CFGBuilder
Function buildFunction(addr_t start, const string& name);
BasicBlock disassembleBlock(addr_t start, addr_t& next);
set<addr_t> findPrologs();
void scanDataPointers();
addr_t skipEndbr64(addr_t addr);
bool isPLTStub(const Function& func) const;
void markVisited(addr_t addr);
bool isVisited(addr_t addr);