admin/mandiant-GoReSym
1
0
Fork 0
mirror of https://github.com/mandiant/GoReSym synced 2026-06-08 15:40:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement Regex Needle Optimization

Browse Source
This commit is contained in:
Stephen Eckels
2023-08-14 15:16:26 -04:00
parent ebc220ad52
commit 11d153df99
5 changed files with 127 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+2 -1
View File
@@ -1,6 +1,6 @@
module github.com/mandiant/GoReSym
go 1.20
go 1.21
require (
github.com/elliotchance/orderedmap v1.4.0
@@ -12,4 +12,5 @@ require (
require (
github.com/felixge/fgprof v0.9.3 // indirect
github.com/google/pprof v0.0.0-20230728192033-2ba5b33183c6 // indirect
golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb
)
go.sum
+2
View File
@@ -24,6 +24,8 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff h1:XmKBi9R6duxOB3lfc72wyrwiOY7X2Jl1wuI+RFOyMDE=
golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4=
golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb h1:mIKbk8weKhSeLH2GmUTrvx8CjkyJmnU1wFmg59CUjFA=
golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
objfile/patterns.go
+70 -106
View File
@@ -5,6 +5,7 @@ import (
"strconv"
"strings"
"golang.org/x/exp/slices"
"rsc.io/binaryregexp"
)
@@ -42,14 +43,14 @@ func isHex(s string) bool {
// although this requires more code, we provide this functionality
// because these patterns are *much* more readable than raw regular expressions,
// we strongly value people being able to understand GoReSym's algorithm.
func RegexpPatternFromYaraPattern(pattern string) (string, error) {
func RegexpPatternFromYaraPattern(pattern string) (*RegexAndNeedle, error) {
if !strings.HasPrefix(pattern, "{") {
return "", errors.New("missing prefix")
return nil, errors.New("missing prefix")
}
if !strings.HasSuffix(pattern, "}") {
return "", errors.New("missing suffix")
return nil, errors.New("missing suffix")
}
pattern = strings.Trim(pattern, "{}")
@@ -58,6 +59,10 @@ func RegexpPatternFromYaraPattern(pattern string) (string, error) {
pattern = strings.ToLower(pattern)
patLen := 0
needle := make([]byte, 0)
tmpNeedle := make([]byte, 0)
var regex_pattern string
for i := 0; i < len(pattern); {
// at the start of this loop,
@@ -71,12 +76,19 @@ func RegexpPatternFromYaraPattern(pattern string) (string, error) {
// output: .
if c == "?" {
if d != "?" {
return "", errors.New("cannot mask the first nibble")
return nil, errors.New("cannot mask the first nibble")
}
regex_pattern += "."
i += 2
patLen += 1
if len(tmpNeedle) > len(needle) {
needle = slices.Clone(tmpNeedle)
tmpNeedle = make([]byte, 0)
} else {
tmpNeedle = make([]byte, 0)
}
continue
}
@@ -85,23 +97,23 @@ func RegexpPatternFromYaraPattern(pattern string) (string, error) {
if c == "[" {
end := strings.Index(pattern[i:], "]")
if end == -1 {
return "", errors.New("unbalanced [")
return nil, errors.New("unbalanced [")
}
chunk := pattern[i+1 : i+end]
low, high, found := strings.Cut(chunk, "-")
if !found {
return "", errors.New("[] didn't contain a dash")
return nil, errors.New("[] didn't contain a dash")
}
_, err := strconv.Atoi(low)
if err != nil {
return "", errors.New("invalid number")
return nil, errors.New("invalid number")
}
_, err = strconv.Atoi(high)
if err != nil {
return "", errors.New("invalid number")
return nil, errors.New("invalid number")
}
regex_pattern += "."
@@ -112,6 +124,14 @@ func RegexpPatternFromYaraPattern(pattern string) (string, error) {
regex_pattern += "}"
i += end + 1
patLen += 1
if len(tmpNeedle) > len(needle) {
needle = slices.Clone(tmpNeedle)
tmpNeedle = make([]byte, 0)
} else {
tmpNeedle = make([]byte, 0)
}
continue
}
@@ -120,7 +140,7 @@ func RegexpPatternFromYaraPattern(pattern string) (string, error) {
if c == "(" {
end := strings.Index(pattern[i:], ")")
if end == -1 {
return "", errors.New("unbalanced (")
return nil, errors.New("unbalanced (")
}
chunk := pattern[i+1 : i+end]
@@ -129,7 +149,7 @@ func RegexpPatternFromYaraPattern(pattern string) (string, error) {
regex_pattern += "("
for j, choice := range choices {
if !isHex(choice) {
return "", errors.New("choice not hex")
return nil, errors.New("choice not hex")
}
if j != 0 {
@@ -141,6 +161,13 @@ func RegexpPatternFromYaraPattern(pattern string) (string, error) {
regex_pattern += ")"
i += end + 1
patLen += len(choices)
if len(tmpNeedle) > len(needle) {
needle = slices.Clone(tmpNeedle)
tmpNeedle = make([]byte, 0)
} else {
tmpNeedle = make([]byte, 0)
}
continue
}
@@ -148,7 +175,7 @@ func RegexpPatternFromYaraPattern(pattern string) (string, error) {
// output: [\x00-\x0F]
if d == "?" {
if !isHex(c) {
return "", errors.New("not hex digit")
return nil, errors.New("not hex digit")
}
regex_pattern += "["
@@ -158,6 +185,13 @@ func RegexpPatternFromYaraPattern(pattern string) (string, error) {
regex_pattern += "]"
i += 2
patLen += 1
if len(tmpNeedle) > len(needle) {
needle = slices.Clone(tmpNeedle)
tmpNeedle = make([]byte, 0)
} else {
tmpNeedle = make([]byte, 0)
}
continue
}
@@ -165,117 +199,47 @@ func RegexpPatternFromYaraPattern(pattern string) (string, error) {
// output: \xAB
if isHex(c) && isHex(d) {
regex_pattern += `\x` + strings.ToUpper(c+d)
byt, err := strconv.ParseInt(c+d, 16, 64)
if err != nil {
return nil, errors.New("not hex digit")
}
tmpNeedle = append(tmpNeedle, byte(byt))
i += 2
patLen += 1
continue
}
return "", errors.New("unexpected value")
return nil, errors.New("unexpected value")
}
return regex_pattern, nil
}
func RegexpFromYaraPattern(pattern string) (*binaryregexp.Regexp, error) {
regex_pattern, e := RegexpPatternFromYaraPattern(pattern)
if e != nil {
return nil, e
if len(tmpNeedle) > len(needle) {
needle = slices.Clone(tmpNeedle)
//tmpNeedle = make([]byte, 0) not needed at exit
}
r := binaryregexp.MustCompile(regex_pattern)
if r == nil {
return nil, errors.New("failed to compile regex")
}
return r, nil
return &RegexAndNeedle{patLen, regex_pattern, r, needle}, nil
}
type BinaryRegexpGroup struct {
patterns map[string]string
re *binaryregexp.Regexp
}
func NewBinaryRegexpGroup(patterns map[string]string) (*BinaryRegexpGroup, error) {
var pattern string
i := 0
pattern += "("
for k, v := range patterns {
if i != 0 {
pattern += "|"
}
i += 1
pattern += "(?P"
pattern += "<" + k + ">"
pattern += v
pattern += ")"
}
pattern += ")"
re := binaryregexp.MustCompile(pattern)
if re == nil {
return nil, errors.New("failed to compile regex")
}
return &BinaryRegexpGroup{
patterns: patterns,
re: re,
}, nil
}
type BinaryRegexGroupMatches struct {
g *BinaryRegexpGroup
matches [][]int
}
func (g *BinaryRegexpGroup) FindAllIndex(buf []byte, n int) *BinaryRegexGroupMatches {
matches := g.re.FindAllIndex(buf, n)
return &BinaryRegexGroupMatches{
g: g,
matches: matches,
}
}
// fetch the index of the subexp for the given regexp.
//
// this is called `(*Regexp) SubexpIndex` in recent Go,
// but doesn't seem to be implemented in binaryregexp.
// https://pkg.go.dev/regexp#Regexp.SubexpIndex
func SubexpIndex(re *binaryregexp.Regexp, name string) int {
for i, n := range re.SubexpNames() {
if n == name {
return i
func FindRegex(data []byte, regexInfo *RegexAndNeedle) []int {
matches := make([]int, 0)
needleMatches := findAllOccurrences(data, [][]byte{regexInfo.needle})
for _, needleMatch := range needleMatches {
for _, reMatch := range regexInfo.re.FindAllIndex(data[needleMatch-regexInfo.len:needleMatch+regexInfo.len], -1) {
start := reMatch[0]
//end := reMatch[1]
matches = append(matches, start)
}
}
return -1
return matches
}
// fetch the [start, end] pairs for the subexp with the given name in the given matches.
func SubexpIndexMatches(re *binaryregexp.Regexp, matches [][]int, name string) [][]int {
index := SubexpIndex(re, name)
var ret [][]int
for _, match := range matches {
start := match[2*index]
end := match[2*index+1]
if start == -1 && end == -1 {
continue
}
ret = append(ret, []int{start, end})
}
return ret
}
// fetch the [start, end] pairs for the subexp with the given name.
func (m *BinaryRegexGroupMatches) MatchesForSubexp(name string) [][]int {
return SubexpIndexMatches(m.g.re, m.matches, name)
type RegexAndNeedle struct {
len int
rawre string
re *binaryregexp.Regexp
needle []byte // longest fixed sub-sequence of regex
}
objfile/patterns_test.go
+21 -56
View File
@@ -1,7 +1,7 @@
package objfile
import (
"fmt"
"bytes"
"testing"
"rsc.io/binaryregexp"
@@ -104,53 +104,6 @@ func TestRegexGrouping(t *testing.T) {
t.Errorf("5")
}
})
t.Run("two groups", func(t *testing.T) {
re := binaryregexp.MustCompile(`((?P<xxx>a(x*)b)|(?P<yyy>c(y*)d))`)
fmt.Printf("aa: %v\n", re)
fmt.Printf("%q\n", re.SubexpNames())
if SubexpIndex(re, "xxx") != 2 {
t.Errorf("xxx index")
}
if SubexpIndex(re, "yyy") != 4 {
t.Errorf("yyy index")
}
if !compare(SubexpIndexMatches(re, re.FindAllStringSubmatchIndex("--ab--", -1), "xxx"), [][]int{{2, 4}}) {
t.Errorf("1")
}
if !compare(SubexpIndexMatches(re, re.FindAllStringSubmatchIndex("--axxb--", -1), "xxx"), [][]int{{2, 6}}) {
t.Errorf("2")
}
if !compare(SubexpIndexMatches(re, re.FindAllStringSubmatchIndex("--ab--", -1), "yyy"), [][]int{}) {
t.Errorf("3: no matches")
}
if !compare(SubexpIndexMatches(re, re.FindAllStringSubmatchIndex("--cd--", -1), "xxx"), [][]int{}) {
t.Errorf("4: no matches")
}
if !compare(SubexpIndexMatches(re, re.FindAllStringSubmatchIndex("--cd--", -1), "yyy"), [][]int{{2, 4}}) {
t.Errorf("5")
}
if !compare(SubexpIndexMatches(re, re.FindAllStringSubmatchIndex("--cyyd--", -1), "yyy"), [][]int{{2, 6}}) {
t.Errorf("6")
}
if !compare(SubexpIndexMatches(re, re.FindAllStringSubmatchIndex("--abcd--", -1), "xxx"), [][]int{{2, 4}}) {
t.Errorf("7")
}
if !compare(SubexpIndexMatches(re, re.FindAllStringSubmatchIndex("--abcd--", -1), "yyy"), [][]int{{4, 6}}) {
t.Errorf("8")
}
})
}
func TestRegexpPatternFromYaraPattern(t *testing.T) {
@@ -159,51 +112,63 @@ func TestRegexpPatternFromYaraPattern(t *testing.T) {
t.Errorf("empty pattern should have errored")
}
p, err := RegexpPatternFromYaraPattern("{}")
reg, err := RegexpPatternFromYaraPattern("{}")
if err != nil {
t.Errorf("empty pattern errored")
}
if p != "" {
if reg.rawre != "" {
t.Errorf("incorrect empty pattern")
}
})
t.Run("x64firstmoduledata", func(t *testing.T) {
p, err := RegexpPatternFromYaraPattern("{ 48 8D 0? ?? ?? ?? ?? EB ?? 48 8? 8? ?? 02 00 00 66 0F 1F 44 00 00 }")
reg, err := RegexpPatternFromYaraPattern("{ 48 8D 0? ?? ?? ?? ?? EB ?? 48 8? 8? ?? 02 00 00 66 0F 1F 44 00 00 }")
if err != nil {
t.Errorf("pattern errored")
}
// manually translated
if p != `\x48\x8D[\x00-\x0F]....\xEB.\x48[\x80-\x8F][\x80-\x8F].\x02\x00\x00\x66\x0F\x1F\x44\x00\x00` {
if reg.rawre != `\x48\x8D[\x00-\x0F]....\xEB.\x48[\x80-\x8F][\x80-\x8F].\x02\x00\x00\x66\x0F\x1F\x44\x00\x00` {
t.Errorf("incorrect pattern")
}
if !bytes.Equal(reg.needle, []byte{02, 00, 00, 0x66, 0x0F, 0x1F, 0x44, 0x00, 0x00}) {
t.Errorf("incorrect needle")
}
})
t.Run("x86sig", func(t *testing.T) {
p, err := RegexpPatternFromYaraPattern("{ 8D ?? ?? ?? ?? ?? EB ?? [0-50] 8B ?? ?? 01 00 00 8B ?? ?? ?? 85 ?? 75 ?? }")
reg, err := RegexpPatternFromYaraPattern("{ 8D ?? ?? ?? ?? ?? EB ?? [0-50] 8B ?? ?? 01 00 00 8B ?? ?? ?? 85 ?? 75 ?? }")
if err != nil {
t.Errorf("pattern errored")
}
// manually translated
if p != `\x8D.....\xEB..{0,50}\x8B..\x01\x00\x00\x8B...\x85.\x75.` {
if reg.rawre != `\x8D.....\xEB..{0,50}\x8B..\x01\x00\x00\x8B...\x85.\x75.` {
t.Errorf("incorrect pattern")
}
if !bytes.Equal(reg.needle, []byte{0x01, 0x00, 0x00, 0x8B}) {
t.Errorf("incorrect needle")
}
})
t.Run("arm64", func(t *testing.T) {
p, err := RegexpPatternFromYaraPattern("{ ?? ?? ?? (90 | b0 | f0 | d0) ?? ?? ?? 91 ?? ?? ?? (14 | 17) ?? ?? 41 F9 ?? ?? ?? B4 }")
reg, err := RegexpPatternFromYaraPattern("{ ?? ?? ?? (90 | b0 | f0 | d0) ?? ?? ?? 91 ?? ?? ?? (14 | 17) ?? ?? 41 F9 ?? ?? ?? B4 }")
if err != nil {
t.Errorf("pattern errored")
}
// manually translated
if p != `...(\x90|\xB0|\xF0|\xD0)...\x91...(\x14|\x17)..\x41\xF9...\xB4` {
if reg.rawre != `...(\x90|\xB0|\xF0|\xD0)...\x91...(\x14|\x17)..\x41\xF9...\xB4` {
t.Errorf("incorrect pattern")
}
if !bytes.Equal(reg.needle, []byte{0x41, 0xF9}) {
t.Errorf("incorrect needle")
}
})
}
objfile/scanner.go
+32 -29
View File
@@ -1,8 +1,6 @@
package objfile
import (
"encoding/binary"
)
import "encoding/binary"
type signatureModuleDataInitx64 struct {
moduleDataPtrLoc uint64 // offset in signature to the location of the pointer to the PCHeader
@@ -81,22 +79,13 @@ var ARM32_sig = signatureModuleDataInitARM32{0, `{ ?? ?? 9F E5 ?? ?? ?? EA ?? ??
func findModuleInitPCHeader(data []byte, sectionBase uint64) []SignatureMatch {
var matches []SignatureMatch = make([]SignatureMatch, 0)
g, e := NewBinaryRegexpGroup(map[string]string{
"x64": x64sig.signature,
"x86": x86sig.signature,
"ARM64": ARM64_sig.signature,
"ARM32": ARM32_sig.signature,
"PPC_BE": PPC_BE_sig.signature,
})
if e != nil {
// programmer error: check the patterns
panic(e)
x64reg, err := RegexpPatternFromYaraPattern(x64sig.signature)
if err != nil {
panic(err)
}
m := g.FindAllIndex(data, -1)
for _, match := range m.MatchesForSubexp("x64") {
sigPtr := uint64(match[0]) // from int
for _, match := range FindRegex(data, x64reg) {
sigPtr := uint64(match) // from int
// this is the pointer offset stored in the instruction
// 0x44E06A: 48 8D 0D 4F F0 24 00 lea rcx, off_69D0C0 (result: 0x24f04f)
@@ -110,8 +99,13 @@ func findModuleInitPCHeader(data []byte, sectionBase uint64) []SignatureMatch {
})
}
for _, match := range m.MatchesForSubexp("x86") {
sigPtr := uint64(match[0]) // from int
x86reg, err := RegexpPatternFromYaraPattern(x86sig.signature)
if err != nil {
panic(err)
}
for _, match := range FindRegex(data, x86reg) {
sigPtr := uint64(match) // from int
moduleDataPtr := uint64(binary.LittleEndian.Uint32(data[sigPtr+x86sig.moduleDataPtrLoc:][:4]))
matches = append(matches, SignatureMatch{
@@ -119,8 +113,13 @@ func findModuleInitPCHeader(data []byte, sectionBase uint64) []SignatureMatch {
})
}
for _, match := range m.MatchesForSubexp("ARM64") {
sigPtr := uint64(match[0]) // from int
arm64reg, err := RegexpPatternFromYaraPattern(ARM64_sig.signature)
if err != nil {
panic(err)
}
for _, match := range FindRegex(data, arm64reg) {
sigPtr := uint64(match) // from int
adrp := binary.LittleEndian.Uint32(data[sigPtr+ARM64_sig.moduleDataPtrADRP:][:4])
add := binary.LittleEndian.Uint32(data[sigPtr+ARM64_sig.moduleDataPtrADD:][:4])
@@ -140,28 +139,32 @@ func findModuleInitPCHeader(data []byte, sectionBase uint64) []SignatureMatch {
})
}
for _, match := range m.MatchesForSubexp("ARM32") {
sigPtr := uint64(match[0]) // from int
arm32reg, err := RegexpPatternFromYaraPattern(ARM32_sig.signature)
if err != nil {
panic(err)
}
for _, match := range FindRegex(data, arm32reg) {
sigPtr := uint64(match) // from int
ldr := binary.LittleEndian.Uint32(data[sigPtr+ARM32_sig.moduleDataPtrLDR:][:4])
// ARM PC relative is always +8 due to legacy nonsense
ldr_pointer_stub := uint64((ldr & 0x00000FFF) + 8)
final := uint64(binary.LittleEndian.Uint32(data[sigPtr+ARM32_sig.moduleDataPtrLDR+ldr_pointer_stub:][:4]))
matches = append(matches, SignatureMatch{
final,
})
}
for _, match := range m.MatchesForSubexp("PPC_BE") {
sigPtr := uint64(match[0]) // from int
ppcBEreg, err := RegexpPatternFromYaraPattern(PPC_BE_sig.signature)
if err != nil {
panic(err)
}
for _, match := range FindRegex(data, ppcBEreg) {
sigPtr := uint64(match) // from int
moduleDataPtrHi := int64(binary.BigEndian.Uint16(data[sigPtr+PPC_BE_sig.moduleDataPtrHi:][:2]))
// addi takes a signed immediate
moduleDataPtrLo := int64(int16(binary.BigEndian.Uint16(data[sigPtr+PPC_BE_sig.moduleDataPtrLo:][:2])))
moduleDataIpOffset := uint64((moduleDataPtrHi << 16) + moduleDataPtrLo)
matches = append(matches, SignatureMatch{
moduleDataIpOffset,