Sync with master and update default package lists

buildid/note.go

@@ -82,15 +82,19 @@ var elfGNUNote = []byte("GNU\x00")
 func readELF(name string, f *os.File, data []byte) (buildid string, err error) {
 	// Assume the note content is in the data, already read.
 	// Rewrite the ELF header to set shnum to 0, so that we can pass
+	// Rewrite the ELF header to set shoff and shnum to 0, so that we can pass
 	// the data to elf.NewFile and it will decode the Prog list but not
 	// try to read the section headers and the string table from disk.
 	// That's a waste of I/O when all we care about is the Prog list
 	// and the one ELF note.
 	switch elf.Class(data[elf.EI_CLASS]) {
 	case elf.ELFCLASS32:
+		data[32], data[33], data[34], data[35] = 0, 0, 0, 0
 		data[48] = 0
 		data[49] = 0
 	case elf.ELFCLASS64:
+		data[40], data[41], data[42], data[43] = 0, 0, 0, 0
+		data[44], data[45], data[46], data[47] = 0, 0, 0, 0
 		data[60] = 0
 		data[61] = 0
 	}

buildinfo/buildinfo.go

@@ -161,7 +161,7 @@ func readRawBuildInfo(r io.ReaderAt) (vers, mod string, err error) {
 			data = data[i:]
 			break
 		}
-		data = data[(i+buildInfoAlign-1)&^buildInfoAlign:]
+		data = data[(i+buildInfoAlign-1)&^(buildInfoAlign-1):]
 	}
 
 	// Decode the blob.
@@ -189,8 +189,10 @@ func readRawBuildInfo(r io.ReaderAt) (vers, mod string, err error) {
 		var readPtr func([]byte) uint64
 		if ptrSize == 4 {
 			readPtr = func(b []byte) uint64 { return uint64(bo.Uint32(b)) }
-		} else {
+		} else if ptrSize == 8 {
 			readPtr = bo.Uint64
+		} else {
+			return "", "", errNotGoExe
 		}
 		vers = readString(x, ptrSize, readPtr, readPtr(data[16:]))
 		mod = readString(x, ptrSize, readPtr, readPtr(data[16+ptrSize:]))

debug/dwarf/buf.go

@@ -67,7 +67,7 @@ func (b *buf) uint8() uint8 {
 }
 
 func (b *buf) bytes(n int) []byte {
-	if len(b.data) < n {
+	if n < 0 || len(b.data) < n {
 		b.error("underflow")
 		return nil
 	}

debug/dwarf/const.go

@@ -427,7 +427,7 @@ const (
 	lneSetDiscriminator = 4
 )
 
-// Line table directory directory and file name entry formats.
+// Line table directory and file name entry formats.
 // These are new in DWARF 5.
 const (
 	lnctPath           = 0x01

debug/dwarf/entry.go

@@ -13,6 +13,7 @@ package dwarf
 import (
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"strconv"
 )
 
@@ -33,7 +34,7 @@ type afield struct {
 // a map from entry format ids to their descriptions
 type abbrevTable map[uint32]abbrev
 
-// ParseAbbrev returns the abbreviation table that starts at byte off
+// parseAbbrev returns the abbreviation table that starts at byte off
 // in the .debug_abbrev section.
 func (d *Data) parseAbbrev(off uint64, vers int) (abbrevTable, error) {
 	if m, ok := d.abbrevCache[off]; ok {
@@ -241,27 +242,27 @@ type Entry struct {
 // A value can be one of several "attribute classes" defined by DWARF.
 // The Go types corresponding to each class are:
 //
-//    DWARF class       Go type        Class
-//    -----------       -------        -----
-//    address           uint64         ClassAddress
-//    block             []byte         ClassBlock
-//    constant          int64          ClassConstant
-//    flag              bool           ClassFlag
-//    reference
-//      to info         dwarf.Offset   ClassReference
-//      to type unit    uint64         ClassReferenceSig
-//    string            string         ClassString
-//    exprloc           []byte         ClassExprLoc
-//    lineptr           int64          ClassLinePtr
-//    loclistptr        int64          ClassLocListPtr
-//    macptr            int64          ClassMacPtr
-//    rangelistptr      int64          ClassRangeListPtr
+//	DWARF class       Go type        Class
+//	-----------       -------        -----
+//	address           uint64         ClassAddress
+//	block             []byte         ClassBlock
+//	constant          int64          ClassConstant
+//	flag              bool           ClassFlag
+//	reference
+//	  to info         dwarf.Offset   ClassReference
+//	  to type unit    uint64         ClassReferenceSig
+//	string            string         ClassString
+//	exprloc           []byte         ClassExprLoc
+//	lineptr           int64          ClassLinePtr
+//	loclistptr        int64          ClassLocListPtr
+//	macptr            int64          ClassMacPtr
+//	rangelistptr      int64          ClassRangeListPtr
 //
 // For unrecognized or vendor-defined attributes, Class may be
 // ClassUnknown.
 type Field struct {
 	Attr  Attr
-	Val   interface{}
+	Val   any
 	Class Class
 }
 
@@ -317,7 +318,7 @@ const (
 	// the "mac" section.
 	ClassMacPtr
 
-	// ClassMacPtr represents values that are an int64 offset into
+	// ClassRangeListPtr represents values that are an int64 offset into
 	// the "rangelist" section.
 	ClassRangeListPtr
 
@@ -355,7 +356,7 @@ const (
 	// into the "loclists" section.
 	ClassLocList
 
-	// ClassRngList represents values that are an int64 offset
+	// ClassRngList represents values that are a uint64 offset
 	// from the base of the "rnglists" section.
 	ClassRngList
 
@@ -380,9 +381,9 @@ func (i Class) GoString() string {
 //
 // A common idiom is to merge the check for nil return with
 // the check that the value has the expected dynamic type, as in:
-//	v, ok := e.Val(AttrSibling).(int64)
 //
-func (e *Entry) Val(a Attr) interface{} {
+//	v, ok := e.Val(AttrSibling).(int64)
+func (e *Entry) Val(a Attr) any {
 	if f := e.AttrField(a); f != nil {
 		return f.Val
 	}
@@ -423,6 +424,76 @@ func (b *buf) entry(cu *Entry, atab abbrevTable, ubase Offset, vers int) *Entry
 		Children: a.children,
 		Field:    make([]Field, len(a.field)),
 	}
+
+	// If we are currently parsing the compilation unit,
+	// we can't evaluate Addrx or Strx until we've seen the
+	// relevant base entry.
+	type delayed struct {
+		idx int
+		off uint64
+		fmt format
+	}
+	var delay []delayed
+
+	resolveStrx := func(strBase, off uint64) string {
+		off += strBase
+		if uint64(int(off)) != off {
+			b.error("DW_FORM_strx offset out of range")
+		}
+
+		b1 := makeBuf(b.dwarf, b.format, "str_offsets", 0, b.dwarf.strOffsets)
+		b1.skip(int(off))
+		is64, _ := b.format.dwarf64()
+		if is64 {
+			off = b1.uint64()
+		} else {
+			off = uint64(b1.uint32())
+		}
+		if b1.err != nil {
+			b.err = b1.err
+			return ""
+		}
+		if uint64(int(off)) != off {
+			b.error("DW_FORM_strx indirect offset out of range")
+		}
+		b1 = makeBuf(b.dwarf, b.format, "str", 0, b.dwarf.str)
+		b1.skip(int(off))
+		val := b1.string()
+		if b1.err != nil {
+			b.err = b1.err
+		}
+		return val
+	}
+
+	resolveRnglistx := func(rnglistsBase, off uint64) uint64 {
+		is64, _ := b.format.dwarf64()
+		if is64 {
+			off *= 8
+		} else {
+			off *= 4
+		}
+		off += rnglistsBase
+		if uint64(int(off)) != off {
+			b.error("DW_FORM_rnglistx offset out of range")
+		}
+
+		b1 := makeBuf(b.dwarf, b.format, "rnglists", 0, b.dwarf.rngLists)
+		b1.skip(int(off))
+		if is64 {
+			off = b1.uint64()
+		} else {
+			off = uint64(b1.uint32())
+		}
+		if b1.err != nil {
+			b.err = b1.err
+			return 0
+		}
+		if uint64(int(off)) != off {
+			b.error("DW_FORM_rnglistx indirect offset out of range")
+		}
+		return rnglistsBase + off
+	}
+
 	for i := range e.Field {
 		e.Field[i].Attr = a.field[i].attr
 		e.Field[i].Class = a.field[i].class
@@ -431,7 +502,7 @@ func (b *buf) entry(cu *Entry, atab abbrevTable, ubase Offset, vers int) *Entry
 			fmt = format(b.uint())
 			e.Field[i].Class = formToClass(fmt, a.field[i].attr, vers, b)
 		}
-		var val interface{}
+		var val any
 		switch fmt {
 		default:
 			b.error("unknown entry attr format 0x" + strconv.FormatInt(int64(fmt), 16))
@@ -467,10 +538,13 @@ func (b *buf) entry(cu *Entry, atab abbrevTable, ubase Offset, vers int) *Entry
 			var addrBase int64
 			if cu != nil {
 				addrBase, _ = cu.Val(AttrAddrBase).(int64)
+			} else if a.tag == TagCompileUnit {
+				delay = append(delay, delayed{i, off, formAddrx})
+				break
 			}
 
 			var err error
-			val, err = b.dwarf.debugAddr(uint64(addrBase), off)
+			val, err = b.dwarf.debugAddr(b.format, uint64(addrBase), off)
 			if err != nil {
 				if b.err == nil {
 					b.err = err
@@ -568,6 +642,7 @@ func (b *buf) entry(cu *Entry, atab abbrevTable, ubase Offset, vers int) *Entry
 			} else {
 				if len(b.dwarf.lineStr) == 0 {
 					b.error("DW_FORM_line_strp with no .debug_line_str section")
+					return nil
 				}
 				b1 = makeBuf(b.dwarf, b.format, "line_str", 0, b.dwarf.lineStr)
 			}
@@ -611,38 +686,16 @@ func (b *buf) entry(cu *Entry, atab abbrevTable, ubase Offset, vers int) *Entry
 			// compilation unit. This won't work if the
 			// program uses Reader.Seek to skip over the
 			// unit. Not much we can do about that.
+			var strBase int64
 			if cu != nil {
-				cuOff, ok := cu.Val(AttrStrOffsetsBase).(int64)
-				if ok {
-					off += uint64(cuOff)
-				}
+				strBase, _ = cu.Val(AttrStrOffsetsBase).(int64)
+			} else if a.tag == TagCompileUnit {
+				delay = append(delay, delayed{i, off, formStrx})
+				break
 			}
 
-			if uint64(int(off)) != off {
-				b.error("DW_FORM_strx offset out of range")
-			}
+			val = resolveStrx(uint64(strBase), off)
 
-			b1 := makeBuf(b.dwarf, b.format, "str_offsets", 0, b.dwarf.strOffsets)
-			b1.skip(int(off))
-			if is64 {
-				off = b1.uint64()
-			} else {
-				off = uint64(b1.uint32())
-			}
-			if b1.err != nil {
-				b.err = b1.err
-				return nil
-			}
-			if uint64(int(off)) != off {
-				b.error("DW_FORM_strx indirect offset out of range")
-			}
-			b1 = makeBuf(b.dwarf, b.format, "str", 0, b.dwarf.str)
-			b1.skip(int(off))
-			val = b1.string()
-			if b1.err != nil {
-				b.err = b1.err
-				return nil
-			}
 		case formStrpSup:
 			is64, known := b.format.dwarf64()
 			if !known {
@@ -687,17 +740,58 @@ func (b *buf) entry(cu *Entry, atab abbrevTable, ubase Offset, vers int) *Entry
 
 		// rnglist
 		case formRnglistx:
-			val = b.uint()
+			off := b.uint()
+
+			// We have to adjust by the rnglists_base of
+			// the compilation unit. This won't work if
+			// the program uses Reader.Seek to skip over
+			// the unit. Not much we can do about that.
+			var rnglistsBase int64
+			if cu != nil {
+				rnglistsBase, _ = cu.Val(AttrRnglistsBase).(int64)
+			} else if a.tag == TagCompileUnit {
+				delay = append(delay, delayed{i, off, formRnglistx})
+				break
+			}
+
+			val = resolveRnglistx(uint64(rnglistsBase), off)
 		}
+
 		e.Field[i].Val = val
 	}
 	if b.err != nil {
 		return nil
 	}
+
+	for _, del := range delay {
+		switch del.fmt {
+		case formAddrx:
+			addrBase, _ := e.Val(AttrAddrBase).(int64)
+			val, err := b.dwarf.debugAddr(b.format, uint64(addrBase), del.off)
+			if err != nil {
+				b.err = err
+				return nil
+			}
+			e.Field[del.idx].Val = val
+		case formStrx:
+			strBase, _ := e.Val(AttrStrOffsetsBase).(int64)
+			e.Field[del.idx].Val = resolveStrx(uint64(strBase), del.off)
+			if b.err != nil {
+				return nil
+			}
+		case formRnglistx:
+			rnglistsBase, _ := e.Val(AttrRnglistsBase).(int64)
+			e.Field[del.idx].Val = resolveRnglistx(uint64(rnglistsBase), del.off)
+			if b.err != nil {
+				return nil
+			}
+		}
+	}
+
 	return e
 }
 
-// A Reader allows reading Entry structures from a DWARF ``info'' section.
+// A Reader allows reading Entry structures from a DWARF “info” section.
 // The Entry structures are arranged in a tree. The Reader's Next function
 // return successive entries from a pre-order traversal of the tree.
 // If an entry has children, its Children field will be true, and the children
@@ -714,7 +808,7 @@ type Reader struct {
 }
 
 // Reader returns a new Reader for Data.
-// The reader is positioned at byte offset 0 in the DWARF ``info'' section.
+// The reader is positioned at byte offset 0 in the DWARF “info” section.
 func (d *Data) Reader() *Reader {
 	r := &Reader{d: d}
 	r.Seek(0)
@@ -877,10 +971,11 @@ func (r *Reader) SeekPC(pc uint64) (*Entry, error) {
 		r.err = nil
 		r.lastChildren = false
 		r.unit = unit
+		r.cu = nil
 		u := &r.d.unit[unit]
 		r.b = makeBuf(r.d, u, "info", u.off, u.data)
 		e, err := r.Next()
-		if err != nil {
+		if err != nil || e == nil || e.Tag == 0 {
 			return nil, err
 		}
 		ranges, err := r.d.Ranges(e)
@@ -946,11 +1041,18 @@ func (d *Data) Ranges(e *Entry) ([][2]uint64, error) {
 			if err != nil {
 				return nil, err
 			}
-			return d.dwarf5Ranges(cu, base, ranges, ret)
+			return d.dwarf5Ranges(u, cu, base, ranges, ret)
 
 		case ClassRngList:
-			// TODO: support DW_FORM_rnglistx
-			return ret, nil
+			rnglist, ok := field.Val.(uint64)
+			if !ok {
+				return ret, nil
+			}
+			cu, base, err := d.baseAddressForEntry(e)
+			if err != nil {
+				return nil, err
+			}
+			return d.dwarf5Ranges(u, cu, base, int64(rnglist), ret)
 
 		default:
 			return ret, nil
@@ -1002,6 +1104,9 @@ func (d *Data) baseAddressForEntry(e *Entry) (*Entry, uint64, error) {
 }
 
 func (d *Data) dwarf2Ranges(u *unit, base uint64, ranges int64, ret [][2]uint64) ([][2]uint64, error) {
+	if ranges < 0 || ranges > int64(len(d.ranges)) {
+		return nil, fmt.Errorf("invalid range offset %d (max %d)", ranges, len(d.ranges))
+	}
 	buf := makeBuf(d, u, "ranges", Offset(ranges), d.ranges[ranges:])
 	for len(buf.data) > 0 {
 		low := buf.addr()
@@ -1021,15 +1126,18 @@ func (d *Data) dwarf2Ranges(u *unit, base uint64, ranges int64, ret [][2]uint64)
 	return ret, nil
 }
 
-// dwarf5Ranges interpets a debug_rnglists sequence, see DWARFv5 section
+// dwarf5Ranges interprets a debug_rnglists sequence, see DWARFv5 section
 // 2.17.3 (page 53).
-func (d *Data) dwarf5Ranges(cu *Entry, base uint64, ranges int64, ret [][2]uint64) ([][2]uint64, error) {
+func (d *Data) dwarf5Ranges(u *unit, cu *Entry, base uint64, ranges int64, ret [][2]uint64) ([][2]uint64, error) {
+	if ranges < 0 || ranges > int64(len(d.rngLists)) {
+		return nil, fmt.Errorf("invalid rnglist offset %d (max %d)", ranges, len(d.ranges))
+	}
 	var addrBase int64
 	if cu != nil {
 		addrBase, _ = cu.Val(AttrAddrBase).(int64)
 	}
 
-	buf := makeBuf(d, d.rngLists, "rnglists", 0, d.rngLists.data)
+	buf := makeBuf(d, u, "rnglists", 0, d.rngLists)
 	buf.skip(int(ranges))
 	for {
 		opcode := buf.uint8()
@@ -1043,7 +1151,7 @@ func (d *Data) dwarf5Ranges(cu *Entry, base uint64, ranges int64, ret [][2]uint6
 		case rleBaseAddressx:
 			baseIdx := buf.uint()
 			var err error
-			base, err = d.debugAddr(uint64(addrBase), baseIdx)
+			base, err = d.debugAddr(u, uint64(addrBase), baseIdx)
 			if err != nil {
 				return nil, err
 			}
@@ -1052,11 +1160,11 @@ func (d *Data) dwarf5Ranges(cu *Entry, base uint64, ranges int64, ret [][2]uint6
 			startIdx := buf.uint()
 			endIdx := buf.uint()
 
-			start, err := d.debugAddr(uint64(addrBase), startIdx)
+			start, err := d.debugAddr(u, uint64(addrBase), startIdx)
 			if err != nil {
 				return nil, err
 			}
-			end, err := d.debugAddr(uint64(addrBase), endIdx)
+			end, err := d.debugAddr(u, uint64(addrBase), endIdx)
 			if err != nil {
 				return nil, err
 			}
@@ -1065,7 +1173,7 @@ func (d *Data) dwarf5Ranges(cu *Entry, base uint64, ranges int64, ret [][2]uint6
 		case rleStartxLength:
 			startIdx := buf.uint()
 			len := buf.uint()
-			start, err := d.debugAddr(uint64(addrBase), startIdx)
+			start, err := d.debugAddr(u, uint64(addrBase), startIdx)
 			if err != nil {
 				return nil, err
 			}
@@ -1093,19 +1201,18 @@ func (d *Data) dwarf5Ranges(cu *Entry, base uint64, ranges int64, ret [][2]uint6
 }
 
 // debugAddr returns the address at idx in debug_addr
-func (d *Data) debugAddr(addrBase, idx uint64) (uint64, error) {
-	off := idx*uint64(d.addr.addrsize()) + addrBase
+func (d *Data) debugAddr(format dataFormat, addrBase, idx uint64) (uint64, error) {
+	off := idx*uint64(format.addrsize()) + addrBase
 
 	if uint64(int(off)) != off {
 		return 0, errors.New("offset out of range")
 	}
 
-	b := makeBuf(d, d.addr, "addr", 0, d.addr.data)
+	b := makeBuf(d, format, "addr", 0, d.addr)
 	b.skip(int(off))
 	val := b.addr()
 	if b.err != nil {
 		return 0, b.err
 	}
-
 	return val, nil
 }

debug/dwarf/line.go

@@ -152,7 +152,7 @@ func (d *Data) LineReader(cu *Entry) (*LineReader, error) {
 		// cu has no line table.
 		return nil, nil
 	}
-	if off > int64(len(d.line)) {
+	if off < 0 || off > int64(len(d.line)) {
 		return nil, errors.New("AttrStmtList value out of range")
 	}
 	// AttrCompDir is optional if all file names are absolute. Use
@@ -215,7 +215,11 @@ func (r *LineReader) readHeader(compDir string) error {
 	} else {
 		headerLength = Offset(buf.uint32())
 	}
-	r.programOffset = buf.off + headerLength
+	programOffset := buf.off + headerLength
+	if programOffset > r.endOffset {
+		return DecodeError{"line", hdrOffset, fmt.Sprintf("malformed line table: program offset %d exceeds end offset %d", programOffset, r.endOffset)}
+	}
+	r.programOffset = programOffset
 	r.minInstructionLength = int(buf.uint8())
 	if r.version >= 4 {
 		// [DWARF4 6.2.4]

debug/dwarf/open.go

@@ -2,9 +2,19 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Package dwarf provides access to DWARF debugging information loaded from
-// executable files, as defined in the DWARF 2.0 Standard at
-// http://dwarfstd.org/doc/dwarf-2.0.0.pdf
+/*
+Package dwarf provides access to DWARF debugging information loaded from
+executable files, as defined in the DWARF 2.0 Standard at
+http://dwarfstd.org/doc/dwarf-2.0.0.pdf.
+
+# Security
+
+This package is not designed to be hardened against adversarial inputs, and is
+outside the scope of https://go.dev/security/policy. In particular, only basic
+validation is done when parsing object files. As such, care should be taken when
+parsing untrusted inputs, as parsing malformed files may consume significant
+resources, or cause panics.
+*/
 package dwarf
 
 import (
@@ -26,10 +36,10 @@ type Data struct {
 	str      []byte
 
 	// New sections added in DWARF 5.
-	addr       *debugAddr
+	addr       []byte
 	lineStr    []byte
 	strOffsets []byte
-	rngLists   *rngLists
+	rngLists   []byte
 
 	// parsed data
 	abbrevCache map[uint64]abbrevTable
@@ -40,21 +50,6 @@ type Data struct {
 	unit        []unit
 }
 
-// rngLists represents the contents of a debug_rnglists section (DWARFv5).
-type rngLists struct {
-	is64  bool
-	asize uint8
-	data  []byte
-	ver   uint16
-}
-
-// debugAddr represents the contents of a debug_addr section (DWARFv5).
-type debugAddr struct {
-	is64  bool
-	asize uint8
-	data  []byte
-}
-
 var errSegmentSelector = errors.New("non-zero segment_selector size not supported")
 
 // New returns a new Data object initialized from the given parameters.
@@ -132,76 +127,14 @@ func (d *Data) AddSection(name string, contents []byte) error {
 	var err error
 	switch name {
 	case ".debug_addr":
-		d.addr, err = d.parseAddrHeader(contents)
+		d.addr = contents
 	case ".debug_line_str":
 		d.lineStr = contents
 	case ".debug_str_offsets":
 		d.strOffsets = contents
 	case ".debug_rnglists":
-		d.rngLists, err = d.parseRngListsHeader(contents)
+		d.rngLists = contents
 	}
 	// Just ignore names that we don't yet support.
 	return err
 }
-
-// parseRngListsHeader reads the header of a debug_rnglists section, see
-// DWARFv5 section 7.28 (page 242).
-func (d *Data) parseRngListsHeader(bytes []byte) (*rngLists, error) {
-	rngLists := &rngLists{data: bytes}
-
-	buf := makeBuf(d, unknownFormat{}, "rnglists", 0, bytes)
-	_, rngLists.is64 = buf.unitLength()
-
-	rngLists.ver = buf.uint16() // version
-
-	rngLists.asize = buf.uint8()
-	segsize := buf.uint8()
-	if segsize != 0 {
-		return nil, errSegmentSelector
-	}
-
-	// Header fields not read: offset_entry_count, offset table
-
-	return rngLists, nil
-}
-
-func (rngLists *rngLists) version() int {
-	return int(rngLists.ver)
-}
-
-func (rngLists *rngLists) dwarf64() (bool, bool) {
-	return rngLists.is64, true
-}
-
-func (rngLists *rngLists) addrsize() int {
-	return int(rngLists.asize)
-}
-
-// parseAddrHeader reads the header of a debug_addr section, see DWARFv5
-// section 7.27 (page 241).
-func (d *Data) parseAddrHeader(bytes []byte) (*debugAddr, error) {
-	addr := &debugAddr{data: bytes}
-
-	buf := makeBuf(d, unknownFormat{}, "addr", 0, bytes)
-	_, addr.is64 = buf.unitLength()
-
-	addr.asize = buf.uint8()
-	segsize := buf.uint8()
-	if segsize != 0 {
-		return nil, errSegmentSelector
-	}
-
-	return addr, nil
-}
-
-func (addr *debugAddr) version() int {
-	return 5
-}
-
-func (addr *debugAddr) dwarf64() (bool, bool) {
-	return addr.is64, true
-}
-
-func (addr *debugAddr) addrsize() int {
-	return int(addr.asize)
-}

debug/dwarf/type.go

@@ -33,10 +33,14 @@ func (c *CommonType) Size() int64 { return c.ByteSize }
 // Basic types
 
 // A BasicType holds fields common to all basic types.
+//
+// See the documentation for StructField for more info on the interpretation of
+// the BitSize/BitOffset/DataBitOffset fields.
 type BasicType struct {
 	CommonType
-	BitSize   int64
-	BitOffset int64
+	BitSize       int64
+	BitOffset     int64
+	DataBitOffset int64
 }
 
 func (b *BasicType) Basic() *BasicType { return b }
@@ -150,13 +154,86 @@ type StructType struct {
 }
 
 // A StructField represents a field in a struct, union, or C++ class type.
+//
+// # Bit Fields
+//
+// The BitSize, BitOffset, and DataBitOffset fields describe the bit
+// size and offset of data members declared as bit fields in C/C++
+// struct/union/class types.
+//
+// BitSize is the number of bits in the bit field.
+//
+// DataBitOffset, if non-zero, is the number of bits from the start of
+// the enclosing entity (e.g. containing struct/class/union) to the
+// start of the bit field. This corresponds to the DW_AT_data_bit_offset
+// DWARF attribute that was introduced in DWARF 4.
+//
+// BitOffset, if non-zero, is the number of bits between the most
+// significant bit of the storage unit holding the bit field to the
+// most significant bit of the bit field. Here "storage unit" is the
+// type name before the bit field (for a field "unsigned x:17", the
+// storage unit is "unsigned"). BitOffset values can vary depending on
+// the endianness of the system. BitOffset corresponds to the
+// DW_AT_bit_offset DWARF attribute that was deprecated in DWARF 4 and
+// removed in DWARF 5.
+//
+// At most one of DataBitOffset and BitOffset will be non-zero;
+// DataBitOffset/BitOffset will only be non-zero if BitSize is
+// non-zero. Whether a C compiler uses one or the other
+// will depend on compiler vintage and command line options.
+//
+// Here is an example of C/C++ bit field use, along with what to
+// expect in terms of DWARF bit offset info. Consider this code:
+//
+//	struct S {
+//		int q;
+//		int j:5;
+//		int k:6;
+//		int m:5;
+//		int n:8;
+//	} s;
+//
+// For the code above, one would expect to see the following for
+// DW_AT_bit_offset values (using GCC 8):
+//
+//	       Little   |     Big
+//	       Endian   |    Endian
+//	                |
+//	"j":     27     |     0
+//	"k":     21     |     5
+//	"m":     16     |     11
+//	"n":     8      |     16
+//
+// Note that in the above the offsets are purely with respect to the
+// containing storage unit for j/k/m/n -- these values won't vary based
+// on the size of prior data members in the containing struct.
+//
+// If the compiler emits DW_AT_data_bit_offset, the expected values
+// would be:
+//
+//	"j":     32
+//	"k":     37
+//	"m":     43
+//	"n":     48
+//
+// Here the value 32 for "j" reflects the fact that the bit field is
+// preceded by other data members (recall that DW_AT_data_bit_offset
+// values are relative to the start of the containing struct). Hence
+// DW_AT_data_bit_offset values can be quite large for structs with
+// many fields.
+//
+// DWARF also allow for the possibility of base types that have
+// non-zero bit size and bit offset, so this information is also
+// captured for base types, but it is worth noting that it is not
+// possible to trigger this behavior using mainstream languages.
 type StructField struct {
-	Name       string
-	Type       Type
-	ByteOffset int64
-	ByteSize   int64 // usually zero; use Type.Size() for normal fields
-	BitOffset  int64 // within the ByteSize bytes at ByteOffset
-	BitSize    int64 // zero if not a bit field
+	Name          string
+	Type          Type
+	ByteOffset    int64
+	ByteSize      int64 // usually zero; use Type.Size() for normal fields
+	BitOffset     int64
+	DataBitOffset int64
+	BitSize       int64 // zero if not a bit field
 }
 
 func (t *StructType) String() string {
@@ -166,6 +243,13 @@ func (t *StructType) String() string {
 	return t.Defn()
 }
 
+func (f *StructField) bitOffset() int64 {
+	if f.BitOffset != 0 {
+		return f.BitOffset
+	}
+	return f.DataBitOffset
+}
+
 func (t *StructType) Defn() string {
 	s := t.Kind
 	if t.StructName != "" {
@@ -184,7 +268,7 @@ func (t *StructType) Defn() string {
 		s += "@" + strconv.FormatInt(f.ByteOffset, 10)
 		if f.BitSize > 0 {
 			s += " : " + strconv.FormatInt(f.BitSize, 10)
-			s += "@" + strconv.FormatInt(f.BitOffset, 10)
+			s += "@" + strconv.FormatInt(f.bitOffset(), 10)
 		}
 	}
 	s += "}"
@@ -287,16 +371,40 @@ type typeReader interface {
 	AddressSize() int
 }
 
-// Type reads the type at off in the DWARF ``info'' section.
+// Type reads the type at off in the DWARF “info” section.
 func (d *Data) Type(off Offset) (Type, error) {
 	return d.readType("info", d.Reader(), off, d.typeCache, nil)
 }
 
+type typeFixer struct {
+	typedefs   []*TypedefType
+	arraytypes []*Type
+}
+
+func (tf *typeFixer) recordArrayType(t *Type) {
+	if t == nil {
+		return
+	}
+	_, ok := (*t).(*ArrayType)
+	if ok {
+		tf.arraytypes = append(tf.arraytypes, t)
+	}
+}
+
+func (tf *typeFixer) apply() {
+	for _, t := range tf.typedefs {
+		t.Common().ByteSize = t.Type.Size()
+	}
+	for _, t := range tf.arraytypes {
+		zeroArray(t)
+	}
+}
+
 // readType reads a type from r at off of name. It adds types to the
 // type cache, appends new typedef types to typedefs, and computes the
 // sizes of types. Callers should pass nil for typedefs; this is used
 // for internal recursion.
-func (d *Data) readType(name string, r typeReader, off Offset, typeCache map[Offset]Type, typedefs *[]*TypedefType) (Type, error) {
+func (d *Data) readType(name string, r typeReader, off Offset, typeCache map[Offset]Type, fixups *typeFixer) (Type, error) {
 	if t, ok := typeCache[off]; ok {
 		return t, nil
 	}
@@ -311,18 +419,16 @@ func (d *Data) readType(name string, r typeReader, off Offset, typeCache map[Off
 	}
 
 	// If this is the root of the recursion, prepare to resolve
-	// typedef sizes once the recursion is done. This must be done
-	// after the type graph is constructed because it may need to
-	// resolve cycles in a different order than readType
-	// encounters them.
-	if typedefs == nil {
-		var typedefList []*TypedefType
+	// typedef sizes and perform other fixups once the recursion is
+	// done. This must be done after the type graph is constructed
+	// because it may need to resolve cycles in a different order than
+	// readType encounters them.
+	if fixups == nil {
+		var fixer typeFixer
 		defer func() {
-			for _, t := range typedefList {
-				t.Common().ByteSize = t.Type.Size()
-			}
+			fixer.apply()
 		}()
-		typedefs = &typedefList
+		fixups = &fixer
 	}
 
 	// Parse type from Entry.
@@ -376,7 +482,7 @@ func (d *Data) readType(name string, r typeReader, off Offset, typeCache map[Off
 		var t Type
 		switch toff := tval.(type) {
 		case Offset:
-			if t, err = d.readType(name, r.clone(), toff, typeCache, typedefs); err != nil {
+			if t, err = d.readType(name, r.clone(), toff, typeCache, fixups); err != nil {
 				return nil
 			}
 		case uint64:
@@ -447,8 +553,12 @@ func (d *Data) readType(name string, r typeReader, off Offset, typeCache map[Off
 		//	AttrName: name of base type in programming language of the compilation unit [required]
 		//	AttrEncoding: encoding value for type (encFloat etc) [required]
 		//	AttrByteSize: size of type in bytes [required]
-		//	AttrBitOffset: for sub-byte types, size in bits
-		//	AttrBitSize: for sub-byte types, bit offset of high order bit in the AttrByteSize bytes
+		//	AttrBitOffset: bit offset of value within containing storage unit
+		//	AttrDataBitOffset: bit offset of value within containing storage unit
+		//	AttrBitSize: size in bits
+		//
+		// For most languages BitOffset/DataBitOffset/BitSize will not be present
+		// for base types.
 		name, _ := e.Val(AttrName).(string)
 		enc, ok := e.Val(AttrEncoding).(int64)
 		if !ok {
@@ -494,7 +604,14 @@ func (d *Data) readType(name string, r typeReader, off Offset, typeCache map[Off
 		}).Basic()
 		t.Name = name
 		t.BitSize, _ = e.Val(AttrBitSize).(int64)
-		t.BitOffset, _ = e.Val(AttrBitOffset).(int64)
+		haveBitOffset := false
+		haveDataBitOffset := false
+		t.BitOffset, haveBitOffset = e.Val(AttrBitOffset).(int64)
+		t.DataBitOffset, haveDataBitOffset = e.Val(AttrDataBitOffset).(int64)
+		if haveBitOffset && haveDataBitOffset {
+			err = DecodeError{name, e.Offset, "duplicate bit offset attributes"}
+			goto Error
+		}
 
 	case TagClassType, TagStructType, TagUnionType:
 		// Structure, union, or class type.  (DWARF v2 §5.5)
@@ -508,6 +625,7 @@ func (d *Data) readType(name string, r typeReader, off Offset, typeCache map[Off
 		//		AttrType: type of member [required]
 		//		AttrByteSize: size in bytes
 		//		AttrBitOffset: bit offset within bytes for bit fields
+		//		AttrDataBitOffset: field bit offset relative to struct start
 		//		AttrBitSize: bit size for bit fields
 		//		AttrDataMemberLoc: location within struct [required for struct, class]
 		// There is much more to handle C++, all ignored for now.
@@ -526,7 +644,8 @@ func (d *Data) readType(name string, r typeReader, off Offset, typeCache map[Off
 		t.Incomplete = e.Val(AttrDeclaration) != nil
 		t.Field = make([]*StructField, 0, 8)
 		var lastFieldType *Type
-		var lastFieldBitOffset int64
+		var lastFieldBitSize int64
+		var lastFieldByteOffset int64
 		for kid := next(); kid != nil; kid = next() {
 			if kid.Tag != TagMember {
 				continue
@@ -553,30 +672,33 @@ func (d *Data) readType(name string, r typeReader, off Offset, typeCache map[Off
 				f.ByteOffset = loc
 			}
 
-			haveBitOffset := false
 			f.Name, _ = kid.Val(AttrName).(string)
 			f.ByteSize, _ = kid.Val(AttrByteSize).(int64)
+			haveBitOffset := false
+			haveDataBitOffset := false
 			f.BitOffset, haveBitOffset = kid.Val(AttrBitOffset).(int64)
+			f.DataBitOffset, haveDataBitOffset = kid.Val(AttrDataBitOffset).(int64)
+			if haveBitOffset && haveDataBitOffset {
+				err = DecodeError{name, e.Offset, "duplicate bit offset attributes"}
+				goto Error
+			}
 			f.BitSize, _ = kid.Val(AttrBitSize).(int64)
 			t.Field = append(t.Field, f)
 
-			bito := f.BitOffset
-			if !haveBitOffset {
-				bito = f.ByteOffset * 8
-			}
-			if bito == lastFieldBitOffset && t.Kind != "union" {
+			if lastFieldBitSize == 0 && lastFieldByteOffset == f.ByteOffset && t.Kind != "union" {
 				// Last field was zero width. Fix array length.
 				// (DWARF writes out 0-length arrays as if they were 1-length arrays.)
-				zeroArray(lastFieldType)
+				fixups.recordArrayType(lastFieldType)
 			}
 			lastFieldType = &f.Type
-			lastFieldBitOffset = bito
+			lastFieldByteOffset = f.ByteOffset
+			lastFieldBitSize = f.BitSize
 		}
 		if t.Kind != "union" {
 			b, ok := e.Val(AttrByteSize).(int64)
-			if ok && b*8 == lastFieldBitOffset {
+			if ok && b == lastFieldByteOffset {
 				// Final field must be zero width. Fix array length.
-				zeroArray(lastFieldType)
+				fixups.recordArrayType(lastFieldType)
 			}
 		}
 
@@ -719,7 +841,7 @@ func (d *Data) readType(name string, r typeReader, off Offset, typeCache map[Off
 				// Record that we need to resolve this
 				// type's size once the type graph is
 				// constructed.
-				*typedefs = append(*typedefs, t)
+				fixups.typedefs = append(fixups.typedefs, t)
 			case *PtrType:
 				b = int64(addressSize)
 			}
@@ -737,11 +859,8 @@ Error:
 }
 
 func zeroArray(t *Type) {
-	if t == nil {
-		return
-	}
-	at, ok := (*t).(*ArrayType)
-	if !ok || at.Type.Size() == 0 {
+	at := (*t).(*ArrayType)
+	if at.Type.Size() == 0 {
 		return
 	}
 	// Make a copy to avoid invalidating typeCache.

debug/dwarf/unit.go

@@ -48,7 +48,9 @@ func (d *Data) parseUnits() ([]unit, error) {
 			break
 		}
 		b.skip(int(len))
-		nunit++
+		if len > 0 {
+			nunit++
+		}
 	}
 	if b.err != nil {
 		return nil, b.err
@@ -61,7 +63,12 @@ func (d *Data) parseUnits() ([]unit, error) {
 		u := &units[i]
 		u.base = b.off
 		var n Offset
-		n, u.is64 = b.unitLength()
+		if b.err != nil {
+			return nil, b.err
+		}
+		for n == 0 {
+			n, u.is64 = b.unitLength()
+		}
 		dataOff := b.off
 		vers := b.uint16()
 		if vers < 2 || vers > 5 {

debug/elf/elf.go

@@ -733,7 +733,7 @@ const (
 )
 
 var compressionStrings = []intName{
-	{0, "COMPRESS_ZLIB"},
+	{1, "COMPRESS_ZLIB"},
 	{0x60000000, "COMPRESS_LOOS"},
 	{0x6fffffff, "COMPRESS_HIOS"},
 	{0x70000000, "COMPRESS_LOPROC"},

debug/elf/file.go

@@ -18,6 +18,7 @@ import (
 	"strings"
 
 	"github.com/mandiant/GoReSym/debug/dwarf"
+	"github.com/mandiant/GoReSym/saferio"
 )
 
 // seekStart, seekCurrent, seekEnd are copies of
@@ -325,6 +326,13 @@ func NewFile(r io.ReaderAt) (*File, error) {
 		shstrndx = int(hdr.Shstrndx)
 	}
 
+	if shoff < 0 {
+		return nil, &FormatError{0, "invalid shoff", shoff}
+	}
+	if phoff < 0 {
+		return nil, &FormatError{0, "invalid phoff", phoff}
+	}
+
 	if shoff == 0 && shnum != 0 {
 		return nil, &FormatError{0, "invalid ELF shnum for shoff=0", shnum}
 	}
@@ -333,6 +341,19 @@ func NewFile(r io.ReaderAt) (*File, error) {
 		return nil, &FormatError{0, "invalid ELF shstrndx", shstrndx}
 	}
 
+	var wantPhentsize, wantShentsize int
+	switch f.Class {
+	case ELFCLASS32:
+		wantPhentsize = 8 * 4
+		wantShentsize = 10 * 4
+	case ELFCLASS64:
+		wantPhentsize = 2*4 + 6*8
+		wantShentsize = 4*4 + 6*8
+	}
+	if phnum > 0 && phentsize < wantPhentsize {
+		return nil, &FormatError{0, "invalid ELF phentsize", phentsize}
+	}
+
 	// Read program headers
 	f.Progs = make([]*Prog, phnum)
 	for i := 0; i < phnum; i++ {
@@ -371,14 +392,74 @@ func NewFile(r io.ReaderAt) (*File, error) {
 				Align:  ph.Align,
 			}
 		}
+		if int64(p.Off) < 0 {
+			return nil, &FormatError{off, "invalid program header offset", p.Off}
+		}
+		if int64(p.Filesz) < 0 {
+			return nil, &FormatError{off, "invalid program header file size", p.Filesz}
+		}
 		p.sr = io.NewSectionReader(r, int64(p.Off), int64(p.Filesz))
 		p.ReaderAt = p.sr
 		f.Progs[i] = p
 	}
 
+	// If the number of sections is greater than or equal to SHN_LORESERVE
+	// (0xff00), shnum has the value zero and the actual number of section
+	// header table entries is contained in the sh_size field of the section
+	// header at index 0.
+	if shoff > 0 && shnum == 0 {
+		var typ, link uint32
+		sr.Seek(shoff, seekStart)
+		switch f.Class {
+		case ELFCLASS32:
+			sh := new(Section32)
+			if err := binary.Read(sr, f.ByteOrder, sh); err != nil {
+				return nil, err
+			}
+			shnum = int(sh.Size)
+			typ = sh.Type
+			link = sh.Link
+		case ELFCLASS64:
+			sh := new(Section64)
+			if err := binary.Read(sr, f.ByteOrder, sh); err != nil {
+				return nil, err
+			}
+			shnum = int(sh.Size)
+			typ = sh.Type
+			link = sh.Link
+		}
+		if SectionType(typ) != SHT_NULL {
+			return nil, &FormatError{shoff, "invalid type of the initial section", SectionType(typ)}
+		}
+
+		if shnum < int(SHN_LORESERVE) {
+			return nil, &FormatError{shoff, "invalid ELF shnum contained in sh_size", shnum}
+		}
+
+		// If the section name string table section index is greater than or
+		// equal to SHN_LORESERVE (0xff00), this member has the value
+		// SHN_XINDEX (0xffff) and the actual index of the section name
+		// string table section is contained in the sh_link field of the
+		// section header at index 0.
+		if shstrndx == int(SHN_XINDEX) {
+			shstrndx = int(link)
+			if shstrndx < int(SHN_LORESERVE) {
+				return nil, &FormatError{shoff, "invalid ELF shstrndx contained in sh_link", shstrndx}
+			}
+		}
+	}
+
+	if shnum > 0 && shentsize < wantShentsize {
+		return nil, &FormatError{0, "invalid ELF shentsize", shentsize}
+	}
+
 	// Read section headers
-	f.Sections = make([]*Section, shnum)
-	names := make([]uint32, shnum)
+	c := saferio.SliceCap((*Section)(nil), uint64(shnum))
+	if c < 0 {
+		return nil, &FormatError{0, "too many sections", shnum}
+	}
+	f.Sections = make([]*Section, 0, c)
+	names := make([]uint32, 0, c)
 	for i := 0; i < shnum; i++ {
 		off := shoff + int64(i)*int64(shentsize)
 		sr.Seek(off, seekStart)
@@ -389,7 +470,7 @@ func NewFile(r io.ReaderAt) (*File, error) {
 			if err := binary.Read(sr, f.ByteOrder, sh); err != nil {
 				return nil, err
 			}
-			names[i] = sh.Name
+			names = append(names, sh.Name)
 			s.SectionHeader = SectionHeader{
 				Type:      SectionType(sh.Type),
 				Flags:     SectionFlag(sh.Flags),
@@ -406,7 +487,7 @@ func NewFile(r io.ReaderAt) (*File, error) {
 			if err := binary.Read(sr, f.ByteOrder, sh); err != nil {
 				return nil, err
 			}
-			names[i] = sh.Name
+			names = append(names, sh.Name)
 			s.SectionHeader = SectionHeader{
 				Type:      SectionType(sh.Type),
 				Flags:     SectionFlag(sh.Flags),
@@ -419,6 +500,12 @@ func NewFile(r io.ReaderAt) (*File, error) {
 				Entsize:   sh.Entsize,
 			}
 		}
+		if int64(s.Offset) < 0 {
+			return nil, &FormatError{off, "invalid section offset", int64(s.Offset)}
+		}
+		if int64(s.FileSize) < 0 {
+			return nil, &FormatError{off, "invalid section size", int64(s.FileSize)}
+		}
 		s.sr = io.NewSectionReader(r, int64(s.Offset), int64(s.FileSize))
 
 		if s.Flags&SHF_COMPRESSED == 0 {
@@ -448,7 +535,7 @@ func NewFile(r io.ReaderAt) (*File, error) {
 			}
 		}
 
-		f.Sections[i] = s
+		f.Sections = append(f.Sections, s)
 	}
 
 	if len(f.Sections) == 0 {
@@ -456,7 +543,16 @@ func NewFile(r io.ReaderAt) (*File, error) {
 	}
 
 	// Load section header string table.
-	shstrtab, err := f.Sections[shstrndx].Data()
+	if shstrndx == 0 {
+		// If the file has no section name string table,
+		// shstrndx holds the value SHN_UNDEF (0).
+		return f, nil
+	}
+	shstr := f.Sections[shstrndx]
+	if shstr.Type != SHT_STRTAB {
+		return nil, &FormatError{shoff + int64(shstrndx*shentsize), "invalid ELF section name string table type", shstr.Type}
+	}
+	shstrtab, err := shstr.Data()
 	if err != nil {
 		return nil, err
 	}

generatepackages.py

@@ -1,37 +1,68 @@
 # Python >= 3.6, tested with Python 3.9.5
 import requests
 
+auth_user = ""
+auth_token = ""
+
 # https://docs.github.com/en/rest/reference/git#get-a-tree
-API_URL = "https://api.github.com/repos/golang/go/git/trees"
+TREE_API_URL = "https://api.github.com/repos/golang/go/git/trees"
+TAG_API_URL = "https://api.github.com/repos/golang/go/git/refs/tags"
 DIR = "src"
 OUTPUT_FILE = "stdpackages.go"
 VAR_NAME = "standardPackages"
 
-
 def get_tree(tree_sha):
-    url = f"{API_URL}/{tree_sha}"
+    url = f"{TREE_API_URL}/{tree_sha}"
     print(f"Getting {url}")
-    r = requests.get(url)
+    r = requests.get(url, auth=(auth_user,auth_token))
     r.raise_for_status()
     return r.json()
 
+def remove_prefix(text, prefix):
+    return text[text.startswith(prefix) and len(prefix):]
 
-r = get_tree("master")
-for leaf in r["tree"]:
-    if leaf["path"] == DIR:
-        sha = leaf["sha"]
-        break
+def get_go_tags():
+    url = TAG_API_URL
+    print(f"Fetching version tags {url}")
+    r = requests.get(url, auth=(auth_user,auth_token))
+    r.raise_for_status()
+    j = r.json()
+    
+    version_tags = ["master"]
+    for obj in j:
+        tag = remove_prefix(obj["ref"], "refs/tags/")
+        if "weekly" in tag:
+            continue
+        
+        version_tags.append(tag)
+    return version_tags
 
-r = get_tree(f"{sha}?recursive=1")
+def filter_path(path):
+    f = remove_prefix(path, "cmd/vendor/")
+    return f
+    
+# enumerates all go version trees by tag (package paths have been re-ordered over time, so we must get all of them)
+paths = []
+for tag in get_go_tags():
+    r = get_tree(tag)
+    for leaf in r["tree"]:
+        if leaf["path"] == DIR:
+            sha = leaf["sha"]
+            break
+    
+    r = get_tree(f"{sha}?recursive=1")
+    
+    if r["truncated"]:
+        raise RuntimeError("Too many paths, needed to fetch one sub-tree at a time")
+    
+    # enumerates the file tree via directory
+    # Use list instead of set to keep order
+    new_paths = [filter_path(leaf["path"]) for leaf in r["tree"] if leaf["type"] == "tree"]
+    paths.extend(x for x in new_paths if x not in paths)
+    
+print(f"Writing paths to {OUTPUT_FILE}")
 
-if r["truncated"]:
-    raise RuntimeError("Too many paths, needed to fetch one sub-tree at a time")
-
-# Use list instead of set to keep order
-paths = [leaf["path"] for leaf in r["tree"] if leaf["type"] == "tree"]
 # paths in the following format: {"path1", "path2", ...}
 paths_str = '{"' + '", "'.join(paths) + '"}'
-
-print(f"Writing paths to {OUTPUT_FILE}")
 with open(OUTPUT_FILE, "w") as f:
     f.write(f"package main\n\nvar {VAR_NAME} = []string{paths_str}")

go.mod

@@ -1,6 +1,6 @@
 module github.com/mandiant/GoReSym
 
-go 1.17
+go 1.20
 
 require (
 	golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff

saferio/io.go

@@ -0,0 +1,135 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package saferio provides I/O functions that avoid allocating large
+// amounts of memory unnecessarily. This is intended for packages that
+// read data from an [io.Reader] where the size is part of the input
+// data but the input may be corrupt, or may be provided by an
+// untrustworthy attacker.
+package saferio
+
+import (
+	"io"
+	"reflect"
+)
+
+// chunk is an arbitrary limit on how much memory we are willing
+// to allocate without concern.
+const chunk = 10 << 20 // 10M
+
+// ReadData reads n bytes from the input stream, but avoids allocating
+// all n bytes if n is large. This avoids crashing the program by
+// allocating all n bytes in cases where n is incorrect.
+//
+// The error is io.EOF only if no bytes were read.
+// If an io.EOF happens after reading some but not all the bytes,
+// ReadData returns io.ErrUnexpectedEOF.
+func ReadData(r io.Reader, n uint64) ([]byte, error) {
+	if int64(n) < 0 || n != uint64(int(n)) {
+		// n is too large to fit in int, so we can't allocate
+		// a buffer large enough. Treat this as a read failure.
+		return nil, io.ErrUnexpectedEOF
+	}
+
+	if n < chunk {
+		buf := make([]byte, n)
+		_, err := io.ReadFull(r, buf)
+		if err != nil {
+			return nil, err
+		}
+		return buf, nil
+	}
+
+	var buf []byte
+	buf1 := make([]byte, chunk)
+	for n > 0 {
+		next := n
+		if next > chunk {
+			next = chunk
+		}
+		_, err := io.ReadFull(r, buf1[:next])
+		if err != nil {
+			if len(buf) > 0 && err == io.EOF {
+				err = io.ErrUnexpectedEOF
+			}
+			return nil, err
+		}
+		buf = append(buf, buf1[:next]...)
+		n -= next
+	}
+	return buf, nil
+}
+
+// ReadDataAt reads n bytes from the input stream at off, but avoids
+// allocating all n bytes if n is large. This avoids crashing the program
+// by allocating all n bytes in cases where n is incorrect.
+func ReadDataAt(r io.ReaderAt, n uint64, off int64) ([]byte, error) {
+	if int64(n) < 0 || n != uint64(int(n)) {
+		// n is too large to fit in int, so we can't allocate
+		// a buffer large enough. Treat this as a read failure.
+		return nil, io.ErrUnexpectedEOF
+	}
+
+	if n < chunk {
+		buf := make([]byte, n)
+		_, err := r.ReadAt(buf, off)
+		if err != nil {
+			// io.SectionReader can return EOF for n == 0,
+			// but for our purposes that is a success.
+			if err != io.EOF || n > 0 {
+				return nil, err
+			}
+		}
+		return buf, nil
+	}
+
+	var buf []byte
+	buf1 := make([]byte, chunk)
+	for n > 0 {
+		next := n
+		if next > chunk {
+			next = chunk
+		}
+		_, err := r.ReadAt(buf1[:next], off)
+		if err != nil {
+			return nil, err
+		}
+		buf = append(buf, buf1[:next]...)
+		n -= next
+		off += int64(next)
+	}
+	return buf, nil
+}
+
+// SliceCap returns the capacity to use when allocating a slice.
+// After the slice is allocated with the capacity, it should be
+// built using append. This will avoid allocating too much memory
+// if the capacity is large and incorrect.
+//
+// A negative result means that the value is always too big.
+//
+// The element type is described by passing a pointer to a value of that type.
+// This would ideally use generics, but this code is built with
+// the bootstrap compiler which need not support generics.
+// We use a pointer so that we can handle slices of interface type.
+func SliceCap(v any, c uint64) int {
+	if int64(c) < 0 || c != uint64(int(c)) {
+		return -1
+	}
+	typ := reflect.TypeOf(v)
+	if typ.Kind() != reflect.Ptr {
+		panic("SliceCap called with non-pointer type")
+	}
+	size := uint64(typ.Elem().Size())
+	if size > 0 && c > (1<<64-1)/size {
+		return -1
+	}
+	if c*size > chunk {
+		c = uint64(chunk / size)
+		if c == 0 {
+			c = 1
+		}
+	}
+	return int(c)
+}