From 8ce8cba533b0f680f66b763fbe02836ef6557ce6 Mon Sep 17 00:00:00 2001 From: Billy Keyes Date: Sat, 13 Apr 2019 22:24:49 -0700 Subject: [PATCH] Fix binary chunk decoding and inflating Git uses a unique Base85 encoding with different characters than the ascii85 encoding implemented by Go, so add a custom decoding function. Once decoded, use zlib instead of the raw DEFLATE algorithm to decompress the data. These issues were caught by some basic parsing tests which are added here as well. --- gitdiff/base85.go | 56 ++++++++++++++++++++++++++++++++ gitdiff/parser.go | 42 +++++++++++------------- gitdiff/parser_binary_test.go | 61 +++++++++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+), 23 deletions(-) create mode 100644 gitdiff/base85.go diff --git a/gitdiff/base85.go b/gitdiff/base85.go new file mode 100644 index 0000000..248cf12 --- /dev/null +++ b/gitdiff/base85.go @@ -0,0 +1,56 @@ +package gitdiff + +import ( + "fmt" +) + +const ( + base85Alphabet = "0123456789" + + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + + "abcdefghijklmnopqrstuvwxyz" + + "!#$%&()*+-;<=>?@^_`{|}~" +) + +var ( + de85 map[byte]byte +) + +func init() { + de85 = make(map[byte]byte) + for i, c := range base85Alphabet { + de85[byte(c)] = byte(i) + } +} + +// base85Decode decodes Base85-encoded data from src into dst. It uses the +// alphabet defined by base85.c in the Git source tree, which appears to be +// unique. src must contain at least len(dst) bytes of encoded data. +func base85Decode(dst, src []byte) error { + var v uint32 + var n, ndst int + for i, b := range src { + if b, ok := de85[b]; ok { + v = 85*v + uint32(b) + n++ + } else { + return fmt.Errorf("invalid base85 byte at index %d: 0x%x", i, b) + } + if n == 5 { + rem := len(dst) - ndst + for j := 0; j < 4 && j < rem; j++ { + dst[ndst] = byte(v >> 24) + ndst++ + v <<= 8 + } + v = 0 + n = 0 + } + } + if n > 0 { + return fmt.Errorf("base85 data terminated by underpadded sequence") + } + if ndst < len(dst) { + return fmt.Errorf("base85 data is too short: %d < %d", ndst, len(dst)) + } + return nil +} diff --git a/gitdiff/parser.go b/gitdiff/parser.go index 9707c9c..9d6e8a5 100644 --- a/gitdiff/parser.go +++ b/gitdiff/parser.go @@ -3,8 +3,7 @@ package gitdiff import ( "bufio" "bytes" - "compress/flate" - "encoding/ascii85" + "compress/zlib" "fmt" "io" "io/ioutil" @@ -423,35 +422,30 @@ func (p *parser) ParseBinaryChunk(frag *BinaryFragment) error { if line == "\n" { break } - if len(line) < len(shortestValidLine) || (len(line)-2)%5 != 0 { return p.Errorf(0, "binary patch: corrupt data line") } - byteCount := int(line[0]) + byteCount, seq := int(line[0]), line[1:len(line)-1] switch { case 'A' <= byteCount && byteCount <= 'Z': byteCount = byteCount - 'A' + 1 case 'a' <= byteCount && byteCount <= 'z': byteCount = byteCount - 'a' + 27 default: - return p.Errorf(0, "binary patch: invalid length byte: %q", line[0]) + return p.Errorf(0, "binary patch: invalid length byte") } // base85 encodes every 4 bytes into 5 characters, with up to 3 bytes of end padding - maxByteCount := (len(line) - 2) / 5 * 4 - if byteCount >= maxByteCount || byteCount < maxByteCount-3 { - return p.Errorf(0, "binary patch: incorrect byte count: %d", byteCount) + maxByteCount := len(seq) / 5 * 4 + if byteCount > maxByteCount || byteCount < maxByteCount-3 { + return p.Errorf(0, "binary patch: incorrect byte count") } - ndst, _, err := ascii85.Decode(buf, []byte(line[1:]), byteCount < maxBytesPerLine) - if err != nil { + if err := base85Decode(buf[:byteCount], []byte(seq)); err != nil { return p.Errorf(0, "binary patch: %v", err) } - if ndst != byteCount { - return p.Errorf(0, "binary patch: %d byte line decoded as %d", byteCount, ndst) - } - data.Write(buf[:ndst]) + data.Write(buf[:byteCount]) if err := p.Next(); err != nil { if err == io.EOF { @@ -472,18 +466,20 @@ func (p *parser) ParseBinaryChunk(frag *BinaryFragment) error { return nil } -func inflateBinaryChunk(frag *BinaryFragment, r io.Reader) (err error) { - inflater := flate.NewReader(r) - defer func() { - if cerr := inflater.Close(); cerr != nil && err == nil { - err = cerr - } - }() - - data, err := ioutil.ReadAll(inflater) +func inflateBinaryChunk(frag *BinaryFragment, r io.Reader) error { + zr, err := zlib.NewReader(r) if err != nil { return err } + + data, err := ioutil.ReadAll(zr) + if err != nil { + return err + } + if err := zr.Close(); err != nil { + return err + } + if int64(len(data)) != frag.Size { return fmt.Errorf("%d byte fragment inflated to %d", frag.Size, len(data)) } diff --git a/gitdiff/parser_binary_test.go b/gitdiff/parser_binary_test.go index d53ca6c..3b120dd 100644 --- a/gitdiff/parser_binary_test.go +++ b/gitdiff/parser_binary_test.go @@ -1,6 +1,7 @@ package gitdiff import ( + "encoding/binary" "io" "reflect" "testing" @@ -108,3 +109,63 @@ func TestParseBinaryFragmentHeader(t *testing.T) { }) } } + +func TestParseBinaryChunk(t *testing.T) { + tests := map[string]struct { + Input string + Fragment BinaryFragment + Output []byte + Err bool + }{ + "newFile": { + Input: "gcmZQzU|?i`U?w2V48*KJ%mKu_Kr9NxNf->s?WfX|B-=Vs{#X~svra7Ekg#T|4s}nH;WnAZ)|1Y*`&cB\n" + + "s(sh?X(Uz6L^!Ou&aF*u`J!eibJifSrv0z>$Q%Hd(^HIJ