Fix invalid encoding on level 9 with single value input (#1115)

* Fix invalid encoding on level 9 with single value input

With single value input and a full block write (>=64K) the indexing function would overflow a uint16 to a 0.

This would make it impossible to generate a valid huffman table for the literal size prediction.

In turn this would mean that the entire block would be output as literals - since the cost of the value would be 0 bits.

This would in turn mean that EOB could not be encoded for the bit writer - since there were no matches. This was previously being satisfied with "filling".

Fixes:

1. First never encode more than `maxFlateBlockTokens` - 32K for the literal estimate table.
2. Always include EOB explicitly - if somehow literals should slip through.
3. Add test that will write big single-value input as regression test. Others were using copy that does smaller writes.

Fixes #1114

* Retract v1.18.1

Author: klauspost

flate/deflate.go

@@ -421,7 +421,9 @@ func (d *compressor) deflateLazy() {
 			d.h = newHuffmanEncoder(maxFlateBlockTokens)
 		}
 		var tmp [256]uint16
-		for _, v := range d.window[s.index:d.windowEnd] {
+		toIndex := d.window[s.index:d.windowEnd]
+		toIndex = toIndex[:min(len(toIndex), maxFlateBlockTokens)]
+		for _, v := range toIndex {
 			tmp[v]++
 		}
 		d.h.generate(tmp[:], 15)

flate/deflate_test.go

@@ -170,6 +170,39 @@ func TestVeryLongSparseChunk(t *testing.T) {
 	t.Log("Length:", buf.Len())
 }
 
+func TestOneMByte(t *testing.T) {
+	var input [1024 * 1024]byte
+
+	var compressedOutput bytes.Buffer
+	for level := HuffmanOnly; level <= BestCompression; level++ {
+		compressedOutput.Reset()
+		compressor, err := NewWriter(&compressedOutput, level)
+		if err != nil {
+			t.Fatalf("create: %s", err)
+		}
+		// Use single write...
+		if _, err := compressor.Write(input[:]); err != nil {
+			t.Fatalf("compress: %s", err)
+		}
+
+		if err := compressor.Close(); err != nil {
+			t.Fatalf("close: %s", err)
+		}
+
+		var decompressedOutput bytes.Buffer
+
+		decompresser := NewReader(&compressedOutput)
+		t.Log("level:", level, "compressed:", compressedOutput.Len())
+		if _, err := io.Copy(&decompressedOutput, decompresser); err != nil {
+			t.Fatalf("decompress: %s", err)
+		}
+
+		if !bytes.Equal(input[:], decompressedOutput.Bytes()) {
+			t.Fatal("input and output do not match")
+		}
+	}
+}
+
 type syncBuffer struct {
 	buf    bytes.Buffer
 	mu     sync.RWMutex

flate/fuzz_test.go

@@ -75,14 +75,11 @@ func FuzzEncoding(f *testing.F) {
 			if !bytes.Equal(data, data2) {
 				t.Fatal(msg + "not equal")
 			}
-			// Do it again...
+			// Do it again... (also uses copy)
 			msg = "level " + strconv.Itoa(level) + " (reset):"
 			buf.Reset()
 			fw.Reset(buf)
-			n, err = fw.Write(data)
-			if n != len(data) {
-				t.Fatal(msg + "short write")
-			}
+			_, err = io.Copy(fw, bytes.NewReader(data))
 			if err != nil {
 				t.Fatal(msg + err.Error())
 			}

flate/huffman_bit_writer.go

@@ -646,7 +646,7 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b
 		w.lastHeader = 0
 	}
 
-	numLiterals, numOffsets := w.indexTokens(tokens, fillReuse && !sync)
+	numLiterals, numOffsets := w.indexTokens(tokens, true)
 	extraBits := 0
 	ssize, storable := w.storedSize(input)
 
@@ -781,7 +781,7 @@ func (w *huffmanBitWriter) fillTokens() {
 // literalFreq and offsetFreq, and generates literalEncoding
 // and offsetEncoding.
 // The number of literal and offset tokens is returned.
-func (w *huffmanBitWriter) indexTokens(t *tokens, filled bool) (numLiterals, numOffsets int) {
+func (w *huffmanBitWriter) indexTokens(t *tokens, alwaysEOB bool) (numLiterals, numOffsets int) {
 	//copy(w.literalFreq[:], t.litHist[:])
 	*(*[256]uint16)(w.literalFreq[:]) = t.litHist
 	//copy(w.literalFreq[256:], t.extraHist[:])
@@ -791,9 +791,10 @@ func (w *huffmanBitWriter) indexTokens(t *tokens, filled bool) (numLiterals, num
 	if t.n == 0 {
 		return
 	}
-	if filled {
-		return maxNumLit, maxNumDist
+	if alwaysEOB {
+		w.literalFreq[endBlockMarker] = 1
 	}
+
 	// get the number of literals
 	numLiterals = len(w.literalFreq)
 	for w.literalFreq[numLiterals-1] == 0 {

flate/testdata/huffman-rand-1k.dyn.expect-noinput

@@ -3,6 +3,9 @@ module github.com/klauspost/compress
 go 1.23
 
 retract (
+	// https://github.com/klauspost/compress/issues/1114
+	v1.18.1
+
 	// https://github.com/klauspost/compress/pull/503
 	v1.14.3
 	v1.14.2