buffer_pool.go

/*
 *
 * Copyright 2024 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package mem

import (
	"fmt"
	"math/bits"
	"slices"
	"sort"
	"sync"

	"google.golang.org/grpc/internal"
)

// BufferPool is a pool of buffers that can be shared and reused, resulting in
// decreased memory allocation.
type BufferPool interface {
	// Get returns a buffer with specified length from the pool.
	Get(length int) *[]byte

	// Put returns a buffer to the pool.
	//
	// The provided pointer must hold a prefix of the buffer obtained via
	// BufferPool.Get to ensure the buffer's entire capacity can be re-used.
	Put(*[]byte)
}

const (
	goPageSizeExponent = 12
	goPageSize         = 1 << goPageSizeExponent // 4KiB. N.B. this must be a power of 2.
)

var (
	defaultBufferPoolSizeExponents = []uint8{
		8,
		goPageSizeExponent,
		14, // 16KB (max HTTP/2 frame size used by gRPC)
		15, // 32KB (default buffer size for io.Copy)
		20, // 1MB
	}
	defaultBufferPool BufferPool
	uintSize          = bits.UintSize // use a variable for mocking during tests.
)

func init() {
	var err error
	defaultBufferPool, err = NewBinaryTieredBufferPool(defaultBufferPoolSizeExponents...)
	if err != nil {
		panic(fmt.Sprintf("Failed to create default buffer pool: %v", err))
	}

	internal.SetDefaultBufferPool = func(pool BufferPool) {
		defaultBufferPool = pool
	}

	internal.SetBufferPoolingThresholdForTesting = func(threshold int) {
		bufferPoolingThreshold = threshold
	}
}

// DefaultBufferPool returns the current default buffer pool. It is a BufferPool
// created with NewBufferPool that uses a set of default sizes optimized for
// expected workflows.
func DefaultBufferPool() BufferPool {
	return defaultBufferPool
}

// NewTieredBufferPool returns a BufferPool implementation that uses multiple
// underlying pools of the given pool sizes.
func NewTieredBufferPool(poolSizes ...int) BufferPool {
	sort.Ints(poolSizes)
	pools := make([]*sizedBufferPool, len(poolSizes))
	for i, s := range poolSizes {
		pools[i] = newSizedBufferPool(s)
	}
	return &tieredBufferPool{
		sizedPools: pools,
	}
}

// tieredBufferPool implements the BufferPool interface with multiple tiers of
// buffer pools for different sizes of buffers.
type tieredBufferPool struct {
	sizedPools   []*sizedBufferPool
	fallbackPool simpleBufferPool
}

func (p *tieredBufferPool) Get(size int) *[]byte {
	return p.getPool(size).Get(size)
}

func (p *tieredBufferPool) Put(buf *[]byte) {
	p.getPool(cap(*buf)).Put(buf)
}

func (p *tieredBufferPool) getPool(size int) BufferPool {
	poolIdx := sort.Search(len(p.sizedPools), func(i int) bool {
		return p.sizedPools[i].defaultSize >= size
	})

	if poolIdx == len(p.sizedPools) {
		return &p.fallbackPool
	}

	return p.sizedPools[poolIdx]
}

type binaryTieredBufferPool struct {
	// exponentToNextLargestPoolMap maps a power-of-two exponent (e.g., 12 for
	// 4KB) to the index of the next largest sizedBufferPool. This is used by
	// Get() to find the smallest pool that can satisfy a request for a given
	// size.
	exponentToNextLargestPoolMap []int
	// exponentToPreviousLargestPoolMap maps a power-of-two exponent to the
	// index of the previous largest sizedBufferPool. This is used by Put()
	// to return a buffer to the most appropriate pool based on its capacity.
	exponentToPreviousLargestPoolMap []int
	sizedPools                       []*sizedBufferPool
	fallbackPool                     simpleBufferPool
	maxPoolCap                       int // Optimization: Cache max capacity
}

// NewBinaryTieredBufferPool returns a BufferPool backed by multiple sub-pools.
// This structure enables O(1) lookup time for Get and Put operations.
//
// The arguments provided are the exponents for the buffer capacities (powers
// of 2), not the raw byte sizes. For example, to create a pool of 16KB buffers
// (2^14 bytes), pass 14 as the argument.
func NewBinaryTieredBufferPool(powerOfTwoExponents ...uint8) (BufferPool, error) {
	slices.Sort(powerOfTwoExponents)
	powerOfTwoExponents = slices.Compact(powerOfTwoExponents)

	// Determine the maximum exponent we need to support. This depends on the
	// word size (32-bit vs 64-bit).
	maxExponent := uintSize - 1
	indexOfNextLargestBit := slices.Repeat([]int{-1}, maxExponent+1)
	indexOfPreviousLargestBit := slices.Repeat([]int{-1}, maxExponent+1)

	maxTier := 0
	pools := make([]*sizedBufferPool, 0, len(powerOfTwoExponents))

	for i, exp := range powerOfTwoExponents {
		// Allocating slices of size > 2^maxExponent isn't possible on
		// maxExponent-bit machines.
		if int(exp) > maxExponent {
			return nil, fmt.Errorf("mem: allocating slice of size 2^%d is not possible", exp)
		}
		tierSize := 1 << exp
		pools = append(pools, newSizedBufferPool(tierSize))
		maxTier = max(maxTier, tierSize)

		// Map the exact power of 2 to this pool index.
		indexOfNextLargestBit[exp] = i
		indexOfPreviousLargestBit[exp] = i
	}

	// Fill gaps for Get() (Next Largest)
	// We iterate backwards. If current is empty, take the value from the right (larger).
	for i := maxExponent - 1; i >= 0; i-- {
		if indexOfNextLargestBit[i] == -1 {
			indexOfNextLargestBit[i] = indexOfNextLargestBit[i+1]
		}
	}

	// Fill gaps for Put() (Previous Largest)
	// We iterate forwards. If current is empty, take the value from the left (smaller).
	for i := 1; i <= maxExponent; i++ {
		if indexOfPreviousLargestBit[i] == -1 {
			indexOfPreviousLargestBit[i] = indexOfPreviousLargestBit[i-1]
		}
	}

	return &binaryTieredBufferPool{
		exponentToNextLargestPoolMap:     indexOfNextLargestBit,
		exponentToPreviousLargestPoolMap: indexOfPreviousLargestBit,
		sizedPools:                       pools,
		maxPoolCap:                       maxTier,
	}, nil
}

func (b *binaryTieredBufferPool) Get(size int) *[]byte {
	return b.poolForGet(size).Get(size)
}

func (b *binaryTieredBufferPool) poolForGet(size int) BufferPool {
	if size == 0 || size > b.maxPoolCap {
		return &b.fallbackPool
	}

	// Calculate the exponent of the smallest power of 2 >= size.
	// We subtract 1 from size to handle exact powers of 2 correctly.
	//
	// Examples:
	// size=16 (0b10000) -> size-1=15 (0b01111) -> bits.Len=4 -> Pool for 2^4
	// size=17 (0b10001) -> size-1=16 (0b10000) -> bits.Len=5 -> Pool for 2^5
	querySize := uint(size - 1)
	poolIdx := b.exponentToNextLargestPoolMap[bits.Len(querySize)]

	return b.sizedPools[poolIdx]
}

func (b *binaryTieredBufferPool) Put(buf *[]byte) {
	// We pass the capacity of the buffer, and not the size of the buffer here.
	// If we did the latter, all buffers would eventually move to the smallest
	// pool.
	b.poolForPut(cap(*buf)).Put(buf)
}

func (b *binaryTieredBufferPool) poolForPut(bCap int) BufferPool {
	if bCap == 0 {
		return NopBufferPool{}
	}
	if bCap > b.maxPoolCap {
		return &b.fallbackPool
	}
	// Find the pool with the largest capacity <= bCap.
	//
	// We calculate the exponent of the largest power of 2 <= bCap.
	// bits.Len(x) returns the minimum number of bits required to represent x;
	// i.e. the number of bits up to and including the most significant bit.
	// Subtracting 1 gives the 0-based index of the most significant bit,
	// which is the exponent of the largest power of 2 <= bCap.
	//
	// Examples:
	// cap=16 (0b10000) -> Len=5 -> 5-1=4 -> 2^4
	// cap=15 (0b01111) -> Len=4 -> 4-1=3 -> 2^3
	largestPowerOfTwo := bits.Len(uint(bCap)) - 1
	poolIdx := b.exponentToPreviousLargestPoolMap[largestPowerOfTwo]
	// The buffer is smaller than the smallest power of 2, discard it.
	if poolIdx == -1 {
		// Buffer is smaller than our smallest pool bucket.
		return NopBufferPool{}
	}
	return b.sizedPools[poolIdx]
}

// sizedBufferPool is a BufferPool implementation that is optimized for specific
// buffer sizes. For example, HTTP/2 frames within gRPC have a default max size
// of 16kb and a sizedBufferPool can be configured to only return buffers with a
// capacity of 16kb. Note that however it does not support returning larger
// buffers and in fact panics if such a buffer is requested. Because of this,
// this BufferPool implementation is not meant to be used on its own and rather
// is intended to be embedded in a tieredBufferPool such that Get is only
// invoked when the required size is smaller than or equal to defaultSize.
type sizedBufferPool struct {
	pool        sync.Pool
	defaultSize int
}

func (p *sizedBufferPool) Get(size int) *[]byte {
	buf, ok := p.pool.Get().(*[]byte)
	if !ok {
		buf := make([]byte, size, p.defaultSize)
		return &buf
	}
	b := *buf
	clear(b[:cap(b)])
	*buf = b[:size]
	return buf
}

func (p *sizedBufferPool) Put(buf *[]byte) {
	if cap(*buf) < p.defaultSize {
		// Ignore buffers that are too small to fit in the pool. Otherwise, when
		// Get is called it will panic as it tries to index outside the bounds
		// of the buffer.
		return
	}
	p.pool.Put(buf)
}

func newSizedBufferPool(size int) *sizedBufferPool {
	return &sizedBufferPool{
		defaultSize: size,
	}
}

var _ BufferPool = (*simpleBufferPool)(nil)

// simpleBufferPool is an implementation of the BufferPool interface that
// attempts to pool buffers with a sync.Pool. When Get is invoked, it tries to
// acquire a buffer from the pool but if that buffer is too small, it returns it
// to the pool and creates a new one.
type simpleBufferPool struct {
	pool sync.Pool
}

func (p *simpleBufferPool) Get(size int) *[]byte {
	bs, ok := p.pool.Get().(*[]byte)
	if ok && cap(*bs) >= size {
		clear((*bs)[:cap(*bs)])
		*bs = (*bs)[:size]
		return bs
	}

	// A buffer was pulled from the pool, but it is too small. Put it back in
	// the pool and create one large enough.
	if ok {
		p.pool.Put(bs)
	}

	// If we're going to allocate, round up to the nearest page. This way if
	// requests frequently arrive with small variation we don't allocate
	// repeatedly if we get unlucky and they increase over time. By default we
	// only allocate here if size > 1MiB. Because goPageSize is a power of 2, we
	// can round up efficiently.
	allocSize := (size + goPageSize - 1) & ^(goPageSize - 1)

	b := make([]byte, size, allocSize)
	return &b
}

func (p *simpleBufferPool) Put(buf *[]byte) {
	p.pool.Put(buf)
}

var _ BufferPool = NopBufferPool{}

// NopBufferPool is a buffer pool that returns new buffers without pooling.
type NopBufferPool struct{}

// Get returns a buffer with specified length from the pool.
func (NopBufferPool) Get(length int) *[]byte {
	b := make([]byte, length)
	return &b
}

// Put returns a buffer to the pool.
func (NopBufferPool) Put(*[]byte) {
}