Skip to content

Commit 8d6ca65

Browse files
authored
feature/s3/manager: Suppress WriterAt which can bypass Windows optimizations. (#1791)
Fixes issue with Go 1.19 discovered in the S3 transfer manager's windows upload and downloader buffer pools. It looks like the update to Go 1.19 changes the code path that is used by the pool because io.NopCloser was updated to forward WriterTo if the underlying Writer implemented it. Looks likes we'd made some assumptions about this behavior not being present. This PR suppresses the WriterTo of the buffer pool to prevent the unexpected usage.
1 parent 876fd54 commit 8d6ca65

File tree

2 files changed

+16
-2
lines changed

2 files changed

+16
-2
lines changed

feature/s3/manager/download.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,8 @@ func WithDownloaderClientOptions(opts ...func(*s3.Options)) func(*Downloader) {
100100
// interface.
101101
//
102102
// Example:
103-
// // Load AWS Config
103+
//
104+
// // Load AWS Config
104105
// cfg, err := config.LoadDefaultConfig(context.TODO())
105106
// if err != nil {
106107
// panic(err)
@@ -153,6 +154,7 @@ func NewDownloader(c DownloadAPIClient, options ...func(*Downloader)) *Downloade
153154
// and GC runs.
154155
//
155156
// Example:
157+
//
156158
// // pre-allocate in memory buffer, where headObject type is *s3.HeadObjectOutput
157159
// buf := make([]byte, int(headObject.ContentLength))
158160
// // wrap with aws.WriteAtBuffer
@@ -397,7 +399,11 @@ func (d *downloader) tryDownloadChunk(params *s3.GetObjectInput, w io.Writer) (i
397399
}
398400
d.setTotalBytes(resp) // Set total if not yet set.
399401

400-
n, err := io.Copy(w, resp.Body)
402+
var src io.Reader = resp.Body
403+
if d.cfg.BufferProvider != nil {
404+
src = &suppressWriterAt{suppressed: src}
405+
}
406+
n, err := io.Copy(w, src)
401407
resp.Body.Close()
402408
if err != nil {
403409
return n, &errReadingBody{err: err}

feature/s3/manager/writer_read_from.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,3 +73,11 @@ func (p *PooledBufferedReadFromProvider) GetReadFrom(writer io.Writer) (r Writer
7373
}
7474
return r, cleanup
7575
}
76+
77+
type suppressWriterAt struct {
78+
suppressed io.Reader
79+
}
80+
81+
func (s *suppressWriterAt) Read(p []byte) (n int, err error) {
82+
return s.suppressed.Read(p)
83+
}

0 commit comments

Comments
 (0)