Skip to content

Commit 5acd7b4

Browse files
MadrigalMadrigal
authored
Add SHA-512 auto checksum calculation for S3 (#3422)
* Add SHA-512 to S3 supported checksums * Add SHA-512 checksum calculation for S3 * add changelog * Add regular s3 to the changelog
1 parent cc6a3d9 commit 5acd7b4

6 files changed

Lines changed: 73 additions & 0 deletions

File tree

‎.changelog/5a03c63707e44718aa1bd8f6ad18ac2b.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"id": "5a03c637-07e4-4718-aa1b-d8f6ad18ac2b",
3+
"type": "feature",
4+
"description": "Add SHA-512 support for s3 operations",
5+
"modules": [
6+
"feature/s3/transfermanager",
7+
"service/s3"
8+
]
9+
}

‎feature/s3/transfermanager/api_op_UploadObject.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,14 @@ type UploadObjectInput struct {
205205
// [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
206206
ChecksumSHA256 *string
207207

208+
// This header can be used as a data integrity check to verify that the data
209+
// received is the same data that was originally sent. This header specifies the
210+
// Base64 encoded, 512-bit SHA512 digest of the object. For more information, see [Checking object integrity]
211+
// in the Amazon S3 User Guide.
212+
//
213+
// [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
214+
ChecksumSHA512 *string
215+
208216
// Size of the body in bytes. This parameter is useful when the size of the body
209217
// cannot be determined automatically. For more information, see [https://www.rfc-editor.org/rfc/rfc9110.html#name-content-length].
210218
//
@@ -475,6 +483,7 @@ func (i UploadObjectInput) mapSingleUploadInput(body io.Reader, checksumAlgorith
475483
ChecksumCRC64NVME: i.ChecksumCRC64NVME,
476484
ChecksumSHA1: i.ChecksumSHA1,
477485
ChecksumSHA256: i.ChecksumSHA256,
486+
ChecksumSHA512: i.ChecksumSHA512,
478487
ContentDisposition: i.ContentDisposition,
479488
ContentEncoding: i.ContentEncoding,
480489
ContentLanguage: i.ContentLanguage,
@@ -561,6 +570,7 @@ func (i UploadObjectInput) mapCompleteMultipartUploadInput(uploadID *string, com
561570
ChecksumCRC64NVME: i.ChecksumCRC64NVME,
562571
ChecksumSHA1: i.ChecksumSHA1,
563572
ChecksumSHA256: i.ChecksumSHA256,
573+
ChecksumSHA512: i.ChecksumSHA512,
564574
ExpectedBucketOwner: i.ExpectedBucketOwner,
565575
IfMatch: i.IfMatch,
566576
IfNoneMatch: i.IfNoneMatch,
@@ -640,6 +650,9 @@ type UploadObjectOutput struct {
640650
// The base64-encoded, 256-bit SHA-256 digest of the object.
641651
ChecksumSHA256 *string
642652

653+
// The base64-encoded, 512-bit SHA-512 digest of the object.
654+
ChecksumSHA512 *string
655+
643656
// This header specifies the checksum type of the object, which determines how
644657
// part-level checksums are combined to create an object-level checksum for
645658
// multipart objects. For PutObject uploads, the checksum type is always
@@ -735,6 +748,7 @@ func (o *UploadObjectOutput) mapFromPutObjectOutput(out *s3.PutObjectOutput, buc
735748
o.ChecksumCRC64NVME = out.ChecksumCRC64NVME
736749
o.ChecksumSHA1 = out.ChecksumSHA1
737750
o.ChecksumSHA256 = out.ChecksumSHA256
751+
o.ChecksumSHA512 = out.ChecksumSHA512
738752
o.ChecksumType = types.ChecksumType(out.ChecksumType)
739753
o.ContentLength = aws.Int64(contentLength)
740754
o.ETag = out.ETag
@@ -761,6 +775,7 @@ func (o *UploadObjectOutput) mapFromCompleteMultipartUploadOutput(out *s3.Comple
761775
o.ChecksumCRC64NVME = out.ChecksumCRC64NVME
762776
o.ChecksumSHA1 = out.ChecksumSHA1
763777
o.ChecksumSHA256 = out.ChecksumSHA256
778+
o.ChecksumSHA512 = out.ChecksumSHA512
764779
o.ChecksumType = types.ChecksumType(out.ChecksumType)
765780
o.ContentLength = aws.Int64(contentLength)
766781
o.ETag = out.ETag

‎feature/s3/transfermanager/mapping_reference_test.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ const mappingReferenceJSON = `
6666
"ChecksumCRC64NVME",
6767
"ChecksumSHA1",
6868
"ChecksumSHA256",
69+
"ChecksumSHA512",
6970
"ContentDisposition",
7071
"ContentEncoding",
7172
"ContentLanguage",
@@ -104,6 +105,7 @@ const mappingReferenceJSON = `
104105
"ChecksumCRC64NVME",
105106
"ChecksumSHA1",
106107
"ChecksumSHA256",
108+
"ChecksumSHA512",
107109
"ChecksumType",
108110
"ETag",
109111
"Expiration",
@@ -197,6 +199,7 @@ const mappingReferenceJSON = `
197199
"ChecksumCRC64NVME",
198200
"ChecksumSHA1",
199201
"ChecksumSHA256",
202+
"ChecksumSHA512",
200203
"ContentDisposition",
201204
"ContentEncoding",
202205
"ContentLanguage",
@@ -274,6 +277,7 @@ const mappingReferenceJSON = `
274277
"ChecksumCRC64NVME",
275278
"ChecksumSHA1",
276279
"ChecksumSHA256",
280+
"ChecksumSHA512",
277281
"ExpectedBucketOwner",
278282
"IfMatch",
279283
"IfNoneMatch",
@@ -299,6 +303,7 @@ const mappingReferenceJSON = `
299303
"ChecksumCRC64NVME",
300304
"ChecksumSHA1",
301305
"ChecksumSHA256",
306+
"ChecksumSHA512",
302307
"ChecksumType",
303308
"ETag",
304309
"Expiration",
@@ -318,6 +323,7 @@ const mappingReferenceJSON = `
318323
"ChecksumCRC64NVME",
319324
"ChecksumSHA1",
320325
"ChecksumSHA256",
326+
"ChecksumSHA512",
321327
"ChecksumType",
322328
"ETag",
323329
"Expiration",

‎feature/s3/transfermanager/types/types.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,7 @@ const (
144144
ChecksumAlgorithmCrc32c = "CRC32C"
145145
ChecksumAlgorithmSha1 = "SHA1"
146146
ChecksumAlgorithmSha256 = "SHA256"
147+
ChecksumAlgorithmSha512 = "SHA512"
147148
)
148149

149150
// ObjectCannedACL defines the canned ACL to apply to the object, see [Canned ACL] in the
@@ -287,6 +288,17 @@ type CompletedPart struct {
287288
// [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html#large-object-checksums
288289
ChecksumSHA256 *string
289290

291+
// The base64-encoded, 512-bit SHA-512 digest of the object. This will only be
292+
// present if it was uploaded with the object. When you use an API operation on an
293+
// object that was uploaded using multipart uploads, this value may not be a direct
294+
// checksum value of the full object. Instead, it's a calculation based on the
295+
// checksum values of each individual part. For more information about how
296+
// checksums are calculated with multipart uploads, see [Checking object integrity]in the Amazon S3 User
297+
// Guide.
298+
//
299+
// [Checking object integrity]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html#large-object-checksums
300+
ChecksumSHA512 *string
301+
290302
// Entity tag returned when the part was uploaded.
291303
ETag *string
292304

@@ -312,6 +324,7 @@ func (cp CompletedPart) MapCompletedPart() types.CompletedPart {
312324
ChecksumCRC32C: cp.ChecksumCRC32C,
313325
ChecksumSHA1: cp.ChecksumSHA1,
314326
ChecksumSHA256: cp.ChecksumSHA256,
327+
ChecksumSHA512: cp.ChecksumSHA512,
315328
ETag: cp.ETag,
316329
PartNumber: cp.PartNumber,
317330
}
@@ -323,6 +336,7 @@ func (cp *CompletedPart) MapFrom(resp *s3.UploadPartOutput, partNum *int32) {
323336
cp.ChecksumCRC32C = resp.ChecksumCRC32C
324337
cp.ChecksumSHA1 = resp.ChecksumSHA1
325338
cp.ChecksumSHA256 = resp.ChecksumSHA256
339+
cp.ChecksumSHA512 = resp.ChecksumSHA512
326340
cp.ETag = resp.ETag
327341
cp.PartNumber = partNum
328342
}

‎service/internal/checksum/algorithms.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"crypto/md5"
55
"crypto/sha1"
66
"crypto/sha256"
7+
"crypto/sha512"
78
"encoding/base64"
89
"encoding/hex"
910
"fmt"
@@ -34,6 +35,9 @@ const (
3435

3536
// AlgorithmCRC64NVME represents CRC64NVME hash algorithm
3637
AlgorithmCRC64NVME Algorithm = "CRC64NVME"
38+
39+
// AlgorithmSHA512 represents SHA512 hash algorithm
40+
AlgorithmSHA512 Algorithm = "SHA512"
3741
)
3842

3943
// inverted NVME polynomial as required by crc64.MakeTable
@@ -45,6 +49,7 @@ var supportedAlgorithms = []Algorithm{
4549
AlgorithmSHA1,
4650
AlgorithmSHA256,
4751
AlgorithmCRC64NVME,
52+
AlgorithmSHA512,
4853
}
4954

5055
func (a Algorithm) String() string { return string(a) }
@@ -99,6 +104,8 @@ func NewAlgorithmHash(v Algorithm) (hash.Hash, error) {
99104
return crc32.New(crc32.MakeTable(crc32.Castagnoli)), nil
100105
case AlgorithmCRC64NVME:
101106
return crc64.New(crc64.MakeTable(crc64NVME)), nil
107+
case AlgorithmSHA512:
108+
return sha512.New(), nil
102109
default:
103110
return nil, fmt.Errorf("unknown checksum algorithm, %v", v)
104111
}
@@ -118,6 +125,8 @@ func AlgorithmChecksumLength(v Algorithm) (int, error) {
118125
return crc32.Size, nil
119126
case AlgorithmCRC64NVME:
120127
return crc64.Size, nil
128+
case AlgorithmSHA512:
129+
return sha512.Size, nil
121130
default:
122131
return 0, fmt.Errorf("unknown checksum algorithm, %v", v)
123132
}

‎service/internal/checksum/algorithms_test.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"bytes"
77
"crypto/sha1"
88
"crypto/sha256"
9+
"crypto/sha512"
910
"encoding/base64"
1011
"fmt"
1112
"hash/crc32"
@@ -75,6 +76,13 @@ func TestComputeChecksumReader(t *testing.T) {
7576
ExpectRead: "hello world",
7677
ExpectChecksum: "jSnVw/bqjr4=",
7778
},
79+
"sha512": {
80+
Input: strings.NewReader("hello world"),
81+
Algorithm: AlgorithmSHA512,
82+
ExpectChecksumLen: base64.StdEncoding.EncodedLen(sha512.Size),
83+
ExpectRead: "hello world",
84+
ExpectChecksum: "MJ7MSJwS1utMxA9QyQLytNDtd+5RGnx6m808qG1M2G+YndNbxf9JlnDaNCVbRbDP2DDoH2Bdz33FVC6TrpzXbw==",
85+
},
7886
}
7987

8088
for name, c := range cases {
@@ -326,6 +334,14 @@ func TestParseAlgorithm(t *testing.T) {
326334
Value: "SHA256",
327335
expectAlgorithm: AlgorithmSHA256,
328336
},
337+
"sha512": {
338+
Value: "sha512",
339+
expectAlgorithm: AlgorithmSHA512,
340+
},
341+
"SHA512": {
342+
Value: "SHA512",
343+
expectAlgorithm: AlgorithmSHA512,
344+
},
329345
"empty": {
330346
Value: "",
331347
expectErr: "unknown checksum algorithm",
@@ -453,6 +469,10 @@ func TestAlgorithmChecksumLength(t *testing.T) {
453469
algorithm: AlgorithmCRC64NVME,
454470
expectLength: crc64.Size,
455471
},
472+
"sha512": {
473+
algorithm: AlgorithmSHA512,
474+
expectLength: sha512.Size,
475+
},
456476
}
457477

458478
for name, c := range cases {

0 commit comments

Comments
 (0)