crypto/aes: dedicated asm version of AES-GCM

The existing implementation didn't use the CLMUL instructions for fast
and constant time binary-field multiplication. With this change, amd64
CPUs that support both AES and CLMUL instructions will use an optimised
asm implementation.

benchmark                 old ns/op     new ns/op     delta
BenchmarkAESGCMSeal8K     91723         3200          -96.51%
BenchmarkAESGCMOpen8K     91487         3324          -96.37%
BenchmarkAESGCMSeal1K     11873         546           -95.40%
BenchmarkAESGCMOpen1K     11833         594           -94.98%

benchmark                 old MB/s     new MB/s     speedup
BenchmarkAESGCMSeal8K     89.31        2559.62      28.66x
BenchmarkAESGCMOpen8K     89.54        2463.78      27.52x
BenchmarkAESGCMSeal1K     86.24        1872.49      21.71x
BenchmarkAESGCMOpen1K     86.53        1721.78      19.90x

Change-Id: Idd63233098356d8b353d16624747b74d0c3f193e
Reviewed-on: https://go-review.googlesource.com/10484
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Adam Langley <agl@golang.org>

Author: vkrasnov

src/crypto/aes/aes_gcm.go

@@ -0,0 +1,160 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build amd64
+
+package aes
+
+import (
+	"crypto/cipher"
+	"crypto/subtle"
+	"errors"
+)
+
+// The following functions are defined in gcm_amd64.s.
+func hasGCMAsm() bool
+func aesEncBlock(dst, src *[16]byte, ks []uint32)
+func gcmAesInit(productTable *[256]byte, ks []uint32)
+func gcmAesData(productTable *[256]byte, data []byte, T *[16]byte)
+func gcmAesEnc(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32)
+func gcmAesDec(productTable *[256]byte, dst, src []byte, ctr, T *[16]byte, ks []uint32)
+func gcmAesFinish(productTable *[256]byte, tagMask, T *[16]byte, pLen, dLen uint64)
+
+const (
+	gcmBlockSize         = 16
+	gcmTagSize           = 16
+	gcmStandardNonceSize = 12
+)
+
+var errOpen = errors.New("cipher: message authentication failed")
+
+// aesCipherGCM implements crypto/cipher.gcmAble so that crypto/cipher.NewGCM
+// will use the optimised implementation in this file when possible. Instances
+// of this type only exist when hasGCMAsm returns true.
+type aesCipherGCM struct {
+	aesCipher
+}
+
+// NewGCM returns the AES cipher wrapped in Galois Counter Mode. This is only
+// called by crypto/cipher.NewGCM via the gcmAble interface.
+func (c *aesCipherGCM) NewGCM(nonceSize int) (cipher.AEAD, error) {
+	g := &gcmAsm{ks: c.enc, nonceSize: nonceSize}
+	gcmAesInit(&g.productTable, g.ks)
+	return g, nil
+}
+
+type gcmAsm struct {
+	// ks is the key schedule, the length of which depends on the size of
+	// the AES key.
+	ks []uint32
+	// productTable contains pre-computed multiples of the binary-field
+	// element used in GHASH.
+	productTable [256]byte
+	// nonceSize contains the expected size of the nonce, in bytes.
+	nonceSize int
+}
+
+func (g *gcmAsm) NonceSize() int {
+	return g.nonceSize
+}
+
+func (*gcmAsm) Overhead() int {
+	return gcmTagSize
+}
+
+// sliceForAppend takes a slice and a requested number of bytes. It returns a
+// slice with the contents of the given slice followed by that many bytes and a
+// second slice that aliases into it and contains only the extra bytes. If the
+// original slice has sufficient capacity then no allocation is performed.
+func sliceForAppend(in []byte, n int) (head, tail []byte) {
+	if total := len(in) + n; cap(in) >= total {
+		head = in[:total]
+	} else {
+		head = make([]byte, total)
+		copy(head, in)
+	}
+	tail = head[len(in):]
+	return
+}
+
+// Seal encrypts and authenticates plaintext. See the cipher.AEAD interface for
+// details.
+func (g *gcmAsm) Seal(dst, nonce, plaintext, data []byte) []byte {
+	if len(nonce) != g.nonceSize {
+		panic("cipher: incorrect nonce length given to GCM")
+	}
+
+	var counter, tagMask [gcmBlockSize]byte
+
+	if len(nonce) == gcmStandardNonceSize {
+		// Init counter to nonce||1
+		copy(counter[:], nonce)
+		counter[gcmBlockSize-1] = 1
+	} else {
+		// Otherwise counter = GHASH(nonce)
+		gcmAesData(&g.productTable, nonce, &counter)
+		gcmAesFinish(&g.productTable, &tagMask, &counter, uint64(len(nonce)), uint64(0))
+	}
+
+	aesEncBlock(&tagMask, &counter, g.ks)
+
+	var tagOut [gcmTagSize]byte
+	gcmAesData(&g.productTable, data, &tagOut)
+
+	ret, out := sliceForAppend(dst, len(plaintext)+gcmTagSize)
+	if len(plaintext) > 0 {
+		gcmAesEnc(&g.productTable, out, plaintext, &counter, &tagOut, g.ks)
+	}
+	gcmAesFinish(&g.productTable, &tagMask, &tagOut, uint64(len(plaintext)), uint64(len(data)))
+	copy(out[len(plaintext):], tagOut[:])
+
+	return ret
+}
+
+// Open authenticates and decrypts ciphertext. See the cipher.AEAD interface
+// for details.
+func (g *gcmAsm) Open(dst, nonce, ciphertext, data []byte) ([]byte, error) {
+	if len(nonce) != g.nonceSize {
+		panic("cipher: incorrect nonce length given to GCM")
+	}
+
+	if len(ciphertext) < gcmTagSize {
+		return nil, errOpen
+	}
+	tag := ciphertext[len(ciphertext)-gcmTagSize:]
+	ciphertext = ciphertext[:len(ciphertext)-gcmTagSize]
+
+	// See GCM spec, section 7.1.
+	var counter, tagMask [gcmBlockSize]byte
+
+	if len(nonce) == gcmStandardNonceSize {
+		// Init counter to nonce||1
+		copy(counter[:], nonce)
+		counter[gcmBlockSize-1] = 1
+	} else {
+		// Otherwise counter = GHASH(nonce)
+		gcmAesData(&g.productTable, nonce, &counter)
+		gcmAesFinish(&g.productTable, &tagMask, &counter, uint64(len(nonce)), uint64(0))
+	}
+
+	aesEncBlock(&tagMask, &counter, g.ks)
+
+	var expectedTag [gcmTagSize]byte
+	gcmAesData(&g.productTable, data, &expectedTag)
+
+	ret, out := sliceForAppend(dst, len(ciphertext))
+	if len(ciphertext) > 0 {
+		gcmAesDec(&g.productTable, out, ciphertext, &counter, &expectedTag, g.ks)
+	}
+	gcmAesFinish(&g.productTable, &tagMask, &expectedTag, uint64(len(ciphertext)), uint64(len(data)))
+
+	if subtle.ConstantTimeCompare(expectedTag[:], tag) != 1 {
+		for i := range out {
+			out[i] = 0
+		}
+		return nil, errOpen
+	}
+
+	return ret, nil
+}

src/crypto/aes/cipher.go

@@ -38,9 +38,14 @@ func NewCipher(key []byte) (cipher.Block, error) {
 	}
 
 	n := k + 28
-	c := &aesCipher{make([]uint32, n), make([]uint32, n)}
+	c := aesCipher{make([]uint32, n), make([]uint32, n)}
 	expandKey(key, c.enc, c.dec)
-	return c, nil
+
+	if hasGCMAsm() {
+		return &aesCipherGCM{c}, nil
+	}
+
+	return &c, nil
 }
 
 func (c *aesCipher) BlockSize() int { return BlockSize }

src/crypto/aes/cipher_generic.go

@@ -17,3 +17,11 @@ func decryptBlock(xk []uint32, dst, src []byte) {
 func expandKey(key []byte, enc, dec []uint32) {
 	expandKeyGo(key, enc, dec)
 }
+
+func hasGCMAsm() bool {
+	return false
+}
+
+type aesCipherGCM struct {
+	aesCipher
+}