Skip to content

Commit e586036

Browse files
klauspostklauspost
authored
Add nopshufb tag (#256)
Add `nopshufb` tag that should remove all code utilizing `PSHUFB` - and similar code on other platforms. # Background > None of section below is legal advice. Seek your own legal counsel. > As stated by the [LICENSE](LICENSE) the authors will not be held reliable for any use of this library. > Users are encouraged to independently verify they comply with all legal requirements. As can be seen in [recent news](https://www.datanami.com/2023/10/16/cloudera-hit-with-240-million-judgement-over-erasure-coding/) there has been lawsuits related to possible patents of aspects of erasure coding functionality. As a possible mitigation it is possible to use the tag `nopshufb` when compiling any code which includes this package. This will remove all inclusion and use of `PSHUFB` and equivalent on other platforms. This is done by adding `-tags=nopshufb` to `go build` and similar commands that produce binary output. The removed code may not be infringing and even after `-tags=nopshufb` there may still be infringing code left.
1 parent b8dc407 commit e586036

29 files changed

+35503
-215
lines changed

‎.github/workflows/go.yml‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ jobs:
1111
build:
1212
strategy:
1313
matrix:
14-
go-version: [1.18.x, 1.19.x, 1.20.x]
14+
go-version: [1.19.x, 1.20.x, 1.21.x]
1515
os: [ubuntu-latest, macos-latest, windows-latest]
1616
env:
1717
CGO_ENABLED: 0
@@ -32,11 +32,15 @@ jobs:
3232
run: go vet ./...
3333

3434
- name: Test
35-
run: go test ./...
35+
run: go test -timeout=15m ./...
3636

3737
- name: Test Noasm
3838
run: go test -tags=noasm -short&&go test -short -no-avx512&&go test -short -no-avx512 -no-avx2&&go test -no-avx512 -no-avx2 -no-ssse3 -short
3939

40+
- name: Test Nopshufb
41+
run: go test -tags=nopshufb -short&&go test -tags=nopshufb -short -no-avx512 -no-gfni&&go test -tags=nopshufb -short&&go test -tags=nopshufb -no-avx512 -no-avx2 -no-ssse3 -no-sse2 -short
42+
43+
4044
- name: Test Race
4145
env:
4246
CGO_ENABLED: 1
@@ -50,7 +54,7 @@ jobs:
5054
- name: Set up Go
5155
uses: actions/setup-go@v2
5256
with:
53-
go-version: 1.19.x
57+
go-version: 1.21.x
5458

5559
- name: Checkout code
5660
uses: actions/checkout@v2
@@ -102,21 +106,21 @@ jobs:
102106
env:
103107
GOOS: linux
104108
GOARCH: arm64
105-
run: go build .&&go build examples/simple-decoder.go&&go build examples/simple-encoder.go&&go build examples/stream-decoder.go&&go build examples/stream-encoder.go
109+
run: go build .&&go build examples/simple-decoder.go&&go build examples/simple-encoder.go&&go build examples/stream-decoder.go&&go build examples/stream-encoder.go&&go build -tags=nopshufb .&&go build -tags=nopshufb examples/simple-decoder.go&&go build examples/simple-encoder.go&&go build -tags=nopshufb examples/stream-decoder.go&&go build examples/stream-encoder.go
106110

107111
- name: Build on PPC64LE
108112
env:
109113
GOOS: linux
110114
GOARCH: ppc64le
111-
run: go build .&&go build examples/simple-decoder.go&&go build examples/simple-encoder.go&&go build examples/stream-decoder.go&&go build examples/stream-encoder.go
115+
run: go build .&&go build examples/simple-decoder.go&&go build examples/simple-encoder.go&&go build examples/stream-decoder.go&&go build examples/stream-encoder.go && go build -tags=nopshufb .&&go build -tags=nopshufb examples/simple-decoder.go&&go build examples/simple-encoder.go&&go build -tags=nopshufb examples/stream-decoder.go&&go build examples/stream-encoder.go
112116

113117
generate:
114118
runs-on: ubuntu-latest
115119
steps:
116120
- name: Set up Go
117121
uses: actions/setup-go@v2
118122
with:
119-
go-version: 1.19.x
123+
go-version: 1.21.x
120124

121125
- name: Checkout code
122126
uses: actions/checkout@v2

‎README.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,6 +534,21 @@ BenchmarkGaloisXor128K-160 862.02 7905.00 9.17x
534534
BenchmarkGaloisXor1M-160 784.60 6296.65 8.03x
535535
```
536536

537+
# Legal
538+
539+
> None of section below is legal advice. Seek your own legal counsel.
540+
> As stated by the [LICENSE](LICENSE) the authors will not be held reliable for any use of this library.
541+
> Users are encouraged to independently verify they comply with all legal requirements.
542+
543+
As can be seen in [recent news](https://www.datanami.com/2023/10/16/cloudera-hit-with-240-million-judgement-over-erasure-coding/)
544+
there has been lawsuits related to possible patents of aspects of erasure coding functionality.
545+
546+
As a possible mitigation it is possible to use the tag `nopshufb` when compiling any code which includes this package.
547+
This will remove all inclusion and use of `PSHUFB` and equivalent on other platforms.
548+
549+
This is done by adding `-tags=nopshufb` to `go build` and similar commands that produce binary output.
550+
551+
The removed code may not be infringing and even after `-tags=nopshufb` there may still be infringing code left.
537552

538553
# Links
539554
* [Backblaze Open Sources Reed-Solomon Erasure Coding Source Code](https://www.backblaze.com/blog/reed-solomon/).

‎_gen/gen.go‎

Lines changed: 150 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
11
//go:build generate
2-
// +build generate
32

43
// Copyright 2022+, Klaus Post. See LICENSE for details.
54

5+
//go:generate go run -tags=generate,nopshufb . -out ../galois_gen_nopshufb_amd64.s -stubs ../galois_gen_nopshufb_amd64.go -pkg=reedsolomon
6+
//go:generate go fmt ../galois_gen_switch_nopshufb_amd64.go
7+
//go:generate go fmt ../galois_gen_nopshufb_amd64.go
8+
//go:generate go run cleanup.go ../galois_gen_nopshufb_amd64.s
9+
610
//go:generate go run -tags=generate . -out ../galois_gen_amd64.s -stubs ../galois_gen_amd64.go -pkg=reedsolomon
711
//go:generate go fmt ../galois_gen_switch_amd64.go
812
//go:generate go fmt ../galois_gen_amd64.go
@@ -41,6 +45,11 @@ func main() {
4145
Constraint(buildtags.Not("appengine").ToConstraint())
4246
Constraint(buildtags.Not("noasm").ToConstraint())
4347
Constraint(buildtags.Not("nogen").ToConstraint())
48+
if pshufb {
49+
Constraint(buildtags.Not("nopshufb").ToConstraint())
50+
} else {
51+
Constraint(buildtags.Opt("nopshufb").ToConstraint())
52+
}
4453
Constraint(buildtags.Term("gc").ToConstraint())
4554

4655
TEXT("_dummy_", 0, "func()")
@@ -54,20 +63,31 @@ func main() {
5463
Comment("#endif")
5564
RET()
5665

66+
genXor()
5767
const perLoopBits = 6
5868
const perLoop = 1 << perLoopBits
5969

6070
for i := 1; i <= inputMax; i++ {
6171
for j := 1; j <= outputMax; j++ {
62-
genMulAvx2(fmt.Sprintf("mulAvxTwo_%dx%d", i, j), i, j, false)
63-
genMulAvx2Sixty64(fmt.Sprintf("mulAvxTwo_%dx%d_64", i, j), i, j, false)
72+
if pshufb {
73+
genMulAvx2(fmt.Sprintf("mulAvxTwo_%dx%d", i, j), i, j, false)
74+
genMulAvx2Sixty64(fmt.Sprintf("mulAvxTwo_%dx%d_64", i, j), i, j, false)
75+
}
6476
genMulAvx512GFNI(fmt.Sprintf("mulGFNI_%dx%d_64", i, j), i, j, false)
6577
genMulAvx512GFNI(fmt.Sprintf("mulGFNI_%dx%d_64Xor", i, j), i, j, true)
66-
genMulAvx2(fmt.Sprintf("mulAvxTwo_%dx%dXor", i, j), i, j, true)
67-
genMulAvx2Sixty64(fmt.Sprintf("mulAvxTwo_%dx%d_64Xor", i, j), i, j, true)
78+
if pshufb {
79+
genMulAvx2(fmt.Sprintf("mulAvxTwo_%dx%dXor", i, j), i, j, true)
80+
genMulAvx2Sixty64(fmt.Sprintf("mulAvxTwo_%dx%d_64Xor", i, j), i, j, true)
81+
}
6882
}
6983
}
70-
f, err := os.Create("../galois_gen_switch_amd64.go")
84+
name := "../galois_gen_switch_amd64.go"
85+
tag := "// +build !nopshufb\n"
86+
if !pshufb {
87+
name = "../galois_gen_switch_nopshufb_amd64.go"
88+
tag = "// +build nopshufb\n"
89+
}
90+
f, err := os.Create(name)
7191
if err != nil {
7292
panic(err)
7393
}
@@ -79,7 +99,8 @@ func main() {
7999
// +build !appengine
80100
// +build !noasm
81101
// +build gc
82-
// +build !nogen
102+
// +build !nogen
103+
` + tag + `
83104
84105
package reedsolomon
85106
@@ -88,32 +109,38 @@ import (
88109
)
89110
90111
`)
91-
92112
w.WriteString(fmt.Sprintf(`const (
93113
avx2CodeGen = true
94114
maxAvx2Inputs = %d
95115
maxAvx2Outputs = %d
96116
minAvx2Size = %d
97117
avxSizeMask = maxInt - (minAvx2Size-1)
98118
)`, inputMax, outputMax, perLoop))
99-
w.WriteString(`
119+
120+
if !pshufb {
121+
w.WriteString("\n\nfunc galMulSlicesAvx2(matrix []byte, in, out [][]byte, start, stop int) int { panic(`no pshufb`)}\n")
122+
w.WriteString("func galMulSlicesAvx2Xor(matrix []byte, in, out [][]byte, start, stop int) int { panic(`no pshufb`)}\n")
123+
}
124+
125+
if pshufb {
126+
w.WriteString(`
100127
101128
func galMulSlicesAvx2(matrix []byte, in, out [][]byte, start, stop int) int {
102129
n := (stop-start) & avxSizeMask
103130
104131
`)
105132

106-
w.WriteString(`switch len(in) {
133+
w.WriteString(`switch len(in) {
107134
`)
108-
for in, defs := range switchDefs[:] {
109-
w.WriteString(fmt.Sprintf(" case %d:\n switch len(out) {\n", in+1))
110-
for out, def := range defs[:] {
111-
w.WriteString(fmt.Sprintf(" case %d:\n", out+1))
112-
w.WriteString(def)
135+
for in, defs := range switchDefs[:] {
136+
w.WriteString(fmt.Sprintf(" case %d:\n switch len(out) {\n", in+1))
137+
for out, def := range defs[:] {
138+
w.WriteString(fmt.Sprintf(" case %d:\n", out+1))
139+
w.WriteString(def)
140+
}
141+
w.WriteString("}\n")
113142
}
114-
w.WriteString("}\n")
115-
}
116-
w.WriteString(`}
143+
w.WriteString(`}
117144
panic(fmt.Sprintf("unhandled size: %dx%d", len(in), len(out)))
118145
}
119146
@@ -122,20 +149,21 @@ func galMulSlicesAvx2Xor(matrix []byte, in, out [][]byte, start, stop int) int {
122149
123150
`)
124151

125-
w.WriteString(`switch len(in) {
152+
w.WriteString(`switch len(in) {
126153
`)
127-
for in, defs := range switchDefsX[:] {
128-
w.WriteString(fmt.Sprintf(" case %d:\n switch len(out) {\n", in+1))
129-
for out, def := range defs[:] {
130-
w.WriteString(fmt.Sprintf(" case %d:\n", out+1))
131-
w.WriteString(def)
154+
for in, defs := range switchDefsX[:] {
155+
w.WriteString(fmt.Sprintf(" case %d:\n switch len(out) {\n", in+1))
156+
for out, def := range defs[:] {
157+
w.WriteString(fmt.Sprintf(" case %d:\n", out+1))
158+
w.WriteString(def)
159+
}
160+
w.WriteString("}\n")
132161
}
133-
w.WriteString("}\n")
134-
}
135-
w.WriteString(`}
162+
w.WriteString(`}
136163
panic(fmt.Sprintf("unhandled size: %dx%d", len(in), len(out)))
137164
}
138165
`)
166+
}
139167

140168
w.WriteString(`
141169
@@ -930,3 +958,98 @@ func genMulAvx512GFNI(name string, inputs int, outputs int, xor bool) {
930958
Label(name + "_end")
931959
RET()
932960
}
961+
962+
func genXor() {
963+
// SSE 2
964+
{
965+
Comment("sSE2XorSlice will XOR in with out and store in out.")
966+
Comment("Processes 16 bytes/loop.")
967+
TEXT("sSE2XorSlice", 0, fmt.Sprintf("func(in, out []byte)"))
968+
Pragma("noescape")
969+
src := Load(Param("in").Base(), GP64())
970+
dst := Load(Param("out").Base(), GP64())
971+
length := Load(Param("in").Len(), GP64())
972+
SHRQ(U8(4), length)
973+
srcX, dstX := XMM(), XMM()
974+
JZ(LabelRef("end"))
975+
Label("loop")
976+
MOVOU(Mem{Base: src}, srcX)
977+
MOVOU(Mem{Base: dst}, dstX)
978+
PXOR(srcX, dstX)
979+
MOVOU(dstX, Mem{Base: dst})
980+
ADDQ(U8(16), src)
981+
ADDQ(U8(16), dst)
982+
DECQ(length)
983+
JNZ(LabelRef("loop"))
984+
Label("end")
985+
RET()
986+
}
987+
988+
// SSE2 64 bytes
989+
{
990+
Comment("sSE2XorSlice_64 will XOR in with out and store in out.")
991+
Comment("Processes 64 bytes/loop.")
992+
TEXT("sSE2XorSlice_64", 0, fmt.Sprintf("func(in, out []byte)"))
993+
Pragma("noescape")
994+
src := Load(Param("in").Base(), GP64())
995+
dst := Load(Param("out").Base(), GP64())
996+
length := Load(Param("in").Len(), GP64())
997+
SHRQ(U8(6), length)
998+
var srcX, dstX [4]reg.VecVirtual
999+
for i := range srcX {
1000+
srcX[i], dstX[i] = XMM(), XMM()
1001+
}
1002+
JZ(LabelRef("end"))
1003+
Label("loop")
1004+
for i := range srcX {
1005+
MOVOU(Mem{Base: src, Disp: 16 * i}, srcX[i])
1006+
}
1007+
for i := range srcX {
1008+
MOVOU(Mem{Base: dst, Disp: 16 * i}, dstX[i])
1009+
}
1010+
for i := range srcX {
1011+
PXOR(srcX[i], dstX[i])
1012+
}
1013+
for i := range srcX {
1014+
MOVOU(dstX[i], Mem{Base: dst, Disp: 16 * i})
1015+
}
1016+
ADDQ(U8(64), src)
1017+
ADDQ(U8(64), dst)
1018+
DECQ(length)
1019+
JNZ(LabelRef("loop"))
1020+
Label("end")
1021+
RET()
1022+
}
1023+
//AVX 2
1024+
{
1025+
Comment("avx2XorSlice_64 will XOR in with out and store in out.")
1026+
Comment("Processes 64 bytes/loop.")
1027+
TEXT("avx2XorSlice_64", 0, fmt.Sprintf("func(in, out []byte)"))
1028+
Pragma("noescape")
1029+
src := Load(Param("in").Base(), GP64())
1030+
dst := Load(Param("out").Base(), GP64())
1031+
length := Load(Param("in").Len(), GP64())
1032+
SHRQ(U8(6), length)
1033+
srcX, dstX := YMM(), YMM()
1034+
srcX2, dstX2 := YMM(), YMM()
1035+
JZ(LabelRef("end"))
1036+
1037+
Label("loop")
1038+
VMOVDQU(Mem{Base: src}, srcX)
1039+
VMOVDQU(Mem{Base: src, Disp: 32}, srcX2)
1040+
VMOVDQU(Mem{Base: dst}, dstX)
1041+
VMOVDQU(Mem{Base: dst, Disp: 32}, dstX2)
1042+
VPXOR(srcX, dstX, dstX)
1043+
VPXOR(srcX2, dstX2, dstX2)
1044+
VMOVDQU(dstX, Mem{Base: dst})
1045+
VMOVDQU(dstX2, Mem{Base: dst, Disp: 32})
1046+
ADDQ(U8(64), src)
1047+
ADDQ(U8(64), dst)
1048+
DECQ(length)
1049+
JNZ(LabelRef("loop"))
1050+
1051+
Label("end")
1052+
VZEROUPPER()
1053+
RET()
1054+
}
1055+
}

‎_gen/gf16.go‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ func genGF16() {
7777
var ctx gf16ctx
7878
// Ported from static void IFFT_DIT2
7979
// https://github.com/catid/leopard/blob/master/LeopardFF16.cpp#L629
80-
{
80+
if pshufb {
8181
TEXT("ifftDIT2_avx2", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[8*16]uint8)"))
8282
Pragma("noescape")
8383
tablePtr := Load(Param("table"), GP64())
@@ -120,7 +120,7 @@ func genGF16() {
120120
VZEROUPPER()
121121
RET()
122122
}
123-
{
123+
if pshufb {
124124
TEXT("fftDIT2_avx2", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[8*16]uint8)"))
125125
Pragma("noescape")
126126
tablePtr := Load(Param("table"), GP64())
@@ -173,7 +173,7 @@ func genGF16() {
173173
RET()
174174
}
175175

176-
{
176+
if pshufb {
177177
TEXT("mulgf16_avx2", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[8*16]uint8)"))
178178
Pragma("noescape")
179179
tablePtr := Load(Param("table"), GP64())
@@ -213,6 +213,9 @@ func genGF16() {
213213
RET()
214214
}
215215
for _, avx512 := range []bool{true, false} {
216+
if !pshufb {
217+
continue
218+
}
216219
x := [8]int{}
217220
for skipMask := range x[:] {
218221
// AVX-512 only uses more registers for tables.
@@ -562,7 +565,7 @@ func genGF16() {
562565

563566
// SSSE3:
564567
ctx.avx512 = false
565-
{
568+
if pshufb {
566569
TEXT("ifftDIT2_ssse3", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[8*16]uint8)"))
567570
Pragma("noescape")
568571
tablePtr := Load(Param("table"), GP64())
@@ -613,7 +616,7 @@ func genGF16() {
613616

614617
RET()
615618
}
616-
{
619+
if pshufb {
617620
TEXT("fftDIT2_ssse3", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[8*16]uint8)"))
618621
Pragma("noescape")
619622
tablePtr := Load(Param("table"), GP64())
@@ -671,7 +674,7 @@ func genGF16() {
671674

672675
RET()
673676
}
674-
{
677+
if pshufb {
675678
TEXT("mulgf16_ssse3", attr.NOSPLIT, fmt.Sprintf("func(x, y []byte, table *[8*16]uint8)"))
676679
Pragma("noescape")
677680
tablePtr := Load(Param("table"), GP64())

0 commit comments

Comments
 (0)