Skip to content

Commit dbe6b19

Browse files
crazy-maxcrazy-max
committed
test: report buildkitd allocated memory
Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com>
1 parent cadd946 commit dbe6b19

4 files changed

Lines changed: 152 additions & 52 deletions

File tree

‎test/build_test.go‎

Lines changed: 61 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,13 @@ RUN cp /etc/foo /etc/bar
7373
fstest.CreateFile("Dockerfile", dockerfile, 0600),
7474
fstest.CreateFile("foo", []byte("foo"), 0600),
7575
)
76-
b.StartTimer()
77-
out, err := buildxBuildCmd(sb, withArgs(dir))
78-
b.StopTimer()
79-
sb.WriteLogFile(b, "buildx", []byte(out))
80-
require.NoError(b, err, out)
76+
reportBuildkitdAlloc(b, sb, func() {
77+
b.StartTimer()
78+
out, err := buildxBuildCmd(sb, withArgs(dir))
79+
b.StopTimer()
80+
sb.WriteLogFile(b, "buildx", []byte(out))
81+
require.NoError(b, err, out)
82+
})
8183
}
8284

8385
func benchmarkBuildMultistage(b *testing.B, sb testutil.Sandbox) {
@@ -94,11 +96,13 @@ COPY --from=base /etc/bar /bar
9496
fstest.CreateFile("Dockerfile", dockerfile, 0600),
9597
fstest.CreateFile("foo", []byte("foo"), 0600),
9698
)
97-
b.StartTimer()
98-
out, err := buildxBuildCmd(sb, withArgs(dir))
99-
b.StopTimer()
100-
sb.WriteLogFile(b, "buildx", []byte(out))
101-
require.NoError(b, err, out)
99+
reportBuildkitdAlloc(b, sb, func() {
100+
b.StartTimer()
101+
out, err := buildxBuildCmd(sb, withArgs(dir))
102+
b.StopTimer()
103+
sb.WriteLogFile(b, "buildx", []byte(out))
104+
require.NoError(b, err, out)
105+
})
102106
}
103107

104108
// https://github.com/docker/buildx/issues/2479
@@ -112,22 +116,26 @@ RUN --mount=type=secret,id=SECRET cat /run/secrets/SECRET
112116
fstest.CreateFile("Dockerfile", dockerfile, 0600),
113117
fstest.CreateFile("secret.txt", []byte("mysecret"), 0600),
114118
)
115-
b.StartTimer()
116-
out, err := buildxBuildCmd(sb, withDir(dir), withArgs("--secret=id=SECRET,src=secret.txt", "."))
117-
b.StopTimer()
118-
sb.WriteLogFile(b, "buildx", []byte(out))
119-
require.NoError(b, err, out)
119+
reportBuildkitdAlloc(b, sb, func() {
120+
b.StartTimer()
121+
out, err := buildxBuildCmd(sb, withDir(dir), withArgs("--secret=id=SECRET,src=secret.txt", "."))
122+
b.StopTimer()
123+
sb.WriteLogFile(b, "buildx", []byte(out))
124+
require.NoError(b, err, out)
125+
})
120126
}
121127

122128
func benchmarkBuildRemote(b *testing.B, sb testutil.Sandbox) {
123-
b.StartTimer()
124-
out, err := buildxBuildCmd(sb, withArgs(
125-
"--build-arg=BUILDKIT_SYNTAX="+dockerfileImagePin,
126-
"https://github.com/dvdksn/buildme.git#eb6279e0ad8a10003718656c6867539bd9426ad8",
127-
))
128-
b.StopTimer()
129-
sb.WriteLogFile(b, "buildx", []byte(out))
130-
require.NoError(b, err, out)
129+
reportBuildkitdAlloc(b, sb, func() {
130+
b.StartTimer()
131+
out, err := buildxBuildCmd(sb, withArgs(
132+
"--build-arg=BUILDKIT_SYNTAX="+dockerfileImagePin,
133+
"https://github.com/dvdksn/buildme.git#eb6279e0ad8a10003718656c6867539bd9426ad8",
134+
))
135+
b.StopTimer()
136+
sb.WriteLogFile(b, "buildx", []byte(out))
137+
require.NoError(b, err, out)
138+
})
131139
}
132140

133141
func benchmarkBuildHighParallelization16x(b *testing.B, sb testutil.Sandbox) {
@@ -165,9 +173,11 @@ RUN cp /etc/foo /etc/bar
165173
require.NoError(b, err, out)
166174
}()
167175
}
168-
b.StartTimer()
169-
wg.Wait()
170-
b.StopTimer()
176+
reportBuildkitdAlloc(b, sb, func() {
177+
b.StartTimer()
178+
wg.Wait()
179+
b.StopTimer()
180+
})
171181
}
172182

173183
func benchmarkBuildFileTransfer(b *testing.B, sb testutil.Sandbox) {
@@ -181,11 +191,13 @@ RUN du -sh . && tree .
181191
fstest.CreateFile("Dockerfile", dockerfile, 0600),
182192
contextDirApplier,
183193
)
184-
b.StartTimer()
185-
out, err := buildxBuildCmd(sb, withArgs(dir))
186-
b.StopTimer()
187-
sb.WriteLogFile(b, "buildx", []byte(out))
188-
require.NoError(b, err, out)
194+
reportBuildkitdAlloc(b, sb, func() {
195+
b.StartTimer()
196+
out, err := buildxBuildCmd(sb, withArgs(dir))
197+
b.StopTimer()
198+
sb.WriteLogFile(b, "buildx", []byte(out))
199+
require.NoError(b, err, out)
200+
})
189201
}
190202

191203
// https://github.com/moby/buildkit/pull/4949
@@ -214,15 +226,17 @@ RUN uname -a
214226
`, busyboxImage))
215227

216228
dir := tmpdir(b, fstest.CreateFile("Dockerfile", dockerfile, 0600))
217-
b.StartTimer()
218-
out, err := buildxBuildCmd(sb, withArgs(
219-
"--build-arg", "BUILDKIT_DOCKERFILE_CHECK=skip=all", // skip all checks (for InvalidBaseImagePlatform): https://docs.docker.com/build/checks/#skip-checks
220-
"--platform", platforms.Format(platform),
221-
dir,
222-
))
223-
b.StopTimer()
224-
sb.WriteLogFile(b, "buildx", []byte(out))
225-
require.NoError(b, err, out)
229+
reportBuildkitdAlloc(b, sb, func() {
230+
b.StartTimer()
231+
out, err := buildxBuildCmd(sb, withArgs(
232+
"--build-arg", "BUILDKIT_DOCKERFILE_CHECK=skip=all", // skip all checks (for InvalidBaseImagePlatform): https://docs.docker.com/build/checks/#skip-checks
233+
"--platform", platforms.Format(platform),
234+
dir,
235+
))
236+
b.StopTimer()
237+
sb.WriteLogFile(b, "buildx", []byte(out))
238+
require.NoError(b, err, out)
239+
})
226240
}
227241

228242
func benchmarkBuildExportUncompressed(b *testing.B, sb testutil.Sandbox) {
@@ -247,9 +261,11 @@ COPY . .
247261
fstest.CreateFile("Dockerfile", dockerfile, 0600),
248262
contextDirApplier,
249263
)
250-
b.StartTimer()
251-
out, err := buildxBuildCmd(sb, withArgs("--output=type=image,compression="+compression, dir))
252-
b.StopTimer()
253-
sb.WriteLogFile(b, "buildx", []byte(out))
254-
require.NoError(b, err, out)
264+
reportBuildkitdAlloc(b, sb, func() {
265+
b.StartTimer()
266+
out, err := buildxBuildCmd(sb, withArgs("--output=type=image,compression="+compression, dir))
267+
b.StopTimer()
268+
sb.WriteLogFile(b, "buildx", []byte(out))
269+
require.NoError(b, err, out)
270+
})
255271
}

‎test/util.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
package test
22

33
import (
4+
"fmt"
5+
"net/http"
46
"os"
57
"os/exec"
68
"path"
79
"testing"
810

911
"github.com/containerd/continuity/fs/fstest"
12+
"github.com/google/pprof/profile"
1013
"github.com/moby/buildkit-bench/util/testutil"
1114
"github.com/stretchr/testify/require"
1215
)
@@ -80,3 +83,47 @@ func buildctlBuildCmd(sb testutil.Sandbox, opts ...cmdOpt) (string, error) {
8083
out, err := cmd.CombinedOutput()
8184
return string(out), err
8285
}
86+
87+
func reportBuildkitdAlloc(b *testing.B, sb testutil.Sandbox, cb func()) {
88+
beforeAlloc, errb := buildkitdAlloc(sb)
89+
cb()
90+
afterAlloc, erra := buildkitdAlloc(sb)
91+
testutil.ReportAlloc(b, afterAlloc-beforeAlloc)
92+
require.NoError(b, errb)
93+
require.NoError(b, erra)
94+
}
95+
96+
func buildkitdAlloc(sb testutil.Sandbox) (int64, error) {
97+
client := &http.Client{}
98+
resp, err := client.Get(fmt.Sprintf("http://%s/debug/pprof/heap?gc=1", sb.DebugAddress()))
99+
if err != nil {
100+
return 0, err
101+
}
102+
defer resp.Body.Close()
103+
104+
p, err := profile.Parse(resp.Body)
105+
if err != nil {
106+
return 0, err
107+
}
108+
109+
return func(prof *profile.Profile, value func(v []int64) int64) int64 {
110+
var total, diff int64
111+
for _, sample := range prof.Sample {
112+
var v int64
113+
v = value(sample.Value)
114+
if v < 0 {
115+
v = -v
116+
}
117+
total += v
118+
if sample.DiffBaseSample() {
119+
diff += v
120+
}
121+
}
122+
if diff > 0 {
123+
total = diff
124+
}
125+
return total
126+
}(p, func(v []int64) int64 {
127+
return v[1] // sample index of alloc_space in heap profiles
128+
}), nil
129+
}

‎testconfig.yml‎

Lines changed: 39 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,97 +11,129 @@ runs:
1111
duration:
1212
description: Time (s)
1313
chart: boxplot
14+
alloc:
15+
description: Allocated memory (bytes)
16+
chart: boxplot
1417
BenchmarkBuildMultistage:
1518
description: Multistage build
1619
count: 4
1720
metrics:
1821
duration:
1922
description: Time (s)
2023
chart: boxplot
24+
alloc:
25+
description: Allocated memory (bytes)
26+
chart: boxplot
2127
BenchmarkBuildSecret:
2228
description: Build with secret
2329
count: 4
2430
metrics:
2531
duration:
2632
description: Time (s)
2733
chart: boxplot
34+
alloc:
35+
description: Allocated memory (bytes)
36+
chart: boxplot
2837
BenchmarkBuildRemote:
2938
description: Build from git context
3039
count: 4
3140
metrics:
3241
duration:
3342
description: Time (s)
3443
chart: boxplot
44+
alloc:
45+
description: Allocated memory (bytes)
46+
chart: boxplot
3547
BenchmarkBuildHighParallelization16x:
3648
description: Build high parallelization (16x)
3749
count: 3
3850
metrics:
3951
duration:
4052
description: Time (s)
4153
chart: boxplot
54+
alloc:
55+
description: Allocated memory (bytes)
56+
chart: boxplot
4257
BenchmarkBuildHighParallelization32x:
4358
description: Build high parallelization (32x)
4459
count: 4
4560
metrics:
4661
duration:
4762
description: Time (s)
4863
chart: boxplot
64+
alloc:
65+
description: Allocated memory (bytes)
66+
chart: boxplot
4967
BenchmarkBuildHighParallelization64x:
5068
description: Build high parallelization (64x)
5169
count: 4
5270
metrics:
5371
duration:
5472
description: Time (s)
5573
chart: boxplot
74+
alloc:
75+
description: Allocated memory (bytes)
76+
chart: boxplot
5677
BenchmarkBuildHighParallelization128x:
5778
description: Build high parallelization (128x)
5879
count: 4
5980
metrics:
6081
duration:
6182
description: Time (s)
6283
chart: boxplot
84+
alloc:
85+
description: Allocated memory (bytes)
86+
chart: boxplot
6387
BenchmarkBuildFileTransfer:
6488
description: Build with substantial file transfer
6589
count: 4
6690
metrics:
6791
duration:
6892
description: Time (s)
6993
chart: boxplot
94+
alloc:
95+
description: Allocated memory (bytes)
96+
chart: boxplot
7097
BenchmarkBuildEmulator:
7198
description: Build with emulator
7299
count: 10
73100
metrics:
74101
duration:
75102
description: Time (s)
76103
chart: boxplot
104+
alloc:
105+
description: Allocated memory (bytes)
106+
chart: boxplot
77107
BenchmarkBuildExportUncompressed:
78108
description: Build export uncompressed
79109
count: 4
80110
metrics:
81111
duration:
82112
description: Time (s)
83113
chart: boxplot
114+
alloc:
115+
description: Allocated memory (bytes)
116+
chart: boxplot
84117
BenchmarkBuildExportGzip:
85118
description: Build export gzip
86119
count: 4
87120
metrics:
88121
duration:
89122
description: Time (s)
90123
chart: boxplot
124+
alloc:
125+
description: Allocated memory (bytes)
126+
chart: boxplot
91127
BenchmarkBuildExportEstargz:
92128
description: Build export estargz
93129
count: 4
94130
metrics:
95131
duration:
96132
description: Time (s)
97133
chart: boxplot
98-
# BenchmarkBuildExportZstd:
99-
# description: Build export zstd
100-
# count: 4
101-
# metrics:
102-
# duration:
103-
# description: Time (s)
104-
# chart: boxplot
134+
alloc:
135+
description: Allocated memory (bytes)
136+
chart: boxplot
105137

106138
BenchmarkDaemon:
107139
BenchmarkDaemonVersion:

‎util/testutil/metric.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,15 @@ import (
88
type MetricUnit string
99

1010
const (
11+
MetricAlloc MetricUnit = "alloc"
1112
MetricBytes MetricUnit = "bytes"
1213
MetricDuration MetricUnit = "duration"
1314
)
1415

16+
func ReportAlloc(b *testing.B, value int64) {
17+
ReportMetric(b, float64(value), MetricAlloc)
18+
}
19+
1520
func ReportMetric(b *testing.B, value float64, unit MetricUnit) {
1621
b.ReportMetric(value, string(unit))
1722
}

0 commit comments

Comments
 (0)