-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmain.go
More file actions
286 lines (245 loc) · 7.25 KB
/
main.go
File metadata and controls
286 lines (245 loc) · 7.25 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
package main
import (
"flag"
"fmt"
"io"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
"github.com/jlbutler/imgmkr/cleanup"
"github.com/jlbutler/imgmkr/mockfs"
"github.com/jlbutler/imgmkr/progress"
"github.com/jlbutler/imgmkr/size"
)
// Command line arguments
var (
layerSizes = flag.String("layer-sizes", "", "Comma-separated list of layer sizes (e.g., 512KB,1MB,2GB,8150)")
tmpdirPrefix = flag.String("tmpdir-prefix", "", "Directory prefix for temporary build files (default: system temp dir)")
maxConcurrent = flag.Int("max-concurrent", 5, "Maximum number of layers to create concurrently")
mockFS = flag.Bool("mock-fs", false, "Create mock filesystem structure instead of single files")
maxDepth = flag.Int("max-depth", 3, "Maximum directory depth for mock filesystem (only used with --mock-fs)")
targetFiles = flag.Int("target-files", 0, "Target number of files per layer for mock filesystem (default: calculated based on layer size)")
)
// createTempDir creates a temporary directory for building the image
func createTempDir(prefix string) (string, error) {
tempDir, err := os.MkdirTemp(prefix, "imgmkr-")
if err != nil {
return "", fmt.Errorf("failed to create temp directory: %w", err)
}
return tempDir, nil
}
// LayerJob represents a layer creation job
type LayerJob struct {
layerNum int
layerDir string
size int64
}
// LayerResult represents the result of a layer creation job
type LayerResult struct {
layerNum int
duration time.Duration
err error
}
// createLayersConcurrently creates multiple layers concurrently using a worker pool
func createLayersConcurrently(buildDir string, sizes []int64, maxWorkers int) error {
// Calculate total size for progress tracking
var totalSize int64
for _, size := range sizes {
totalSize += size
}
// Create progress tracker
tracker := progress.New(len(sizes), totalSize)
jobs := make(chan LayerJob, len(sizes))
results := make(chan LayerResult, len(sizes))
// Start workers
var wg sync.WaitGroup
for w := 0; w < maxWorkers; w++ {
wg.Add(1)
go func() {
defer wg.Done()
for job := range jobs {
startTime := time.Now()
var err error
if *mockFS {
err = mockfs.Create(job.layerDir, job.size, *maxDepth, *targetFiles)
} else {
err = createLayerFile(job.layerDir, job.size)
}
results <- LayerResult{
layerNum: job.layerNum,
duration: time.Since(startTime),
err: err,
}
}
}()
}
// Send jobs
go func() {
defer close(jobs)
for i, size := range sizes {
layerDir := filepath.Join(buildDir, fmt.Sprintf("layer%d", i+1))
jobs <- LayerJob{
layerNum: i + 1,
layerDir: layerDir,
size: size,
}
}
}()
// Collect results
go func() {
wg.Wait()
close(results)
}()
// Process results and report progress
completed := make(map[int]LayerResult)
for result := range results {
if result.err != nil {
return fmt.Errorf("error creating layer %d: %w", result.layerNum, result.err)
}
completed[result.layerNum] = result
tracker.Update(result.layerNum, sizes[result.layerNum-1], result.duration)
}
// Finish progress display
tracker.Finish()
return nil
}
// createLayerFile creates a file of the specified size filled with random data
func createLayerFile(layerDir string, fileSize int64) error {
// Create the layer directory if it doesn't exist
if err := os.MkdirAll(layerDir, 0755); err != nil {
return fmt.Errorf("failed to create layer directory: %w", err)
}
// Create a file with the size as part of the name
fileName := fmt.Sprintf("%s-file", size.Format(fileSize))
filePath := filepath.Join(layerDir, fileName)
file, err := os.Create(filePath)
if err != nil {
return fmt.Errorf("failed to create file: %w", err)
}
defer file.Close()
// Fill the file with data in chunks
const chunkSize = 10 * size.MB
remaining := fileSize
for remaining > 0 {
writeSize := remaining
if writeSize > chunkSize {
writeSize = chunkSize
}
// Create a buffer with data
data := make([]byte, writeSize)
_, err := io.ReadFull(strings.NewReader(strings.Repeat("x", int(writeSize))), data)
if err != nil {
return fmt.Errorf("failed to generate data: %w", err)
}
// Write the data to the file
_, err = file.Write(data)
if err != nil {
return fmt.Errorf("failed to write data to file: %w", err)
}
remaining -= writeSize
}
return nil
}
// createDockerfile creates a Dockerfile that adds each layer
func createDockerfile(buildDir string, numLayers int) error {
dockerfilePath := filepath.Join(buildDir, "Dockerfile")
file, err := os.Create(dockerfilePath)
if err != nil {
return fmt.Errorf("failed to create Dockerfile: %w", err)
}
defer file.Close()
// Start with a scratch image
_, err = file.WriteString("FROM scratch\n")
if err != nil {
return fmt.Errorf("failed to write to Dockerfile: %w", err)
}
// Add each layer
for i := 1; i <= numLayers; i++ {
layerDir := fmt.Sprintf("layer%d", i)
_, err = file.WriteString(fmt.Sprintf("ADD %s /\n", layerDir))
if err != nil {
return fmt.Errorf("failed to write to Dockerfile: %w", err)
}
}
return nil
}
// buildImage builds the Docker image using finch or docker
func buildImage(buildDir string, repoTag string) error {
// Try finch first, fallback to docker if not available
var cmdName string
_, err := exec.LookPath("finch")
if err == nil {
cmdName = "finch"
} else {
_, err = exec.LookPath("docker")
if err == nil {
cmdName = "docker"
} else {
return fmt.Errorf("neither finch nor docker command found")
}
}
// Build the image
cmd := exec.Command(cmdName, "build", "-t", repoTag, ".")
cmd.Dir = buildDir
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
fmt.Printf("Building image with %s...\n", cmdName)
err = cmd.Run()
if err != nil {
return fmt.Errorf("failed to build image: %w", err)
}
return nil
}
func main() {
// Parse command line flags
flag.Parse()
// Validate required flags
if *layerSizes == "" {
log.Fatal("--layer-sizes is required")
}
// Get the repository:tag argument
args := flag.Args()
if len(args) != 1 {
log.Fatal("Repository:tag argument is required")
}
repoTag := args[0]
// Parse layer sizes
sizes, err := size.ParseList(*layerSizes)
if err != nil {
log.Fatalf("Error parsing layer sizes: %v", err)
}
// Number of layers is inferred from the layer sizes
numLayers := len(sizes)
// Create a temporary build directory
fmt.Println("Creating temporary build directory...")
buildDir, err := createTempDir(*tmpdirPrefix)
if err != nil {
log.Fatalf("Error creating temporary directory: %v", err)
}
// Setup cleanup manager and signal handling
cleanupManager := cleanup.New(buildDir)
cleanupManager.SetupSignalHandling()
defer cleanupManager.GracefulCleanup()
// Create layer files
fmt.Printf("Creating layer files (max %d concurrent)...\n", *maxConcurrent)
err = createLayersConcurrently(buildDir, sizes, *maxConcurrent)
if err != nil {
log.Fatalf("Error creating layer files: %v", err)
}
// Create Dockerfile
fmt.Println("Creating Dockerfile...")
err = createDockerfile(buildDir, numLayers)
if err != nil {
log.Fatalf("Error creating Dockerfile: %v", err)
}
// Build the image
err = buildImage(buildDir, repoTag)
if err != nil {
log.Fatalf("Error building image: %v", err)
}
fmt.Printf("Successfully built image %s\n", repoTag)
}