From df48bb6882a3290d29ad4a50ece3aa6aef5dc79a Mon Sep 17 00:00:00 2001 From: Andrew Gillis <11790789+gammazero@users.noreply.github.com> Date: Wed, 1 Oct 2025 01:10:49 -0700 Subject: [PATCH 1/4] upgrade go-ds-pebble to v0.5.2 (#11000) * upgrade go-ds-pebble to v0.5.2 * Change pebble option `MaxConcurrentCompactions` to `CompactionConcurrencyRange` (cherry picked from commit 9370004b5fcbc4168e49d13634dfe3ea0ec34cc1) --- docs/examples/kubo-as-a-library/go.mod | 12 ++++++----- docs/examples/kubo-as-a-library/go.sum | 28 +++++++++++++++----------- go.mod | 12 ++++++----- go.sum | 28 +++++++++++++++----------- plugin/plugins/pebbleds/pebbleds.go | 2 +- test/dependencies/go.mod | 10 +++++---- test/dependencies/go.sum | 24 +++++++++++++--------- 7 files changed, 67 insertions(+), 49 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 81c2a147b8f..9b3cf3dfd2a 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -16,8 +16,10 @@ require ( require ( bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc // indirect github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect - github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect + github.com/DataDog/zstd v1.5.7 // indirect github.com/Jorropo/jsync v1.0.1 // indirect + github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 // indirect + github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/benbjohnson/clock v1.3.5 // indirect @@ -29,11 +31,10 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/ceramicnetwork/go-dag-jose v0.1.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 // indirect + github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect github.com/cockroachdb/errors v1.11.3 // indirect - github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect - github.com/cockroachdb/pebble/v2 v2.0.6 // indirect + github.com/cockroachdb/pebble/v2 v2.1.0 // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect @@ -81,7 +82,7 @@ require ( github.com/ipfs/go-ds-flatfs v0.5.5 // indirect github.com/ipfs/go-ds-leveldb v0.5.2 // indirect github.com/ipfs/go-ds-measure v0.2.2 // indirect - github.com/ipfs/go-ds-pebble v0.5.1 // indirect + github.com/ipfs/go-ds-pebble v0.5.2 // indirect github.com/ipfs/go-dsqueue v0.0.5 // indirect github.com/ipfs/go-fs-lock v0.1.1 // indirect github.com/ipfs/go-ipfs-cmds v0.15.0 // indirect @@ -132,6 +133,7 @@ require ( github.com/miekg/dns v1.1.68 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect + github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index eb90aa12ad5..3d8c6c960f1 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -29,11 +29,15 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e h1:ZIWapoIRN1VqT8GR8jAwb1Ie9GyehWjVcGh32Y2MznE= -github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= +github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU= github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 h1:8XBWWQD+vFF+JqOsm16t0Kab1a7YWV8+GISVEP8AuZ8= +github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= @@ -81,20 +85,18 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 h1:bvJv505UUfjzbaIPdNS4AEkHreDqQk6yuNpsdRHpwFA= -github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= -github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056 h1:slXychO2uDM6hYRu4c0pD0udNI8uObfeKN6UInWViS8= -github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= -github.com/cockroachdb/pebble/v2 v2.0.6 h1:eL54kX2AKp1ePJ/8vq4IO3xIEPpvVjlSP12dlLYilyE= -github.com/cockroachdb/pebble/v2 v2.0.6/go.mod h1:un1DXG73PKw3F7Ndd30YactyvsFviI9Fuhe0tENdnyA= +github.com/cockroachdb/pebble/v2 v2.1.0 h1:6KZvjSpWcEXZUvlLzTRC7T1A2G7r+bFskIzggklxixo= +github.com/cockroachdb/pebble/v2 v2.1.0/go.mod h1:Aza05DCCc05ghIJZkB4Q/axv/JK9wx5cFwWcnhG0eGw= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 h1:Nua446ru3juLHLZd4AwKNzClZgL1co3pUPGv3o8FlcA= @@ -319,8 +321,8 @@ github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= github.com/ipfs/go-ds-measure v0.2.2 h1:4kwvBGbbSXNYe4ANlg7qTIYoZU6mNlqzQHdVqICkqGI= github.com/ipfs/go-ds-measure v0.2.2/go.mod h1:b/87ak0jMgH9Ylt7oH0+XGy4P8jHx9KG09Qz+pOeTIs= -github.com/ipfs/go-ds-pebble v0.5.1 h1:p0FAE0zw9J/3T1VkGB9s98jWmfKmw2t0iEwfMUv8iSQ= -github.com/ipfs/go-ds-pebble v0.5.1/go.mod h1:LsmQx4w+0o9znl4hTxYo1Y2lnBTzNCwc4kNpD3wWXM0= +github.com/ipfs/go-ds-pebble v0.5.2 h1:85kWoZ15RcFfy7Oh+OmRvQuREOq/EYsWFHPHt1FUbvk= +github.com/ipfs/go-ds-pebble v0.5.2/go.mod h1:pn2bxYkAE7JRkbAF7D8xuEEFD3oOQ7QqQZPWkAVBs58= github.com/ipfs/go-dsqueue v0.0.5 h1:TUOk15TlCJ/NKV8Yk2W5wgkEjDa44Nem7a7FGIjsMNU= github.com/ipfs/go-dsqueue v0.0.5/go.mod h1:i/jAlpZjBbQJLioN+XKbFgnd+u9eAhGZs9IrqIzTd9g= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= @@ -490,6 +492,8 @@ github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKo github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= diff --git a/go.mod b/go.mod index a1a2a3ab195..8a487b8472f 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 github.com/ceramicnetwork/go-dag-jose v0.1.1 github.com/cheggaaa/pb v1.0.29 - github.com/cockroachdb/pebble/v2 v2.0.6 + github.com/cockroachdb/pebble/v2 v2.1.0 github.com/coreos/go-systemd/v22 v22.5.0 github.com/dustin/go-humanize v1.0.1 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 @@ -32,7 +32,7 @@ require ( github.com/ipfs/go-ds-flatfs v0.5.5 github.com/ipfs/go-ds-leveldb v0.5.2 github.com/ipfs/go-ds-measure v0.2.2 - github.com/ipfs/go-ds-pebble v0.5.1 + github.com/ipfs/go-ds-pebble v0.5.2 github.com/ipfs/go-fs-lock v0.1.1 github.com/ipfs/go-ipfs-cmds v0.15.0 github.com/ipfs/go-ipld-cbor v0.2.1 @@ -97,8 +97,10 @@ require ( require ( github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect - github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect + github.com/DataDog/zstd v1.5.7 // indirect github.com/Jorropo/jsync v1.0.1 // indirect + github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 // indirect + github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/benbjohnson/clock v1.3.5 // indirect @@ -107,9 +109,8 @@ require ( github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 // indirect + github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect github.com/cockroachdb/errors v1.11.3 // indirect - github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 // indirect @@ -180,6 +181,7 @@ require ( github.com/mholt/acmez/v3 v3.1.2 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect + github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect diff --git a/go.sum b/go.sum index 5dd61e2dc8d..3e969c930fd 100644 --- a/go.sum +++ b/go.sum @@ -47,12 +47,16 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e h1:ZIWapoIRN1VqT8GR8jAwb1Ie9GyehWjVcGh32Y2MznE= -github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= +github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU= github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 h1:8XBWWQD+vFF+JqOsm16t0Kab1a7YWV8+GISVEP8AuZ8= +github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= @@ -112,20 +116,18 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 h1:bvJv505UUfjzbaIPdNS4AEkHreDqQk6yuNpsdRHpwFA= -github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= -github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056 h1:slXychO2uDM6hYRu4c0pD0udNI8uObfeKN6UInWViS8= -github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= -github.com/cockroachdb/pebble/v2 v2.0.6 h1:eL54kX2AKp1ePJ/8vq4IO3xIEPpvVjlSP12dlLYilyE= -github.com/cockroachdb/pebble/v2 v2.0.6/go.mod h1:un1DXG73PKw3F7Ndd30YactyvsFviI9Fuhe0tENdnyA= +github.com/cockroachdb/pebble/v2 v2.1.0 h1:6KZvjSpWcEXZUvlLzTRC7T1A2G7r+bFskIzggklxixo= +github.com/cockroachdb/pebble/v2 v2.1.0/go.mod h1:Aza05DCCc05ghIJZkB4Q/axv/JK9wx5cFwWcnhG0eGw= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 h1:Nua446ru3juLHLZd4AwKNzClZgL1co3pUPGv3o8FlcA= @@ -386,8 +388,8 @@ github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= github.com/ipfs/go-ds-measure v0.2.2 h1:4kwvBGbbSXNYe4ANlg7qTIYoZU6mNlqzQHdVqICkqGI= github.com/ipfs/go-ds-measure v0.2.2/go.mod h1:b/87ak0jMgH9Ylt7oH0+XGy4P8jHx9KG09Qz+pOeTIs= -github.com/ipfs/go-ds-pebble v0.5.1 h1:p0FAE0zw9J/3T1VkGB9s98jWmfKmw2t0iEwfMUv8iSQ= -github.com/ipfs/go-ds-pebble v0.5.1/go.mod h1:LsmQx4w+0o9znl4hTxYo1Y2lnBTzNCwc4kNpD3wWXM0= +github.com/ipfs/go-ds-pebble v0.5.2 h1:85kWoZ15RcFfy7Oh+OmRvQuREOq/EYsWFHPHt1FUbvk= +github.com/ipfs/go-ds-pebble v0.5.2/go.mod h1:pn2bxYkAE7JRkbAF7D8xuEEFD3oOQ7QqQZPWkAVBs58= github.com/ipfs/go-dsqueue v0.0.5 h1:TUOk15TlCJ/NKV8Yk2W5wgkEjDa44Nem7a7FGIjsMNU= github.com/ipfs/go-dsqueue v0.0.5/go.mod h1:i/jAlpZjBbQJLioN+XKbFgnd+u9eAhGZs9IrqIzTd9g= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= @@ -586,6 +588,8 @@ github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKo github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= diff --git a/plugin/plugins/pebbleds/pebbleds.go b/plugin/plugins/pebbleds/pebbleds.go index fab1cc16a88..141eff74b2b 100644 --- a/plugin/plugins/pebbleds/pebbleds.go +++ b/plugin/plugins/pebbleds/pebbleds.go @@ -134,7 +134,7 @@ func (*pebbledsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap { WALBytesPerSync: walBytesPerSync, } if maxConcurrentCompactions != 0 { - c.pebbleOpts.MaxConcurrentCompactions = func() int { return maxConcurrentCompactions } + c.pebbleOpts.CompactionConcurrencyRange = func() (int, int) { return 1, maxConcurrentCompactions } } if walMinSyncSec != 0 { c.pebbleOpts.WALMinSyncInterval = func() time.Duration { return time.Duration(walMinSyncSec) * time.Second } diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 65d3151aac7..268ccae2722 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -28,12 +28,14 @@ require ( github.com/Antonboom/testifylint v1.5.2 // indirect github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect github.com/Crocmagnon/fatcontext v0.7.1 // indirect - github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect + github.com/DataDog/zstd v1.5.7 // indirect github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.1 // indirect github.com/Jorropo/jsync v1.0.1 // indirect github.com/Masterminds/semver/v3 v3.3.0 // indirect github.com/OpenPeeDeeP/depguard/v2 v2.2.1 // indirect + github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 // indirect + github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect github.com/alecthomas/go-check-sumtype v0.3.1 // indirect github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect github.com/alexkohler/nakedret/v2 v2.0.5 // indirect @@ -60,11 +62,10 @@ require ( github.com/charithe/durationcheck v0.0.10 // indirect github.com/chavacava/garif v0.1.0 // indirect github.com/ckaznocha/intrange v0.3.0 // indirect - github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 // indirect + github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect github.com/cockroachdb/errors v1.11.3 // indirect - github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect - github.com/cockroachdb/pebble/v2 v2.0.6 // indirect + github.com/cockroachdb/pebble/v2 v2.1.0 // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect @@ -202,6 +203,7 @@ require ( github.com/mgechev/revive v1.7.0 // indirect github.com/mholt/acmez/v3 v3.1.2 // indirect github.com/miekg/dns v1.1.68 // indirect + github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index aec72c23d20..4c3e98cce4d 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -27,8 +27,8 @@ github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/k github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Crocmagnon/fatcontext v0.7.1 h1:SC/VIbRRZQeQWj/TcQBS6JmrXcfA+BU4OGSVUt54PjM= github.com/Crocmagnon/fatcontext v0.7.1/go.mod h1:1wMvv3NXEBJucFGfwOJBxSVWcoIO6emV215SMkW9MFU= -github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e h1:ZIWapoIRN1VqT8GR8jAwb1Ie9GyehWjVcGh32Y2MznE= -github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= +github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 h1:sHglBQTwgx+rWPdisA5ynNEsoARbiCBOyGcJM4/OzsM= github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.1 h1:Sz1JIXEcSfhz7fUi7xHnhpIE0thVASYjvosApmHuD2k= @@ -41,6 +41,10 @@ github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+ github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/OpenPeeDeeP/depguard/v2 v2.2.1 h1:vckeWVESWp6Qog7UZSARNqfu/cZqvki8zsuj3piCMx4= github.com/OpenPeeDeeP/depguard/v2 v2.2.1/go.mod h1:q4DKzC4UcVaAvcfd41CZh0PWpGgzrVxUYBlgKNGquUo= +github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 h1:8XBWWQD+vFF+JqOsm16t0Kab1a7YWV8+GISVEP8AuZ8= +github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk= +github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo= github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= @@ -104,20 +108,18 @@ github.com/chavacava/garif v0.1.0/go.mod h1:XMyYCkEL58DF0oyW4qDjjnPWONs2HBqYKI+U github.com/ckaznocha/intrange v0.3.0 h1:VqnxtK32pxgkhJgYQEeOArVidIPg+ahLP7WBOXZd5ZY= github.com/ckaznocha/intrange v0.3.0/go.mod h1:+I/o2d2A1FBHgGELbGxzIcyd3/9l9DuwjM8FsbSS3Lo= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 h1:bvJv505UUfjzbaIPdNS4AEkHreDqQk6yuNpsdRHpwFA= -github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= -github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056 h1:slXychO2uDM6hYRu4c0pD0udNI8uObfeKN6UInWViS8= -github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU= +github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI= +github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= -github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= -github.com/cockroachdb/pebble/v2 v2.0.6 h1:eL54kX2AKp1ePJ/8vq4IO3xIEPpvVjlSP12dlLYilyE= -github.com/cockroachdb/pebble/v2 v2.0.6/go.mod h1:un1DXG73PKw3F7Ndd30YactyvsFviI9Fuhe0tENdnyA= +github.com/cockroachdb/pebble/v2 v2.1.0 h1:6KZvjSpWcEXZUvlLzTRC7T1A2G7r+bFskIzggklxixo= +github.com/cockroachdb/pebble/v2 v2.1.0/go.mod h1:Aza05DCCc05ghIJZkB4Q/axv/JK9wx5cFwWcnhG0eGw= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 h1:Nua446ru3juLHLZd4AwKNzClZgL1co3pUPGv3o8FlcA= @@ -522,6 +524,8 @@ github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKo github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw= +github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= From e7c5804e3830db56692aaea29b92244af3a5c829 Mon Sep 17 00:00:00 2001 From: Andrew Gillis <11790789+gammazero@users.noreply.github.com> Date: Fri, 3 Oct 2025 23:33:40 -0700 Subject: [PATCH 2/4] Upgrade go-ds-pebble to v0.5.3 (#11011) This fixes a panic that happens when reusing a batch go-datastore Batch. (cherry picked from commit 2b5adeedcc1277aea186b166f26374e673532244) --- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 9b3cf3dfd2a..e383e1f253c 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -82,7 +82,7 @@ require ( github.com/ipfs/go-ds-flatfs v0.5.5 // indirect github.com/ipfs/go-ds-leveldb v0.5.2 // indirect github.com/ipfs/go-ds-measure v0.2.2 // indirect - github.com/ipfs/go-ds-pebble v0.5.2 // indirect + github.com/ipfs/go-ds-pebble v0.5.3 // indirect github.com/ipfs/go-dsqueue v0.0.5 // indirect github.com/ipfs/go-fs-lock v0.1.1 // indirect github.com/ipfs/go-ipfs-cmds v0.15.0 // indirect diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 3d8c6c960f1..55304498701 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -321,8 +321,8 @@ github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= github.com/ipfs/go-ds-measure v0.2.2 h1:4kwvBGbbSXNYe4ANlg7qTIYoZU6mNlqzQHdVqICkqGI= github.com/ipfs/go-ds-measure v0.2.2/go.mod h1:b/87ak0jMgH9Ylt7oH0+XGy4P8jHx9KG09Qz+pOeTIs= -github.com/ipfs/go-ds-pebble v0.5.2 h1:85kWoZ15RcFfy7Oh+OmRvQuREOq/EYsWFHPHt1FUbvk= -github.com/ipfs/go-ds-pebble v0.5.2/go.mod h1:pn2bxYkAE7JRkbAF7D8xuEEFD3oOQ7QqQZPWkAVBs58= +github.com/ipfs/go-ds-pebble v0.5.3 h1:4esRt82+LkenUnIWyUCghR1gzRfqeCYGGKX/hRmabro= +github.com/ipfs/go-ds-pebble v0.5.3/go.mod h1:pn2bxYkAE7JRkbAF7D8xuEEFD3oOQ7QqQZPWkAVBs58= github.com/ipfs/go-dsqueue v0.0.5 h1:TUOk15TlCJ/NKV8Yk2W5wgkEjDa44Nem7a7FGIjsMNU= github.com/ipfs/go-dsqueue v0.0.5/go.mod h1:i/jAlpZjBbQJLioN+XKbFgnd+u9eAhGZs9IrqIzTd9g= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= diff --git a/go.mod b/go.mod index 8a487b8472f..48702ed0f28 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/ipfs/go-ds-flatfs v0.5.5 github.com/ipfs/go-ds-leveldb v0.5.2 github.com/ipfs/go-ds-measure v0.2.2 - github.com/ipfs/go-ds-pebble v0.5.2 + github.com/ipfs/go-ds-pebble v0.5.3 github.com/ipfs/go-fs-lock v0.1.1 github.com/ipfs/go-ipfs-cmds v0.15.0 github.com/ipfs/go-ipld-cbor v0.2.1 diff --git a/go.sum b/go.sum index 3e969c930fd..0cd74a1d1cf 100644 --- a/go.sum +++ b/go.sum @@ -388,8 +388,8 @@ github.com/ipfs/go-ds-leveldb v0.5.2 h1:6nmxlQ2zbp4LCNdJVsmHfs9GP0eylfBNxpmY1csp github.com/ipfs/go-ds-leveldb v0.5.2/go.mod h1:2fAwmcvD3WoRT72PzEekHBkQmBDhc39DJGoREiuGmYo= github.com/ipfs/go-ds-measure v0.2.2 h1:4kwvBGbbSXNYe4ANlg7qTIYoZU6mNlqzQHdVqICkqGI= github.com/ipfs/go-ds-measure v0.2.2/go.mod h1:b/87ak0jMgH9Ylt7oH0+XGy4P8jHx9KG09Qz+pOeTIs= -github.com/ipfs/go-ds-pebble v0.5.2 h1:85kWoZ15RcFfy7Oh+OmRvQuREOq/EYsWFHPHt1FUbvk= -github.com/ipfs/go-ds-pebble v0.5.2/go.mod h1:pn2bxYkAE7JRkbAF7D8xuEEFD3oOQ7QqQZPWkAVBs58= +github.com/ipfs/go-ds-pebble v0.5.3 h1:4esRt82+LkenUnIWyUCghR1gzRfqeCYGGKX/hRmabro= +github.com/ipfs/go-ds-pebble v0.5.3/go.mod h1:pn2bxYkAE7JRkbAF7D8xuEEFD3oOQ7QqQZPWkAVBs58= github.com/ipfs/go-dsqueue v0.0.5 h1:TUOk15TlCJ/NKV8Yk2W5wgkEjDa44Nem7a7FGIjsMNU= github.com/ipfs/go-dsqueue v0.0.5/go.mod h1:i/jAlpZjBbQJLioN+XKbFgnd+u9eAhGZs9IrqIzTd9g= github.com/ipfs/go-fs-lock v0.1.1 h1:TecsP/Uc7WqYYatasreZQiP9EGRy4ZnKoG4yXxR33nw= From b48f7ae429cf2a2195f1061cf854a0b35b8567d9 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 8 Oct 2025 18:02:04 +0200 Subject: [PATCH 3/4] fix: migrations for Windows (#11010) * test: add migration tests for Windows and macOS - add dedicated CI workflow for migration tests on Windows/macOS - workflow triggers on migration-related file changes only * build: remove redundant go version checks - remove GO_MIN_VERSION and check_go_version scripts - go.mod already enforces minimum version (go 1.25) - fixes make build on Windows * fix: windows migration panic by reading config into memory fixes migration panic on Windows when upgrading from v0.37 to v0.38 by reading the entire config file into memory before performing atomic operations. this avoids file locking issues on Windows where open files cannot be renamed. also fixes: - TestRepoDir to set USERPROFILE on Windows (not just HOME) - CLI migration tests to sanitize directory names (remove colons) minimal fix that solves the "panic: error can't be dealt with transactionally: Access is denied" error without adding unnecessary platform-specific complexity. * fix: set PATH for CLI migration tests in CI the CLI tests need the built ipfs binary to be in PATH * fix: use ipfs shutdown for graceful daemon termination in tests replaces platform-specific signal handling with ipfs shutdown command which works consistently across all platforms including Windows * fix: isolate PATH modifications in parallel migration tests tests running in parallel with t.Parallel() were interfering with each other through global PATH modifications via os.Setenv(). this caused tests to download real migration binaries instead of using mocks, leading to Windows failures due to path separator issues in external tools. now each test builds its own custom PATH and passes it explicitly to commands, preventing interference between parallel tests. * chore: improve error messages in WithBackup * fix: Windows CI migration test failures - add .exe extension to mock migration binaries on Windows - handle repo lock file properly in mock migration binary - ensure lock is created and removed to prevent conflicts * refactor: align atomicfile error handling with fs-repo-migrations - check close error in Abort() before attempting removal - leave temp file on rename failure for debugging (like fs-repo-15-to-16) - improves consistency with external migration implementations * fix: use req.Context in repo migrate to avoid double-lock The repo migrate command was calling cctx.Context() which has a hidden side effect: it lazily constructs the IPFS node by calling GetNode(), which opens the repository and acquires repo.lock. When migrations then tried to acquire the same lock, it failed with "lock is already held by us" because go4.org/lock tracks locks per-process in a global map. The fix uses req.Context instead, which is a plain context.Context with no side effects. This provides what migrations need (cancellation handling) without triggering node construction or repo opening. Context types explained: - req.Context: Standard Go context for request lifetime, cancellation, and timeouts. No side effects. - cctx.Context(): Kubo-specific method that lazily constructs the full IPFS node (opens repo, acquires lock, initializes subsystems). Returns the node's internal context. Why req.Context is correct here: - Migrations work on raw filesystem (only need ConfigRoot path) - Command has SetDoesNotUseRepo(true) - doesn't need running node - Migrations handle their own locking via lockfile.Lock() - Need cancellation support but not node lifecycle The bug only appeared with embedded migrations (v16+) because they run in-process. External migrations (pre-v16) were separate processes, so each had isolated state. Sequential migrations (forward then backward) in the same process exposed this latent double-lock issue. Also adds repo.lock acquisition to RunEmbeddedMigrations to prevent concurrent migration access, and removes the now-unnecessary daemon lock check from the migrate command handler. * fix: use req.Context for migrations and autoconf in daemon startup daemon.go was incorrectly using cctx.Context() in two critical places: 1. Line 337: migrations call - cctx.Context() triggers GetNode() which opens the repo and acquires repo.lock BEFORE migrations run, causing "lock is already held by us" errors when migrations try to lock 2. Line 390: autoconf client.Start() - uses context for HTTP timeouts and background updater lifecycle, doesn't need node construction Both now use req.Context (plain Go context) which provides: - request lifetime and cancellation - no side effects (doesn't construct node or open repo) - correct lifecycle for HTTP requests and background goroutines (cherry picked from commit f4834e797db174e0630aeee4b6dc68d133106f48) --- .github/workflows/test-migrations.yml | 85 +++++++ bin/check_go_version | 44 ---- bin/check_version | 77 ------- cmd/ipfs/kubo/daemon.go | 6 +- core/commands/repo.go | 13 +- docs/changelogs/v0.38.md | 18 ++ mk/golang.mk | 6 +- .../migrations/atomicfile/atomicfile.go | 29 ++- .../migrations/atomicfile/atomicfile_test.go | 208 ++++++++++++++++++ repo/fsrepo/migrations/common/utils.go | 47 ++-- repo/fsrepo/migrations/embedded.go | 8 + repo/fsrepo/migrations/ipfsdir_test.go | 2 + .../migrations/migration_16_to_latest_test.go | 169 +++++++++++++- .../migrations/migration_concurrent_test.go | 55 +++++ .../migration_mixed_15_to_latest_test.go | 193 ++++++++-------- 15 files changed, 693 insertions(+), 267 deletions(-) create mode 100644 .github/workflows/test-migrations.yml delete mode 100755 bin/check_go_version delete mode 100755 bin/check_version create mode 100644 repo/fsrepo/migrations/atomicfile/atomicfile_test.go create mode 100644 test/cli/migrations/migration_concurrent_test.go diff --git a/.github/workflows/test-migrations.yml b/.github/workflows/test-migrations.yml new file mode 100644 index 00000000000..1def94ff7c1 --- /dev/null +++ b/.github/workflows/test-migrations.yml @@ -0,0 +1,85 @@ +name: Migrations + +on: + workflow_dispatch: + pull_request: + paths: + # Migration implementation files + - 'repo/fsrepo/migrations/**' + - 'test/cli/migrations/**' + # Config and repo handling + - 'repo/fsrepo/**' + # This workflow file itself + - '.github/workflows/test-migrations.yml' + push: + branches: + - 'master' + - 'release-*' + paths: + - 'repo/fsrepo/migrations/**' + - 'test/cli/migrations/**' + - 'repo/fsrepo/**' + - '.github/workflows/test-migrations.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: true + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + env: + TEST_VERBOSE: 1 + IPFS_CHECK_RCMGR_DEFAULTS: 1 + defaults: + run: + shell: bash + steps: + - name: Check out Kubo + uses: actions/checkout@v5 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + + - name: Build kubo binary + run: | + make build + echo "Built ipfs binary at $(pwd)/cmd/ipfs/" + + - name: Add kubo to PATH + run: | + echo "$(pwd)/cmd/ipfs" >> $GITHUB_PATH + + - name: Verify ipfs in PATH + run: | + which ipfs || echo "ipfs not in PATH" + ipfs version || echo "Failed to run ipfs version" + + - name: Run migration unit tests + run: | + go test ./repo/fsrepo/migrations/... + + - name: Run CLI migration tests + env: + IPFS_PATH: ${{ runner.temp }}/ipfs-test + run: | + export PATH="${{ github.workspace }}/cmd/ipfs:$PATH" + which ipfs || echo "ipfs not found in PATH" + ipfs version || echo "Failed to run ipfs version" + go test ./test/cli/migrations/... + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }}-test-results + path: | + test/**/*.log + ${{ runner.temp }}/ipfs-test/ diff --git a/bin/check_go_version b/bin/check_go_version deleted file mode 100755 index 74320010bc6..00000000000 --- a/bin/check_go_version +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -# -# Check that the go version is at least equal to a minimum version -# number. -# -# Call it for example like this: -# -# $ check_go_version "1.5.2" -# - -USAGE="$0 GO_MIN_VERSION" - -die() { - printf >&2 "fatal: %s\n" "$@" - exit 1 -} - -# Get arguments - -test "$#" -eq "1" || die "This program must be passed exactly 1 arguments" "Usage: $USAGE" - -GO_MIN_VERSION="$1" - -UPGRADE_MSG="Please take a look at https://golang.org/doc/install to install or upgrade go." - -# Get path to the directory containing this file -# If $0 has no slashes, uses "./" -PREFIX=$(expr "$0" : "\(.*\/\)") || PREFIX='./' -# Include the 'check_at_least_version' function -. ${PREFIX}check_version - -# Check that the go binary exists and is in the path - -GOCC=${GOCC="go"} - -type ${GOCC} >/dev/null 2>&1 || die_upgrade "go is not installed or not in the PATH!" - -# Check the go binary version - -VERS_STR=$(${GOCC} version 2>&1) || die "'go version' failed with output: $VERS_STR" - -GO_CUR_VERSION=$(expr "$VERS_STR" : ".*go version.* go\([^[:space:]]*\) .*") || die "Invalid 'go version' output: $VERS_STR" - -check_at_least_version "$GO_MIN_VERSION" "$GO_CUR_VERSION" "${GOCC}" diff --git a/bin/check_version b/bin/check_version deleted file mode 100755 index 25007002c3b..00000000000 --- a/bin/check_version +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/sh - -if test "x$UPGRADE_MSG" = "x"; then - printf >&2 "fatal: Please set '"'$UPGRADE_MSG'"' before sourcing this script\n" - exit 1 -fi - -die_upgrade() { - printf >&2 "fatal: %s\n" "$@" - printf >&2 "=> %s\n" "$UPGRADE_MSG" - exit 1 -} - -major_number() { - vers="$1" - - # Hack around 'expr' exiting with code 1 when it outputs 0 - case "$vers" in - 0) echo "0" ;; - 0.*) echo "0" ;; - *) expr "$vers" : "\([^.]*\).*" || return 1 - esac -} - -check_at_least_version() { - MIN_VERS="$1" - CUR_VERS="$2" - PROG_NAME="$3" - - # Get major, minor and fix numbers for each version - MIN_MAJ=$(major_number "$MIN_VERS") || die "No major version number in '$MIN_VERS' for '$PROG_NAME'" - CUR_MAJ=$(major_number "$CUR_VERS") || die "No major version number in '$CUR_VERS' for '$PROG_NAME'" - - # We expect a version to be of form X.X.X - # if the second dot doesn't match, we consider it a prerelease - - if MIN_MIN=$(expr "$MIN_VERS" : "[^.]*\.\([0-9][0-9]*\)"); then - # this captured digit is necessary, since expr returns code 1 if the output is empty - if expr "$MIN_VERS" : "[^.]*\.[0-9]*\([0-9]\.\|[0-9]\$\)" >/dev/null; then - MIN_PRERELEASE="0" - else - MIN_PRERELEASE="1" - fi - MIN_FIX=$(expr "$MIN_VERS" : "[^.]*\.[0-9][0-9]*[^0-9][^0-9]*\([0-9][0-9]*\)") || MIN_FIX="0" - else - MIN_MIN="0" - MIN_PRERELEASE="0" - MIN_FIX="0" - fi - if CUR_MIN=$(expr "$CUR_VERS" : "[^.]*\.\([0-9][0-9]*\)"); then - # this captured digit is necessary, since expr returns code 1 if the output is empty - if expr "$CUR_VERS" : "[^.]*\.[0-9]*\([0-9]\.\|[0-9]\$\)" >/dev/null; then - CUR_PRERELEASE="0" - else - CUR_PRERELEASE="1" - fi - CUR_FIX=$(expr "$CUR_VERS" : "[^.]*\.[0-9][0-9]*[^0-9][^0-9]*\([0-9][0-9]*\)") || CUR_FIX="0" - else - CUR_MIN="0" - CUR_PRERELEASE="0" - CUR_FIX="0" - fi - - # Compare versions - VERS_LEAST="$PROG_NAME version '$CUR_VERS' should be at least '$MIN_VERS'" - test "$CUR_MAJ" -lt "$MIN_MAJ" && die_upgrade "$VERS_LEAST" - test "$CUR_MAJ" -gt "$MIN_MAJ" || { - test "$CUR_MIN" -lt "$MIN_MIN" && die_upgrade "$VERS_LEAST" - test "$CUR_MIN" -gt "$MIN_MIN" || { - test "$CUR_PRERELEASE" -gt "$MIN_PRERELEASE" && die_upgrade "$VERS_LEAST" - test "$CUR_PRERELEASE" -lt "$MIN_PRERELEASE" || { - test "$CUR_FIX" -lt "$MIN_FIX" && die_upgrade "$VERS_LEAST" - true - } - } - } -} diff --git a/cmd/ipfs/kubo/daemon.go b/cmd/ipfs/kubo/daemon.go index 7daa66ee76a..fa89bf632b4 100644 --- a/cmd/ipfs/kubo/daemon.go +++ b/cmd/ipfs/kubo/daemon.go @@ -334,7 +334,8 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment } // Use hybrid migration strategy that intelligently combines external and embedded migrations - err = migrations.RunHybridMigrations(cctx.Context(), version.RepoVersion, cctx.ConfigRoot, false) + // Use req.Context instead of cctx.Context() to avoid attempting repo open before migrations complete + err = migrations.RunHybridMigrations(req.Context, version.RepoVersion, cctx.ConfigRoot, false) if err != nil { fmt.Println("Repository migration failed:") fmt.Printf(" %s\n", err) @@ -387,7 +388,8 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment log.Errorf("failed to create autoconf client: %v", err) } else { // Start primes cache and starts background updater - if _, err := client.Start(cctx.Context()); err != nil { + // Use req.Context for background updater lifecycle (node doesn't exist yet) + if _, err := client.Start(req.Context); err != nil { log.Errorf("failed to start autoconf updater: %v", err) } } diff --git a/core/commands/repo.go b/core/commands/repo.go index 01714312740..622e92d7ed0 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -423,19 +423,12 @@ migration. Versions below 16 require external migration tools. return fmt.Errorf("downgrade from version %d to %d requires --allow-downgrade flag", currentVersion, targetVersion) } - // Check if repo is locked by daemon before running migration - locked, err := fsrepo.LockedByOtherProcess(cctx.ConfigRoot) - if err != nil { - return fmt.Errorf("could not check repo lock: %w", err) - } - if locked { - return fmt.Errorf("cannot run migration while daemon is running (repo.lock exists)") - } - fmt.Printf("Migrating repository from version %d to %d...\n", currentVersion, targetVersion) // Use hybrid migration strategy that intelligently combines external and embedded migrations - err = migrations.RunHybridMigrations(cctx.Context(), targetVersion, cctx.ConfigRoot, allowDowngrade) + // Use req.Context instead of cctx.Context() to avoid opening the repo before migrations run, + // which would acquire the lock that migrations need + err = migrations.RunHybridMigrations(req.Context, targetVersion, cctx.ConfigRoot, allowDowngrade) if err != nil { fmt.Println("Repository migration failed:") fmt.Printf(" %s\n", err) diff --git a/docs/changelogs/v0.38.md b/docs/changelogs/v0.38.md index 2edb31adf87..a0342438070 100644 --- a/docs/changelogs/v0.38.md +++ b/docs/changelogs/v0.38.md @@ -5,6 +5,7 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team. - [v0.38.0](#v0380) +- [v0.38.1](#v0381) ## v0.38.0 @@ -290,3 +291,20 @@ The new [`Internal.MFSNoFlushLimit`](https://github.com/ipfs/kubo/blob/master/do | Jakub Sztandera | 1 | +67/-15 | 3 | | Masih H. Derkani | 1 | +1/-2 | 2 | | Dominic Della Valle | 1 | +2/-1 | 1 | + +## v0.38.1 + +Fixes migration panic on Windows when upgrading from v0.37 to v0.38 ("panic: error can't be dealt with transactionally: Access is denied"). + +### Changelog + +
+Full Changelog + +
+ +### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors + +| Contributor | Commits | Lines ยฑ | Files Changed | +|-------------|---------|---------|---------------| +| TBD | | | | diff --git a/mk/golang.mk b/mk/golang.mk index 4f4cd4fed34..b50179a0a04 100644 --- a/mk/golang.mk +++ b/mk/golang.mk @@ -1,5 +1,4 @@ # golang utilities -GO_MIN_VERSION = 1.25 export GO111MODULE=on @@ -74,11 +73,8 @@ test_go_lint: test/bin/golangci-lint test_go: $(TEST_GO) -check_go_version: - @$(GOCC) version - bin/check_go_version $(GO_MIN_VERSION) +# Version check is no longer needed - go.mod enforces minimum version .PHONY: check_go_version -DEPS_GO += check_go_version TEST += $(TEST_GO) TEST_SHORT += test_go_fmt test_go_short diff --git a/repo/fsrepo/migrations/atomicfile/atomicfile.go b/repo/fsrepo/migrations/atomicfile/atomicfile.go index 87704196d8d..209b8c368cb 100644 --- a/repo/fsrepo/migrations/atomicfile/atomicfile.go +++ b/repo/fsrepo/migrations/atomicfile/atomicfile.go @@ -1,6 +1,7 @@ package atomicfile import ( + "fmt" "io" "os" "path/filepath" @@ -34,23 +35,27 @@ func New(path string, mode os.FileMode) (*File, error) { // Close atomically replaces the target file with the temporary file func (f *File) Close() error { - if err := f.File.Close(); err != nil { - os.Remove(f.File.Name()) - return err + closeErr := f.File.Close() + if closeErr != nil { + // Try to cleanup temp file, but prioritize close error + _ = os.Remove(f.File.Name()) + return closeErr } - - if err := os.Rename(f.File.Name(), f.path); err != nil { - os.Remove(f.File.Name()) - return err - } - - return nil + return os.Rename(f.File.Name(), f.path) } // Abort removes the temporary file without replacing the target func (f *File) Abort() error { - f.File.Close() - return os.Remove(f.File.Name()) + closeErr := f.File.Close() + removeErr := os.Remove(f.File.Name()) + + if closeErr != nil && removeErr != nil { + return fmt.Errorf("abort failed: close: %w, remove: %v", closeErr, removeErr) + } + if closeErr != nil { + return closeErr + } + return removeErr } // ReadFrom reads from the given reader into the atomic file diff --git a/repo/fsrepo/migrations/atomicfile/atomicfile_test.go b/repo/fsrepo/migrations/atomicfile/atomicfile_test.go new file mode 100644 index 00000000000..668045d1281 --- /dev/null +++ b/repo/fsrepo/migrations/atomicfile/atomicfile_test.go @@ -0,0 +1,208 @@ +package atomicfile + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestNew_Success verifies atomic file creation +func TestNew_Success(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + + af, err := New(path, 0644) + require.NoError(t, err) + defer func() { _ = af.Abort() }() + + // Verify temp file exists + assert.FileExists(t, af.File.Name()) + + // Verify temp file is in same directory + assert.Equal(t, dir, filepath.Dir(af.File.Name())) +} + +// TestClose_Success verifies atomic replacement +func TestClose_Success(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + + af, err := New(path, 0644) + require.NoError(t, err) + + content := []byte("test content") + _, err = af.Write(content) + require.NoError(t, err) + + tempName := af.File.Name() + + require.NoError(t, af.Close()) + + // Verify target file exists with correct content + data, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, content, data) + + // Verify temp file removed + assert.NoFileExists(t, tempName) +} + +// TestAbort_Success verifies cleanup +func TestAbort_Success(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + + af, err := New(path, 0644) + require.NoError(t, err) + + tempName := af.File.Name() + + require.NoError(t, af.Abort()) + + // Verify temp file removed + assert.NoFileExists(t, tempName) + + // Verify target not created + assert.NoFileExists(t, path) +} + +// TestAbort_ErrorHandling tests error capture +func TestAbort_ErrorHandling(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + + af, err := New(path, 0644) + require.NoError(t, err) + + // Close file to force close error + af.File.Close() + + // Remove temp file to force remove error + os.Remove(af.File.Name()) + + err = af.Abort() + // Should get both errors + require.Error(t, err) + assert.Contains(t, err.Error(), "abort failed") +} + +// TestClose_CloseError verifies cleanup on close failure +func TestClose_CloseError(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + + af, err := New(path, 0644) + require.NoError(t, err) + + tempName := af.File.Name() + + // Close file to force close error + af.File.Close() + + err = af.Close() + require.Error(t, err) + + // Verify temp file cleaned up even on error + assert.NoFileExists(t, tempName) +} + +// TestReadFrom verifies io.Copy integration +func TestReadFrom(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + + af, err := New(path, 0644) + require.NoError(t, err) + defer func() { _ = af.Abort() }() + + content := []byte("test content from reader") + n, err := af.ReadFrom(bytes.NewReader(content)) + require.NoError(t, err) + assert.Equal(t, int64(len(content)), n) +} + +// TestFilePermissions verifies mode is set correctly +func TestFilePermissions(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + + af, err := New(path, 0600) + require.NoError(t, err) + + _, err = af.Write([]byte("test")) + require.NoError(t, err) + + require.NoError(t, af.Close()) + + info, err := os.Stat(path) + require.NoError(t, err) + + // On Unix, check exact permissions + if runtime.GOOS != "windows" { + mode := info.Mode().Perm() + assert.Equal(t, os.FileMode(0600), mode) + } +} + +// TestMultipleAbortsSafe verifies calling Abort multiple times is safe +func TestMultipleAbortsSafe(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.txt") + + af, err := New(path, 0644) + require.NoError(t, err) + + tempName := af.File.Name() + + // First abort should succeed + require.NoError(t, af.Abort()) + assert.NoFileExists(t, tempName, "temp file should be removed after first abort") + + // Second abort should handle gracefully (file already gone) + err = af.Abort() + // Error is acceptable since file is already removed, but it should not panic + t.Logf("Second Abort() returned: %v", err) +} + +// TestNoTempFilesAfterOperations verifies no .tmp-* files remain after operations +func TestNoTempFilesAfterOperations(t *testing.T) { + const testIterations = 5 + + tests := []struct { + name string + operation func(*File) error + }{ + {"close", (*File).Close}, + {"abort", (*File).Abort}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + + // Perform multiple operations + for i := 0; i < testIterations; i++ { + path := filepath.Join(dir, fmt.Sprintf("test%d.txt", i)) + + af, err := New(path, 0644) + require.NoError(t, err) + + _, err = af.Write([]byte("test data")) + require.NoError(t, err) + + require.NoError(t, tt.operation(af)) + } + + // Check for any .tmp-* files + tmpFiles, err := filepath.Glob(filepath.Join(dir, ".tmp-*")) + require.NoError(t, err) + assert.Empty(t, tmpFiles, "should be no temp files after %s", tt.name) + }) + } +} diff --git a/repo/fsrepo/migrations/common/utils.go b/repo/fsrepo/migrations/common/utils.go index 217da609fcf..e7d704dad94 100644 --- a/repo/fsrepo/migrations/common/utils.go +++ b/repo/fsrepo/migrations/common/utils.go @@ -1,6 +1,7 @@ package common import ( + "bytes" "encoding/json" "fmt" "io" @@ -40,47 +41,51 @@ func Must(err error) { // WithBackup performs a config file operation with automatic backup and rollback on error func WithBackup(configPath string, backupSuffix string, fn func(in io.ReadSeeker, out io.Writer) error) error { - in, err := os.Open(configPath) + // Read the entire file into memory first + // This allows us to close the file before doing atomic operations, + // which is necessary on Windows where open files can't be renamed + data, err := os.ReadFile(configPath) if err != nil { - return err + return fmt.Errorf("failed to read config file %s: %w", configPath, err) } - defer in.Close() - // Create backup - backup, err := atomicfile.New(configPath+backupSuffix, 0600) + // Create an in-memory reader for the data + in := bytes.NewReader(data) + + // Create backup atomically to prevent partial backup on interruption + backupPath := configPath + backupSuffix + backup, err := atomicfile.New(backupPath, 0600) if err != nil { - return err + return fmt.Errorf("failed to create backup file for %s: %w", backupPath, err) } - - // Copy to backup - if _, err := backup.ReadFrom(in); err != nil { + if _, err := backup.Write(data); err != nil { Must(backup.Abort()) - return err + return fmt.Errorf("failed to write backup data: %w", err) } - - // Reset input for reading - if _, err := in.Seek(0, io.SeekStart); err != nil { + if err := backup.Close(); err != nil { Must(backup.Abort()) - return err + return fmt.Errorf("failed to finalize backup: %w", err) } - // Create output file + // Create output file atomically out, err := atomicfile.New(configPath, 0600) if err != nil { - Must(backup.Abort()) - return err + // Clean up backup on error + os.Remove(backupPath) + return fmt.Errorf("failed to create atomic file for %s: %w", configPath, err) } // Run the conversion function if err := fn(in, out); err != nil { Must(out.Abort()) - Must(backup.Abort()) - return err + // Clean up backup on error + os.Remove(backupPath) + return fmt.Errorf("config conversion failed: %w", err) } - // Close everything on success + // Close the output file atomically Must(out.Close()) - Must(backup.Close()) + // Backup remains for potential revert return nil } diff --git a/repo/fsrepo/migrations/embedded.go b/repo/fsrepo/migrations/embedded.go index a2aa4d2523f..a8218be6378 100644 --- a/repo/fsrepo/migrations/embedded.go +++ b/repo/fsrepo/migrations/embedded.go @@ -6,6 +6,7 @@ import ( "log" "os" + lockfile "github.com/ipfs/go-fs-lock" "github.com/ipfs/kubo/repo/fsrepo/migrations/common" mg16 "github.com/ipfs/kubo/repo/fsrepo/migrations/fs-repo-16-to-17/migration" mg17 "github.com/ipfs/kubo/repo/fsrepo/migrations/fs-repo-17-to-18/migration" @@ -109,6 +110,13 @@ func RunEmbeddedMigrations(ctx context.Context, targetVer int, ipfsDir string, a return err } + // Acquire lock once for all embedded migrations to prevent concurrent access + lk, err := lockfile.Lock(ipfsDir, "repo.lock") + if err != nil { + return fmt.Errorf("failed to acquire repo lock: %w", err) + } + defer lk.Close() + fromVer, err := RepoVersion(ipfsDir) if err != nil { return fmt.Errorf("could not get repo version: %w", err) diff --git a/repo/fsrepo/migrations/ipfsdir_test.go b/repo/fsrepo/migrations/ipfsdir_test.go index c94ebc58671..c18721baed2 100644 --- a/repo/fsrepo/migrations/ipfsdir_test.go +++ b/repo/fsrepo/migrations/ipfsdir_test.go @@ -11,6 +11,8 @@ import ( func TestRepoDir(t *testing.T) { fakeHome := t.TempDir() t.Setenv("HOME", fakeHome) + // On Windows, os.UserHomeDir() uses USERPROFILE, not HOME + t.Setenv("USERPROFILE", fakeHome) fakeIpfs := filepath.Join(fakeHome, ".ipfs") t.Setenv(config.EnvDir, fakeIpfs) diff --git a/test/cli/migrations/migration_16_to_latest_test.go b/test/cli/migrations/migration_16_to_latest_test.go index 521b3164671..97a1ec1ffe3 100644 --- a/test/cli/migrations/migration_16_to_latest_test.go +++ b/test/cli/migrations/migration_16_to_latest_test.go @@ -21,6 +21,7 @@ import ( ipfs "github.com/ipfs/kubo" "github.com/ipfs/kubo/test/cli/harness" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -52,6 +53,13 @@ func TestMigration16ToLatest(t *testing.T) { // Comparison tests using 'ipfs repo migrate' command t.Run("repo migrate: forward migration with auto values", testRepoMigrationWithAuto) t.Run("repo migrate: backward migration", testRepoBackwardMigration) + + // Temp file and backup cleanup tests + t.Run("daemon migrate: no temp files after successful migration", testNoTempFilesAfterSuccessfulMigration) + t.Run("daemon migrate: no temp files after failed migration", testNoTempFilesAfterFailedMigration) + t.Run("daemon migrate: backup files persist after successful migration", testBackupFilesPersistAfterSuccessfulMigration) + t.Run("repo migrate: backup files can revert migration", testBackupFilesCanRevertMigration) + t.Run("repo migrate: conversion failure cleans up temp files", testConversionFailureCleanup) } // ============================================================================= @@ -392,10 +400,15 @@ func setupStaticV16Repo(t *testing.T) *harness.Node { v16FixturePath := "testdata/v16-repo" // Create a temporary test directory - each test gets its own copy - // Use ./tmp.DELETEME/ as requested by user instead of /tmp/ - tmpDir := filepath.Join("tmp.DELETEME", "migration-test-"+t.Name()) + // Sanitize test name for Windows - replace invalid characters + sanitizedName := strings.Map(func(r rune) rune { + if strings.ContainsRune(`<>:"/\|?*`, r) { + return '_' + } + return r + }, t.Name()) + tmpDir := filepath.Join(t.TempDir(), "migration-test-"+sanitizedName) require.NoError(t, os.MkdirAll(tmpDir, 0755)) - t.Cleanup(func() { os.RemoveAll(tmpDir) }) // Convert to absolute path for harness absTmpDir, err := filepath.Abs(tmpDir) @@ -559,6 +572,8 @@ func testRepoBackwardMigration(t *testing.T) { // First run forward migration to get to v17 result := node.RunIPFS("repo", "migrate") + t.Logf("Forward migration stdout:\n%s", result.Stdout.String()) + t.Logf("Forward migration stderr:\n%s", result.Stderr.String()) require.Empty(t, result.Stderr.String(), "Forward migration should succeed") // Verify we're at the latest version @@ -569,6 +584,8 @@ func testRepoBackwardMigration(t *testing.T) { // Now run reverse migration back to v16 result = node.RunIPFS("repo", "migrate", "--to=16", "--allow-downgrade") + t.Logf("Backward migration stdout:\n%s", result.Stdout.String()) + t.Logf("Backward migration stderr:\n%s", result.Stderr.String()) require.Empty(t, result.Stderr.String(), "Reverse migration should succeed") // Verify version was downgraded to 16 @@ -753,3 +770,149 @@ func runDaemonWithMultipleMigrationMonitoring(t *testing.T, node *harness.Node, } } } + +// ============================================================================= +// TEMP FILE AND BACKUP CLEANUP TESTS +// ============================================================================= + +// Helper functions for test cleanup assertions +func assertNoTempFiles(t *testing.T, dir string, msgAndArgs ...interface{}) { + t.Helper() + tmpFiles, err := filepath.Glob(filepath.Join(dir, ".tmp-*")) + require.NoError(t, err) + assert.Empty(t, tmpFiles, msgAndArgs...) +} + +func backupPath(configPath string, fromVer, toVer int) string { + return fmt.Sprintf("%s.%d-to-%d.bak", configPath, fromVer, toVer) +} + +func setupDaemonCmd(ctx context.Context, node *harness.Node, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, node.IPFSBin, args...) + cmd.Dir = node.Dir + for k, v := range node.Runner.Env { + cmd.Env = append(cmd.Env, k+"="+v) + } + return cmd +} + +func testNoTempFilesAfterSuccessfulMigration(t *testing.T) { + node := setupStaticV16Repo(t) + + // Run successful migration + _, migrationSuccess := runDaemonMigrationWithMonitoring(t, node) + require.True(t, migrationSuccess, "migration should succeed") + + assertNoTempFiles(t, node.Dir, "no temp files should remain after successful migration") +} + +func testNoTempFilesAfterFailedMigration(t *testing.T) { + node := setupStaticV16Repo(t) + + // Corrupt config to force migration failure + configPath := filepath.Join(node.Dir, "config") + corruptedJson := `{"Bootstrap": ["auto",` // Invalid JSON + require.NoError(t, os.WriteFile(configPath, []byte(corruptedJson), 0644)) + + // Attempt migration (should fail) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + cmd := setupDaemonCmd(ctx, node, "daemon", "--migrate") + output, _ := cmd.CombinedOutput() + t.Logf("Failed migration output: %s", output) + + assertNoTempFiles(t, node.Dir, "no temp files should remain after failed migration") +} + +func testBackupFilesPersistAfterSuccessfulMigration(t *testing.T) { + node := setupStaticV16Repo(t) + + // Run migration from v16 to latest (v18) + _, migrationSuccess := runDaemonMigrationWithMonitoring(t, node) + require.True(t, migrationSuccess, "migration should succeed") + + // Check for backup files from each migration step + configPath := filepath.Join(node.Dir, "config") + backup16to17 := backupPath(configPath, 16, 17) + backup17to18 := backupPath(configPath, 17, 18) + + // Both backup files should exist + assert.FileExists(t, backup16to17, "16-to-17 backup should exist") + assert.FileExists(t, backup17to18, "17-to-18 backup should exist") + + // Verify backup files contain valid JSON + data16to17, err := os.ReadFile(backup16to17) + require.NoError(t, err) + var config16to17 map[string]interface{} + require.NoError(t, json.Unmarshal(data16to17, &config16to17), "16-to-17 backup should be valid JSON") + + data17to18, err := os.ReadFile(backup17to18) + require.NoError(t, err) + var config17to18 map[string]interface{} + require.NoError(t, json.Unmarshal(data17to18, &config17to18), "17-to-18 backup should be valid JSON") +} + +func testBackupFilesCanRevertMigration(t *testing.T) { + node := setupStaticV16Repo(t) + + configPath := filepath.Join(node.Dir, "config") + versionPath := filepath.Join(node.Dir, "version") + + // Read original v16 config + originalConfig, err := os.ReadFile(configPath) + require.NoError(t, err) + + // Migrate to v17 only + result := node.RunIPFS("repo", "migrate", "--to=17") + require.Empty(t, result.Stderr.String(), "migration to v17 should succeed") + + // Verify backup exists + backup16to17 := backupPath(configPath, 16, 17) + assert.FileExists(t, backup16to17, "16-to-17 backup should exist") + + // Manually revert using backup + backupData, err := os.ReadFile(backup16to17) + require.NoError(t, err) + require.NoError(t, os.WriteFile(configPath, backupData, 0600)) + require.NoError(t, os.WriteFile(versionPath, []byte("16"), 0644)) + + // Verify config matches original + revertedConfig, err := os.ReadFile(configPath) + require.NoError(t, err) + assert.JSONEq(t, string(originalConfig), string(revertedConfig), "reverted config should match original") + + // Verify version is back to 16 + versionData, err := os.ReadFile(versionPath) + require.NoError(t, err) + assert.Equal(t, "16", strings.TrimSpace(string(versionData)), "version should be reverted to 16") +} + +func testConversionFailureCleanup(t *testing.T) { + // This test verifies that when a migration's conversion function fails, + // all temporary files are cleaned up properly + node := setupStaticV16Repo(t) + + configPath := filepath.Join(node.Dir, "config") + + // Create a corrupted config that will cause conversion to fail during JSON parsing + // The migration will read this, attempt to parse as JSON, and fail + corruptedJson := `{"Bootstrap": ["auto",` // Invalid JSON - missing closing bracket + require.NoError(t, os.WriteFile(configPath, []byte(corruptedJson), 0644)) + + // Attempt migration (should fail during conversion) + result := node.RunIPFS("repo", "migrate") + require.NotEmpty(t, result.Stderr.String(), "migration should fail with error") + + assertNoTempFiles(t, node.Dir, "no temp files should remain after conversion failure") + + // Verify no backup files were created (failure happened before backup) + backupFiles, err := filepath.Glob(filepath.Join(node.Dir, "config.*.bak")) + require.NoError(t, err) + assert.Empty(t, backupFiles, "no backup files should be created on conversion failure") + + // Verify corrupted config is unchanged (atomic operations prevented overwrite) + currentConfig, err := os.ReadFile(configPath) + require.NoError(t, err) + assert.Equal(t, corruptedJson, string(currentConfig), "corrupted config should remain unchanged") +} diff --git a/test/cli/migrations/migration_concurrent_test.go b/test/cli/migrations/migration_concurrent_test.go new file mode 100644 index 00000000000..8c716f51c5e --- /dev/null +++ b/test/cli/migrations/migration_concurrent_test.go @@ -0,0 +1,55 @@ +package migrations + +// NOTE: These concurrent migration tests require the local Kubo binary (built with 'make build') to be in PATH. +// +// To run these tests successfully: +// export PATH="$(pwd)/cmd/ipfs:$PATH" +// go test ./test/cli/migrations/ + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const daemonStartupWait = 2 * time.Second + +// TestConcurrentMigrations tests concurrent daemon --migrate attempts +func TestConcurrentMigrations(t *testing.T) { + t.Parallel() + + t.Run("concurrent daemon migrations prevented by lock", testConcurrentDaemonMigrations) +} + +func testConcurrentDaemonMigrations(t *testing.T) { + node := setupStaticV16Repo(t) + + // Start first daemon --migrate in background (holds repo.lock) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + firstDaemon := setupDaemonCmd(ctx, node, "daemon", "--migrate") + require.NoError(t, firstDaemon.Start()) + defer func() { + // Shutdown first daemon + shutdownCmd := setupDaemonCmd(context.Background(), node, "shutdown") + _ = shutdownCmd.Run() + _ = firstDaemon.Wait() + }() + + // Wait for first daemon to start and acquire lock + time.Sleep(daemonStartupWait) + + // Attempt second daemon --migrate (should fail due to lock) + secondDaemon := setupDaemonCmd(context.Background(), node, "daemon", "--migrate") + output, err := secondDaemon.CombinedOutput() + t.Logf("Second daemon output: %s", output) + + // Should fail with lock error + require.Error(t, err, "second daemon should fail when first daemon holds lock") + require.Contains(t, string(output), "lock", "error should mention lock") + + assertNoTempFiles(t, node.Dir, "no temp files should be created when lock fails") +} diff --git a/test/cli/migrations/migration_mixed_15_to_latest_test.go b/test/cli/migrations/migration_mixed_15_to_latest_test.go index 9f1a482f81d..6ee96b93946 100644 --- a/test/cli/migrations/migration_mixed_15_to_latest_test.go +++ b/test/cli/migrations/migration_mixed_15_to_latest_test.go @@ -23,8 +23,9 @@ import ( "os" "os/exec" "path/filepath" + "runtime" + "slices" "strings" - "syscall" "testing" "time" @@ -61,7 +62,8 @@ func testDaemonMigration15ToLatest(t *testing.T) { node := setupStaticV15Repo(t) // Create mock migration binary for 15โ†’16 (16โ†’17 will use embedded migration) - createMockMigrationBinary(t, "15", "16") + mockBinDir := createMockMigrationBinary(t, "15", "16") + customPath := buildCustomPath(mockBinDir) configPath := filepath.Join(node.Dir, "config") versionPath := filepath.Join(node.Dir, "version") @@ -80,7 +82,7 @@ func testDaemonMigration15ToLatest(t *testing.T) { originalPeerID := getNestedValue(originalConfig, "Identity.PeerID") // Run dual migration using daemon --migrate - stdoutOutput, migrationSuccess := runDaemonWithLegacyMigrationMonitoring(t, node) + stdoutOutput, migrationSuccess := runDaemonWithLegacyMigrationMonitoring(t, node, customPath) // Debug output t.Logf("Daemon output:\n%s", stdoutOutput) @@ -124,7 +126,8 @@ func testRepoMigration15ToLatest(t *testing.T) { node := setupStaticV15Repo(t) // Create mock migration binary for 15โ†’16 (16โ†’17 will use embedded migration) - createMockMigrationBinary(t, "15", "16") + mockBinDir := createMockMigrationBinary(t, "15", "16") + customPath := buildCustomPath(mockBinDir) configPath := filepath.Join(node.Dir, "config") versionPath := filepath.Join(node.Dir, "version") @@ -135,16 +138,7 @@ func testRepoMigration15ToLatest(t *testing.T) { require.Equal(t, "15", strings.TrimSpace(string(versionData)), "Should start at version 15") // Run migration using 'ipfs repo migrate' with custom PATH - result := node.Runner.Run(harness.RunRequest{ - Path: node.IPFSBin, - Args: []string{"repo", "migrate"}, - CmdOpts: []harness.CmdOpt{ - func(cmd *exec.Cmd) { - // Ensure the command inherits our modified PATH with mock binaries - cmd.Env = append(cmd.Env, "PATH="+os.Getenv("PATH")) - }, - }, - }) + result := runMigrationWithCustomPath(node, customPath, "repo", "migrate") require.Empty(t, result.Stderr.String(), "Migration should succeed without errors") // Verify final version is latest @@ -184,10 +178,10 @@ func setupStaticV15Repo(t *testing.T) *harness.Node { } // runDaemonWithLegacyMigrationMonitoring monitors for hybrid migration patterns -func runDaemonWithLegacyMigrationMonitoring(t *testing.T, node *harness.Node) (string, bool) { +func runDaemonWithLegacyMigrationMonitoring(t *testing.T, node *harness.Node, customPath string) (string, bool) { // Monitor for hybrid migration completion - use "Hybrid migration completed successfully" as success pattern stdoutOutput, daemonStarted := runDaemonWithMigrationMonitoringCustomEnv(t, node, "Using hybrid migration strategy", "Hybrid migration completed successfully", map[string]string{ - "PATH": os.Getenv("PATH"), // Pass current PATH which includes our mock binaries + "PATH": customPath, // Pass custom PATH with our mock binaries }) // Check for hybrid migration patterns in output @@ -271,17 +265,59 @@ func runDaemonWithMigrationMonitoringCustomEnv(t *testing.T, node *harness.Node, t.Log("Daemon startup timed out") } - // Stop the daemon + // Stop the daemon using ipfs shutdown command for graceful shutdown if cmd.Process != nil { - _ = cmd.Process.Signal(syscall.SIGTERM) + shutdownCmd := exec.Command(node.IPFSBin, "shutdown") + shutdownCmd.Dir = node.Dir + for k, v := range node.Runner.Env { + shutdownCmd.Env = append(shutdownCmd.Env, k+"="+v) + } + + if err := shutdownCmd.Run(); err != nil { + // If graceful shutdown fails, force kill + _ = cmd.Process.Kill() + } + + // Wait for process to exit _ = cmd.Wait() } return outputBuffer.String(), daemonReady && migrationStarted && migrationCompleted } -// createMockMigrationBinary creates a platform-agnostic Go binary for migration on PATH -func createMockMigrationBinary(t *testing.T, fromVer, toVer string) { +// buildCustomPath creates a custom PATH with mock migration binaries prepended. +// This is necessary for test isolation when running tests in parallel with t.Parallel(). +// Without isolated PATH handling, parallel tests can interfere with each other through +// global PATH modifications, causing tests to download real migration binaries instead +// of using the test mocks. +func buildCustomPath(mockBinDirs ...string) string { + // Prepend mock directories to ensure they're found first + pathElements := append(mockBinDirs, os.Getenv("PATH")) + return strings.Join(pathElements, string(filepath.ListSeparator)) +} + +// runMigrationWithCustomPath runs a migration command with a custom PATH environment. +// This ensures the migration uses our mock binaries instead of downloading real ones. +func runMigrationWithCustomPath(node *harness.Node, customPath string, args ...string) *harness.RunResult { + return node.Runner.Run(harness.RunRequest{ + Path: node.IPFSBin, + Args: args, + CmdOpts: []harness.CmdOpt{ + func(cmd *exec.Cmd) { + // Remove existing PATH entries using slices.DeleteFunc + cmd.Env = slices.DeleteFunc(cmd.Env, func(s string) bool { + return strings.HasPrefix(s, "PATH=") + }) + // Add custom PATH + cmd.Env = append(cmd.Env, "PATH="+customPath) + }, + }, + }) +} + +// createMockMigrationBinary creates a platform-agnostic Go binary for migration testing. +// Returns the directory containing the binary to be added to PATH. +func createMockMigrationBinary(t *testing.T, fromVer, toVer string) string { // Create bin directory for migration binaries binDir := t.TempDir() @@ -289,73 +325,60 @@ func createMockMigrationBinary(t *testing.T, fromVer, toVer string) { scriptName := fmt.Sprintf("fs-repo-%s-to-%s", fromVer, toVer) sourceFile := filepath.Join(binDir, scriptName+".go") binaryPath := filepath.Join(binDir, scriptName) + if runtime.GOOS == "windows" { + binaryPath += ".exe" + } + // Generate minimal mock migration binary code goSource := fmt.Sprintf(`package main - -import ( - "fmt" - "os" - "path/filepath" - "strings" -) - +import ("fmt"; "os"; "path/filepath"; "strings"; "time") func main() { - // Parse command line arguments - real migration binaries expect -path= - var repoPath string + var path string var revert bool - for _, arg := range os.Args[1:] { - if strings.HasPrefix(arg, "-path=") { - repoPath = strings.TrimPrefix(arg, "-path=") - } else if arg == "-revert" { - revert = true - } + for _, a := range os.Args[1:] { + if strings.HasPrefix(a, "-path=") { path = a[6:] } + if a == "-revert" { revert = true } } - - if repoPath == "" { - fmt.Fprintf(os.Stderr, "Usage: %%s -path= [-verbose=true] [-revert]\n", os.Args[0]) + if path == "" { fmt.Fprintln(os.Stderr, "missing -path="); os.Exit(1) } + + from, to := "%s", "%s" + if revert { from, to = to, from } + fmt.Printf("fake applying %%s-to-%%s repo migration\n", from, to) + + // Create and immediately remove lock file to simulate proper locking behavior + lockPath := filepath.Join(path, "repo.lock") + lockFile, err := os.Create(lockPath) + if err != nil && !os.IsExist(err) { + fmt.Fprintf(os.Stderr, "Error creating lock: %%v\n", err) os.Exit(1) } - - // Determine source and target versions based on revert flag - var sourceVer, targetVer string - if revert { - // When reverting, we go backwards: fs-repo-15-to-16 with -revert goes 16โ†’15 - sourceVer = "%s" - targetVer = "%s" - } else { - // Normal forward migration: fs-repo-15-to-16 goes 15โ†’16 - sourceVer = "%s" - targetVer = "%s" + if lockFile != nil { + lockFile.Close() + defer os.Remove(lockPath) } - - // Print migration message (same format as real migrations) - fmt.Printf("fake applying %%s-to-%%s repo migration\n", sourceVer, targetVer) - - // Update version file - versionFile := filepath.Join(repoPath, "version") - err := os.WriteFile(versionFile, []byte(targetVer), 0644) - if err != nil { - fmt.Fprintf(os.Stderr, "Error updating version: %%v\n", err) + + // Small delay to simulate migration work + time.Sleep(10 * time.Millisecond) + + if err := os.WriteFile(filepath.Join(path, "version"), []byte(to), 0644); err != nil { + fmt.Fprintf(os.Stderr, "Error: %%v\n", err) os.Exit(1) } -} -`, toVer, fromVer, fromVer, toVer) +}`, fromVer, toVer) require.NoError(t, os.WriteFile(sourceFile, []byte(goSource), 0644)) // Compile the Go binary - require.NoError(t, os.Setenv("CGO_ENABLED", "0")) // Ensure static binary - require.NoError(t, exec.Command("go", "build", "-o", binaryPath, sourceFile).Run()) - - // Add bin directory to PATH for this test - currentPath := os.Getenv("PATH") - newPath := binDir + string(filepath.ListSeparator) + currentPath - require.NoError(t, os.Setenv("PATH", newPath)) - t.Cleanup(func() { os.Setenv("PATH", currentPath) }) + cmd := exec.Command("go", "build", "-o", binaryPath, sourceFile) + cmd.Env = append(os.Environ(), "CGO_ENABLED=0") // Ensure static binary + require.NoError(t, cmd.Run()) // Verify the binary exists and is executable _, err := os.Stat(binaryPath) require.NoError(t, err, "Mock binary should exist") + + // Return the bin directory to be added to PATH + return binDir } // expectedMigrationSteps generates the expected migration step strings for a version range. @@ -416,26 +439,19 @@ func testRepoReverseHybridMigrationLatestTo15(t *testing.T) { // Start with v15 fixture and migrate forward to latest to create proper backup files node := setupStaticV15Repo(t) - // Create mock migration binary for 15โ†’16 (needed for forward migration) - createMockMigrationBinary(t, "15", "16") - // Create mock migration binary for 16โ†’15 (needed for downgrade) - createMockMigrationBinary(t, "16", "15") + // Create mock migration binaries for both forward and reverse migrations + mockBinDirs := []string{ + createMockMigrationBinary(t, "15", "16"), // for forward migration + createMockMigrationBinary(t, "16", "15"), // for downgrade + } + customPath := buildCustomPath(mockBinDirs...) configPath := filepath.Join(node.Dir, "config") versionPath := filepath.Join(node.Dir, "version") // Step 1: Forward migration from v15 to latest to create backup files t.Logf("Step 1: Forward migration v15 โ†’ v%d", ipfs.RepoVersion) - result := node.Runner.Run(harness.RunRequest{ - Path: node.IPFSBin, - Args: []string{"repo", "migrate"}, - CmdOpts: []harness.CmdOpt{ - func(cmd *exec.Cmd) { - // Ensure the command inherits our modified PATH with mock binaries - cmd.Env = append(cmd.Env, "PATH="+os.Getenv("PATH")) - }, - }, - }) + result := runMigrationWithCustomPath(node, customPath, "repo", "migrate") // Debug: print the output to see what happened t.Logf("Forward migration stdout:\n%s", result.Stdout.String()) @@ -459,16 +475,7 @@ func testRepoReverseHybridMigrationLatestTo15(t *testing.T) { // Step 2: Reverse hybrid migration from latest to v15 t.Logf("Step 2: Reverse hybrid migration v%d โ†’ v15", ipfs.RepoVersion) - result = node.Runner.Run(harness.RunRequest{ - Path: node.IPFSBin, - Args: []string{"repo", "migrate", "--to=15", "--allow-downgrade"}, - CmdOpts: []harness.CmdOpt{ - func(cmd *exec.Cmd) { - // Ensure the command inherits our modified PATH with mock binaries - cmd.Env = append(cmd.Env, "PATH="+os.Getenv("PATH")) - }, - }, - }) + result = runMigrationWithCustomPath(node, customPath, "repo", "migrate", "--to=15", "--allow-downgrade") require.Empty(t, result.Stderr.String(), "Reverse hybrid migration should succeed without errors") // Debug output From fb14754c53d209b252282cbb18f6fe7bb172fbe6 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 8 Oct 2025 18:29:38 +0200 Subject: [PATCH 4/4] chore: v0.38.1 --- docs/changelogs/v0.38.md | 22 ++++++++++++++++++---- version.go | 2 +- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/docs/changelogs/v0.38.md b/docs/changelogs/v0.38.md index a0342438070..2bffa58850c 100644 --- a/docs/changelogs/v0.38.md +++ b/docs/changelogs/v0.38.md @@ -296,10 +296,23 @@ The new [`Internal.MFSNoFlushLimit`](https://github.com/ipfs/kubo/blob/master/do Fixes migration panic on Windows when upgrading from v0.37 to v0.38 ("panic: error can't be dealt with transactionally: Access is denied"). -### Changelog +Updates go-ds-pebble to v0.5.3 (pebble v2.1.0). -
-Full Changelog +### ๐Ÿ“ Changelog + +
Full Changelog + +- github.com/ipfs/kubo: + - chore: v0.38.1 + - fix: migrations for Windows (#11010) ([ipfs/kubo#11010](https://github.com/ipfs/kubo/pull/11010)) + - Upgrade go-ds-pebble to v0.5.3 (#11011) ([ipfs/kubo#11011](https://github.com/ipfs/kubo/pull/11011)) + - upgrade go-ds-pebble to v0.5.2 (#11000) ([ipfs/kubo#11000](https://github.com/ipfs/kubo/pull/11000)) +- github.com/ipfs/go-ds-pebble (v0.5.1 -> v0.5.3): + - new version (#62) ([ipfs/go-ds-pebble#62](https://github.com/ipfs/go-ds-pebble/pull/62)) + - fix panic when batch is reused after commit (#61) ([ipfs/go-ds-pebble#61](https://github.com/ipfs/go-ds-pebble/pull/61)) + - new version (#60) ([ipfs/go-ds-pebble#60](https://github.com/ipfs/go-ds-pebble/pull/60)) + - Upgrade to pebble v2.1.0 (#59) ([ipfs/go-ds-pebble#59](https://github.com/ipfs/go-ds-pebble/pull/59)) + - update readme (#57) ([ipfs/go-ds-pebble#57](https://github.com/ipfs/go-ds-pebble/pull/57))
@@ -307,4 +320,5 @@ Fixes migration panic on Windows when upgrading from v0.37 to v0.38 ("panic: err | Contributor | Commits | Lines ยฑ | Files Changed | |-------------|---------|---------|---------------| -| TBD | | | | +| Marcin Rataj | 2 | +613/-267 | 15 | +| Andrew Gillis | 6 | +148/-22 | 8 | diff --git a/version.go b/version.go index eb1dd3850b3..9ebe9f187fc 100644 --- a/version.go +++ b/version.go @@ -11,7 +11,7 @@ import ( var CurrentCommit string // CurrentVersionNumber is the current application's version literal. -const CurrentVersionNumber = "0.38.0" +const CurrentVersionNumber = "0.38.1" const ApiVersion = "/kubo/" + CurrentVersionNumber + "/" //nolint