fix(jsonutils): fixed data race with lexer

Signed-off-by: Frederic BIDON <[email protected]>

Author: fredbi

.github/workflows/go-test.yml

@@ -102,10 +102,6 @@ jobs:
 
     steps:
     - uses: actions/checkout@v5
-    #- name: Ensure cache is available
-    # run: |
-    #   mkdir -p /home/runner/go/pkg/mod
-    #   mkdir -p /home/runner/.cache/go.build
     - uses: actions/setup-go@v6
       with:
         go-version: '${{ matrix.go_version }}'
@@ -119,6 +115,7 @@ jobs:
         # *.coverage.* pattern is automatically detected by codecov
         COVER_PROFILE:  'all_modules.coverage.${{ matrix.os }}.${{ matrix.go_version }}.out'
       run: |
+        # when go1.25 becomes the oldstable, we may replace this bash with "go work test"
         declare -a ALL_MODULES
         BASH_MAJOR=$(echo $BASH_VERSION|cut -d'.' -f1)
         if [[ "${BASH_MAJOR}" -ge 4 ]] ; then

TODO.md

@@ -0,0 +1 @@
+fix data race https://github.com/go-openapi/swag/actions/runs/17989156861/job/51174860188

hack/tag_modules.sh

@@ -18,9 +18,10 @@ echo "Tagging all modules in repo ${root##*/}..."
 
 while read module_location ; do
   relative_location=${module_location#"$root"/}
+  relative_location=${relative_location#"$root"}
   module_dir=${relative_location%"/go.mod"}
   base_tag="${module_dir#"./"}"
-  if [[ "${base_tag}" ==  "." ]] ; then
+  if [[ "${base_tag}" ==  "" ]] ; then
     module_tag="${tag}" # e.g. "v0.24.0"
   else
     module_tag="${base_tag}/${tag}" # e.g. "mangling/v0.24.0"

jsonutils/adapters/stdlib/json/lexer.go

@@ -143,23 +143,18 @@ func newLexer(data []byte) *jlexer {
 		// current: undefToken,
 		next: undefToken,
 	}
-	if data != nil {
-		l.buf = &bytesReader{
-			buf: data,
-		}
-		l.dec = stdjson.NewDecoder(l.buf) // unfortunately, cannot pool this
+	l.buf = &bytesReader{
+		buf: data,
 	}
+	l.dec = stdjson.NewDecoder(l.buf) // unfortunately, cannot pool this
 
 	return l
 }
 
 func (l *jlexer) Reset() {
-	l.dec = nil
 	l.err = nil
 	l.next = undefToken
-	if l.buf != nil {
-		l.buf.Reset()
-	}
+	// leave l.dec and l.buf alone, since they are replaced at every Borrow
 }
 
 func (l *jlexer) Error() error {

jsonutils/adapters/stdlib/json/ordered_map_test.go

@@ -16,8 +16,12 @@ package json
 
 import (
 	stdjson "encoding/json"
+	"fmt"
+	"io"
+	"sync"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	fixtures "github.com/go-openapi/swag/jsonutils/fixtures_test"
@@ -127,3 +131,35 @@ func TestLexerErrors(t *testing.T) {
 		})
 	}
 }
+
+func TestReproDataRace(t *testing.T) {
+	t.Parallel()
+	const parallelRoutines = 1000
+
+	// NOTE: with go1.25, use synctest.Test
+	var wg sync.WaitGroup
+
+	for range parallelRoutines {
+		wg.Add(1)
+		go func() {
+			defer func() {
+				wg.Done()
+			}()
+
+			toks := make([]token, 0, 4)
+			buf := []byte(`{"test":"data"}`)
+			l := poolOfLexers.Borrow(buf)
+
+			for tok := l.NextToken(); tok != eofToken; tok = l.NextToken() {
+				toks = append(toks, tok)
+			}
+			assert.Len(t, toks, 4)
+			fmt.Fprintf(io.Discard, "%d", len(toks))
+			defer func() {
+				poolOfLexers.Redeem(l)
+			}()
+		}()
+	}
+
+	wg.Wait()
+}

jsonutils/adapters/stdlib/json/pool.go

@@ -62,19 +62,18 @@ func (p *lexersPool) Borrow(data []byte) *jlexer {
 	ptr := p.Get()
 
 	l := ptr.(*jlexer)
-	l.Reset()
-
 	l.buf = poolOfReaders.Borrow(data)
 	l.dec = json.NewDecoder(l.buf) // cannot pool, not exposed by the encoding/json API
+	l.Reset()
 
 	return l
 }
 
 func (p *lexersPool) Redeem(l *jlexer) {
 	l.dec = nil
-	if l.buf != nil {
-		poolOfReaders.Redeem(l.buf)
-	}
+	discard := l.buf
+	l.buf = nil
+	poolOfReaders.Redeem(discard)
 	p.Put(l)
 }