perf(lexers): build the registry from generated metadata instead of parsing XML at init (#1356)

Importing the lexers package decodes the `<config>` element of all 285
embedded XML definitions with encoding/xml at init. This generates that
metadata into Go literals with a new go:generate tool
(internal/metadatagen, output printed with repr), so building the
registry parses no XML at all; each lexer's rules are still loaded
lazily from its XML on first use, as before. A test verifies the
generated metadata matches the XML and fails with a hint to re-run go
generate when they drift.

Analyser regexes are now compiled on the first AnalyseText call, with
their validity checked at generation time instead. Registering a fresh
lexer also no longer scans the whole registry slice, which was quadratic
over a full registration pass.

| `GODEBUG=inittrace=1` | init clock | init heap |
|---|---|---|
| before | 9.4 ms | 3.0 MB / 34,714 allocs |
| after | 0.92 ms | 463 KB / 2,341 allocs |

Part of a series reducing the cost of importing chroma (lexers + styles
+ formatters) from ~15.4 ms to ~1.1 ms.

Author: mattn

.github/workflows/ci.yml

@@ -12,6 +12,13 @@ jobs:
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
       - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
       - run: go test ./...
+  check-generated:
+    name: Check generated code
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+      - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1
+      - run: just check-generated
   lint:
     name: Lint
     runs-on: ubuntu-latest

Justfile

@@ -21,6 +21,18 @@ tokentype-string:
 format-js:
     biome format --write cmd/chromad/static/index.js cmd/chromad/static/chroma.js
 
+# Regenerate lexers/lexers_gen.go from the embedded lexer definitions
+lexer-metadata:
+    GOOS= GOARCH= go generate ./lexers
+
+# Check that generated code is in sync with the embedded lexer definitions
+check-generated: lexer-metadata
+    #!/usr/bin/env bash
+    git diff --exit-code lexers/lexers_gen.go || {
+        echo "lexers/lexers_gen.go is stale; run 'just lexer-metadata' and commit the result." >&2
+        exit 1
+    }
+
 # Tidy Go modules
 tidy:
     find . -name 'go.mod' -execdir go mod tidy \;

internal/metadatagen/main.go

@@ -0,0 +1,85 @@
+// Command metadatagen generates a Go source file containing the <config>
+// section of every embedded lexer XML definition, so that the lexer registry
+// can be constructed without parsing any XML at init time.
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"go/format"
+	"io/fs"
+	"log"
+	"os"
+
+	"github.com/alecthomas/chroma/v3"
+	"github.com/alecthomas/repr"
+	"github.com/dlclark/regexp2/v2"
+)
+
+func main() {
+	log.SetFlags(0)
+	log.SetPrefix("metadatagen: ")
+	dir := flag.String("dir", "embedded", "directory containing lexer XML definitions")
+	out := flag.String("out", "lexers_gen.go", "output file")
+	flag.Parse()
+
+	fsys := os.DirFS(*dir)
+	paths, err := fs.Glob(fsys, "*.xml")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	buf := &bytes.Buffer{}
+	fmt.Fprintf(buf, `// Code generated by "go generate" (internal/metadatagen); DO NOT EDIT.
+
+package lexers
+
+import (
+	"github.com/alecthomas/chroma/v3"
+)
+
+// embeddedLexers mirrors the <config> element of every XML lexer definition
+// under embedded/, so that the registry can be built without parsing XML at
+// init time. TestEmbeddedLexerMetadata verifies it is up to date.
+var embeddedLexers = []struct {
+	path   string
+	config chroma.Config
+}{
+`)
+	for _, path := range paths {
+		lexer, err := chroma.NewXMLLexer(fsys, path)
+		if err != nil {
+			log.Fatalf("%s: %s", path, err)
+		}
+		config := lexer.Config()
+		validateAnalysers(path, config)
+		fmt.Fprintf(buf, "\t{%q, %s},\n", "embedded/"+path, repr.String(*config, repr.Indent("\t")))
+	}
+	fmt.Fprintf(buf, "}\n")
+
+	source, err := format.Source(buf.Bytes())
+	if err != nil {
+		log.Fatalf("formatting generated source: %s", err)
+	}
+	if err := os.WriteFile(*out, source, 0o600); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// validateAnalysers compiles the analyser regexes so that an invalid pattern
+// fails at generation time; at runtime they are compiled lazily.
+func validateAnalysers(path string, config *chroma.Config) {
+	if config.Analyse == nil {
+		return
+	}
+	flags := regexp2.None
+	if config.CaseInsensitive {
+		flags = regexp2.IgnoreCase
+	}
+	for _, ra := range config.Analyse.Regexes {
+		if _, err := regexp2.Compile(ra.Pattern, flags); err != nil {
+			log.Fatalf("%s: %q is not a valid analyser regex: %s", path, ra.Pattern, err)
+		}
+	}
+}

lexers/lexers.go

@@ -2,23 +2,29 @@ package lexers
 
 import (
 	"embed"
-	"io/fs"
 
 	"github.com/alecthomas/chroma/v3"
 )
 
+//go:generate go run github.com/alecthomas/chroma/v3/internal/metadatagen -dir embedded -out lexers_gen.go
+
 //go:embed embedded
 var embedded embed.FS
 
 // GlobalLexerRegistry is the global LexerRegistry of Lexers.
+//
+// It is populated from generated metadata mirroring the embedded XML
+// definitions, so that registering every lexer does not require parsing any
+// XML; each lexer's rules are still loaded from its XML on first use.
 var GlobalLexerRegistry = func() *chroma.LexerRegistry {
 	reg := chroma.NewLexerRegistry()
-	paths, err := fs.Glob(embedded, "embedded/*.xml")
-	if err != nil {
-		panic(err)
-	}
-	for _, path := range paths {
-		reg.Register(chroma.MustNewXMLLexer(embedded, path))
+	for i := range embeddedLexers {
+		entry := &embeddedLexers[i]
+		lexer, err := chroma.NewXMLLexerFromConfig(&entry.config, embedded, entry.path)
+		if err != nil {
+			panic(err)
+		}
+		reg.Register(lexer)
 	}
 	return reg
 }()