perf(lexers): build the registry from generated metadata instead of parsing XML at init

Importing the lexers package decoded the <config> element of all 285
embedded XML definitions with encoding/xml during init, which cost about
9.4 ms and 35k allocations before main starts. Generate that metadata
into Go literals with a new go:generate tool (internal/metadatagen), so
building the registry parses no XML at all; each lexer's rules are still
loaded lazily from its XML on first use, as before. A test verifies the
generated metadata matches the XML definitions and fails with a hint to
re-run go generate when they drift.

The analyser regexes are now compiled on the first call to AnalyseText
rather than at registration time, with their validity checked at
generation time instead. Registering a fresh lexer no longer scans the
whole registry slice, which was quadratic across a full registration
pass.

Author: mattn

internal/metadatagen/main.go

@@ -0,0 +1,85 @@
+// Command metadatagen generates a Go source file containing the <config>
+// section of every embedded lexer XML definition, so that the lexer registry
+// can be constructed without parsing any XML at init time.
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"go/format"
+	"io/fs"
+	"log"
+	"os"
+
+	"github.com/alecthomas/chroma/v3"
+	"github.com/alecthomas/repr"
+	"github.com/dlclark/regexp2/v2"
+)
+
+func main() {
+	log.SetFlags(0)
+	log.SetPrefix("metadatagen: ")
+	dir := flag.String("dir", "embedded", "directory containing lexer XML definitions")
+	out := flag.String("out", "lexers_gen.go", "output file")
+	flag.Parse()
+
+	fsys := os.DirFS(*dir)
+	paths, err := fs.Glob(fsys, "*.xml")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	buf := &bytes.Buffer{}
+	fmt.Fprintf(buf, `// Code generated by "go generate" (internal/metadatagen); DO NOT EDIT.
+
+package lexers
+
+import (
+	"github.com/alecthomas/chroma/v3"
+)
+
+// embeddedLexers mirrors the <config> element of every XML lexer definition
+// under embedded/, so that the registry can be built without parsing XML at
+// init time. TestEmbeddedLexerMetadata verifies it is up to date.
+var embeddedLexers = []struct {
+	path   string
+	config chroma.Config
+}{
+`)
+	for _, path := range paths {
+		lexer, err := chroma.NewXMLLexer(fsys, path)
+		if err != nil {
+			log.Fatalf("%s: %s", path, err)
+		}
+		config := lexer.Config()
+		validateAnalysers(path, config)
+		fmt.Fprintf(buf, "\t{%q, %s},\n", "embedded/"+path, repr.String(*config, repr.Indent("\t")))
+	}
+	fmt.Fprintf(buf, "}\n")
+
+	source, err := format.Source(buf.Bytes())
+	if err != nil {
+		log.Fatalf("formatting generated source: %s", err)
+	}
+	if err := os.WriteFile(*out, source, 0o600); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// validateAnalysers compiles the analyser regexes so that an invalid pattern
+// fails at generation time; at runtime they are compiled lazily.
+func validateAnalysers(path string, config *chroma.Config) {
+	if config.Analyse == nil {
+		return
+	}
+	flags := regexp2.None
+	if config.CaseInsensitive {
+		flags = regexp2.IgnoreCase
+	}
+	for _, ra := range config.Analyse.Regexes {
+		if _, err := regexp2.Compile(ra.Pattern, flags); err != nil {
+			log.Fatalf("%s: %q is not a valid analyser regex: %s", path, ra.Pattern, err)
+		}
+	}
+}

lexers/lexers.go

@@ -2,23 +2,29 @@ package lexers
 
 import (
 	"embed"
-	"io/fs"
 
 	"github.com/alecthomas/chroma/v3"
 )
 
+//go:generate go run github.com/alecthomas/chroma/v3/internal/metadatagen -dir embedded -out lexers_gen.go
+
 //go:embed embedded
 var embedded embed.FS
 
 // GlobalLexerRegistry is the global LexerRegistry of Lexers.
+//
+// It is populated from generated metadata mirroring the embedded XML
+// definitions, so that registering every lexer does not require parsing any
+// XML; each lexer's rules are still loaded from its XML on first use.
 var GlobalLexerRegistry = func() *chroma.LexerRegistry {
 	reg := chroma.NewLexerRegistry()
-	paths, err := fs.Glob(embedded, "embedded/*.xml")
-	if err != nil {
-		panic(err)
-	}
-	for _, path := range paths {
-		reg.Register(chroma.MustNewXMLLexer(embedded, path))
+	for i := range embeddedLexers {
+		entry := &embeddedLexers[i]
+		lexer, err := chroma.NewXMLLexerFromConfig(&entry.config, embedded, entry.path)
+		if err != nil {
+			panic(err)
+		}
+		reg.Register(lexer)
 	}
 	return reg
 }()