refactor: remove lexer serialization

XML lexer definitions are only consumed at runtime; the writer was kept solely for a hidden migration command. Remove the unused public round-trip API and CLI exporter while preserving XML loading.

Author: alecthomas

cmd/chroma/main.go

@@ -10,7 +10,6 @@ import (
 	"os/signal"
 	"path"
 	"path/filepath"
-	"regexp"
 	"runtime"
 	"runtime/pprof"
 	"sort"
@@ -48,7 +47,6 @@ command, for Go.
 		Check      bool             `help:"Do not format, check for tokenisation errors instead."`
 		Filename   string           `help:"Filename to use for selecting a lexer when reading from stdin."`
 		Fail       bool             `help:"Exit silently with status 1 if no specific lexer was found."`
-		XML        string           `hidden:"" help:"Generate XML lexer definitions." type:"existingdir" placeholder:"DIR"`
 
 		Lexer string `group:"select" help:"Lexer to use when formatting or path to an XML file to load." default:"autodetect" short:"l"`
 		Style string `group:"select" help:"Style to use for formatting or path to an XML file to load." default:"swapoff" short:"s"`
@@ -153,11 +151,6 @@ func main() {
 		"select": "Select lexer and style:",
 		"html":   "HTML formatter options:",
 	})
-	if cli.XML != "" {
-		err := dumpXMLLexerDefinitions(cli.XML)
-		ctx.FatalIfErrorf(err)
-		return
-	}
 	if cli.List {
 		listAll()
 		return
@@ -432,34 +425,3 @@ func check(filename string, it iter.Seq[chroma.Token]) bool {
 	}
 	return failed
 }
-
-var nameCleanRe = regexp.MustCompile(`[^A-Za-z0-9_#+-]`)
-
-func dumpXMLLexerDefinitions(dir string) error {
-	for _, name := range lexers.Names(false) {
-		lex := lexers.Get(name)
-		if rlex, ok := lex.(*chroma.RegexLexer); ok {
-			data, err := chroma.Marshal(rlex)
-			if err != nil {
-				if errors.Is(err, chroma.ErrNotSerialisable) {
-					fmt.Fprintf(os.Stderr, "warning: %q: %s\n", name, err)
-					continue
-				}
-				return err
-			}
-			name := strings.ToLower(nameCleanRe.ReplaceAllString(lex.Config().Name, "_"))
-			filename := filepath.Join(dir, name) + ".xml"
-			// fmt.Println(name)
-			_, err = os.Stat(filename)
-			if err == nil {
-				fmt.Fprintf(os.Stderr, "warning: %s already exists\n", filename)
-				continue
-			}
-			err = os.WriteFile(filename, data, 0600)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}

emitters.go

@@ -17,12 +17,6 @@ type ValidatingEmitter interface {
 	ValidateEmitter(rule *CompiledRule) error
 }
 
-// SerialisableEmitter is an Emitter that can be serialised and deserialised to/from JSON.
-type SerialisableEmitter interface {
-	Emitter
-	EmitterKind() string
-}
-
 // EmitterFunc is a function that is an Emitter.
 type EmitterFunc func(groups []string, state *LexerState) iter.Seq[Token]
 
@@ -44,8 +38,6 @@ func ByGroups(emitters ...Emitter) Emitter {
 	return &byGroupsEmitter{Emitters: emitters}
 }
 
-func (b *byGroupsEmitter) EmitterKind() string { return "bygroups" }
-
 func (b *byGroupsEmitter) ValidateEmitter(rule *CompiledRule) error {
 	if len(rule.Regexp.GetGroupNumbers())-1 != len(b.Emitters) {
 		return fmt.Errorf("number of groups %d does not match number of emitters %d", len(rule.Regexp.GetGroupNumbers())-1, len(b.Emitters))
@@ -146,7 +138,6 @@ type usingByGroup struct {
 	Emitters          Emitters `xml:"emitters"`
 }
 
-func (u *usingByGroup) EmitterKind() string { return "usingbygroup" }
 func (u *usingByGroup) Emit(groups []string, state *LexerState) iter.Seq[Token] {
 	// bounds check
 	if len(u.Emitters) != len(groups)-1 {
@@ -173,7 +164,7 @@ func (u *usingByGroup) Emit(groups []string, state *LexerState) iter.Seq[Token]
 
 // UsingLexer returns an Emitter that uses a given Lexer for parsing and emitting.
 //
-// This Emitter is not serialisable.
+// This Emitter cannot be represented in an XML lexer definition.
 func UsingLexer(lexer Lexer) Emitter {
 	return EmitterFunc(func(groups []string, _ *LexerState) iter.Seq[Token] {
 		it, err := lexer.Tokenise(&TokeniseOptions{State: "root", Nested: true}, groups[0])
@@ -188,8 +179,6 @@ type usingEmitter struct {
 	Lexer string `xml:"lexer,attr"`
 }
 
-func (u *usingEmitter) EmitterKind() string { return "using" }
-
 func (u *usingEmitter) Emit(groups []string, state *LexerState) iter.Seq[Token] {
 	if state.Registry == nil {
 		panic(fmt.Sprintf("no LexerRegistry available for Using(%q)", u.Lexer))
@@ -216,8 +205,6 @@ type usingSelfEmitter struct {
 	State string `xml:"state,attr"`
 }
 
-func (u *usingSelfEmitter) EmitterKind() string { return "usingself" }
-
 func (u *usingSelfEmitter) Emit(groups []string, state *LexerState) iter.Seq[Token] {
 	it, err := state.Lexer.Tokenise(&TokeniseOptions{State: u.State, Nested: true}, groups[0])
 	if err != nil {

mutators.go

@@ -12,12 +12,6 @@ type Mutator interface {
 	Mutate(state *LexerState) error
 }
 
-// SerialisableMutator is a Mutator that can be serialised and deserialised.
-type SerialisableMutator interface {
-	Mutator
-	MutatorKind() string
-}
-
 // ValidatingMutator is a Mutator that can validate itself against the compiled rules.
 //
 // Validation occurs once, after all LexerMutators have been applied.
@@ -64,21 +58,6 @@ func (m *multiMutator) UnmarshalXML(d *xml.Decoder, start xml.StartElement) erro
 	}
 }
 
-func (m *multiMutator) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
-	name := xml.Name{Local: "mutators"}
-	if err := e.EncodeToken(xml.StartElement{Name: name}); err != nil {
-		return err
-	}
-	for _, m := range m.Mutators {
-		if err := marshalMutator(e, m); err != nil {
-			return err
-		}
-	}
-	return e.EncodeToken(xml.EndElement{Name: name})
-}
-
-func (m *multiMutator) MutatorKind() string { return "mutators" }
-
 func (m *multiMutator) ValidateMutator(rules CompiledRules) error {
 	for _, mutator := range m.Mutators {
 		if validate, ok := mutator.(ValidatingMutator); ok {
@@ -113,8 +92,6 @@ func Include(state string) Rule {
 	return Rule{Mutator: &includeMutator{state}}
 }
 
-func (i *includeMutator) MutatorKind() string { return "include" }
-
 func (i *includeMutator) Mutate(s *LexerState) error {
 	return fmt.Errorf("should never reach here Include(%q)", i.State)
 }
@@ -132,8 +109,6 @@ type combinedMutator struct {
 	States []string `xml:"state,attr"`
 }
 
-func (c *combinedMutator) MutatorKind() string { return "combined" }
-
 // Combined creates a new anonymous state from the given states, and pushes that state.
 func Combined(states ...string) Mutator {
 	return &combinedMutator{states}
@@ -169,8 +144,6 @@ var (
 	_ ValidatingMutator = (*multiMutator)(nil)
 )
 
-func (p *pushMutator) MutatorKind() string { return "push" }
-
 func (p *pushMutator) ValidateMutator(rules CompiledRules) error {
 	for _, state := range p.States {
 		if state == "#pop" {
@@ -209,8 +182,6 @@ type popMutator struct {
 	Depth int `xml:"depth,attr"`
 }
 
-func (p *popMutator) MutatorKind() string { return "pop" }
-
 func (p *popMutator) Mutate(state *LexerState) error {
 	if len(state.Stack) == 0 {
 		return fmt.Errorf("nothing to pop")

serialise_test.go

@@ -371,5 +371,3 @@ func (t TokenType) InSubCategory(other TokenType) bool {
 func (t TokenType) Emit(groups []string, _ *LexerState) iter.Seq[Token] {
 	return Literator(Token{Type: t, Value: groups[0]})
 }
-
-func (t TokenType) EmitterKind() string { return "token" }