fix: four lexers push onto states that do not exist

lexers/embedded/psl.xml

@@ -164,6 +164,11 @@
         <token type="CommentSingle"/>
       </rule>
     </state>
+    <state name="warn-directive">
+      <rule pattern=".+$">
+        <token type="CommentSingle"/>
+      </rule>
+    </state>
     <state name="classdef-directive">
       <rule pattern="\s+">
         <token type="Text"/>

lexers/embedded/sourcepawn.xml

@@ -31,8 +31,6 @@
       <rule pattern="(bool|float|void|int|char)\b"><token type="KeywordType"/></rule>
       <rule pattern="(true|false)\b"><token type="KeywordConstant"/></rule>
       <rule pattern="[a-zA-Z_]\w*"><token type="Name"/></rule>
-      <rule pattern="((?:[\w*\s])+?(?:\s|[*]))([a-zA-Z_]\w*)(\s*\([^;]*?\))([^;{]*)(\{)"><bygroups><usingself state="root"/><token type="NameFunction"/><usingself state="root"/><usingself state="root"/><token type="Punctuation"/></bygroups><push state="function"/></rule>
-      <rule pattern="((?:[\w*\s])+?(?:\s|[*]))([a-zA-Z_]\w*)(\s*\([^;]*?\))([^;]*)(;)"><bygroups><usingself state="root"/><token type="NameFunction"/><usingself state="root"/><usingself state="root"/><token type="Punctuation"/></bygroups></rule>
     </state>
     <state name="string">
       <rule pattern="&quot;"><token type="LiteralString"/><pop depth="1"/></rule>

lexers/embedded/spade.xml

@@ -20,6 +20,14 @@
         <pop depth="1"/>
       </rule>
     </state>
+    <state name="lifetime">
+      <rule pattern="[a-zA-Z_]+\w*">
+        <token type="NameAttribute"/>
+      </rule>
+      <rule>
+        <pop depth="1"/>
+      </rule>
+    </state>
     <state name="number_lit">
       <rule pattern="[ui]\d*">
         <token type="Keyword"/>

lexers/embedded/yang.xml

@@ -85,7 +85,7 @@
       </rule>
       <rule pattern="/\*">
         <token type="CommentMultiline"/>
-        <push state="comment"/>
+        <push/>
       </rule>
       <rule pattern="\*/">
         <token type="CommentMultiline"/>

lexers/testdata/psl.actual

@@ -16,4 +16,6 @@ public Number SumMethod() {
     set x = x + $$getMore^ProcedureName("AAA")
     do finishTask(x, "AAA")
 
-    quit
+    quit
+    #WARN unused variable
+    #INFO informational

lexers/testdata/psl.expected

@@ -148,5 +148,12 @@
   {"type":"LiteralString","value":"\"AAA\""},
   {"type":"Punctuation","value":")"},
   {"type":"Text","value":"\n\n    "},
-  {"type":"Keyword","value":"quit"}
+  {"type":"Keyword","value":"quit"},
+  {"type":"Text","value":"\n    "},
+  {"type":"KeywordPseudo","value":"#WARN"},
+  {"type":"CommentSingle","value":" unused variable"},
+  {"type":"Text","value":"\n    "},
+  {"type":"KeywordPseudo","value":"#INFO"},
+  {"type":"CommentSingle","value":" informational"},
+  {"type":"Text","value":"\n"}
 ]

lexers/testdata/spade.actual

@@ -184,3 +184,7 @@ fn named_args() {
 	call$(a, b: true)
 }
 
+
+type T {
+    x: 'a int,
+}

lexers/testdata/spade.expected

@@ -1085,5 +1085,20 @@
   {"type":"Punctuation","value":")"},
   {"type":"TextWhitespace","value":"\n"},
   {"type":"Punctuation","value":"}"},
-  {"type":"TextWhitespace","value":"\n\n"}
+  {"type":"TextWhitespace","value":"\n\n\n"},
+  {"type":"Name","value":"type"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Name","value":"T"},
+  {"type":"TextWhitespace","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"TextWhitespace","value":"\n    "},
+  {"type":"Name","value":"x"},
+  {"type":"Text","value":": "},
+  {"type":"NameAttribute","value":"'a"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Punctuation","value":","},
+  {"type":"TextWhitespace","value":"\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"TextWhitespace","value":"\n"}
 ]

lexers/testdata/yang.actual

@@ -62,3 +62,5 @@ module server-system {
         }
     }
 }
+
+/* nested /* comment */ still comment */

lexers/testdata/yang.expected

@@ -216,5 +216,7 @@
   {"type":"Punctuation","value":"}"},
   {"type":"TextWhitespace","value":"\n"},
   {"type":"Punctuation","value":"}"},
+  {"type":"TextWhitespace","value":"\n\n"},
+  {"type":"CommentMultiline","value":"/* nested /* comment */ still comment */"},
   {"type":"TextWhitespace","value":"\n"}
 ]

mutators.go

@@ -18,6 +18,14 @@ type SerialisableMutator interface {
 	MutatorKind() string
 }
 
+// ValidatingMutator is a Mutator that can validate itself against the compiled rules.
+//
+// Validation occurs once, after all LexerMutators have been applied.
+type ValidatingMutator interface {
+	Mutator
+	ValidateMutator(rules CompiledRules) error
+}
+
 // A LexerMutator is an additional interface that a Mutator can implement
 // to modify the lexer when it is compiled.
 type LexerMutator interface {
@@ -71,6 +79,17 @@ func (m *multiMutator) MarshalXML(e *xml.Encoder, start xml.StartElement) error
 
 func (m *multiMutator) MutatorKind() string { return "mutators" }
 
+func (m *multiMutator) ValidateMutator(rules CompiledRules) error {
+	for _, mutator := range m.Mutators {
+		if validate, ok := mutator.(ValidatingMutator); ok {
+			if err := validate.ValidateMutator(rules); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 func (m *multiMutator) Mutate(state *LexerState) error {
 	for _, modifier := range m.Mutators {
 		if err := modifier.Mutate(state); err != nil {
@@ -145,8 +164,25 @@ type pushMutator struct {
 	States []string `xml:"state,attr"`
 }
 
+var (
+	_ ValidatingMutator = (*pushMutator)(nil)
+	_ ValidatingMutator = (*multiMutator)(nil)
+)
+
 func (p *pushMutator) MutatorKind() string { return "push" }
 
+func (p *pushMutator) ValidateMutator(rules CompiledRules) error {
+	for _, state := range p.States {
+		if state == "#pop" {
+			continue
+		}
+		if _, ok := rules[state]; !ok {
+			return fmt.Errorf("invalid push state %q", state)
+		}
+	}
+	return nil
+}
+
 func (p *pushMutator) Mutate(s *LexerState) error {
 	if len(p.States) == 0 {
 		s.Stack = append(s.Stack, s.State)

regexp.go

@@ -392,7 +392,7 @@ restart:
 			}
 		}
 	}
-	// Validate emitters
+	// Validate emitters and mutators
 	for state := range r.rules {
 		for i := range len(r.rules[state]) {
 			rule := r.rules[state][i]
@@ -401,6 +401,11 @@ restart:
 					return fmt.Errorf("%s: %s: %s: %w", r.config.Name, state, rule.Pattern, err)
 				}
 			}
+			if validate, ok := rule.Mutator.(ValidatingMutator); ok {
+				if err := validate.ValidateMutator(r.rules); err != nil {
+					return fmt.Errorf("%s: %s: %s: %w", r.config.Name, state, rule.Pattern, err)
+				}
+			}
 		}
 	}
 	r.compiled = true

regexp_test.go

@@ -216,3 +216,30 @@ func TestIgnoreToken(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, []Token{{Keyword, "hello"}, {TextWhitespace, "\n"}}, slices.Collect(it))
 }
+
+func TestPushToUndefinedStateIsAnError(t *testing.T) {
+	lexer, err := NewLexer(&Config{Name: "test"}, func() Rules {
+		return Rules{
+			"root": {
+				{`a`, Text, Push("nonexistent")},
+			},
+		}
+	})
+	assert.NoError(t, err)
+	_, err = lexer.Tokenise(nil, "a")
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), `invalid push state "nonexistent"`)
+
+	// #pop is a valid pseudo-state and must not be rejected.
+	lexer = mustNewLexer(t, &Config{Name: "test"}, Rules{ // nolint: forbidigo
+		"root": {
+			{`a`, Text, Push("other")},
+		},
+		"other": {
+			{`b`, Text, Push("#pop")},
+		},
+	})
+	it, err := lexer.Tokenise(nil, "ab")
+	assert.NoError(t, err)
+	assert.Equal(t, []Token{{Text, "a"}, {Text, "b"}}, slices.Collect(it))
+}