diff --git a/example_test.go b/example_test.go index 752d373..b329323 100644 --- a/example_test.go +++ b/example_test.go @@ -30,7 +30,7 @@ function b() { // var a = 3; // // // b is a cool function - // function b() { + // function b() { // return 7; // } } diff --git a/highlight.go b/highlight.go index 9ff794c..49d06d3 100644 --- a/highlight.go +++ b/highlight.go @@ -194,17 +194,31 @@ var DefaultHTMLConfig = HTMLConfig{ } func Print(s *scanner.Scanner, w io.Writer, p Printer) error { + accum := "" + lastKind := Whitespace tok := s.Scan() for tok != scanner.EOF { - tokText := s.TokenText() - err := p.Print(w, tokenKind(tok, tokText), tokText) - if err != nil { - return err + text := s.TokenText() + kind := tokenKind(tok, text) + if kind != lastKind { + if accum != "" { + if err := p.Print(w, lastKind, accum); err != nil { + return err + } + accum = "" + } + lastKind = kind } - + accum += text tok = s.Scan() } + if accum != "" { + if err := p.Print(w, lastKind, accum); err != nil { + return err + } + } + return nil } @@ -214,20 +228,38 @@ func Annotate(src []byte, a Annotator) (annotate.Annotations, error) { var anns annotate.Annotations read := 0 + accum := "" + lastKind := Whitespace tok := s.Scan() for tok != scanner.EOF { - tokText := s.TokenText() + text := s.TokenText() + kind := tokenKind(tok, text) + if kind != lastKind { + if accum != "" { + ann, err := a.Annotate(read, lastKind, accum) + if err != nil { + return nil, err + } + read += len(accum) + if ann != nil { + anns = append(anns, ann) + } + accum = "" + } + lastKind = kind + } + accum += text + tok = s.Scan() + } - ann, err := a.Annotate(read, tokenKind(tok, tokText), tokText) + if accum != "" { + ann, err := a.Annotate(read, lastKind, accum) if err != nil { return nil, err } - read += len(tokText) if ann != nil { anns = append(anns, ann) } - - tok = s.Scan() } return anns, nil diff --git a/highlight_test.go b/highlight_test.go index 9b651b5..7c38eaa 100644 --- a/highlight_test.go +++ b/highlight_test.go @@ -88,8 +88,7 @@ func TestAnnotate(t *testing.T) { src := []byte(`a:=2`) want := annotate.Annotations{ {Start: 0, End: 1, Left: []byte(``), Right: []byte("")}, - {Start: 1, End: 2, Left: []byte(``), Right: []byte("")}, - {Start: 2, End: 3, Left: []byte(``), Right: []byte("")}, + {Start: 1, End: 3, Left: []byte(``), Right: []byte("")}, {Start: 3, End: 4, Left: []byte(``), Right: []byte("")}, } got, err := Annotate(src, HTMLAnnotator(DefaultHTMLConfig)) diff --git a/testdata/consolidate_tokens.js b/testdata/consolidate_tokens.js new file mode 100644 index 0000000..4a7a97b --- /dev/null +++ b/testdata/consolidate_tokens.js @@ -0,0 +1 @@ +>= \ No newline at end of file diff --git a/testdata/consolidate_tokens.js.html b/testdata/consolidate_tokens.js.html new file mode 100755 index 0000000..45733f1 --- /dev/null +++ b/testdata/consolidate_tokens.js.html @@ -0,0 +1 @@ +>= \ No newline at end of file diff --git a/testdata/consolidate_tokens.js.ol.html b/testdata/consolidate_tokens.js.ol.html new file mode 100755 index 0000000..7ffa4de --- /dev/null +++ b/testdata/consolidate_tokens.js.ol.html @@ -0,0 +1,3 @@ +
    +
  1. >=
    2. +
\ No newline at end of file diff --git a/testdata/simple.c.html b/testdata/simple.c.html index 16e62e2..d99b53c 100755 --- a/testdata/simple.c.html +++ b/testdata/simple.c.html @@ -2,5 +2,5 @@ int main(void) { - printf("hello, world\n"); + printf("hello, world\n"); } diff --git a/testdata/simple.c.ol.html b/testdata/simple.c.ol.html index 05fbbcc..2b5772a 100755 --- a/testdata/simple.c.ol.html +++ b/testdata/simple.c.ol.html @@ -3,7 +3,7 @@
  • int main(void)
  • {
    • -
  • printf("hello, world\n");
    • +
  • printf("hello, world\n");
  • }
  • \ No newline at end of file diff --git a/testdata/simple.go.html b/testdata/simple.go.html index 8aba451..27c7153 100755 --- a/testdata/simple.go.html +++ b/testdata/simple.go.html @@ -1,8 +1,8 @@ // +build ignore package foo -func Bar(baz string, qux *Zip) (*Zap, Zop) { - ziz := mop(3, "hello world") +func Bar(baz string, qux *Zip) (*Zap, Zop) { + ziz := mop(3, "hello world") } type Qaz struct { diff --git a/testdata/simple.go.ol.html b/testdata/simple.go.ol.html index a797d49..932ea15 100755 --- a/testdata/simple.go.ol.html +++ b/testdata/simple.go.ol.html @@ -2,8 +2,8 @@
  • // +build ignore
  • package foo
  • -
  • func Bar(baz string, qux *Zip) (*Zap, Zop) {
    • -
  • ziz := mop(3, "hello world")
    • +
  • func Bar(baz string, qux *Zip) (*Zap, Zop) {
    • +
  • ziz := mop(3, "hello world")
  • }
  • type Qaz struct {
    • diff --git a/testdata/simple.js.html b/testdata/simple.js.html index b93fba3..580c971 100755 --- a/testdata/simple.js.html +++ b/testdata/simple.js.html @@ -1,10 +1,10 @@ // foo is a cool function -function foo() {} +function foo() {} /* bar is a cool var */ var bar = 3; -A.prototype.foo = function() { - this.noise || '<chirp>'; +A.prototype.foo = function() { + this.noise || '<chirp>'; return 'Hello from ' + this.name; } diff --git a/testdata/simple.js.ol.html b/testdata/simple.js.ol.html index 0d38efb..0eac9d2 100755 --- a/testdata/simple.js.ol.html +++ b/testdata/simple.js.ol.html @@ -1,12 +1,12 @@
    1. // foo is a cool function
      2. -
    2. function foo() {}
      3. +
    3. function foo() {}
    5. /* bar is a cool var */
    6. var bar = 3;
    7. -
    8. A.prototype.foo = function() {
      9. -
    9. this.noise || '<chirp>';
      10. +
    10. A.prototype.foo = function() {
      11. +
    11. this.noise || '<chirp>';
    12. return 'Hello from ' + this.name;
    13. }
    14. diff --git a/testdata/simple.py.html b/testdata/simple.py.html index 398e8eb..d435a9e 100755 --- a/testdata/simple.py.html +++ b/testdata/simple.py.html @@ -1,4 +1,4 @@ from foo import bar -def f(self, a, b): +def f(self, a, b): print('hello!') diff --git a/testdata/simple.py.ol.html b/testdata/simple.py.ol.html index 210a7d2..0796125 100755 --- a/testdata/simple.py.ol.html +++ b/testdata/simple.py.ol.html @@ -1,7 +1,7 @@
      1. from foo import bar
      2. -
      3. def f(self, a, b):
        4. +
      4. def f(self, a, b):
      5. print('hello!')
      \ No newline at end of file diff --git a/testdata/simple.rb.html b/testdata/simple.rb.html index 43178f7..2c14054 100755 --- a/testdata/simple.rb.html +++ b/testdata/simple.rb.html @@ -8,5 +8,5 @@ def foo(a, b) puts a - A::B + A::B end diff --git a/testdata/simple.rb.ol.html b/testdata/simple.rb.ol.html index 7eee0ae..c6bd8fb 100755 --- a/testdata/simple.rb.ol.html +++ b/testdata/simple.rb.ol.html @@ -9,7 +9,7 @@
    16. def foo(a, b)
    17. puts a
      18. -
    18. A::B
      19. +
    19. A::B
    20. end
    \ No newline at end of file diff --git a/testdata/underscore.go.html b/testdata/underscore.go.html index 0dcc1bd..675c32d 100755 --- a/testdata/underscore.go.html +++ b/testdata/underscore.go.html @@ -2,7 +2,7 @@ package foo_bar -func foo() { - for _, a := range foo { +func foo() { + for _, a := range foo { } } diff --git a/testdata/underscore.go.ol.html b/testdata/underscore.go.ol.html index f5e4fd1..06b6903 100755 --- a/testdata/underscore.go.ol.html +++ b/testdata/underscore.go.ol.html @@ -3,8 +3,8 @@
  • package foo_bar
  • -
  • func foo() {
    • -
  • for _, a := range foo {
    • +
  • func foo() {
    • +
  • for _, a := range foo {
  • }
  • }
  • diff --git a/testdata/utf8.go.html b/testdata/utf8.go.html index 9518531..58cffaf 100755 --- a/testdata/utf8.go.html +++ b/testdata/utf8.go.html @@ -11,6 +11,6 @@ // ᚷᛁᚠ᛫ᚻᛖ᛫ᚹᛁᛚᛖ᛫ᚠᚩᚱ᛫ᛞᚱᛁᚻᛏᚾᛖ᛫ᛞᚩᛗᛖᛋ᛫ᚻᛚᛇᛏᚪᚾ᛬ var B = "Τὴ γλῶσσα μοῦ ἔδωσαν ἑλληνικὴ" -func F() { +func F() { fmt.Println(A, B) } diff --git a/testdata/utf8.go.ol.html b/testdata/utf8.go.ol.html index cd4c7ba..6ff517b 100755 --- a/testdata/utf8.go.ol.html +++ b/testdata/utf8.go.ol.html @@ -12,7 +12,7 @@
  • // ᚷᛁᚠ᛫ᚻᛖ᛫ᚹᛁᛚᛖ᛫ᚠᚩᚱ᛫ᛞᚱᛁᚻᛏᚾᛖ᛫ᛞᚩᛗᛖᛋ᛫ᚻᛚᛇᛏᚪᚾ᛬
  • var B = "Τὴ γλῶσσα μοῦ ἔδωσαν ἑλληνικὴ"
  • -
  • func F() {
    • +
  • func F() {
  • fmt.Println(A, B)
  • }