fix(sentence): support continuous whitespace nodes

Author: azu

src/sentence-splitter.js

@@ -63,6 +63,7 @@ export function split(text, options = {}) {
     let startPoint = 0;
     let currentIndex = 0;
     let isSplitPoint = false;
+    let isInSentence = false;
     const newLineCharactersLength = newLineCharacters.length;
     for (; currentIndex < text.length; currentIndex++) {
         let char = text[currentIndex];
@@ -90,12 +91,16 @@ export function split(text, options = {}) {
                 // reset stat
                 startPoint = currentIndex;
                 isSplitPoint = false;
-                // Sentence<WhiteSpace>Sentence
-                if (whiteSpaceCharacters.indexOf(char) !== -1) {
-                    results.push(createNode(Syntax.WhiteSpace, currentIndex, currentIndex + 1));
-                    startPoint++;
-                    currentIndex++;
-                }
+                isInSentence = false;
+            }
+            // Sentence<WhiteSpace>*Sentence
+            if (isInSentence === false && whiteSpaceCharacters.indexOf(char) !== -1) {
+                // Add WhiteSpace
+                results.push(createNode(Syntax.WhiteSpace, startPoint, currentIndex + 1));
+                startPoint++;
+            } else {
+                // New sentence start
+                isInSentence = true;
             }
         }
     }

test/sentence-utils-test.js

@@ -108,6 +108,42 @@ describe("sentence-utils", function() {
         assert.deepEqual(sentence2.loc.start, { line: 1, column: 10 });
         assert.deepEqual(sentence2.loc.end, { line: 1, column: 18 });
     });
+    it("should return sentences split by multiple whitespaces", function() {
+        const sentences = splitSentences("1st text.   2nd text");
+        assert.equal(sentences.length, 2 + 3);
+        const [sentence0, whitespace0, whitespace1, whitespace2, sentence1] = sentences;
+        assert.strictEqual(sentence0.raw, "1st text.");
+        assert.deepEqual(sentence0.range, [0, 9]);
+        assert.strictEqual(whitespace0.type, Syntax.WhiteSpace);
+        assert.strictEqual(whitespace0.value, " ");
+        assert.deepEqual(whitespace0.range, [9, 10]);
+        assert.strictEqual(whitespace1.type, Syntax.WhiteSpace);
+        assert.strictEqual(whitespace1.value, " ");
+        assert.deepEqual(whitespace1.range, [10, 11]);
+        assert.strictEqual(whitespace2.type, Syntax.WhiteSpace);
+        assert.strictEqual(whitespace2.value, " ");
+        assert.deepEqual(whitespace2.range, [11, 12]);
+        assert.strictEqual(sentence1.raw, "2nd text");
+        assert.deepEqual(sentence1.range, [12, 20]);
+    });
+    it("should return sentences split by text and whitespaces, and new line", function() {
+        const sentences = splitSentences("1st text. \n 2nd text");
+        assert.equal(sentences.length, 2 + 2 + 1);
+        const [sentence0, whitespace0, lineBreak, whitespace1, sentence1] = sentences;
+        assert.strictEqual(sentence0.raw, "1st text.");
+        assert.deepEqual(sentence0.range, [0, 9]);
+        assert.strictEqual(whitespace0.type, Syntax.WhiteSpace);
+        assert.strictEqual(whitespace0.value, " ");
+        assert.deepEqual(whitespace0.range, [9, 10]);
+        assert.strictEqual(lineBreak.type, Syntax.WhiteSpace);
+        assert.strictEqual(lineBreak.value, "\n");
+        assert.deepEqual(lineBreak.range, [10, 11]);
+        assert.strictEqual(whitespace1.type, Syntax.WhiteSpace);
+        assert.strictEqual(whitespace1.value, " ");
+        assert.deepEqual(whitespace1.range, [11, 12]);
+        assert.strictEqual(sentence1.raw, "2nd text");
+        assert.deepEqual(sentence1.range, [12, 20]);
+    });
     it("should return sentences split by !?", function() {
         let sentences = splitSentences("text!?text");
         assert.equal(sentences.length, 2);