feat(setence): add Whitespace Node between Sentence (#7)

Author: azu

bin/cmd.js

@@ -1,16 +1,18 @@
 #!/usr/bin/env node
-var split = require('../').split;
-var Syntax = require('../').Syntax;
-var concat = require('concat-stream');
-var fs = require('fs');
+var split = require("../").split;
+var Syntax = require("../").Syntax;
+var concat = require("concat-stream");
+var fs = require("fs");
 var file = process.argv[2];
-var input = file && file !== '-'
-    ? fs.createReadStream(process.argv[2])
-    : process.stdin;
-input.pipe(concat(function(buf) {
-    split(buf.toString('utf8')).filter(function(node) {
-        return node.type === Syntax.Sentence;
-    }).forEach(function(sentence) {
-        console.log(sentence.value);
-    });
-}));
+var input = file && file !== "-" ? fs.createReadStream(process.argv[2]) : process.stdin;
+input.pipe(
+    concat(function(buf) {
+        split(buf.toString("utf8"))
+            .filter(function(node) {
+                return node.type === Syntax.Sentence;
+            })
+            .forEach(function(sentence) {
+                console.log(sentence.value);
+            });
+    })
+);

package.json

@@ -29,7 +29,10 @@
     "build": "NODE_ENV=production babel src --out-dir lib --source-maps",
     "watch": "babel src --out-dir lib --watch --source-maps",
     "prepublish": "npm run --if-present build",
-    "test": "mocha"
+    "test": "mocha",
+    "prettier": "prettier --write '**/*.{js,jsx,ts,tsx,css}'",
+    "precommit": "lint-staged",
+    "postcommit": "git reset"
   },
   "keywords": [
     "text",
@@ -47,7 +50,20 @@
     "babel-preset-es2015": "^6.1.18",
     "babel-preset-power-assert": "^1.0.0",
     "babel-register": "^6.18.0",
+    "husky": "^0.14.3",
+    "lint-staged": "^4.3.0",
     "mocha": "^2.3.3",
-    "power-assert": "^1.4.2"
+    "power-assert": "^1.4.2",
+    "prettier": "^1.7.4"
+  },
+  "prettier": {
+    "printWidth": 120,
+    "tabWidth": 4
+  },
+  "lint-staged": {
+    "*.{js,jsx,ts,tsx,css}": [
+      "prettier --write",
+      "git add"
+    ]
   }
-}
+}

src/sentence-splitter.js

@@ -2,45 +2,52 @@
 "use strict";
 const assert = require("assert");
 import StructureSource from "structured-source";
+
 const defaultOptions = {
     // charRegExp is deprecated
     charRegExp: /[\.。\?\!？！]/,
     // separator char list
     separatorChars: [".", "。", "?", "!", "？", "！"],
-    newLineCharacters: "\n"
+    newLineCharacters: "\n",
+    whiteSpaceCharacters: [" ", "　"]
 };
 export const Syntax = {
-    "WhiteSpace": "WhiteSpace",
-    "Sentence": "Sentence"
+    WhiteSpace: "WhiteSpace",
+    Sentence: "Sentence"
 };
+
 /**
  * @param {string} text
  * @param {{
  *      charRegExp: ?Object,
  *      separatorChars: ?string[],
- *      newLineCharacters: ?String
+ *      newLineCharacters: ?String,
+ *      whiteSpaceCharacters: ?string[]
  *  }} options
  * @returns {Array}
  */
 export function split(text, options = {}) {
     const charRegExp = options.charRegExp;
     const separatorChars = options.separatorChars || defaultOptions.separatorChars;
-    assert(!(options.charRegExp && options.separatorChars), "should use either one `charRegExp` or `separatorChars`.\n"
-        + "`charRegExp` is deprecated.");
+    const whiteSpaceCharacters = options.whiteSpaceCharacters || defaultOptions.whiteSpaceCharacters;
+    assert(
+        !(options.charRegExp && options.separatorChars),
+        "should use either one `charRegExp` or `separatorChars`.\n" + "`charRegExp` is deprecated."
+    );
     /**
      * Is the `char` separator symbol?
      * @param {string} char
      * @returns {boolean}
      */
-    const testCharIsSeparator = (char) => {
+    const testCharIsSeparator = char => {
         if (charRegExp) {
             return charRegExp.test(char);
         }
         return separatorChars.indexOf(char) !== -1;
     };
     const newLineCharacters = options.newLineCharacters || defaultOptions.newLineCharacters;
     const src = new StructureSource(text);
-    let createNode = (type, start, end) => {
+    const createNode = (type, start, end) => {
         let range = [start, end];
         let location = src.rangeToLocation(range);
         let slicedText = text.slice(start, end);
@@ -84,6 +91,12 @@ export function split(text, options = {}) {
                 startPoint = currentIndex;
                 isSplitPoint = false;
             }
+            // Sentence<WhiteSpace>Sentence
+            if (whiteSpaceCharacters.indexOf(char) !== -1) {
+                results.push(createNode(Syntax.WhiteSpace, currentIndex, currentIndex + 1));
+                startPoint++;
+                currentIndex++;
+            }
         }
     }
 
@@ -92,6 +105,7 @@ export function split(text, options = {}) {
     }
     return results;
 }
+
 /**
  * @param {string} text
  * @param {Object} loc
@@ -105,8 +119,9 @@ export function createWhiteSpaceNode(text, loc, range) {
         value: text,
         loc: loc,
         range: range
-    }
+    };
 }
+
 /**
  * @param {string} text
  * @param {Object} loc
@@ -120,5 +135,5 @@ export function createSentenceNode(text, loc, range) {
         value: text,
         loc: loc,
         range: range
-    }
-}
+    };
+}