fix G9Bc rule check to fix Hindi break (#55)

* fix G9Bc rule check to fix Hindi break

* update bundle stats

* add changeset

* clarify a bit

* what a compression...

* add notes

Author: cometkim

.changeset/young-readers-act.md

@@ -0,0 +1,5 @@
+---
+"unicode-segmenter": patch
+---
+
+Fix Hindi text segmentation

README.md

@@ -233,7 +233,7 @@ See more on [benchmark](benchmark).
 
 | Name                         | Unicode® | ESM? |   Size    | Size (min) | Size (min+gzip) | Size (min+br) |
 |------------------------------|----------|------|----------:|-----------:|----------------:|--------------:|
-| `unicode-segmenter/grapheme` |   15.1.0 |    ✔️ |    28,288 |     24,616 |           6,592 |         4,433 |
+| `unicode-segmenter/grapheme` |   15.1.0 |    ✔️ |    28,337 |     24,623 |           6,599 |         4,360 |
 | `graphemer`                  |   15.0.0 |    ✖️ ️|   410,424 |     95,104 |          15,752 |        10,660 |
 | `grapheme-splitter`          |   10.0.0 |    ✖️ |   122,241 |     23,680 |           7,852 |         4,841 |
 | `unicode-segmentation`*      |   15.0.0 |    ✔️ |    51,251 |     51,251 |          22,545 |        16,614 |

src/grapheme.js

@@ -108,9 +108,12 @@ export function* graphemeSegments(input) {
       catBegin = catBefore;
     }
 
+    // Note: Lazily update `consonant` and `linker` state
+    // which is a extra overhead only for Hindi text.
     if (!consonant && catBefore === 0) {
       consonant = isIndicConjunctCosonant(cp);
-    } else if (catBefore === 3) {
+    } else if (catBefore === 3 /* Extend */) {
+      // Note: \p{InCB=Linker} is a subset of \p{Extend}
       linker = isIndicConjunctLinker(cp);
     }
 
@@ -130,7 +133,7 @@ export function* graphemeSegments(input) {
       return;
     }
 
-    if (catBefore === 10 /* Regional_Indicator*/) {
+    if (catBefore === 10 /* Regional_Indicator */) {
       risCount += 1;
     } else {
       risCount = 0;
@@ -140,9 +143,11 @@ export function* graphemeSegments(input) {
       ) {
         emoji = true;
 
-      // Put GB9c rule checking here to reduce.
+      // Note: Put GB9c rule checking here to reduce.
       } else if (catAfter === 0 /* Any */) {
-        incb = consonant && linker && isIndicConjunctCosonant(cp);
+        incb = consonant && linker && (consonant = isIndicConjunctCosonant(cp));
+        // It cannot be both a linker and a consonant.
+        linker = linker && !consonant;
       }
     }
 
@@ -160,8 +165,7 @@ export function* graphemeSegments(input) {
       index = cursor;
       segment = '';
       emoji = false;
-      consonant = false;
-      linker = false;
+      incb = false;
       catBegin = catAfter;
     }
   }
@@ -227,7 +231,14 @@ function isIndicConjunctCosonant(cp) {
  * @return {boolean}
  */
 function isIndicConjunctLinker(cp) {
-  return (cp === 0x094D || cp === 0x09CD || cp === 0x0ACD || cp === 0x0B4D || cp === 0x0C4D || cp === 0x0D4D);
+  return (
+    cp === 2381 /* 0x094D */ ||
+    cp === 2509 /* 0x09CD */ ||
+    cp === 2765 /* 0x0ACD */ ||
+    cp === 2893 /* 0x0B4D */ ||
+    cp === 3149 /* 0x0C4D */ ||
+    cp === 3405 /* 0x0D4D */
+  );
 }
 
 /**
@@ -246,12 +257,13 @@ function isBoundary(catBefore, catAfter, risCount, emoji, incb) {
     return false;
   }
 
-  if (
-    // GB4
-    (catBefore === 1 || catBefore === 2 || catBefore === 6) ||
-    // GB5
-    (catAfter === 1 || catAfter === 2 || catAfter === 6)
-  ) {
+  // GB4
+  if (catBefore === 1 || catBefore === 2 || catBefore === 6) {
+    return true;
+  }
+
+  // GB5
+  if (catAfter === 1 || catAfter === 2 || catAfter === 6) {
     return true;
   }
 
@@ -279,8 +291,13 @@ function isBoundary(catBefore, catAfter, risCount, emoji, incb) {
     return false;
   }
 
-  // GB9. GB9a
-  if (catAfter === 3 || catAfter === 11 || catAfter === 14) {
+  // GB9
+  if (catAfter === 3 || catAfter === 14) {
+    return false;
+  }
+
+  // GB9a
+  if (catAfter === 11) {
     return false;
   }
 

test/grapheme.js

@@ -178,6 +178,8 @@ test('counterexamples', async t => {
     '🇷🇸A',
     '👩‍🦰👩‍👩‍👦‍👦🏳️‍🌈',
     'अनुच्छेद',
+    'पक्षियों',
+    'क्‍त',
     '് ',
     '्क',
   ];