fix: improve slug generation

Author: sy-records

src/core/render/compiler/heading.js

@@ -6,9 +6,9 @@ import {
 import { slugify } from '../slugify.js';
 
 export const headingCompiler = ({ renderer, router, compiler }) =>
-  (renderer.heading = function ({ tokens, depth }) {
-    const text = this.parser.parseInline(tokens);
-    let { str, config } = getAndRemoveConfig(text);
+  (renderer.heading = function ({ tokens, depth, text }) {
+    const parsedText = this.parser.parseInline(tokens);
+    let { str, config } = getAndRemoveConfig(parsedText);
     const nextToc = { depth, title: str };
 
     const { content, ignoreAllSubs, ignoreSubHeading } =
@@ -18,7 +18,7 @@ export const headingCompiler = ({ renderer, router, compiler }) =>
     nextToc.title = removeAtag(str);
     nextToc.ignoreAllSubs = ignoreAllSubs;
     nextToc.ignoreSubHeading = ignoreSubHeading;
-    const slug = slugify(config.id || str);
+    const slug = slugify(config.id || text);
     const url = router.toURL(router.getCurrentPath(), { id: slug });
     nextToc.slug = url;
     compiler.toc.push(nextToc);

src/core/render/slugify.js

@@ -12,11 +12,12 @@ export function slugify(str) {
 
   let slug = str
     .trim()
+    .normalize('NFKD')
+    .replace(/[\p{Emoji_Presentation}\p{Extended_Pictographic}]/gu, '')
     .replace(/[A-Z]+/g, lower)
     .replace(/<[^>]+>/g, '')
     .replace(re, '')
     .replace(/\s/g, '-')
-    .replace(/-+/g, '-')
     .replace(/^(\d)/, '_$1');
   let count = cache[slug];
 

src/plugins/search/search.js

@@ -1,6 +1,7 @@
 import {
   getAndRemoveConfig,
   getAndRemoveDocsifyIgnoreConfig,
+  removeAtag,
 } from '../../core/render/utils.js';
 import { markdownToTxt } from './markdown-to-txt.js';
 import Dexie from 'dexie';
@@ -110,16 +111,11 @@ export function genIndex(path, content = '', router, depth, indexKey) {
     if (token.type === 'heading' && token.depth <= depth) {
       const { str, config } = getAndRemoveConfig(token.text);
 
-      const text = getAndRemoveDocsifyIgnoreConfig(token.text).content;
-
-      if (config.id) {
-        slug = router.toURL(path, { id: slugify(config.id) });
-      } else {
-        slug = router.toURL(path, { id: slugify(escapeHtml(text)) });
-      }
+      slug = router.toURL(path, { id: slugify(config.id || token.text) });
 
       if (str) {
         title = getAndRemoveDocsifyIgnoreConfig(str).content;
+        title = removeAtag(title.trim());
       }
 
       index[slug] = {

test/unit/render-util.test.js

@@ -168,13 +168,49 @@ describe('core/render/tpl', () => {
 
 describe('core/render/slugify', () => {
   test('slugify()', () => {
-    const result = slugify(
+    const htmlStrippedSlug = slugify(
       'Bla bla bla <svg aria-label="broken" class="broken" viewPort="0 0 1 1"><circle cx="0.5" cy="0.5"/></svg>',
     );
-    const result2 = slugify(
+    expect(htmlStrippedSlug).toBe('bla-bla-bla-');
+
+    const nestedHtmlStrippedSlug = slugify(
       'Another <span style="font-size: 1.2em" class="foo bar baz">broken <span class="aaa">example</span></span>',
     );
-    expect(result).toBe('bla-bla-bla-');
-    expect(result2).toBe('another-broken-example');
+    expect(nestedHtmlStrippedSlug).toBe('another-broken-example');
+
+    const emojiRemovedSlug = slugify('emoji test ⚠️🔥✅');
+    expect(emojiRemovedSlug).toBe('emoji-test-️');
+
+    const multiSpaceSlug = slugify('Title    with   multiple spaces');
+    expect(multiSpaceSlug).toBe('title----with---multiple-spaces');
+
+    const numberLeadingSlug = slugify('123abc');
+    expect(numberLeadingSlug).toBe('_123abc');
+
+    const firstDuplicate = slugify('duplicate');
+    expect(firstDuplicate).toBe('duplicate');
+
+    const secondDuplicate = slugify('duplicate');
+    expect(secondDuplicate).toBe('duplicate-1');
+
+    const thirdDuplicate = slugify('duplicate');
+    expect(thirdDuplicate).toBe('duplicate-2');
+
+    const mixedCaseSlug = slugify('This Is Mixed CASE');
+    expect(mixedCaseSlug).toBe('this-is-mixed-case');
+
+    const chinesePreservedSlug = slugify('你好 world');
+    expect(chinesePreservedSlug).toBe('你好-world');
+
+    const specialCharSlug = slugify('C++ vs. Java & Python!');
+    expect(specialCharSlug).toBe('c-vs-java--python');
+
+    const docsifyIgnoreSlug = slugify(
+      'Ignore Heading <!-- {docsify-ignore} -->',
+    );
+    expect(docsifyIgnoreSlug).toBe('ignore-heading-');
+
+    const quoteCleanedSlug = slugify('"The content"');
+    expect(quoteCleanedSlug).toBe('the-content');
   });
 });