cherry-pick(#18768): chore: verify tab groups in docs during lint (#18837)

This extracts the logic from playwright.dev so that we get early
warnings.

Co-authored-by: Dmitry Gozman <[email protected]>

Author: aslushnikov

packages/playwright-core/types/types.d.ts

@@ -13501,7 +13501,6 @@ export interface APIRequest {
  * If you want API requests to not interfere with the browser cookies you should create a new [APIRequestContext] by
  * calling [apiRequest.newContext([options])](https://playwright.dev/docs/api/class-apirequest#api-request-new-context).
  * Such `APIRequestContext` object will have its own isolated cookie storage.
- *
  */
 export interface APIRequestContext {
   /**
@@ -14206,7 +14205,6 @@ export interface APIRequestContext {
  * [APIResponse] class represents responses returned by
  * [apiRequestContext.get(url[, options])](https://playwright.dev/docs/api/class-apirequestcontext#api-request-context-get)
  * and similar methods.
- *
  */
 export interface APIResponse {
   /**

utils/doclint/cli.js

@@ -89,7 +89,6 @@ async function run() {
 
   // Patch docker version in docs
   {
-    const regex = new RegExp("(mcr.microsoft.com/playwright[^: ]*):?([^ ]*)");
     for (const filePath of getAllMarkdownFiles(path.join(PROJECT_DIR, 'docs'))) {
       let content = fs.readFileSync(filePath).toString();
       content = content.replace(new RegExp('(mcr.microsoft.com/playwright[^:]*):([\\w\\d-.]+)', 'ig'), (match, imageName, imageVersion) => {
@@ -165,6 +164,9 @@ async function run() {
 
         // This validates member links.
         documentation.setLinkRenderer(() => undefined);
+        // This validates code snippet groups in comments.
+        documentation.setCodeGroupsTransformer(lang, tabs => tabs.map(tab => tab.spec));
+        documentation.generateSourceCodeComments();
 
         const relevantMarkdownFiles = new Set([...getAllMarkdownFiles(documentationRoot)
           // filter out language specific files
@@ -185,9 +187,12 @@ async function run() {
           if (langs.some(other => other !== lang && filePath.endsWith(`-${other}.md`)))
             continue;
           const data = fs.readFileSync(filePath, 'utf-8');
-          const rootNode = md.filterNodesForLanguage(md.parse(data), lang);
+          let rootNode = md.filterNodesForLanguage(md.parse(data), lang);
+          // Validates code snippet groups.
+          rootNode = md.processCodeGroups(rootNode, lang, tabs => tabs.map(tab => tab.spec));
+          // Renders links.
           documentation.renderLinksInText(rootNode);
-          // Validate links
+          // Validate links.
           {
             md.visitAll(rootNode, node => {
               if (!node.text)

utils/doclint/documentation.js

@@ -165,9 +165,23 @@ class Documentation {
     this._patchLinks?.(null, nodes);
   }
 
+  /**
+   * @param {string} lang
+   * @param {import('../markdown').CodeGroupTransformer} transformer
+   */
+  setCodeGroupsTransformer(lang, transformer) {
+    this._codeGroupsTransformer = { lang, transformer };
+  }
+
   generateSourceCodeComments() {
-    for (const clazz of this.classesArray)
-      clazz.visit(item => item.comment = generateSourceCodeComment(item.spec));
+    for (const clazz of this.classesArray) {
+      clazz.visit(item => {
+        let spec = item.spec;
+        if (spec && this._codeGroupsTransformer)
+          spec = md.processCodeGroups(spec, this._codeGroupsTransformer.lang, this._codeGroupsTransformer.transformer);
+        item.comment = generateSourceCodeComment(spec);
+      });
+    }
   }
 
   clone() {
@@ -814,8 +828,6 @@ function patchLinks(classOrMember, spec, classesMap, membersMap, linkRenderer) {
 function generateSourceCodeComment(spec) {
   const comments = (spec || []).filter(n => !n.type.startsWith('h') && (n.type !== 'li' ||  n.liType !== 'default')).map(c => md.clone(c));
   md.visitAll(comments, node => {
-    if (node.codeLang && node.codeLang.includes('tab=js-js'))
-      node.type = 'null';
     if (node.type === 'li' && node.liType === 'bullet')
       node.liType = 'default';
     if (node.type === 'note') {

utils/generate_types/index.js

@@ -16,7 +16,6 @@
 
 //@ts-check
 const path = require('path');
-const os = require('os');
 const toKebabCase = require('lodash/kebabCase')
 const devices = require('../../packages/playwright-core/lib/server/deviceDescriptors');
 const Documentation = require('../doclint/documentation');
@@ -87,6 +86,7 @@ class TypesGenerator {
         return createMarkdownLink(member, `${className}${member.alias}`);
       throw new Error('Unknown member kind ' + member.kind);
     });
+    this.documentation.setCodeGroupsTransformer('js', tabs => tabs.filter(tab => tab.value === 'ts').map(tab => tab.spec));
     this.documentation.generateSourceCodeComments();
 
     const handledClasses = new Set();

utils/markdown.js

@@ -49,7 +49,7 @@
 
 /** @typedef {MarkdownBaseNode & {
  *    type: 'note',
- *    text: string,          
+ *    text: string,
  *    noteType: string,
  *  }} MarkdownNoteNode */
 
@@ -62,6 +62,12 @@
  *    lines: string[],
  *  }} MarkdownPropsNode */
 
+/** @typedef {{
+ *    value: string, groupId: string, spec: MarkdownNode
+ * }} CodeGroup */
+
+/** @typedef {function(CodeGroup[]): MarkdownNode[]} CodeGroupTransformer */
+
 /** @typedef {MarkdownTextNode | MarkdownLiNode | MarkdownCodeNode | MarkdownNoteNode | MarkdownHeaderNode | MarkdownNullNode | MarkdownPropsNode } MarkdownNode */
 
 function flattenWrappedLines(content) {
@@ -307,7 +313,7 @@ function innerRenderMdNode(indent, node, lastNode, result, maxColumns) {
     if (process.env.API_JSON_MODE)
       result.push(`${indent}\`\`\`${node.codeLang}`);
     else
-      result.push(`${indent}\`\`\`${codeLangToHighlighter(node.codeLang)}`);
+      result.push(`${indent}\`\`\`${node.codeLang ? parseCodeLang(node.codeLang).highlighter : ''}`);
     for (const line of node.lines)
       result.push(indent + line);
     result.push(`${indent}\`\`\``);
@@ -469,13 +475,82 @@ function filterNodesForLanguage(nodes, language) {
 
 /**
  * @param {string} codeLang
- * @return {string}
+ * @return {{ highlighter: string, language: string|undefined, codeGroup: string|undefined}}
  */
-function codeLangToHighlighter(codeLang) {
-  const [lang] = codeLang.split(' ');
-  if (lang === 'python')
-    return 'py';
-  return lang;
+function parseCodeLang(codeLang) {
+  if (codeLang === 'python async')
+    return { highlighter: 'py', codeGroup: 'python-async', language: 'python' };
+  if (codeLang === 'python sync')
+    return { highlighter: 'py', codeGroup: 'python-sync', language: 'python' };
+
+  const [highlighter] = codeLang.split(' ');
+  if (!highlighter)
+    throw new Error(`Cannot parse code block lang: "${codeLang}"`);
+
+  const languageMatch = codeLang.match(/ lang=([\w\d]+)/);
+  let language = languageMatch ? languageMatch[1] : undefined;
+  if (!language) {
+    if (highlighter === 'ts')
+      language = 'js';
+    else if (['js', 'python', 'csharp', 'java'].includes(highlighter))
+      language = highlighter;
+  }
+
+  const tabMatch = codeLang.match(/ tab=([\w\d-]+)/);
+  return { highlighter, language, codeGroup: tabMatch ? tabMatch[1] : '' };
+}
+
+/**
+ * @param {MarkdownNode[]} spec
+ * @param {string} language
+ * @param {CodeGroupTransformer} transformer
+ * @returns {MarkdownNode[]}
+ */
+function processCodeGroups(spec, language, transformer) {
+  /** @type {MarkdownNode[]} */
+  const newSpec = [];
+  for (let i = 0; i < spec.length; ++i) {
+    /** @type {{value: string, groupId: string, spec: MarkdownNode}[]} */
+    const tabs = [];
+    for (;i < spec.length; i++) {
+      const codeLang = spec[i].codeLang;
+      if (!codeLang)
+        break;
+      let parsed;
+      try {
+        parsed = parseCodeLang(codeLang);
+      } catch (e) {
+        throw new Error(e.message + '\n while processing:\n' + render([spec[i]]));
+      }
+      if (!parsed.codeGroup)
+        break;
+      if (parsed.language && parsed.language !== language)
+        continue;
+      const [groupId, value] = parsed.codeGroup.split('-');
+      tabs.push({ groupId, value, spec: spec[i] });
+    }
+    if (tabs.length) {
+      if (tabs.length === 1)
+        throw new Error(`Lonely tab "${tabs[0].spec.codeLang}". Make sure there are at least two tabs in the group.\n` + render([tabs[0].spec]));
+
+      // Validate group consistency.
+      const groupId = tabs[0].groupId;
+      const values = new Set();
+      for (const tab of tabs) {
+        if (tab.groupId !== groupId)
+          throw new Error('Mixed group ids: ' + render(spec));
+        if (values.has(tab.value))
+          throw new Error(`Duplicated tab "${tab.value}"\n` + render(tabs.map(tab => tab.spec)));
+        values.add(tab.value);
+      }
+
+      // Append transformed nodes.
+      newSpec.push(...transformer(tabs));
+    }
+    if (i < spec.length)
+      newSpec.push(spec[i]);
+  }
+  return newSpec;
 }
 
-module.exports = { parse, render, clone, visitAll, visit, generateToc, filterNodesForLanguage, codeLangToHighlighter };
+module.exports = { parse, render, clone, visitAll, visit, generateToc, filterNodesForLanguage, parseCodeLang, processCodeGroups };