feat: Add plaintext format detection (#15)

Author: Yahiewi

src/detect.ts

@@ -0,0 +1,141 @@
+// ---------------------------------------------------------------------------
+// Format detection
+// ---------------------------------------------------------------------------
+
+import { isMyST } from "./parseMd";
+
+export type PlainbFormat = "percent" | "sphinx-gallery" | "classic" | "myst";
+
+/**
+ * Tracks quote states to determine if lines/delimiters are within strings or docstrings.
+ */
+class StringParser {
+  private single: string | null = null;
+  private triple: string | null = null;
+  private tripleStart = -1;
+
+  isQuoted(): boolean {
+    return this.single !== null || this.triple !== null;
+  }
+
+  readLine(line: string): void {
+    // Comment lines outside of strings do not change state.
+    if (!this.isQuoted() && line.trimStart().startsWith("#")) {
+      return;
+    }
+    this.tripleStart = -1;
+    for (let i = 0; i < line.length; i++) {
+      const char = line[i];
+      // '#' outside of any string starts a comment.
+      if (this.single === null && this.triple === null && char === "#") {
+        break;
+      }
+      if (char !== '"' && char !== "'") {
+        continue;
+      }
+      // Escaped quote.
+      if (line[i - 1] === "\\") {
+        continue;
+      }
+      if (this.single === char) {
+        this.single = null;
+        continue;
+      }
+      if (this.single !== null) {
+        continue;
+      }
+      // Triple quote start or end.
+      if (i >= this.tripleStart + 3 && line.slice(i - 2, i + 1) === char.repeat(3)) {
+        if (this.triple === char) {
+          this.triple = null;
+          this.tripleStart = i;
+          continue;
+        }
+        if (this.triple !== null) {
+          continue;
+        }
+        this.triple = char;
+        this.tripleStart = i;
+        continue;
+      }
+      if (this.triple !== null) {
+        continue;
+      }
+      // Single/double quoted string start.
+      this.single = char;
+    }
+    // Single-line quotes do not carry over to the next line in Python.
+    this.single = null;
+  }
+}
+
+const DOUBLE_PERCENT_RE = /^#\s*%%/;
+const TWENTY_HASH_RE = /^#( ?)#{19,}\s*$/;
+
+/** Check if the file starts with a module docstring, ignoring front matter. */
+function hasLeadingDocstring(lines: string[]): boolean {
+  let i = 0;
+  // Skip commented YAML front matter.
+  if (lines[0]?.trim() === "# ---") {
+    i = 1;
+    while (i < lines.length && lines[i].trim() !== "# ---") {
+      i++;
+    }
+    i++; // skip the closing delimiter
+  }
+  // A module docstring may only be preceded by blank lines and comments.
+  while (i < lines.length && (lines[i].trim() === "" || lines[i].trimStart().startsWith("#"))) {
+    i++;
+  }
+  return /^[rbuf]*("""|''')/i.test((lines[i] ?? "").trimStart());
+}
+
+/** Detect whether a Python script uses percent or Sphinx Gallery format. */
+export function detectPy(text: string): "percent" | "sphinx-gallery" {
+  const lines = text.replace(/\r\n/g, "\n").split("\n");
+  const parser = new StringParser();
+  let doublePercent = 0;
+  let twentyHash = 0;
+  let hasPercentMd = false;
+
+  for (const line of lines) {
+    parser.readLine(line);
+    if (parser.isQuoted()) {
+      continue;
+    }
+    if (DOUBLE_PERCENT_RE.test(line)) {
+      doublePercent++;
+      if (/# %%\s*\[(markdown|md|raw)\]/i.test(line)) {
+        hasPercentMd = true;
+      }
+    }
+    if (TWENTY_HASH_RE.test(line)) {
+      twentyHash++;
+    }
+  }
+
+  // A Sphinx Gallery script starts with a module docstring, and separates
+  // cells using either twenty hashes or `# %%` without percent-format cell tags.
+  if (hasLeadingDocstring(lines) && !hasPercentMd) {
+    return "sphinx-gallery";
+  }
+
+  if (doublePercent >= 1) {
+    return "percent";
+  }
+  if (twentyHash >= 2) {
+    return "sphinx-gallery";
+  }
+  return "percent";
+}
+
+/**
+ * Detect the plainb format of a file from its text and extension.
+ */
+export function detectFormat(text: string, ext: string): PlainbFormat {
+  const normExt = ext.toLowerCase().replace(/^\./, "");
+  if (normExt === "md" || normExt === "markdown") {
+    return isMyST(text) ? "myst" : "classic";
+  }
+  return detectPy(text);
+}

src/index.ts

@@ -4,29 +4,84 @@ export * from "./parseClassicMd";
 export * from "./parseMystMd";
 export * from "./parseSphinxGallery";
 export * from "./notebook";
+export * from "./detect";
 export * from "./toPy";
 export * from "./toClassicMd";
 export * from "./toMystMd";
 export * from "./toSphinxGallery";
 
 import { parsePy } from "./parsePy";
 import { parseMd } from "./parseMd";
+import { parseClassicMd } from "./parseClassicMd";
+import { parseMystMd } from "./parseMystMd";
 import { parseSphinxGallery } from "./parseSphinxGallery";
 import { toPy } from "./toPy";
+import { toClassicMd } from "./toClassicMd";
 import { toMystMd } from "./toMystMd";
 import { toSphinxGallery } from "./toSphinxGallery";
+import type { PlainbFormat } from "./detect";
 import type { Notebook } from "./notebook";
 
+/**
+ * Parse a file by explicitly providing the format.
+ *
+ * @param text - the file contents
+ * @param format - the format of the file, "py", "md", or "sphinx-gallery"
+ */
 export function parse(text: string, format: "py" | "md" | "sphinx-gallery"): Notebook {
   if (format === "py") return parsePy(text);
   if (format === "md") return parseMd(text);
   if (format === "sphinx-gallery") return parseSphinxGallery(text);
   throw new Error(`Unknown format: "${format}". Expected "py", "md", or "sphinx-gallery".`);
 }
 
+/**
+ * Serialize a notebook to a file by explicitly providing the format.
+ *
+ * @param notebook - the notebook to serialize
+ * @param format - the format to serialize to, "py", "md", or "sphinx-gallery"
+ */
 export function serialize(notebook: Notebook, format: "py" | "md" | "sphinx-gallery"): string {
   if (format === "py") return toPy(notebook);
   if (format === "md") return toMystMd(notebook);
   if (format === "sphinx-gallery") return toSphinxGallery(notebook);
   throw new Error(`Unknown format: "${format}". Expected "py", "md", or "sphinx-gallery".`);
 }
+
+/**
+ * Parse a file given its detected {@link PlainbFormat}.
+ *
+ * @param text - the file contents
+ * @param format - the format returned by {@link detectFormat}
+ */
+export function parseFormat(text: string, format: PlainbFormat): Notebook {
+  switch (format) {
+    case "percent":
+      return parsePy(text);
+    case "sphinx-gallery":
+      return parseSphinxGallery(text);
+    case "myst":
+      return parseMystMd(text);
+    case "classic":
+      return parseClassicMd(text);
+  }
+}
+
+/**
+ * Serialize a notebook to a given {@link PlainbFormat}.
+ *
+ * @param notebook - the notebook to serialize
+ * @param format - the target format
+ */
+export function serializeFormat(notebook: Notebook, format: PlainbFormat): string {
+  switch (format) {
+    case "percent":
+      return toPy(notebook);
+    case "sphinx-gallery":
+      return toSphinxGallery(notebook);
+    case "myst":
+      return toMystMd(notebook);
+    case "classic":
+      return toClassicMd(notebook);
+  }
+}

src/parseMd.ts

@@ -3,7 +3,7 @@ import { parseMystMd } from "./parseMystMd";
 import type { Notebook } from "./notebook";
 
 /** Detect MyST notebook format by scanning for {code-cell}/{raw-cell} directives or +++ breaks. */
-function isMyST(text: string): boolean {
+export function isMyST(text: string): boolean {
   const lines = text.split("\n");
   const limit = Math.min(lines.length, 100);
   for (let i = 0; i < limit; i++) {

test/detect.test.ts

@@ -0,0 +1,86 @@
+import { test, describe } from "node:test";
+import assert from "node:assert/strict";
+import { detectFormat, detectPy } from "../src/detect.js";
+
+// ---------------------------------------------------------------------------
+// Python scripts
+// ---------------------------------------------------------------------------
+
+describe("detectPy", () => {
+  test("`# %%` delimiter → percent", () => {
+    assert.equal(detectPy("# %%\nx = 1\n# %%\ny = 2"), "percent");
+  });
+
+  test("`#%%` without space → percent", () => {
+    assert.equal(detectPy("#%%\nx = 1"), "percent");
+  });
+
+  test("two 20-hash runs → sphinx-gallery", () => {
+    const text = "####################\nx = 1\n####################\ny = 2";
+    assert.equal(detectPy(text), "sphinx-gallery");
+  });
+
+  test("`# %%` wins over hash runs (percent precedence)", () => {
+    const text = "####################\n# %%\nx = 1\n####################";
+    assert.equal(detectPy(text), "percent");
+  });
+
+  test("leading module docstring, no delimiters → sphinx-gallery", () => {
+    assert.equal(detectPy('"""Module doc."""\nimport os\nx = 1'), "sphinx-gallery");
+  });
+
+  test("r-prefixed docstring → sphinx-gallery", () => {
+    assert.equal(detectPy('r"""\nTitle\n=====\n"""\nx = 1'), "sphinx-gallery");
+  });
+
+  test("plain script, no docstring, no delimiters → percent", () => {
+    assert.equal(detectPy("import os\nx = 1\nprint(x)"), "percent");
+  });
+
+  test("`# %%` inside a docstring is not counted", () => {
+    // The only `# %%` lives inside the docstring; file should fall back to the
+    // leading-docstring rule → sphinx-gallery, not percent.
+    const text = '"""\nExample:\n# %%\n"""\nimport os';
+    assert.equal(detectPy(text), "sphinx-gallery");
+  });
+
+  test("a single hash run is not enough for sphinx", () => {
+    assert.equal(detectPy("####################\nimport os\nx = 1"), "percent");
+  });
+
+  test("leading docstring + `# %%` cell separators → sphinx-gallery", () => {
+    const text = '"""\nLeading docstring\n"""\n# %%\nx = 1\n# %%\ny = 2';
+    assert.equal(detectPy(text), "sphinx-gallery");
+  });
+
+  test("leading docstring + `# %% [markdown]` → percent", () => {
+    const text = '"""\nLeading docstring\n"""\n# %% [markdown]\n# Some text\n# %%\nx = 1';
+    assert.equal(detectPy(text), "percent");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Extension dispatch
+// ---------------------------------------------------------------------------
+
+describe("detectFormat", () => {
+  test(".md with {code-cell} → myst", () => {
+    assert.equal(detectFormat("```{code-cell}\nx = 1\n```", ".md"), "myst");
+  });
+
+  test(".md with +++ → myst", () => {
+    assert.equal(detectFormat("First.\n+++\nSecond.", ".md"), "myst");
+  });
+
+  test(".md plain prose → classic", () => {
+    assert.equal(detectFormat("# Title\n\n```python\nx = 1\n```", ".md"), "classic");
+  });
+
+  test("extension without a leading dot is accepted", () => {
+    assert.equal(detectFormat("# %%\nx = 1", "py"), "percent");
+  });
+
+  test(".py routes to detectPy", () => {
+    assert.equal(detectFormat("# %%\nx = 1", ".py"), "percent");
+  });
+});