build(docs-gen): add ast raw data extractor

infra/docs-gen/EXTRACTION_PIPELINE.md

@@ -0,0 +1,48 @@
+# Extension Extraction Pipeline
+
+```mermaid
+flowchart TD
+    CLI["extract-extension-data.mjs<br/>Parse CLI options"] --> Extractor["ExtensionExtractor<br/>src/extractor/index.mjs"]
+    Extractor --> EntryPoints["Extension entry points<br/>EXTENSION_ENTRY_POINTS"]
+    EntryPoints --> Collect["collectExtensionRefs()<br/>src/extractor/extension-refs.mjs"]
+    Collect --> AllRefs["Full extension ref list"]
+    AllRefs --> Filter["filterExtensionRefs()<br/>Apply blacklist and --only"]
+    Filter --> Scan["scanExtension()<br/>src/extractor/scan.mjs"]
+
+    Scan --> FieldConfig["Docs field config<br/>EXTENSION_DOC_FIELD_CONFIG"]
+    Scan --> Files["readExtensionSources()<br/>src/extractor/extension-sources.mjs"]
+    Files --> Selectors["Source file selectors<br/>src/extractor/source-files.mjs"]
+    Selectors --> SourceText["Production source text"]
+    Selectors --> TestFiles["Test files"]
+    Selectors --> SerializerFiles["Serializer and Specs files"]
+
+    SourceText --> Constants["extractConstants()<br/>src/extractor/constants.mjs"]
+    SourceText --> AstBarrel["AST extractor API<br/>src/extractor/ast.mjs"]
+    AstBarrel --> AstScanners["Focused AST scanners<br/>src/extractor/ast/*.mjs"]
+    SourceText --> Options["Option declarations<br/>src/extractor/options.mjs"]
+    SerializerFiles --> SerializerHints["Serializer hints"]
+    TestFiles --> MarkupExamples["Markup examples<br/>src/extractor/examples.mjs"]
+
+    Constants --> Schema["Schema names<br/>src/extractor/schema.mjs"]
+    AstScanners --> ExtractedFields["Actions, keymaps, input rules,<br/>plugins, md plugins"]
+    Options --> ExtractedFields
+    FieldConfig --> IR["Extension IR record"]
+    Schema --> IR
+    ExtractedFields --> IR
+    SerializerHints --> IR
+    MarkupExamples --> IR
+
+    Extractor --> Presets["Preset membership<br/>src/extractor/presets.mjs"]
+    Presets --> IR
+    IR --> JsonOut["extensions.json<br/>src/extractor/output.mjs"]
+    IR --> MarkdownOut["raw/*.md<br/>output.mjs -> markdown-gen.mjs"]
+```
+
+The extractor keeps orchestration and parsing separate:
+
+- `index.mjs` asks `extension-refs.mjs` to collect all configured extension entry points,
+  filters them by blacklist and `--only`, and only then scans each remaining extension.
+- `scan.mjs` coordinates one extension scan; `extension-sources.mjs`, `schema.mjs`, and `record-fields.mjs` own the focused extraction steps.
+- `source-files.mjs` owns file selection rules.
+- `ast.mjs` exposes focused AST scanners from `ast/*.mjs`; `options.mjs`, `examples.mjs`, and `constants.mjs` own their TypeScript AST parsing details.
+- `output.mjs` and `markdown-gen.mjs` own generated artifacts.

infra/docs-gen/README.md

@@ -0,0 +1,55 @@
+# Docs Generator
+
+`infra/docs-gen` contains tooling for two documentation flows:
+
+- building Diplodoc input from `docs/*.md`;
+- extracting raw extension metadata from configured extension entry points.
+
+## Commands
+
+- `pnpm --filter @markdown-editor/docs-gen run build` creates `tmp/docs-src` and builds `dist/docs`.
+- `pnpm --filter @markdown-editor/docs-gen run extract` creates `tmp/docs-gen/extensions.json` and `tmp/docs-gen/raw/*.md`.
+- `pnpm --filter @markdown-editor/docs-gen run test` runs the colocated Node unit tests.
+
+## Files
+
+- `package.json` defines the local docs-gen package, scripts, and Nx target.
+- `EXTRACTION_PIPELINE.md` documents the raw extension extraction flow with a Mermaid diagram.
+- `src/config.mjs` stores shared paths, extension entry points, extension categories, extension blacklist entries, raw docs field sources, preset definitions, and generator constants.
+- `src/extract-extension-data.mjs` provides the CLI entry point for raw extension extraction.
+- `src/generate-docs.mjs` builds the Diplodoc source tree from repository markdown files.
+- `src/logger.mjs` contains the small console logger used by docs-gen CLIs.
+- `src/utils.mjs` contains filesystem helpers shared by the build and extraction flows.
+
+## Extractor Files
+
+- `src/extractor/index.mjs` contains `ExtensionExtractor`, the high-level orchestrator that collects configured extension entry points, enriches records with presets, and writes output.
+- `src/extractor/README.md` maps extractor modules and field ownership in English and Russian.
+- `src/extractor/extension-refs.mjs` collects all configured extension references and applies blacklist and `--only` filters before scanning.
+- `src/extractor/scan.mjs` scans one filtered extension directory and assembles the raw extension IR record from `EXTENSION_DOC_FIELD_CONFIG`.
+- `src/extractor/extension-sources.mjs` reads extension files and prepares source text groups.
+- `src/extractor/schema.mjs` resolves schema node and mark names from spec files.
+- `src/extractor/record-fields.mjs` maps raw IR fields to their extractor functions.
+- `src/extractor/source-files.mjs` selects source, spec, serializer, root index, specs index, and test files from an extension directory.
+- `src/extractor/output.mjs` writes extracted JSON and raw Markdown artifacts.
+- `src/extractor/markdown-gen.mjs` renders one raw extension Markdown file from extracted metadata.
+- `src/extractor/presets.mjs` parses editor preset files and resolves inherited preset membership.
+- `src/extractor/constants.mjs` extracts string constants, enum values, object scalar members, and resolves references between them.
+- `src/extractor/options.mjs` extracts extension option fields through focused modules in `src/extractor/options/`.
+- `src/extractor/examples.mjs` extracts serializer test markup examples from `same(...)` calls and simple local string expressions.
+- `src/extractor/ast.mjs` re-exports all AST helpers and source scanners.
+- `src/extractor/ast/core.mjs` contains generic TypeScript AST utilities.
+- `src/extractor/ast/builder.mjs` recognizes extension builder call chains.
+- `src/extractor/ast/actions.mjs`, `schema.mjs`, `keymaps.mjs`, `plugins.mjs`, `input-rules.mjs`, `md-plugins.mjs`, and `serializer.mjs` extract one metadata family each.
+- `src/extractor/actions.test.mjs` covers action extraction behavior.
+- `src/extractor/cli.test.mjs` covers extraction CLI argument parsing.
+- `src/extractor/constants.test.mjs` covers constant extraction and reference resolution behavior.
+- `src/extractor/examples.test.mjs` covers markup example extraction behavior.
+- `src/extractor/keymaps.test.mjs` covers keymap extraction behavior.
+- `src/extractor/options.test.mjs` covers option declaration extraction behavior.
+
+## Output
+
+- `tmp/docs-gen/extensions.json` is the machine-readable raw IR for all extracted extensions.
+- `tmp/docs-gen/raw/*.md` is the raw Markdown view of each extension record.
+- `tmp/docs-src` is the generated Diplodoc input tree used by the docs build.

infra/docs-gen/package.json

@@ -14,9 +14,12 @@
     }
   },
   "scripts": {
-    "build": "node src/generate-docs.mjs && yfm -i ../../tmp/docs-src -o ../../dist/docs"
+    "build": "node src/generate-docs.mjs && yfm -i ../../tmp/docs-src -o ../../dist/docs",
+    "extract": "node src/extract-extension-data.mjs",
+    "test": "node --test src/extractor/*.test.mjs"
   },
   "dependencies": {
-    "@diplodoc/cli": "5.43.0"
+    "@diplodoc/cli": "5.43.0",
+    "typescript": "catalog:ts"
   }
 }

infra/docs-gen/src/config.mjs

@@ -0,0 +1,99 @@
+/**
+ * English: Shared docs-gen paths, field contracts, extension filters, and preset definitions.
+ *
+ * Русский: Общие пути docs-gen, контракт полей, фильтры расширений и определения пресетов.
+ */
+import {dirname, join, resolve} from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+export const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../..');
+export const DOCS_GEN_DIR = join(REPO_ROOT, 'tmp/docs-gen');
+export const EDITOR_PKG_DIR = join(REPO_ROOT, 'packages/editor');
+export const PAGE_CONSTRUCTOR_EXTENSION_PKG_DIR = join(
+    REPO_ROOT,
+    'packages/page-constructor-extension',
+);
+
+export const EXTENSION_CATEGORIES = ['base', 'behavior', 'markdown', 'yfm', 'additional'];
+
+export const EXTENSION_ENTRY_POINTS = [
+    {
+        id: 'editor',
+        packageName: '@gravity-ui/markdown-editor',
+        packageDir: EDITOR_PKG_DIR,
+        kind: 'category-dirs',
+        extensionsDir: 'src/extensions',
+        categories: EXTENSION_CATEGORIES,
+        presetsDir: 'src/presets',
+    },
+    {
+        id: 'page-constructor-extension',
+        packageName: '@gravity-ui/markdown-editor-page-constructor-extension',
+        packageDir: PAGE_CONSTRUCTOR_EXTENSION_PKG_DIR,
+        kind: 'single-extension',
+        extensionName: 'YfmPageConstructorExtension',
+        extensionDir: 'src/extension',
+        category: 'external',
+    },
+];
+
+export const INTERNAL_EXTENSION_BLACKLIST = [
+    'BaseInputRules',
+    'BaseKeymap',
+    'BaseStyles',
+    'ReactRenderer',
+    'SharedState',
+];
+
+export const EXTENSION_BLACKLIST = [
+    // Example blacklist entry for generated extension docs.
+    'YfmCut',
+];
+
+export const EXTENSION_DOC_FIELD_CONFIG = {
+    name: {from: 'extension directory name'},
+    packageName: {from: 'configured extension entry point package name'},
+    entryPoint: {from: 'configured extension entry point id'},
+    sourcePath: {from: 'extension directory path relative to repository root'},
+    category: {from: 'configured extension category directory'},
+    nodes: {from: 'extension spec files: addNode and addNodeSpec calls'},
+    marks: {from: 'extension spec files: addMark and addMarkSpec calls'},
+    actions: {from: 'extension source files: addAction calls'},
+    keymaps: {from: 'extension source files: addKeymap callback return values'},
+    inputRules: {from: 'extension source files: input-rule factory calls'},
+    plugins: {from: 'extension source files: addPlugin calls'},
+    mdPlugins: {from: 'extension source files: markdown-it md.use calls'},
+    serializerHints: {from: 'serializer source files: state.write and state.text calls'},
+    options: {from: 'extension source files: local *Options TypeScript declarations'},
+    markupExamples: {from: 'extension test files: same(...) serializer examples'},
+    presets: {from: 'editor preset files: preset builder use(...) calls'},
+};
+
+export const PRESET_DEFS = [
+    {name: 'ZeroPreset', file: 'zero.ts', parent: null},
+    {name: 'CommonMarkPreset', file: 'commonmark.ts', parent: 'ZeroPreset'},
+    {name: 'DefaultPreset', file: 'default.ts', parent: 'CommonMarkPreset'},
+    {name: 'YfmPreset', file: 'yfm.ts', parent: 'DefaultPreset'},
+    {name: 'FullPreset', file: 'full.ts', parent: 'YfmPreset'},
+];
+
+/**
+ * Checks whether an extension should be skipped by the raw data extractor.
+ */
+export function isBlacklistedExtension(name) {
+    return [...INTERNAL_EXTENSION_BLACKLIST, ...EXTENSION_BLACKLIST].includes(name);
+}
+
+/**
+ * Creates extension entry points with optional CLI path overrides.
+ */
+export function createExtensionEntryPoints({editorPkg = EDITOR_PKG_DIR} = {}) {
+    return EXTENSION_ENTRY_POINTS.map((entryPoint) => {
+        if (entryPoint.id !== 'editor') return entryPoint;
+
+        return {
+            ...entryPoint,
+            packageDir: editorPkg,
+        };
+    });
+}

infra/docs-gen/src/extract-extension-data.mjs

@@ -0,0 +1,139 @@
+#!/usr/bin/env node
+/**
+ * English: CLI entry point for raw extension metadata extraction.
+ *
+ * Русский: CLI-точка входа для извлечения сырых метаданных расширений.
+ */
+import {isAbsolute, join} from 'node:path';
+import process from 'node:process';
+import {fileURLToPath} from 'node:url';
+
+import {DOCS_GEN_DIR, EDITOR_PKG_DIR, REPO_ROOT, createExtensionEntryPoints} from './config.mjs';
+import {ExtensionExtractor} from './extractor/index.mjs';
+import {logger} from './logger.mjs';
+
+/**
+ * Resolves a path from the repository root.
+ */
+function resolveFromRoot(path) {
+    return isAbsolute(path) ? path : join(REPO_ROOT, path);
+}
+
+/**
+ * Creates default CLI options.
+ */
+function createDefaultOptions() {
+    return {
+        editorPkg: EDITOR_PKG_DIR,
+        outDir: DOCS_GEN_DIR,
+        only: null,
+    };
+}
+
+/**
+ * Reads a required option value.
+ */
+function readOptionValue(args, index, optionName) {
+    const value = args[index + 1];
+    if (!value || value.startsWith('--')) {
+        throw new Error(`Missing value for ${optionName}`);
+    }
+
+    return value;
+}
+
+/**
+ * Parses comma-separated extension names.
+ */
+function parseOnlyOption(value) {
+    return value
+        .split(',')
+        .map((item) => item.trim())
+        .filter(Boolean);
+}
+
+/**
+ * Applies one CLI option to a new options object.
+ */
+function applyOption(opts, args, index) {
+    const arg = args[index];
+
+    switch (arg) {
+        case '--editor-pkg':
+            return {
+                nextIndex: index + 1,
+                opts: {...opts, editorPkg: resolveFromRoot(readOptionValue(args, index, arg))},
+            };
+        case '--out-dir':
+            return {
+                nextIndex: index + 1,
+                opts: {...opts, outDir: resolveFromRoot(readOptionValue(args, index, arg))},
+            };
+        case '--only':
+            return {
+                nextIndex: index + 1,
+                opts: {...opts, only: parseOnlyOption(readOptionValue(args, index, arg))},
+            };
+        case '--help':
+            return {nextIndex: index, opts: {...opts, help: true}};
+        default:
+            throw new Error(`Unknown option: ${arg}`);
+    }
+}
+
+/**
+ * Parses CLI options for extension data extraction.
+ */
+export function parseArgs(args = process.argv.slice(2)) {
+    let opts = createDefaultOptions();
+
+    for (let index = 0; index < args.length; index++) {
+        const arg = args[index];
+        if (arg === '--') return opts;
+
+        const parsedOption = applyOption(opts, args, index);
+        opts = parsedOption.opts;
+        index = parsedOption.nextIndex;
+    }
+
+    return opts;
+}
+
+/**
+ * Prints command usage information.
+ */
+function printHelp() {
+    logger.info('Usage: pnpm --filter @markdown-editor/docs-gen run extract [options]');
+    logger.info('');
+    logger.info('Options:');
+    logger.info('  --only Bold,Link       Extract selected extension names');
+    logger.info('  --out-dir tmp/docs-gen Override output directory');
+    logger.info('  --editor-pkg path      Override configured editor package path');
+}
+
+/**
+ * Runs extension data extraction from CLI arguments.
+ */
+export function main(args = process.argv.slice(2)) {
+    const opts = parseArgs(args);
+    if (opts.help) {
+        printHelp();
+        return;
+    }
+
+    new ExtensionExtractor({
+        entryPoints: createExtensionEntryPoints({editorPkg: opts.editorPkg}),
+        outDir: opts.outDir,
+        repoRoot: REPO_ROOT,
+        versionPackageDir: opts.editorPkg,
+    }).run({only: opts.only});
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
+    try {
+        main();
+    } catch (error) {
+        logger.error(error);
+        process.exit(1);
+    }
+}