diff --git a/infra/docs-gen/EXTRACTION_PIPELINE.md b/infra/docs-gen/EXTRACTION_PIPELINE.md new file mode 100644 index 00000000..aeda0325 --- /dev/null +++ b/infra/docs-gen/EXTRACTION_PIPELINE.md @@ -0,0 +1,48 @@ +# Extension Extraction Pipeline + +```mermaid +flowchart TD + CLI["extract-extension-data.mjs
Parse CLI options"] --> Extractor["ExtensionExtractor
src/extractor/index.mjs"] + Extractor --> EntryPoints["Extension entry points
EXTENSION_ENTRY_POINTS"] + EntryPoints --> Collect["collectExtensionRefs()
src/extractor/extension-refs.mjs"] + Collect --> AllRefs["Full extension ref list"] + AllRefs --> Filter["filterExtensionRefs()
Apply blacklist and --only"] + Filter --> Scan["scanExtension()
src/extractor/scan.mjs"] + + Scan --> FieldConfig["Docs field config
EXTENSION_DOC_FIELD_CONFIG"] + Scan --> Files["readExtensionSources()
src/extractor/extension-sources.mjs"] + Files --> Selectors["Source file selectors
src/extractor/source-files.mjs"] + Selectors --> SourceText["Production source text"] + Selectors --> TestFiles["Test files"] + Selectors --> SerializerFiles["Serializer and Specs files"] + + SourceText --> Constants["extractConstants()
src/extractor/constants.mjs"] + SourceText --> AstBarrel["AST extractor API
src/extractor/ast.mjs"] + AstBarrel --> AstScanners["Focused AST scanners
src/extractor/ast/*.mjs"] + SourceText --> Options["Option declarations
src/extractor/options.mjs"] + SerializerFiles --> SerializerHints["Serializer hints"] + TestFiles --> MarkupExamples["Markup examples
src/extractor/examples.mjs"] + + Constants --> Schema["Schema names
src/extractor/schema.mjs"] + AstScanners --> ExtractedFields["Actions, keymaps, input rules,
plugins, md plugins"] + Options --> ExtractedFields + FieldConfig --> IR["Extension IR record"] + Schema --> IR + ExtractedFields --> IR + SerializerHints --> IR + MarkupExamples --> IR + + Extractor --> Presets["Preset membership
src/extractor/presets.mjs"] + Presets --> IR + IR --> JsonOut["extensions.json
src/extractor/output.mjs"] + IR --> MarkdownOut["raw/*.md
output.mjs -> markdown-gen.mjs"] +``` + +The extractor keeps orchestration and parsing separate: + +- `index.mjs` asks `extension-refs.mjs` to collect all configured extension entry points, + filters them by blacklist and `--only`, and only then scans each remaining extension. +- `scan.mjs` coordinates one extension scan; `extension-sources.mjs`, `schema.mjs`, and `record-fields.mjs` own the focused extraction steps. +- `source-files.mjs` owns file selection rules. +- `ast.mjs` exposes focused AST scanners from `ast/*.mjs`; `options.mjs`, `examples.mjs`, and `constants.mjs` own their TypeScript AST parsing details. +- `output.mjs` and `markdown-gen.mjs` own generated artifacts. diff --git a/infra/docs-gen/README.md b/infra/docs-gen/README.md new file mode 100644 index 00000000..48e7b893 --- /dev/null +++ b/infra/docs-gen/README.md @@ -0,0 +1,55 @@ +# Docs Generator + +`infra/docs-gen` contains tooling for two documentation flows: + +- building Diplodoc input from `docs/*.md`; +- extracting raw extension metadata from configured extension entry points. + +## Commands + +- `pnpm --filter @markdown-editor/docs-gen run build` creates `tmp/docs-src` and builds `dist/docs`. +- `pnpm --filter @markdown-editor/docs-gen run extract` creates `tmp/docs-gen/extensions.json` and `tmp/docs-gen/raw/*.md`. +- `pnpm --filter @markdown-editor/docs-gen run test` runs the colocated Node unit tests. + +## Files + +- `package.json` defines the local docs-gen package, scripts, and Nx target. +- `EXTRACTION_PIPELINE.md` documents the raw extension extraction flow with a Mermaid diagram. +- `src/config.mjs` stores shared paths, extension entry points, extension categories, extension blacklist entries, raw docs field sources, preset definitions, and generator constants. +- `src/extract-extension-data.mjs` provides the CLI entry point for raw extension extraction. +- `src/generate-docs.mjs` builds the Diplodoc source tree from repository markdown files. +- `src/logger.mjs` contains the small console logger used by docs-gen CLIs. +- `src/utils.mjs` contains filesystem helpers shared by the build and extraction flows. + +## Extractor Files + +- `src/extractor/index.mjs` contains `ExtensionExtractor`, the high-level orchestrator that collects configured extension entry points, enriches records with presets, and writes output. +- `src/extractor/README.md` maps extractor modules and field ownership in English and Russian. +- `src/extractor/extension-refs.mjs` collects all configured extension references and applies blacklist and `--only` filters before scanning. +- `src/extractor/scan.mjs` scans one filtered extension directory and assembles the raw extension IR record from `EXTENSION_DOC_FIELD_CONFIG`. +- `src/extractor/extension-sources.mjs` reads extension files and prepares source text groups. +- `src/extractor/schema.mjs` resolves schema node and mark names from spec files. +- `src/extractor/record-fields.mjs` maps raw IR fields to their extractor functions. +- `src/extractor/source-files.mjs` selects source, spec, serializer, root index, specs index, and test files from an extension directory. +- `src/extractor/output.mjs` writes extracted JSON and raw Markdown artifacts. +- `src/extractor/markdown-gen.mjs` renders one raw extension Markdown file from extracted metadata. +- `src/extractor/presets.mjs` parses editor preset files and resolves inherited preset membership. +- `src/extractor/constants.mjs` extracts string constants, enum values, object scalar members, and resolves references between them. +- `src/extractor/options.mjs` extracts extension option fields through focused modules in `src/extractor/options/`. +- `src/extractor/examples.mjs` extracts serializer test markup examples from `same(...)` calls and simple local string expressions. +- `src/extractor/ast.mjs` re-exports all AST helpers and source scanners. +- `src/extractor/ast/core.mjs` contains generic TypeScript AST utilities. +- `src/extractor/ast/builder.mjs` recognizes extension builder call chains. +- `src/extractor/ast/actions.mjs`, `schema.mjs`, `keymaps.mjs`, `plugins.mjs`, `input-rules.mjs`, `md-plugins.mjs`, and `serializer.mjs` extract one metadata family each. +- `src/extractor/actions.test.mjs` covers action extraction behavior. +- `src/extractor/cli.test.mjs` covers extraction CLI argument parsing. +- `src/extractor/constants.test.mjs` covers constant extraction and reference resolution behavior. +- `src/extractor/examples.test.mjs` covers markup example extraction behavior. +- `src/extractor/keymaps.test.mjs` covers keymap extraction behavior. +- `src/extractor/options.test.mjs` covers option declaration extraction behavior. + +## Output + +- `tmp/docs-gen/extensions.json` is the machine-readable raw IR for all extracted extensions. +- `tmp/docs-gen/raw/*.md` is the raw Markdown view of each extension record. +- `tmp/docs-src` is the generated Diplodoc input tree used by the docs build. diff --git a/infra/docs-gen/package.json b/infra/docs-gen/package.json index de6aa64c..404f34ac 100644 --- a/infra/docs-gen/package.json +++ b/infra/docs-gen/package.json @@ -14,9 +14,12 @@ } }, "scripts": { - "build": "node src/generate-docs.mjs && yfm -i ../../tmp/docs-src -o ../../dist/docs" + "build": "node src/generate-docs.mjs && yfm -i ../../tmp/docs-src -o ../../dist/docs", + "extract": "node src/extract-extension-data.mjs", + "test": "node --test src/extractor/*.test.mjs" }, "dependencies": { - "@diplodoc/cli": "5.43.0" + "@diplodoc/cli": "5.43.0", + "typescript": "catalog:ts" } } diff --git a/infra/docs-gen/src/config.mjs b/infra/docs-gen/src/config.mjs new file mode 100644 index 00000000..15e16424 --- /dev/null +++ b/infra/docs-gen/src/config.mjs @@ -0,0 +1,99 @@ +/** + * English: Shared docs-gen paths, field contracts, extension filters, and preset definitions. + * + * Русский: Общие пути docs-gen, контракт полей, фильтры расширений и определения пресетов. + */ +import {dirname, join, resolve} from 'node:path'; +import {fileURLToPath} from 'node:url'; + +export const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +export const DOCS_GEN_DIR = join(REPO_ROOT, 'tmp/docs-gen'); +export const EDITOR_PKG_DIR = join(REPO_ROOT, 'packages/editor'); +export const PAGE_CONSTRUCTOR_EXTENSION_PKG_DIR = join( + REPO_ROOT, + 'packages/page-constructor-extension', +); + +export const EXTENSION_CATEGORIES = ['base', 'behavior', 'markdown', 'yfm', 'additional']; + +export const EXTENSION_ENTRY_POINTS = [ + { + id: 'editor', + packageName: '@gravity-ui/markdown-editor', + packageDir: EDITOR_PKG_DIR, + kind: 'category-dirs', + extensionsDir: 'src/extensions', + categories: EXTENSION_CATEGORIES, + presetsDir: 'src/presets', + }, + { + id: 'page-constructor-extension', + packageName: '@gravity-ui/markdown-editor-page-constructor-extension', + packageDir: PAGE_CONSTRUCTOR_EXTENSION_PKG_DIR, + kind: 'single-extension', + extensionName: 'YfmPageConstructorExtension', + extensionDir: 'src/extension', + category: 'external', + }, +]; + +export const INTERNAL_EXTENSION_BLACKLIST = [ + 'BaseInputRules', + 'BaseKeymap', + 'BaseStyles', + 'ReactRenderer', + 'SharedState', +]; + +export const EXTENSION_BLACKLIST = [ + // Example blacklist entry for generated extension docs. + 'YfmCut', +]; + +export const EXTENSION_DOC_FIELD_CONFIG = { + name: {from: 'extension directory name'}, + packageName: {from: 'configured extension entry point package name'}, + entryPoint: {from: 'configured extension entry point id'}, + sourcePath: {from: 'extension directory path relative to repository root'}, + category: {from: 'configured extension category directory'}, + nodes: {from: 'extension spec files: addNode and addNodeSpec calls'}, + marks: {from: 'extension spec files: addMark and addMarkSpec calls'}, + actions: {from: 'extension source files: addAction calls'}, + keymaps: {from: 'extension source files: addKeymap callback return values'}, + inputRules: {from: 'extension source files: input-rule factory calls'}, + plugins: {from: 'extension source files: addPlugin calls'}, + mdPlugins: {from: 'extension source files: markdown-it md.use calls'}, + serializerHints: {from: 'serializer source files: state.write and state.text calls'}, + options: {from: 'extension source files: local *Options TypeScript declarations'}, + markupExamples: {from: 'extension test files: same(...) serializer examples'}, + presets: {from: 'editor preset files: preset builder use(...) calls'}, +}; + +export const PRESET_DEFS = [ + {name: 'ZeroPreset', file: 'zero.ts', parent: null}, + {name: 'CommonMarkPreset', file: 'commonmark.ts', parent: 'ZeroPreset'}, + {name: 'DefaultPreset', file: 'default.ts', parent: 'CommonMarkPreset'}, + {name: 'YfmPreset', file: 'yfm.ts', parent: 'DefaultPreset'}, + {name: 'FullPreset', file: 'full.ts', parent: 'YfmPreset'}, +]; + +/** + * Checks whether an extension should be skipped by the raw data extractor. + */ +export function isBlacklistedExtension(name) { + return [...INTERNAL_EXTENSION_BLACKLIST, ...EXTENSION_BLACKLIST].includes(name); +} + +/** + * Creates extension entry points with optional CLI path overrides. + */ +export function createExtensionEntryPoints({editorPkg = EDITOR_PKG_DIR} = {}) { + return EXTENSION_ENTRY_POINTS.map((entryPoint) => { + if (entryPoint.id !== 'editor') return entryPoint; + + return { + ...entryPoint, + packageDir: editorPkg, + }; + }); +} diff --git a/infra/docs-gen/src/extract-extension-data.mjs b/infra/docs-gen/src/extract-extension-data.mjs new file mode 100644 index 00000000..6fb03bfd --- /dev/null +++ b/infra/docs-gen/src/extract-extension-data.mjs @@ -0,0 +1,139 @@ +#!/usr/bin/env node +/** + * English: CLI entry point for raw extension metadata extraction. + * + * Русский: CLI-точка входа для извлечения сырых метаданных расширений. + */ +import {isAbsolute, join} from 'node:path'; +import process from 'node:process'; +import {fileURLToPath} from 'node:url'; + +import {DOCS_GEN_DIR, EDITOR_PKG_DIR, REPO_ROOT, createExtensionEntryPoints} from './config.mjs'; +import {ExtensionExtractor} from './extractor/index.mjs'; +import {logger} from './logger.mjs'; + +/** + * Resolves a path from the repository root. + */ +function resolveFromRoot(path) { + return isAbsolute(path) ? path : join(REPO_ROOT, path); +} + +/** + * Creates default CLI options. + */ +function createDefaultOptions() { + return { + editorPkg: EDITOR_PKG_DIR, + outDir: DOCS_GEN_DIR, + only: null, + }; +} + +/** + * Reads a required option value. + */ +function readOptionValue(args, index, optionName) { + const value = args[index + 1]; + if (!value || value.startsWith('--')) { + throw new Error(`Missing value for ${optionName}`); + } + + return value; +} + +/** + * Parses comma-separated extension names. + */ +function parseOnlyOption(value) { + return value + .split(',') + .map((item) => item.trim()) + .filter(Boolean); +} + +/** + * Applies one CLI option to a new options object. + */ +function applyOption(opts, args, index) { + const arg = args[index]; + + switch (arg) { + case '--editor-pkg': + return { + nextIndex: index + 1, + opts: {...opts, editorPkg: resolveFromRoot(readOptionValue(args, index, arg))}, + }; + case '--out-dir': + return { + nextIndex: index + 1, + opts: {...opts, outDir: resolveFromRoot(readOptionValue(args, index, arg))}, + }; + case '--only': + return { + nextIndex: index + 1, + opts: {...opts, only: parseOnlyOption(readOptionValue(args, index, arg))}, + }; + case '--help': + return {nextIndex: index, opts: {...opts, help: true}}; + default: + throw new Error(`Unknown option: ${arg}`); + } +} + +/** + * Parses CLI options for extension data extraction. + */ +export function parseArgs(args = process.argv.slice(2)) { + let opts = createDefaultOptions(); + + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg === '--') return opts; + + const parsedOption = applyOption(opts, args, index); + opts = parsedOption.opts; + index = parsedOption.nextIndex; + } + + return opts; +} + +/** + * Prints command usage information. + */ +function printHelp() { + logger.info('Usage: pnpm --filter @markdown-editor/docs-gen run extract [options]'); + logger.info(''); + logger.info('Options:'); + logger.info(' --only Bold,Link Extract selected extension names'); + logger.info(' --out-dir tmp/docs-gen Override output directory'); + logger.info(' --editor-pkg path Override configured editor package path'); +} + +/** + * Runs extension data extraction from CLI arguments. + */ +export function main(args = process.argv.slice(2)) { + const opts = parseArgs(args); + if (opts.help) { + printHelp(); + return; + } + + new ExtensionExtractor({ + entryPoints: createExtensionEntryPoints({editorPkg: opts.editorPkg}), + outDir: opts.outDir, + repoRoot: REPO_ROOT, + versionPackageDir: opts.editorPkg, + }).run({only: opts.only}); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + try { + main(); + } catch (error) { + logger.error(error); + process.exit(1); + } +} diff --git a/infra/docs-gen/src/extractor/README.md b/infra/docs-gen/src/extractor/README.md new file mode 100644 index 00000000..17398663 --- /dev/null +++ b/infra/docs-gen/src/extractor/README.md @@ -0,0 +1,67 @@ +# Extractor Module Guide + +## English + +`infra/docs-gen/src/extractor` collects raw extension metadata for documentation. The +pipeline intentionally keeps each step narrow: + +- `index.mjs` orchestrates discovery, filtering, scanning, preset enrichment, and output. +- `extension-refs.mjs` collects all configured package entry points, then applies blacklist + and `--only` filters before any extension scan starts. +- `scan.mjs` builds one extension record from `EXTENSION_DOC_FIELD_CONFIG`. +- `extension-sources.mjs` reads extension source files and builds source text groups. +- `schema.mjs` resolves schema node and mark names from spec source. +- `record-fields.mjs` maps configured raw fields to focused extractor functions. +- `source-files.mjs` chooses source, spec, serializer, and test files. +- `ast.mjs` is the public barrel for AST-based extraction helpers. +- `ast/core.mjs` contains generic TypeScript AST utilities. +- `ast/builder.mjs` recognizes extension builder call chains. +- `ast/actions.mjs`, `ast/schema.mjs`, `ast/keymaps.mjs`, `ast/plugins.mjs`, + `ast/input-rules.mjs`, `ast/md-plugins.mjs`, and `ast/serializer.mjs` extract one + metadata family each. +- `constants.mjs` resolves local string constants, enums, and scalar object members. +- `options.mjs` is the public entry for local `*Options` TypeScript declarations. +- `options/declarations.mjs`, `options/fields.mjs`, and `options/resolve.mjs` split option + parsing, field shaping, and reference resolution. +- `examples.mjs` extracts serializer examples from `same(...)` test helpers. +- `presets.mjs` maps extensions to editor presets. +- `markdown-gen.mjs` and `output.mjs` render and write the raw artifacts. + +`EXTENSION_ENTRY_POINTS` in `../config.mjs` defines where extension entry points come +from. It currently covers the editor package category directories and the +page-constructor extension package. `EXTENSION_DOC_FIELD_CONFIG` is the contract for +which fields are written and where each field comes from. Add a field there first, then +add the matching extractor in `record-fields.mjs`. + +## Русский + +`infra/docs-gen/src/extractor` собирает сырые метаданные расширений для документации. +Пайплайн специально разбит на узкие шаги: + +- `index.mjs` управляет обнаружением, фильтрацией, сканированием, пресетами и выводом. +- `extension-refs.mjs` собирает все сконфигурированные package entry points, затем + применяет blacklist и `--only` до запуска scan по расширениям. +- `scan.mjs` собирает одну запись расширения по `EXTENSION_DOC_FIELD_CONFIG`. +- `extension-sources.mjs` читает source files расширения и собирает группы source text. +- `schema.mjs` резолвит имена schema node и mark из spec source. +- `record-fields.mjs` связывает сконфигурированные raw fields с узкими extractor-функциями. +- `source-files.mjs` выбирает source, spec, serializer и test файлы. +- `ast.mjs` публично экспортирует AST-based helper-ы. +- `ast/core.mjs` содержит общие утилиты TypeScript AST. +- `ast/builder.mjs` распознает цепочки вызовов extension builder. +- `ast/actions.mjs`, `ast/schema.mjs`, `ast/keymaps.mjs`, `ast/plugins.mjs`, + `ast/input-rules.mjs`, `ast/md-plugins.mjs` и `ast/serializer.mjs` извлекают по + одной группе метаданных. +- `constants.mjs` резолвит локальные строковые константы, enum-ы и scalar object members. +- `options.mjs` публично запускает разбор локальных TypeScript declarations вида `*Options`. +- `options/declarations.mjs`, `options/fields.mjs` и `options/resolve.mjs` разделяют + parsing declarations, формирование fields и reference resolution. +- `examples.mjs` извлекает serializer examples из test helper-ов `same(...)`. +- `presets.mjs` сопоставляет расширения с editor presets. +- `markdown-gen.mjs` и `output.mjs` рендерят и записывают raw artifacts. + +`EXTENSION_ENTRY_POINTS` в `../config.mjs` задает, откуда брать extension entry points. +Сейчас там есть category directories пакета editor и отдельный пакет page-constructor +extension. `EXTENSION_DOC_FIELD_CONFIG` задает контракт: какие поля пишутся и откуда +каждое поле берется. Новое поле сначала добавляется туда, затем для него добавляется +extractor в `record-fields.mjs`. diff --git a/infra/docs-gen/src/extractor/actions.test.mjs b/infra/docs-gen/src/extractor/actions.test.mjs new file mode 100644 index 00000000..46c9269c --- /dev/null +++ b/infra/docs-gen/src/extractor/actions.test.mjs @@ -0,0 +1,30 @@ +/** + * English: Unit coverage for AST-based action extraction. + * + * Русский: Unit-покрытие AST-based извлечения actions. + */ +import assert from 'node:assert/strict'; +import {test} from 'node:test'; + +import {extractActions} from './ast.mjs'; + +test('extractActions captures direct and chained builder.addAction calls', () => { + const content = [ + "builder.addAction('bold', () => boldAction);", + 'builder', + ' .addAction(BoldAction.Toggle, () => toggle)', + ' .addAction(BoldAction.Off, () => off);', + ].join('\n'); + + assert.deepEqual(extractActions(content), ['bold', 'BoldAction.Toggle', 'BoldAction.Off']); +}); + +test('extractActions ignores non-builder addAction calls', () => { + const content = [ + "tr.addAction('shouldNotMatch', cb);", + "service.addAction('alsoSkip', cb);", + "builder.addAction('keepMe', cb);", + ].join('\n'); + + assert.deepEqual(extractActions(content), ['keepMe']); +}); diff --git a/infra/docs-gen/src/extractor/ast.mjs b/infra/docs-gen/src/extractor/ast.mjs new file mode 100644 index 00000000..52525c46 --- /dev/null +++ b/infra/docs-gen/src/extractor/ast.mjs @@ -0,0 +1,22 @@ +/** + * English: Barrel module that exposes all AST-based extractor helpers. + * + * Русский: Barrel-модуль, экспортирующий все AST-based helper-ы extractor-а. + */ +export {extractActions} from './ast/actions.mjs'; +export { + forEachNode, + getCallPropertyName, + getExpressionName, + getStaticPropertyName, + getStringValue, + parseSource, + unique, + unwrapExpression, +} from './ast/core.mjs'; +export {extractInputRules} from './ast/input-rules.mjs'; +export {extractKeymaps} from './ast/keymaps.mjs'; +export {extractMdPlugins} from './ast/md-plugins.mjs'; +export {extractPlugins} from './ast/plugins.mjs'; +export {extractSerializerSyntax} from './ast/serializer.mjs'; +export {extractAddMark, extractAddNode, extractMarkSpecs, extractNodeSpecs} from './ast/schema.mjs'; diff --git a/infra/docs-gen/src/extractor/ast/actions.mjs b/infra/docs-gen/src/extractor/ast/actions.mjs new file mode 100644 index 00000000..24194403 --- /dev/null +++ b/infra/docs-gen/src/extractor/ast/actions.mjs @@ -0,0 +1,13 @@ +/** + * English: AST scanner for editor action identifiers registered by extensions. + * + * Русский: AST-сканер идентификаторов editor actions, регистрируемых расширениями. + */ +import {extractBuilderCallFirstArgs} from './builder.mjs'; + +/** + * Extracts editor action identifiers from builder calls. + */ +export function extractActions(content) { + return extractBuilderCallFirstArgs(content, new Set(['addAction'])); +} diff --git a/infra/docs-gen/src/extractor/ast/builder.mjs b/infra/docs-gen/src/extractor/ast/builder.mjs new file mode 100644 index 00000000..937f9b2d --- /dev/null +++ b/infra/docs-gen/src/extractor/ast/builder.mjs @@ -0,0 +1,69 @@ +/** + * English: Helpers for recognizing extension builder call chains in TypeScript AST. + * + * Русский: Хелперы для распознавания цепочек вызовов extension builder в TypeScript AST. + */ +import ts from 'typescript'; + +import {forEachNode, getExpressionName, parseSource, unique, unwrapExpression} from './core.mjs'; + +const BUILDER_ROOT_NAME = 'builder'; + +const BUILDER_CHAIN_METHODS = new Set([ + 'addAction', + 'addKeymap', + 'addMark', + 'addMarkSpec', + 'addMarkdownTokenParserSpec', + 'addNode', + 'addNodeSerializerSpec', + 'addNodeSpec', + 'addPlugin', + 'configureMd', +]); + +/** + * Checks whether an expression is the root builder or a builder call chain. + */ +function isBuilderExpression(expression) { + const current = unwrapExpression(expression); + + if (ts.isIdentifier(current)) return current.text === BUILDER_ROOT_NAME; + + if (!ts.isCallExpression(current)) return false; + if (!ts.isPropertyAccessExpression(current.expression)) return false; + if (!BUILDER_CHAIN_METHODS.has(current.expression.name.text)) return false; + + return isBuilderExpression(current.expression.expression); +} + +/** + * Checks whether a call is made on the extension builder chain. + */ +export function isBuilderMethodCall(callExpression, methodNames) { + const expression = unwrapExpression(callExpression.expression); + + return ( + ts.isPropertyAccessExpression(expression) && + methodNames.has(expression.name.text) && + isBuilderExpression(expression.expression) + ); +} + +/** + * Extracts first arguments from extension builder method calls. + */ +export function extractBuilderCallFirstArgs(content, methodNames) { + const sourceFile = parseSource(content); + const names = []; + + forEachNode(sourceFile, (node) => { + if (!ts.isCallExpression(node) || !isBuilderMethodCall(node, methodNames)) return; + + const firstArg = node.arguments[0]; + const name = firstArg ? getExpressionName(firstArg, sourceFile) : null; + if (name) names.push(name); + }); + + return unique(names); +} diff --git a/infra/docs-gen/src/extractor/ast/core.mjs b/infra/docs-gen/src/extractor/ast/core.mjs new file mode 100644 index 00000000..80f880d3 --- /dev/null +++ b/infra/docs-gen/src/extractor/ast/core.mjs @@ -0,0 +1,111 @@ +/** + * English: Shared TypeScript AST primitives used by docs-gen source scanners. + * + * Русский: Общие примитивы TypeScript AST для сканеров исходников docs-gen. + */ +import ts from 'typescript'; + +/** + * Parses TypeScript or TSX source into a traversable AST. + */ +export function parseSource(content, fileName = 'source.tsx') { + return ts.createSourceFile(fileName, content, ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX); +} + +/** + * Visits every AST node depth-first. + */ +export function forEachNode(root, callback) { + const visit = (node) => { + callback(node); + ts.forEachChild(node, visit); + }; + + visit(root); +} + +/** + * Deduplicates values while preserving source order. + */ +export function unique(values) { + return [...new Set(values.filter(Boolean))]; +} + +/** + * Removes syntax wrappers that do not affect a static expression value. + */ +export function unwrapExpression(expression) { + let current = expression; + + while ( + ts.isParenthesizedExpression(current) || + ts.isAsExpression(current) || + ts.isSatisfiesExpression(current) || + ts.isNonNullExpression(current) || + ts.isTypeAssertionExpression(current) + ) { + current = current.expression; + } + + return current; +} + +/** + * Reads a static property name from an object-like AST node. + */ +export function getStaticPropertyName(name, {allowComputedLiteral = false} = {}) { + if (ts.isIdentifier(name) || ts.isStringLiteral(name) || ts.isNumericLiteral(name)) { + return name.text; + } + + if ( + allowComputedLiteral && + ts.isComputedPropertyName(name) && + ts.isStringLiteralLike(unwrapExpression(name.expression)) + ) { + return unwrapExpression(name.expression).text; + } + + return null; +} + +/** + * Resolves syntax that names a constant, enum member, or string literal. + */ +export function getExpressionName(expression, sourceFile) { + const current = unwrapExpression(expression); + + if (ts.isStringLiteralLike(current)) return current.text; + if (ts.isIdentifier(current)) return current.text; + + if (ts.isPropertyAccessExpression(current)) { + const baseName = getExpressionName(current.expression, sourceFile); + return baseName ? `${baseName}.${current.name.text}` : current.getText(sourceFile).trim(); + } + + if ( + ts.isElementAccessExpression(current) && + ts.isStringLiteralLike(current.argumentExpression) + ) { + const baseName = getExpressionName(current.expression, sourceFile); + return baseName ? `${baseName}.${current.argumentExpression.text}` : null; + } + + return null; +} + +/** + * Resolves a literal string expression. + */ +export function getStringValue(expression) { + const current = unwrapExpression(expression); + return ts.isStringLiteralLike(current) ? current.text : null; +} + +/** + * Returns the called property name for a call expression. + */ +export function getCallPropertyName(callExpression) { + const expression = unwrapExpression(callExpression.expression); + return ts.isPropertyAccessExpression(expression) ? expression.name.text : null; +} diff --git a/infra/docs-gen/src/extractor/ast/factory.mjs b/infra/docs-gen/src/extractor/ast/factory.mjs new file mode 100644 index 00000000..ca28b957 --- /dev/null +++ b/infra/docs-gen/src/extractor/ast/factory.mjs @@ -0,0 +1,48 @@ +/** + * English: Static naming helpers for plugin and factory callback expressions. + * + * Русский: Хелперы статического именования plugin/factory callback выражений. + */ +import ts from 'typescript'; + +import {getExpressionName, unwrapExpression} from './core.mjs'; + +/** + * Reads a returned expression from a function-like callback. + */ +function getStaticReturnExpression(callback) { + const body = callback.body; + if (!ts.isBlock(body)) return unwrapExpression(body); + + for (const statement of body.statements) { + if (ts.isReturnStatement(statement)) return statement.expression || null; + } + + return null; +} + +/** + * Describes a factory callback or plugin expression with a stable identifier. + */ +export function describeFactoryExpression(expression, sourceFile) { + const current = unwrapExpression(expression); + + if (ts.isIdentifier(current) || ts.isPropertyAccessExpression(current)) { + return getExpressionName(current, sourceFile); + } + + if (ts.isCallExpression(current)) { + return getExpressionName(current.expression, sourceFile); + } + + if (ts.isNewExpression(current)) { + return getExpressionName(current.expression, sourceFile); + } + + if (ts.isArrowFunction(current) || ts.isFunctionExpression(current)) { + const returned = getStaticReturnExpression(current); + return returned ? describeFactoryExpression(returned, sourceFile) : null; + } + + return null; +} diff --git a/infra/docs-gen/src/extractor/ast/input-rules.mjs b/infra/docs-gen/src/extractor/ast/input-rules.mjs new file mode 100644 index 00000000..336311c1 --- /dev/null +++ b/infra/docs-gen/src/extractor/ast/input-rules.mjs @@ -0,0 +1,69 @@ +/** + * English: AST scanner for markdown input-rule syntax hints. + * + * Русский: AST-сканер markdown input-rule подсказок синтаксиса. + */ +import ts from 'typescript'; + +import { + forEachNode, + getStaticPropertyName, + getStringValue, + parseSource, + unique, + unwrapExpression, +} from './core.mjs'; + +const INPUT_RULE_FACTORIES = new Set([ + 'inlineNodeInputRule', + 'markInputRule', + 'nodeInputRule', + 'textblockTypeInputRule', + 'wrappingInputRule', +]); + +/** + * Extracts one input-rule syntax descriptor from a factory call. + */ +function describeInputRuleCall(callExpression) { + const firstArg = callExpression.arguments[0]; + if (!firstArg) return null; + + const current = unwrapExpression(firstArg); + if (ts.isObjectLiteralExpression(current)) { + let open = null; + let close = null; + + for (const property of current.properties) { + if (!ts.isPropertyAssignment(property)) continue; + + const key = getStaticPropertyName(property.name); + if (key === 'open') open = getStringValue(property.initializer); + if (key === 'close') close = getStringValue(property.initializer); + } + + return open !== null && close !== null ? `${open}...${close}` : null; + } + + return null; +} + +/** + * Extracts input-rule syntax patterns. + */ +export function extractInputRules(content) { + const sourceFile = parseSource(content); + const rules = []; + + forEachNode(sourceFile, (node) => { + if (!ts.isCallExpression(node)) return; + + const expression = unwrapExpression(node.expression); + if (!ts.isIdentifier(expression) || !INPUT_RULE_FACTORIES.has(expression.text)) return; + + const rule = describeInputRuleCall(node); + if (rule) rules.push(rule); + }); + + return unique(rules); +} diff --git a/infra/docs-gen/src/extractor/ast/keymaps.mjs b/infra/docs-gen/src/extractor/ast/keymaps.mjs new file mode 100644 index 00000000..b03e7455 --- /dev/null +++ b/infra/docs-gen/src/extractor/ast/keymaps.mjs @@ -0,0 +1,153 @@ +/** + * English: AST scanner for static key bindings returned from addKeymap callbacks. + * + * Русский: AST-сканер статичных key bindings из callback-ов addKeymap. + */ +import ts from 'typescript'; + +import {isBuilderMethodCall} from './builder.mjs'; +import { + forEachNode, + getStaticPropertyName, + parseSource, + unique, + unwrapExpression, +} from './core.mjs'; + +/** + * Extracts static keys from an object literal. + */ +function extractObjectLiteralKeys(objectLiteral, knownObjects = new Map()) { + const keys = []; + + for (const property of objectLiteral.properties) { + if (ts.isSpreadAssignment(property)) { + const spreadName = ts.isIdentifier(property.expression) + ? property.expression.text + : null; + if (spreadName && knownObjects.has(spreadName)) { + keys.push(...knownObjects.get(spreadName)); + } + continue; + } + + if (!ts.isPropertyAssignment(property) && !ts.isShorthandPropertyAssignment(property)) { + continue; + } + + const key = getStaticPropertyName(property.name); + if (key) keys.push(key); + } + + return keys; +} + +/** + * Reads object-literal keys from a return expression or local object reference. + */ +function extractKeymapExpressionKeys(expression, knownObjects = new Map()) { + const current = unwrapExpression(expression); + + if (ts.isObjectLiteralExpression(current)) { + return extractObjectLiteralKeys(current, knownObjects); + } + + if (ts.isIdentifier(current) && knownObjects.has(current.text)) { + return knownObjects.get(current.text); + } + + return []; +} + +/** + * Registers one top-level object literal binding from a callback block. + */ +function collectObjectBinding(statement, knownObjects) { + if (!ts.isVariableStatement(statement)) return; + + for (const declaration of statement.declarationList.declarations) { + if (!ts.isIdentifier(declaration.name) || !declaration.initializer) continue; + + const initializer = unwrapExpression(declaration.initializer); + if (ts.isObjectLiteralExpression(initializer)) { + knownObjects.set( + declaration.name.text, + unique(extractObjectLiteralKeys(initializer, knownObjects)), + ); + } + } +} + +/** + * Registers a static key assignment into a known object binding. + */ +function collectObjectAssignment(statement, knownObjects) { + if (!ts.isExpressionStatement(statement)) return; + + const expression = unwrapExpression(statement.expression); + if ( + !ts.isBinaryExpression(expression) || + expression.operatorToken.kind !== ts.SyntaxKind.EqualsToken + ) { + return; + } + + const left = unwrapExpression(expression.left); + let objectName = null; + let keyName = null; + + if (ts.isPropertyAccessExpression(left)) { + objectName = ts.isIdentifier(left.expression) ? left.expression.text : null; + keyName = left.name.text; + } else if (ts.isElementAccessExpression(left)) { + objectName = ts.isIdentifier(left.expression) ? left.expression.text : null; + keyName = ts.isStringLiteralLike(left.argumentExpression) + ? left.argumentExpression.text + : null; + } + + if (!objectName || !keyName || !knownObjects.has(objectName)) return; + + knownObjects.set(objectName, unique([...knownObjects.get(objectName), keyName])); +} + +/** + * Extracts static key bindings from an addKeymap callback. + */ +function extractKeymapCallbackKeys(callback) { + const body = unwrapExpression(callback.body); + if (!ts.isBlock(body)) return extractKeymapExpressionKeys(body); + + const knownObjects = new Map(); + for (const statement of body.statements) { + collectObjectBinding(statement, knownObjects); + collectObjectAssignment(statement, knownObjects); + + if (ts.isReturnStatement(statement) && statement.expression) { + return extractKeymapExpressionKeys(statement.expression, knownObjects); + } + } + + return []; +} + +/** + * Extracts static key bindings from addKeymap callbacks. + */ +export function extractKeymaps(content) { + const sourceFile = parseSource(content); + const keymaps = []; + + forEachNode(sourceFile, (node) => { + if (!ts.isCallExpression(node) || !isBuilderMethodCall(node, new Set(['addKeymap']))) { + return; + } + + const callback = unwrapExpression(node.arguments[0]); + if (callback && (ts.isArrowFunction(callback) || ts.isFunctionExpression(callback))) { + keymaps.push(...extractKeymapCallbackKeys(callback)); + } + }); + + return unique(keymaps); +} diff --git a/infra/docs-gen/src/extractor/ast/md-plugins.mjs b/infra/docs-gen/src/extractor/ast/md-plugins.mjs new file mode 100644 index 00000000..88b7410b --- /dev/null +++ b/infra/docs-gen/src/extractor/ast/md-plugins.mjs @@ -0,0 +1,32 @@ +/** + * English: AST scanner for markdown-it plugin registrations inside configureMd callbacks. + * + * Русский: AST-сканер регистраций markdown-it plugins внутри callback-ов configureMd. + */ +import ts from 'typescript'; + +import {forEachNode, getCallPropertyName, parseSource, unique, unwrapExpression} from './core.mjs'; +import {describeFactoryExpression} from './factory.mjs'; + +/** + * Extracts markdown-it plugin registrations. + */ +export function extractMdPlugins(content) { + const sourceFile = parseSource(content); + const plugins = []; + + forEachNode(sourceFile, (node) => { + if (!ts.isCallExpression(node) || getCallPropertyName(node) !== 'use') return; + + const expression = unwrapExpression(node.expression); + if (!ts.isPropertyAccessExpression(expression) || !ts.isIdentifier(expression.expression)) + return; + if (expression.expression.text !== 'md') return; + + const firstArg = node.arguments[0]; + const pluginName = firstArg ? describeFactoryExpression(firstArg, sourceFile) : null; + if (pluginName) plugins.push(pluginName); + }); + + return unique(plugins); +} diff --git a/infra/docs-gen/src/extractor/ast/plugins.mjs b/infra/docs-gen/src/extractor/ast/plugins.mjs new file mode 100644 index 00000000..486b3b8a --- /dev/null +++ b/infra/docs-gen/src/extractor/ast/plugins.mjs @@ -0,0 +1,30 @@ +/** + * English: AST scanner for ProseMirror plugin registrations. + * + * Русский: AST-сканер регистраций ProseMirror plugins. + */ +import ts from 'typescript'; + +import {isBuilderMethodCall} from './builder.mjs'; +import {forEachNode, parseSource, unique} from './core.mjs'; +import {describeFactoryExpression} from './factory.mjs'; + +/** + * Extracts ProseMirror plugin factory names. + */ +export function extractPlugins(content) { + const sourceFile = parseSource(content); + const plugins = []; + + forEachNode(sourceFile, (node) => { + if (!ts.isCallExpression(node) || !isBuilderMethodCall(node, new Set(['addPlugin']))) { + return; + } + + const firstArg = node.arguments[0]; + const pluginName = firstArg ? describeFactoryExpression(firstArg, sourceFile) : null; + if (pluginName) plugins.push(pluginName); + }); + + return unique(plugins); +} diff --git a/infra/docs-gen/src/extractor/ast/schema.mjs b/infra/docs-gen/src/extractor/ast/schema.mjs new file mode 100644 index 00000000..deddccbc --- /dev/null +++ b/infra/docs-gen/src/extractor/ast/schema.mjs @@ -0,0 +1,34 @@ +/** + * English: AST scanners for schema node and mark registrations. + * + * Русский: AST-сканеры регистраций schema node и mark. + */ +import {extractBuilderCallFirstArgs} from './builder.mjs'; + +/** + * Extracts ProseMirror node registrations from builder calls. + */ +export function extractAddNode(content) { + return extractBuilderCallFirstArgs(content, new Set(['addNode'])); +} + +/** + * Extracts ProseMirror mark registrations from builder calls. + */ +export function extractAddMark(content) { + return extractBuilderCallFirstArgs(content, new Set(['addMark'])); +} + +/** + * Extracts node names from granular node spec registrations. + */ +export function extractNodeSpecs(content) { + return extractBuilderCallFirstArgs(content, new Set(['addNodeSpec'])); +} + +/** + * Extracts mark names from granular mark spec registrations. + */ +export function extractMarkSpecs(content) { + return extractBuilderCallFirstArgs(content, new Set(['addMarkSpec'])); +} diff --git a/infra/docs-gen/src/extractor/ast/serializer.mjs b/infra/docs-gen/src/extractor/ast/serializer.mjs new file mode 100644 index 00000000..21c7b91d --- /dev/null +++ b/infra/docs-gen/src/extractor/ast/serializer.mjs @@ -0,0 +1,31 @@ +/** + * English: AST scanner for serializer markdown output hints. + * + * Русский: AST-сканер markdown-подсказок из serializer output. + */ +import ts from 'typescript'; + +import {forEachNode, getStringValue, parseSource, unique, unwrapExpression} from './core.mjs'; + +/** + * Extracts serializer output snippets. + */ +export function extractSerializerSyntax(content) { + const sourceFile = parseSource(content); + const snippets = []; + + forEachNode(sourceFile, (node) => { + if (!ts.isCallExpression(node)) return; + + const expression = unwrapExpression(node.expression); + if (!ts.isPropertyAccessExpression(expression)) return; + if (!ts.isIdentifier(expression.expression) || expression.expression.text !== 'state') + return; + if (expression.name.text !== 'write' && expression.name.text !== 'text') return; + + const snippet = node.arguments[0] ? getStringValue(node.arguments[0]) : null; + if (snippet?.trim()) snippets.push(snippet); + }); + + return unique(snippets); +} diff --git a/infra/docs-gen/src/extractor/cli.test.mjs b/infra/docs-gen/src/extractor/cli.test.mjs new file mode 100644 index 00000000..cd0d2dee --- /dev/null +++ b/infra/docs-gen/src/extractor/cli.test.mjs @@ -0,0 +1,26 @@ +/** + * English: Unit coverage for extension extraction CLI argument parsing. + * + * Русский: Unit-покрытие парсинга аргументов CLI извлечения расширений. + */ +import assert from 'node:assert/strict'; +import {test} from 'node:test'; + +import {parseArgs} from '../extract-extension-data.mjs'; + +test('parseArgs parses selected extensions', () => { + assert.deepEqual(parseArgs(['--only', 'Bold, Link']).only, ['Bold', 'Link']); +}); + +test('parseArgs stops option parsing after separator', () => { + assert.equal(parseArgs(['--help', '--', '--unknown']).help, true); +}); + +test('parseArgs rejects missing option values', () => { + assert.throws(() => parseArgs(['--out-dir']), { + message: 'Missing value for --out-dir', + }); + assert.throws(() => parseArgs(['--only', '--help']), { + message: 'Missing value for --only', + }); +}); diff --git a/infra/docs-gen/src/extractor/constants.mjs b/infra/docs-gen/src/extractor/constants.mjs new file mode 100644 index 00000000..ddfab6f9 --- /dev/null +++ b/infra/docs-gen/src/extractor/constants.mjs @@ -0,0 +1,159 @@ +/** + * English: Resolves local constants used by AST extractor fields. + * + * Русский: Резолвит локальные константы, используемые полями AST extractor-а. + */ +import ts from 'typescript'; + +import { + forEachNode, + getExpressionName, + getStaticPropertyName, + getStringValue, + parseSource, + unwrapExpression, +} from './ast.mjs'; + +/** + * Resolves a scalar initializer into a literal value or constant reference. + */ +function readScalarInitializer(initializer, sourceFile) { + const stringValue = getStringValue(initializer); + if (stringValue !== null) return stringValue; + + const current = unwrapExpression(initializer); + if (ts.isIdentifier(current) || ts.isPropertyAccessExpression(current)) { + return getExpressionName(current, sourceFile); + } + + return null; +} + +/** + * Extracts top-level scalar members from an object literal. + */ +function extractObjectScalarMembers(objectName, objectLiteral, constants, sourceFile) { + for (const property of objectLiteral.properties) { + if (!ts.isPropertyAssignment(property)) continue; + + const key = getStaticPropertyName(property.name); + if (!key) continue; + + const value = readScalarInitializer(property.initializer, sourceFile); + if (value !== null) { + constants.set(`${objectName}.${key}`, value); + } + } +} + +/** + * Extracts a scalar const declaration. + */ +function extractVariableDeclaration(declaration, constants, sourceFile) { + if (!ts.isIdentifier(declaration.name) || !declaration.initializer) return; + + const name = declaration.name.text; + const initializer = unwrapExpression(declaration.initializer); + + if (ts.isObjectLiteralExpression(initializer)) { + extractObjectScalarMembers(name, initializer, constants, sourceFile); + return; + } + + const value = readScalarInitializer(initializer, sourceFile); + if (value !== null) { + constants.set(name, value); + } +} + +/** + * Extracts string-valued enum members. + */ +function extractEnumDeclaration(enumDeclaration, constants) { + const enumName = enumDeclaration.name.text; + + for (const member of enumDeclaration.members) { + const memberName = getStaticPropertyName(member.name); + if (!memberName || !member.initializer) continue; + + const value = getStringValue(member.initializer); + if (value !== null) { + constants.set(`${enumName}.${memberName}`, value); + } + } +} + +/** + * Resolves aliases between constants after all candidates are collected. + */ +function resolveConstantAliases(constants) { + for (let pass = 0; pass < 5; pass++) { + for (const [key, value] of constants) { + if (typeof value === 'string' && constants.has(value) && key !== value) { + constants.set(key, constants.get(value)); + } + } + } +} + +/** + * Extracts string-valued constants, enums, and scalar object members. + */ +export function extractConstants(content) { + const sourceFile = parseSource(content); + const constants = new Map(); + + forEachNode(sourceFile, (node) => { + if (ts.isVariableDeclaration(node)) { + extractVariableDeclaration(node, constants, sourceFile); + } else if (ts.isEnumDeclaration(node)) { + extractEnumDeclaration(node, constants); + } + }); + + resolveConstantAliases(constants); + + return constants; +} + +/** + * Resolves one raw identifier through the constants map. + */ +export function resolveConstant(raw, constants) { + if (!raw) return raw; + if (constants.has(raw)) return constants.get(raw); + + for (const [key, value] of constants) { + if (key.endsWith(`.${raw}`) || key === raw) return value; + } + + return raw; +} + +/** + * Resolves a list of raw identifiers and expands constant namespaces. + */ +export function resolveAllConstants(rawList, constants) { + const resolved = []; + + for (const raw of rawList) { + const value = resolveConstant(raw, constants); + + if (value === raw && constants.size > 0) { + const prefix = raw + '.'; + const members = []; + for (const [key, memberValue] of constants) { + if (key.startsWith(prefix)) members.push(resolveConstant(memberValue, constants)); + } + + if (members.length > 0) { + resolved.push(...members); + continue; + } + } + + resolved.push(value); + } + + return [...new Set(resolved)]; +} diff --git a/infra/docs-gen/src/extractor/constants.test.mjs b/infra/docs-gen/src/extractor/constants.test.mjs new file mode 100644 index 00000000..838fc3ea --- /dev/null +++ b/infra/docs-gen/src/extractor/constants.test.mjs @@ -0,0 +1,52 @@ +/** + * English: Unit coverage for constant extraction and reference resolution. + * + * Русский: Unit-покрытие извлечения констант и резолва ссылок. + */ +import assert from 'node:assert/strict'; +import {test} from 'node:test'; + +import {extractConstants, resolveAllConstants} from './constants.mjs'; + +test('extractConstants captures top-level scalar object props with nested objects', () => { + const content = [ + 'export const CLASSNAMES = {', + " Inline: { Container: 'a', Sharp: 'b' },", + " Block: 'mathblock',", + " Display: 'display',", + '} as const;', + ].join('\n'); + + const map = extractConstants(content); + + assert.equal(map.get('CLASSNAMES.Block'), 'mathblock'); + assert.equal(map.get('CLASSNAMES.Display'), 'display'); +}); + +test('extractConstants handles enums after objects with nested braces', () => { + const content = [ + 'const NESTED = { Inner: { Foo: "bar" }, Outer: "value" };', + 'export enum NodeName { Para = "paragraph", Doc = "doc" }', + ].join('\n'); + + const map = extractConstants(content); + + assert.equal(map.get('NESTED.Outer'), 'value'); + assert.equal(map.get('NodeName.Para'), 'paragraph'); + assert.equal(map.get('NodeName.Doc'), 'doc'); +}); + +test('extractConstants resolves identifier-valued props through reference chains', () => { + const content = ["const NAME = 'bold';", 'export const Specs = { Bold: NAME };'].join('\n'); + + const map = extractConstants(content); + + assert.equal(map.get('Specs.Bold'), 'bold'); +}); + +test('resolveAllConstants expands a prefix into all known members', () => { + const content = 'enum NodeName { Para = "paragraph", Doc = "doc" }'; + const map = extractConstants(content); + + assert.deepEqual(resolveAllConstants(['NodeName'], map).sort(), ['doc', 'paragraph']); +}); diff --git a/infra/docs-gen/src/extractor/examples.mjs b/infra/docs-gen/src/extractor/examples.mjs new file mode 100644 index 00000000..8df050dc --- /dev/null +++ b/infra/docs-gen/src/extractor/examples.mjs @@ -0,0 +1,154 @@ +/** + * English: Extracts markdown examples from serializer tests using TypeScript AST. + * + * Русский: Извлекает markdown examples из serializer tests через TypeScript AST. + */ +import ts from 'typescript'; + +import {forEachNode, parseSource, unwrapExpression} from './ast.mjs'; + +/** + * Applies a known string method to a value. + */ +function applyStringMethod(value, method) { + if (method === 'trim') return value.trim(); + if (method === 'trimStart') return value.trimStart(); + if (method === 'trimEnd') return value.trimEnd(); + return value; +} + +/** + * Resolves a template literal expression. + */ +function resolveTemplateExpression(expression, bindings) { + let result = expression.head.text; + + for (const span of expression.templateSpans) { + const value = resolveStringExpression(span.expression, bindings); + if (value === null) return null; + + result += value + span.literal.text; + } + + return result; +} + +/** + * Resolves an array join expression. + */ +function resolveArrayJoinExpression(callExpression, bindings) { + const callee = unwrapExpression(callExpression.expression); + if (!ts.isPropertyAccessExpression(callee) || callee.name.text !== 'join') return null; + + const arrayExpression = unwrapExpression(callee.expression); + if (!ts.isArrayLiteralExpression(arrayExpression)) return null; + + const delimiterExpression = callExpression.arguments[0]; + const delimiter = delimiterExpression + ? resolveStringExpression(delimiterExpression, bindings) + : ','; + if (delimiter === null) return null; + + const parts = arrayExpression.elements.map((element) => + resolveStringExpression(element, bindings), + ); + if (parts.some((part) => part === null)) return null; + + return parts.join(delimiter); +} + +/** + * Resolves a string method call. + */ +function resolveStringMethodCall(callExpression, bindings) { + const callee = unwrapExpression(callExpression.expression); + if (!ts.isPropertyAccessExpression(callee)) return null; + + const method = callee.name.text; + if (method === 'join') return resolveArrayJoinExpression(callExpression, bindings); + if (method !== 'trim' && method !== 'trimStart' && method !== 'trimEnd') return null; + + const value = resolveStringExpression(callee.expression, bindings); + return value === null ? null : applyStringMethod(value, method); +} + +/** + * Resolves supported string expressions. + */ +function resolveStringExpression(expression, bindings) { + const current = unwrapExpression(expression); + + if (ts.isStringLiteralLike(current)) return current.text; + if (ts.isTemplateExpression(current)) return resolveTemplateExpression(current, bindings); + if (ts.isIdentifier(current)) return bindings.get(current.text) ?? null; + if (ts.isCallExpression(current)) return resolveStringMethodCall(current, bindings); + + if (ts.isBinaryExpression(current) && current.operatorToken.kind === ts.SyntaxKind.PlusToken) { + const left = resolveStringExpression(current.left, bindings); + const right = resolveStringExpression(current.right, bindings); + return left === null || right === null ? null : left + right; + } + + return null; +} + +/** + * Collects resolvable string bindings declared before a source position. + */ +function collectStringBindingsBefore(sourceFile, position) { + const declarations = []; + + forEachNode(sourceFile, (node) => { + if (!ts.isVariableDeclaration(node) || !ts.isIdentifier(node.name) || !node.initializer) { + return; + } + + if (node.getStart(sourceFile) < position) { + declarations.push(node); + } + }); + + declarations.sort((left, right) => left.getStart(sourceFile) - right.getStart(sourceFile)); + + const bindings = new Map(); + for (const declaration of declarations) { + const value = resolveStringExpression(declaration.initializer, bindings); + if (value !== null) { + bindings.set(declaration.name.text, value); + } + } + + return bindings; +} + +/** + * Checks whether a call expression invokes same(...). + */ +function isSameCall(callExpression) { + const callee = unwrapExpression(callExpression.expression); + + if (ts.isIdentifier(callee)) return callee.text === 'same'; + return ts.isPropertyAccessExpression(callee) && callee.name.text === 'same'; +} + +/** + * Extracts markdown examples from serializer test helpers. + */ +export function extractTestExamples(content) { + const sourceFile = parseSource(content); + const examples = []; + + forEachNode(sourceFile, (node) => { + if (!ts.isCallExpression(node) || !isSameCall(node) || node.arguments.length === 0) { + return; + } + + const bindings = collectStringBindingsBefore(sourceFile, node.getStart(sourceFile)); + const example = resolveStringExpression(node.arguments[0], bindings); + if (example !== null) { + examples.push(example); + } + }); + + return examples; +} diff --git a/infra/docs-gen/src/extractor/examples.test.mjs b/infra/docs-gen/src/extractor/examples.test.mjs new file mode 100644 index 00000000..647d760f --- /dev/null +++ b/infra/docs-gen/src/extractor/examples.test.mjs @@ -0,0 +1,51 @@ +/** + * English: Unit coverage for serializer markdown example extraction. + * + * Русский: Unit-покрытие извлечения markdown examples из serializer tests. + */ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {test} from 'node:test'; + +import {extractTestExamples} from './examples.mjs'; + +/** + * Reads a repository file as UTF-8 text. + */ +function readRepoFile(relativePath) { + return readFileSync(new URL(`../../../../${relativePath}`, import.meta.url), 'utf-8'); +} + +test('extractTestExamples resolves literals, local bindings, joins, and template bindings', () => { + const content = [ + "const formula = 'x + y';", + 'const markup = `', + '# title', + '`.trimStart();', + "same(['Term', ': Description'].join('\\n'), doc());", + 'checker.same(markup, doc());', + 'same(`$$${formula}$$\\n\\n`, doc());', + ].join('\n'); + + assert.deepEqual(extractTestExamples(content), [ + 'Term\n: Description', + '# title\n', + '$$x + y$$\n\n', + ]); +}); + +test('extractTestExamples captures joined Deflist markup', () => { + const examples = extractTestExamples( + readRepoFile('packages/editor/src/extensions/markdown/Deflist/Deflist.test.ts'), + ); + + assert.ok(examples.includes('Term\n: Description')); +}); + +test('extractTestExamples captures markup stored in local constants', () => { + const examples = extractTestExamples( + readRepoFile('packages/editor/src/extensions/yfm/YfmTable/YfmTable.test.ts'), + ); + + assert.ok(examples.some((example) => example.includes('nested table'))); +}); diff --git a/infra/docs-gen/src/extractor/extension-refs.mjs b/infra/docs-gen/src/extractor/extension-refs.mjs new file mode 100644 index 00000000..f628d15a --- /dev/null +++ b/infra/docs-gen/src/extractor/extension-refs.mjs @@ -0,0 +1,85 @@ +/** + * English: Collects configured extension entry points into scan references. + * + * Русский: Собирает сконфигурированные entry points расширений в scan references. + */ +import {join} from 'node:path'; + +import {isBlacklistedExtension} from '../config.mjs'; +import {listDirs} from '../utils.mjs'; + +/** + * Creates one scan reference for an extension directory. + */ +function createExtensionRef(entryPoint, {name, category, extDir}) { + return { + name, + packageName: entryPoint.packageName, + entryPoint: entryPoint.id, + category, + extDir, + }; +} + +/** + * Collects extension references from category-based editor package directories. + */ +function collectCategoryExtensionRefs(entryPoint) { + const refs = []; + + for (const category of entryPoint.categories) { + const categoryDir = join(entryPoint.packageDir, entryPoint.extensionsDir, category); + for (const dirName of listDirs(categoryDir)) { + refs.push( + createExtensionRef(entryPoint, { + name: dirName, + category, + extDir: join(categoryDir, dirName), + }), + ); + } + } + + return refs; +} + +/** + * Collects a package that exposes one extension entry point. + */ +function collectSingleExtensionRef(entryPoint) { + return [ + createExtensionRef(entryPoint, { + name: entryPoint.extensionName, + category: entryPoint.category, + extDir: join(entryPoint.packageDir, entryPoint.extensionDir), + }), + ]; +} + +/** + * Collects all extension references from configured entry points. + */ +export function collectExtensionRefs(entryPoints) { + return entryPoints.flatMap((entryPoint) => { + switch (entryPoint.kind) { + case 'category-dirs': + return collectCategoryExtensionRefs(entryPoint); + case 'single-extension': + return collectSingleExtensionRef(entryPoint); + default: + throw new Error(`Unknown extension entry point kind: ${entryPoint.kind}`); + } + }); +} + +/** + * Applies configured filters after the full extension list is known. + */ +export function filterExtensionRefs(refs, {only} = {}) { + const onlySet = only?.length ? new Set(only) : null; + + return refs.filter((ref) => { + if (isBlacklistedExtension(ref.name)) return false; + return !onlySet || onlySet.has(ref.name); + }); +} diff --git a/infra/docs-gen/src/extractor/extension-sources.mjs b/infra/docs-gen/src/extractor/extension-sources.mjs new file mode 100644 index 00000000..3b59c662 --- /dev/null +++ b/infra/docs-gen/src/extractor/extension-sources.mjs @@ -0,0 +1,29 @@ +/** + * English: Reads extension files and prepares source groups for metadata scanning. + * + * Русский: Читает файлы расширения и готовит группы source files для metadata scanning. + */ +import {readAllTsFiles} from '../utils.mjs'; + +import {joinContents, selectSourceFiles, selectSpecFiles} from './source-files.mjs'; + +/** + * Reads extension files and separates production sources from tests. + */ +export function readExtensionSources(extDir) { + const allFiles = readAllTsFiles(extDir); + const sourceFiles = selectSourceFiles(allFiles); + + return { + allFiles, + sourceFiles, + allContent: joinContents(sourceFiles), + }; +} + +/** + * Builds the source text used for schema extraction. + */ +export function buildSchemaContent(sourceFiles, extDir) { + return joinContents(selectSpecFiles(sourceFiles, extDir)); +} diff --git a/infra/docs-gen/src/extractor/index.mjs b/infra/docs-gen/src/extractor/index.mjs new file mode 100644 index 00000000..af1fe68b --- /dev/null +++ b/infra/docs-gen/src/extractor/index.mjs @@ -0,0 +1,95 @@ +/** + * English: Orchestrates extension discovery, filtering, scanning, enrichment, and output. + * + * Русский: Управляет обнаружением, фильтрацией, сканированием, обогащением и выводом расширений. + */ +import {existsSync, mkdirSync, rmSync} from 'node:fs'; +import {join} from 'node:path'; + +import {logger} from '../logger.mjs'; +import {readText} from '../utils.mjs'; + +import {collectExtensionRefs, filterExtensionRefs} from './extension-refs.mjs'; +import {writeExtensionsJson, writeRawMarkdownFiles} from './output.mjs'; +import {getPresetsForExtension, parsePresets} from './presets.mjs'; +import {scanExtension} from './scan.mjs'; + +export class ExtensionExtractor { + /** + * Creates an extension extractor for configured source paths. + */ + constructor({entryPoints, outDir, repoRoot, versionPackageDir}) { + this.entryPoints = entryPoints; + this.repoRoot = repoRoot; + this.versionPackageDir = versionPackageDir; + this.outDir = outDir; + this.rawDir = join(outDir, 'raw'); + } + + /** + * Scans one extension directory into raw metadata. + */ + scan(ref) { + return scanExtension({...ref, repoRoot: this.repoRoot}); + } + + /** + * Scans all configured extension entry points. + */ + scanAll({only} = {}) { + const allRefs = collectExtensionRefs(this.entryPoints); + const extensionRefs = filterExtensionRefs(allRefs, {only}); + + return { + totalCount: allRefs.length, + extensions: extensionRefs.map((ref) => this.scan(ref)), + }; + } + + /** + * Parses presets from the configured editor entry point. + */ + getPresetMap() { + const presetEntryPoint = this.entryPoints.find((entryPoint) => entryPoint.presetsDir); + if (!presetEntryPoint) return new Map(); + + return parsePresets(join(presetEntryPoint.packageDir, presetEntryPoint.presetsDir)); + } + + /** + * Writes extracted JSON IR and raw Markdown files. + */ + run({only} = {}) { + logger.info('Extracting raw extension data...'); + + if (existsSync(this.rawDir)) { + rmSync(this.rawDir, {recursive: true, force: true}); + } + mkdirSync(this.rawDir, {recursive: true}); + + const version = this.getEditorVersion(); + const {totalCount, extensions} = this.scanAll({only}); + const presetMap = this.getPresetMap(); + + for (const extension of extensions) { + extension.presets = getPresetsForExtension(presetMap, extension.name); + } + + writeExtensionsJson(this.outDir, version, extensions); + writeRawMarkdownFiles(this.rawDir, extensions, presetMap, version); + + logger.success(`Raw data written to ${this.outDir}`); + logger.info(`Discovered extensions: ${totalCount}`); + logger.info(`Extensions: ${extensions.length}`); + + return {version, extensions}; + } + + /** + * Reads the editor package version. + */ + getEditorVersion() { + const pkg = JSON.parse(readText(join(this.versionPackageDir, 'package.json'))); + return pkg.version; + } +} diff --git a/infra/docs-gen/src/extractor/keymaps.test.mjs b/infra/docs-gen/src/extractor/keymaps.test.mjs new file mode 100644 index 00000000..ebdfe1ff --- /dev/null +++ b/infra/docs-gen/src/extractor/keymaps.test.mjs @@ -0,0 +1,69 @@ +/** + * English: Unit coverage for AST-based keymap extraction. + * + * Русский: Unit-покрытие AST-based извлечения keymap. + */ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {test} from 'node:test'; + +import {extractKeymaps} from './ast.mjs'; + +/** + * Reads a repository file as UTF-8 text. + */ +function readRepoFile(relativePath) { + return readFileSync(new URL(`../../../../${relativePath}`, import.meta.url), 'utf-8'); +} + +test('extractKeymaps handles direct object returns and ignores computed keys', () => { + const content = [ + 'builder.addKeymap(() => ({', + ' Tab: handleTab,', + " 'Shift-Tab': handleShiftTab,", + ' [dynamicKey]: ignoreMe,', + '}));', + ].join('\n'); + + assert.deepEqual(extractKeymaps(content), ['Tab', 'Shift-Tab']); +}); + +test('extractKeymaps merges returned object literals with spread bindings', () => { + const content = [ + 'builder.addKeymap(() => {', + ' const bindings: Keymap = {Backspace: resetHeading};', + ' return {', + ' Tab: handleTab,', + ' ...bindings,', + " 'Shift-Tab': handleShiftTab,", + ' };', + '});', + ].join('\n'); + + assert.deepEqual(extractKeymaps(content), ['Tab', 'Backspace', 'Shift-Tab']); +}); + +test('extractKeymaps captures static keymaps from the Lists extension', () => { + const content = readRepoFile('packages/editor/src/extensions/markdown/Lists/index.ts'); + + assert.deepEqual(extractKeymaps(content), [ + 'Tab', + 'Shift-Tab', + 'Backspace', + 'Mod-[', + 'Mod-]', + 'Enter', + ]); +}); + +test('extractKeymaps captures static block-body bindings and ignores dynamic ones', () => { + const headingContent = readRepoFile('packages/editor/src/extensions/markdown/Heading/index.ts'); + const historyContent = readRepoFile('packages/editor/src/extensions/behavior/History/index.ts'); + const editorModeContent = readRepoFile( + 'packages/editor/src/extensions/behavior/EditorModeKeymap/index.ts', + ); + + assert.deepEqual(extractKeymaps(headingContent), ['Backspace']); + assert.deepEqual(extractKeymaps(historyContent), []); + assert.deepEqual(extractKeymaps(editorModeContent), []); +}); diff --git a/infra/docs-gen/src/extractor/markdown-gen.mjs b/infra/docs-gen/src/extractor/markdown-gen.mjs new file mode 100644 index 00000000..68159023 --- /dev/null +++ b/infra/docs-gen/src/extractor/markdown-gen.mjs @@ -0,0 +1,96 @@ +/** + * English: Renders raw extension metadata records into markdown files. + * + * Русский: Рендерит сырые metadata records расширений в markdown-файлы. + */ +import {getPresetsForExtension} from './presets.mjs'; + +/** + * Formats a value as inline Markdown code. + */ +function code(value) { + return `\`${String(value).split('|').join('\\|').split('\n').join('\\n')}\``; +} + +/** + * Adds a Markdown list section when values are present. + */ +function addList(lines, title, values) { + if (values.length === 0) return; + + lines.push(`## ${title}`, ''); + for (const value of values) { + lines.push(`- ${code(value)}`); + } + lines.push(''); +} + +/** + * Generates a raw Markdown page for extracted extension data. + */ +export function generateRawMd(extension, presetMap, version) { + const presets = getPresetsForExtension(presetMap, extension.name); + const lines = [ + '---', + `extension: ${extension.name}`, + `version: ${version}`, + `packageName: ${JSON.stringify(extension.packageName)}`, + `entryPoint: ${JSON.stringify(extension.entryPoint)}`, + `category: ${extension.category}`, + `source: ${extension.sourcePath}`, + '---', + '', + `# ${extension.name}`, + '', + '## Source', + '', + `- Package: ${code(extension.packageName)}`, + `- Entry point: ${code(extension.entryPoint)}`, + `- Path: ${code(extension.sourcePath)}`, + '', + '## Presets', + '', + ]; + + if (presets.length > 0) { + for (const preset of presets) lines.push(`- ${preset}`); + } else { + lines.push('Not included in standard presets.'); + } + lines.push(''); + + if (extension.nodes.length > 0 || extension.marks.length > 0) { + lines.push('## Schema', ''); + for (const node of extension.nodes) { + lines.push(`- Node: ${code(node)}`); + } + for (const mark of extension.marks) { + lines.push(`- Mark: ${code(mark)}`); + } + lines.push(''); + } + + addList(lines, 'Actions', extension.actions); + addList(lines, 'Keymaps', extension.keymaps); + addList(lines, 'Input Rules', extension.inputRules); + addList(lines, 'Markdown-It Plugins', extension.mdPlugins); + addList(lines, 'ProseMirror Plugins', extension.plugins); + addList(lines, 'Serializer Hints', extension.serializerHints); + + if (extension.options.length > 0) { + lines.push('## Options', '', '| Option | Type |', '|--------|------|'); + for (const option of extension.options) { + lines.push(`| ${code(option.name)} | ${code(option.type)} |`); + } + lines.push(''); + } + + if (extension.markupExamples.length > 0) { + lines.push('## Markup Examples', ''); + for (const example of extension.markupExamples.slice(0, 10)) { + lines.push('```markdown', example, '```', ''); + } + } + + return lines.join('\n'); +} diff --git a/infra/docs-gen/src/extractor/options.mjs b/infra/docs-gen/src/extractor/options.mjs new file mode 100644 index 00000000..a25fedb0 --- /dev/null +++ b/infra/docs-gen/src/extractor/options.mjs @@ -0,0 +1,29 @@ +/** + * English: Public entry point for extracting local extension option declarations. + * + * Русский: Публичная точка входа для извлечения локальных option declarations расширений. + */ +import {parseOptionDeclarations} from './options/declarations.mjs'; +import {resolveOptionDeclaration} from './options/resolve.mjs'; + +/** + * Selects preferred option declaration names. + */ +function selectOptionNames(declarations, preferredNames) { + const existingPreferredNames = preferredNames.filter((name) => declarations.has(name)); + return existingPreferredNames.length > 0 ? existingPreferredNames : [...declarations.keys()]; +} + +/** + * Extracts exported extension options fields. + */ +export function extractOptionsType(content, preferredNames = []) { + const declarations = parseOptionDeclarations(content); + + for (const name of selectOptionNames(declarations, preferredNames)) { + const fields = resolveOptionDeclaration(name, declarations); + if (fields.length > 0) return fields; + } + + return []; +} diff --git a/infra/docs-gen/src/extractor/options.test.mjs b/infra/docs-gen/src/extractor/options.test.mjs new file mode 100644 index 00000000..e8c2d12a --- /dev/null +++ b/infra/docs-gen/src/extractor/options.test.mjs @@ -0,0 +1,84 @@ +/** + * English: Unit coverage for TypeScript option declaration extraction. + * + * Русский: Unit-покрытие извлечения TypeScript option declarations. + */ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {test} from 'node:test'; + +import {extractOptionsType} from './options.mjs'; + +/** + * Reads a repository file as UTF-8 text. + */ +function readRepoFile(relativePath) { + return readFileSync(new URL(`../../../../${relativePath}`, import.meta.url), 'utf-8'); +} + +/** + * Maps option fields by name. + */ +function mapFields(fields) { + return new Map(fields.map((field) => [field.name, field.type])); +} + +test('extractOptionsType resolves local aliases', () => { + const content = [ + readRepoFile('packages/editor/src/extensions/markdown/Image/index.ts'), + readRepoFile('packages/editor/src/extensions/markdown/Image/imageUrlPaste/index.ts'), + ].join('\n'); + + assert.deepEqual(extractOptionsType(content, ['ImageOptions']), [ + {name: 'parseInsertedUrlAsImage', type: 'ParseInsertedUrlAsImage'}, + ]); +}); + +test('extractOptionsType resolves Pick utility types', () => { + const content = [ + readRepoFile('packages/editor/src/extensions/behavior/Clipboard/index.ts'), + readRepoFile('packages/editor/src/extensions/behavior/Clipboard/clipboard.ts'), + ].join('\n'); + + assert.deepEqual(extractOptionsType(content, ['ClipboardOptions']), [ + {name: 'pasteFileHandler', type: '(file: File) => void'}, + ]); +}); + +test('extractOptionsType keeps nested object option types intact', () => { + const fields = mapFields( + extractOptionsType( + readRepoFile('packages/editor/src/extensions/additional/Mermaid/index.ts'), + ['MermaidOptions'], + ), + ); + + assert.equal(fields.get('autoSave'), '{ enabled: boolean; delay?: number; }'); + assert.equal( + fields.get('theme'), + "{ dark: MermaidConfig['theme']; light: MermaidConfig['theme']; }", + ); +}); + +test('extractOptionsType reads interface declarations with local fields', () => { + const fields = mapFields( + extractOptionsType( + readRepoFile('packages/editor/src/extensions/additional/YfmHtmlBlock/index.ts'), + ['YfmHtmlBlockOptions'], + ), + ); + + assert.equal(fields.get('useConfig'), '() => IHTMLIFrameElementConfig | undefined'); + assert.equal(fields.get('autoSave'), '{ enabled: boolean; delay?: number; }'); +}); + +test('extractOptionsType merges intersections without corrupting nested fields', () => { + const content = [ + readRepoFile('packages/editor/src/extensions/markdown/CodeBlock/index.ts'), + readRepoFile('packages/editor/src/extensions/markdown/CodeBlock/CodeBlockSpecs/index.ts'), + ].join('\n'); + const fields = mapFields(extractOptionsType(content, ['CodeBlockOptions'])); + + assert.equal(fields.get('codeBlockKey'), 'string | null'); + assert.equal(fields.get('lineWrapping'), '{ enabled?: boolean; }'); +}); diff --git a/infra/docs-gen/src/extractor/options/declarations.mjs b/infra/docs-gen/src/extractor/options/declarations.mjs new file mode 100644 index 00000000..5222b496 --- /dev/null +++ b/infra/docs-gen/src/extractor/options/declarations.mjs @@ -0,0 +1,47 @@ +/** + * English: Parses local TypeScript declarations that can describe extension options. + * + * Русский: Парсит локальные TypeScript declarations, описывающие options расширений. + */ +import ts from 'typescript'; + +import {parseSource} from '../ast.mjs'; + +/** + * Creates a type declaration record. + */ +function createDeclaration(name, kind, node, sourceFile) { + return { + name, + kind, + node, + sourceFile, + }; +} + +/** + * Parses local *Options declarations from TypeScript source. + */ +export function parseOptionDeclarations(content) { + const sourceFile = parseSource(content); + const declarations = new Map(); + + for (const statement of sourceFile.statements) { + if ( + (ts.isInterfaceDeclaration(statement) || ts.isTypeAliasDeclaration(statement)) && + statement.name.text.endsWith('Options') + ) { + declarations.set( + statement.name.text, + createDeclaration( + statement.name.text, + ts.isInterfaceDeclaration(statement) ? 'interface' : 'type', + statement, + sourceFile, + ), + ); + } + } + + return declarations; +} diff --git a/infra/docs-gen/src/extractor/options/fields.mjs b/infra/docs-gen/src/extractor/options/fields.mjs new file mode 100644 index 00000000..9a41de11 --- /dev/null +++ b/infra/docs-gen/src/extractor/options/fields.mjs @@ -0,0 +1,112 @@ +/** + * English: Reads and transforms option field descriptors from TypeScript type nodes. + * + * Русский: Читает и преобразует descriptors option fields из TypeScript type nodes. + */ +import ts from 'typescript'; + +import {getStaticPropertyName} from '../ast.mjs'; + +/** + * Checks whether a character is a whitespace separator. + */ +function isWhitespace(char) { + return char === ' ' || char === '\n' || char === '\r' || char === '\t' || char === '\f'; +} + +/** + * Normalizes whitespace in extracted type snippets. + */ +function normalizeWhitespace(content) { + const trimmed = content.trim(); + let result = ''; + let isInsideWhitespace = false; + + for (const char of trimmed) { + if (isWhitespace(char)) { + if (!isInsideWhitespace) { + result += ' '; + } + isInsideWhitespace = true; + } else { + result += char; + isInsideWhitespace = false; + } + } + + return result; +} + +/** + * Deduplicates option fields by name. + */ +export function uniqueOptionFields(fields) { + const result = new Map(); + + for (const field of fields) { + if (!result.has(field.name)) { + result.set(field.name, field); + } + } + + return [...result.values()]; +} + +/** + * Reads a field type as source text. + */ +function readFieldType(member, sourceFile) { + return member.type ? normalizeWhitespace(member.type.getText(sourceFile)) : 'unknown'; +} + +/** + * Reads field names from Pick/Omit string literal unions. + */ +export function readStringLiteralUnion(typeNode) { + if (ts.isLiteralTypeNode(typeNode) && ts.isStringLiteralLike(typeNode.literal)) { + return [typeNode.literal.text]; + } + + if (ts.isUnionTypeNode(typeNode)) { + return typeNode.types.flatMap(readStringLiteralUnion); + } + + return []; +} + +/** + * Reads fields declared inside an inline object type. + */ +export function readTypeLiteralFields(typeLiteral, sourceFile) { + const fields = []; + + for (const member of typeLiteral.members) { + if (!ts.isPropertySignature(member) || !member.name) continue; + + const name = getStaticPropertyName(member.name); + if (!name) continue; + + fields.push({ + name, + type: readFieldType(member, sourceFile), + }); + } + + return fields; +} + +/** + * Filters fields to names from a utility type. + */ +export function pickOptionFields(fields, names) { + const allowedNames = new Set(names); + return fields.filter((field) => allowedNames.has(field.name)); +} + +/** + * Excludes fields by names from a utility type. + */ +export function omitOptionFields(fields, names) { + const omittedNames = new Set(names); + return fields.filter((field) => !omittedNames.has(field.name)); +} diff --git a/infra/docs-gen/src/extractor/options/resolve.mjs b/infra/docs-gen/src/extractor/options/resolve.mjs new file mode 100644 index 00000000..e202ffeb --- /dev/null +++ b/infra/docs-gen/src/extractor/options/resolve.mjs @@ -0,0 +1,104 @@ +/** + * English: Resolves local option declarations into flat field lists. + * + * Русский: Резолвит локальные option declarations в плоские списки fields. + */ +import ts from 'typescript'; + +import { + omitOptionFields, + pickOptionFields, + readStringLiteralUnion, + readTypeLiteralFields, + uniqueOptionFields, +} from './fields.mjs'; + +/** + * Resolves a type reference to another local declaration or utility type. + */ +function resolveTypeReference(typeNode, declarations, seen, sourceFile) { + const typeName = typeNode.typeName.getText(sourceFile); + const typeArguments = typeNode.typeArguments || []; + + if ((typeName === 'Pick' || typeName === 'Omit') && typeArguments.length >= 2) { + const fields = resolveTypeNode(typeArguments[0], declarations, seen, sourceFile); + const names = readStringLiteralUnion(typeArguments[1]); + + return typeName === 'Pick' + ? pickOptionFields(fields, names) + : omitOptionFields(fields, names); + } + + if (!declarations.has(typeName)) return []; + + return resolveOptionDeclaration(typeName, declarations, seen); +} + +/** + * Resolves supported TypeScript option type nodes. + */ +function resolveTypeNode(typeNode, declarations, seen, sourceFile) { + if (!typeNode) return []; + + if (ts.isParenthesizedTypeNode(typeNode)) { + return resolveTypeNode(typeNode.type, declarations, seen, sourceFile); + } + + if (ts.isTypeLiteralNode(typeNode)) { + return readTypeLiteralFields(typeNode, sourceFile); + } + + if (ts.isIntersectionTypeNode(typeNode)) { + return typeNode.types.flatMap((part) => + resolveTypeNode(part, declarations, seen, sourceFile), + ); + } + + if (ts.isTypeReferenceNode(typeNode)) { + return resolveTypeReference(typeNode, declarations, seen, sourceFile); + } + + return []; +} + +/** + * Resolves fields inherited by an interface declaration. + */ +function resolveInterfaceHeritage(interfaceNode, declarations, seen, sourceFile) { + const clauses = interfaceNode.heritageClauses || []; + const fields = []; + + for (const clause of clauses) { + if (clause.token !== ts.SyntaxKind.ExtendsKeyword) continue; + + for (const type of clause.types) { + const typeName = type.expression.getText(sourceFile); + if (!declarations.has(typeName)) continue; + + fields.push(...resolveOptionDeclaration(typeName, declarations, seen)); + } + } + + return fields; +} + +/** + * Resolves fields from a declaration by name. + */ +export function resolveOptionDeclaration(name, declarations, seen = new Set()) { + if (seen.has(name)) return []; + seen.add(name); + + const declaration = declarations.get(name); + if (!declaration) return []; + + const {node, sourceFile} = declaration; + const inheritedFields = ts.isInterfaceDeclaration(node) + ? resolveInterfaceHeritage(node, declarations, seen, sourceFile) + : resolveTypeNode(node.type, declarations, seen, sourceFile); + const ownFields = ts.isInterfaceDeclaration(node) + ? readTypeLiteralFields(node, sourceFile) + : []; + + return uniqueOptionFields([...inheritedFields, ...ownFields]); +} diff --git a/infra/docs-gen/src/extractor/output.mjs b/infra/docs-gen/src/extractor/output.mjs new file mode 100644 index 00000000..7977da74 --- /dev/null +++ b/infra/docs-gen/src/extractor/output.mjs @@ -0,0 +1,31 @@ +/** + * English: Writes extracted raw JSON and markdown artifacts to disk. + * + * Русский: Записывает извлеченные raw JSON и markdown artifacts на диск. + */ +import {writeFileSync} from 'node:fs'; +import {join} from 'node:path'; + +import {generateRawMd} from './markdown-gen.mjs'; + +/** + * Writes extension IR as JSON. + */ +export function writeExtensionsJson(outDir, version, extensions) { + writeFileSync( + join(outDir, 'extensions.json'), + JSON.stringify({version, extensions}, null, 2) + '\n', + ); +} + +/** + * Writes raw Markdown files for extensions. + */ +export function writeRawMarkdownFiles(rawDir, extensions, presetMap, version) { + for (const extension of extensions) { + writeFileSync( + join(rawDir, `${extension.name}.md`), + generateRawMd(extension, presetMap, version), + ); + } +} diff --git a/infra/docs-gen/src/extractor/presets.mjs b/infra/docs-gen/src/extractor/presets.mjs new file mode 100644 index 00000000..2219a362 --- /dev/null +++ b/infra/docs-gen/src/extractor/presets.mjs @@ -0,0 +1,72 @@ +/** + * English: Maps extensions to editor presets by scanning preset source files. + * + * Русский: Сопоставляет расширения с editor presets через сканирование исходников пресетов. + */ +import {existsSync} from 'node:fs'; +import {join} from 'node:path'; + +import ts from 'typescript'; + +import {PRESET_DEFS} from '../config.mjs'; +import {readText} from '../utils.mjs'; + +import {forEachNode, getExpressionName, parseSource, unwrapExpression} from './ast.mjs'; + +/** + * Extracts direct extension uses from a preset source file. + */ +function extractPresetUses(content) { + const sourceFile = parseSource(content); + const directUses = []; + + forEachNode(sourceFile, (node) => { + if (!ts.isCallExpression(node)) return; + + const expression = unwrapExpression(node.expression); + if (!ts.isPropertyAccessExpression(expression) || expression.name.text !== 'use') return; + + const firstArg = node.arguments[0]; + const extensionName = firstArg ? getExpressionName(firstArg, sourceFile) : null; + if (!extensionName) return; + + if (!extensionName.endsWith('Preset') && !extensionName.endsWith('Specs')) { + directUses.push(extensionName); + } + }); + + return directUses; +} + +/** + * Parses preset files into inherited extension membership. + */ +export function parsePresets(presetsDir) { + const presetMap = new Map(); + + for (const def of PRESET_DEFS) { + const filePath = join(presetsDir, def.file); + if (!existsSync(filePath)) continue; + + const directUses = extractPresetUses(readText(filePath)); + const inherited = def.parent ? presetMap.get(def.parent) || [] : []; + presetMap.set(def.name, [...new Set([...inherited, ...directUses])]); + } + + return presetMap; +} + +/** + * Finds presets that include an extension. + */ +export function getPresetsForExtension(presetMap, extensionName) { + const presets = []; + + for (const [presetName, extensions] of presetMap) { + if (extensions.includes(extensionName)) { + presets.push(presetName); + } + } + + return presets; +} diff --git a/infra/docs-gen/src/extractor/record-fields.mjs b/infra/docs-gen/src/extractor/record-fields.mjs new file mode 100644 index 00000000..96793ca2 --- /dev/null +++ b/infra/docs-gen/src/extractor/record-fields.mjs @@ -0,0 +1,134 @@ +/** + * English: Maps raw extension record fields to focused extractor functions. + * + * Русский: Связывает поля raw-записи расширения с узкими extractor-функциями. + */ +import {EXTENSION_DOC_FIELD_CONFIG} from '../config.mjs'; + +import { + extractActions, + extractInputRules, + extractKeymaps, + extractMdPlugins, + extractPlugins, + extractSerializerSyntax, +} from './ast.mjs'; +import {resolveAllConstants} from './constants.mjs'; +import {extractTestExamples} from './examples.mjs'; +import {extractOptionsType} from './options.mjs'; +import { + findRootIndexFile, + findSpecsIndexFile, + isTestFile, + joinContents, + selectSerializerFiles, +} from './source-files.mjs'; + +/** + * Deduplicates extracted values while keeping the source order. + */ +function unique(values) { + return [...new Set(values)]; +} + +/** + * Builds option declaration names preferred for an extension. + */ +function buildPreferredOptionNames(extensionName) { + return [`${extensionName}Options`, `${extensionName}SpecsOptions`]; +} + +/** + * Extracts extension options from local source declarations. + */ +export function extractOptions(sourceFiles, extDir, extensionName) { + const rootIndexFile = findRootIndexFile(sourceFiles, extDir); + const specsIndexFile = findSpecsIndexFile(sourceFiles); + const preferredNames = buildPreferredOptionNames(extensionName); + const allOptions = extractOptionsType(joinContents(sourceFiles), preferredNames); + + if (allOptions.length > 0) return allOptions; + if (rootIndexFile) return extractOptionsType(rootIndexFile.content, preferredNames); + if (specsIndexFile) return extractOptionsType(specsIndexFile.content, preferredNames); + + return []; +} + +/** + * Extracts unique markup examples from test files. + */ +export function extractMarkupExamples(files) { + return [ + ...new Set( + files + .filter((file) => isTestFile(file.path)) + .flatMap((file) => extractTestExamples(file.content)), + ), + ]; +} + +/** + * Extracts raw action identifiers. + */ +function extractActionNames(content, constants) { + return resolveAllConstants(extractActions(content), constants); +} + +/** + * Extracts unique ProseMirror plugin names. + */ +function extractPluginNames(content) { + return unique(extractPlugins(content)); +} + +/** + * Extracts unique markdown-it plugin names. + */ +function extractMdPluginNames(content) { + return unique(extractMdPlugins(content)); +} + +/** + * Extracts unique serializer output snippets. + */ +function extractSerializerHints(sourceFiles) { + const serializerContent = joinContents(selectSerializerFiles(sourceFiles)); + return unique(extractSerializerSyntax(serializerContent)); +} + +const FIELD_EXTRACTORS = { + name: ({name}) => name, + packageName: ({packageName}) => packageName, + entryPoint: ({entryPoint}) => entryPoint, + sourcePath: ({sourcePath}) => sourcePath, + category: ({category}) => category, + nodes: ({schema}) => schema.nodes, + marks: ({schema}) => schema.marks, + actions: ({allContent, constants}) => extractActionNames(allContent, constants), + keymaps: ({allContent}) => extractKeymaps(allContent), + inputRules: ({allContent}) => extractInputRules(allContent), + plugins: ({allContent}) => extractPluginNames(allContent), + mdPlugins: ({allContent}) => extractMdPluginNames(allContent), + serializerHints: ({sourceFiles}) => extractSerializerHints(sourceFiles), + options: ({sourceFiles, extDir, name}) => extractOptions(sourceFiles, extDir, name), + markupExamples: ({allFiles}) => extractMarkupExamples(allFiles), + presets: () => [], +}; + +/** + * Builds the final extension IR record. + */ +export function createExtensionRecord(context) { + const record = {}; + + for (const fieldName of Object.keys(EXTENSION_DOC_FIELD_CONFIG)) { + const extractField = FIELD_EXTRACTORS[fieldName]; + if (!extractField) { + throw new Error(`Missing docs-gen field extractor: ${fieldName}`); + } + + record[fieldName] = extractField(context); + } + + return record; +} diff --git a/infra/docs-gen/src/extractor/scan.mjs b/infra/docs-gen/src/extractor/scan.mjs new file mode 100644 index 00000000..52705482 --- /dev/null +++ b/infra/docs-gen/src/extractor/scan.mjs @@ -0,0 +1,41 @@ +/** + * English: Builds one raw extension record from filtered extension source files. + * + * Русский: Собирает одну raw-запись расширения из отфильтрованных source files. + */ +import {basename, relative} from 'node:path'; + +import {extractConstants} from './constants.mjs'; +import {buildSchemaContent, readExtensionSources} from './extension-sources.mjs'; +import {createExtensionRecord} from './record-fields.mjs'; +import {extractSchema} from './schema.mjs'; + +/** + * Scans one extension directory into raw metadata. + */ +export function scanExtension({ + extDir, + name = basename(extDir), + packageName, + entryPoint, + category, + repoRoot, +}) { + const {allFiles, sourceFiles, allContent} = readExtensionSources(extDir); + const constants = extractConstants(allContent); + const schema = extractSchema(buildSchemaContent(sourceFiles, extDir), constants); + + return createExtensionRecord({ + extDir, + name, + packageName, + entryPoint, + sourcePath: relative(repoRoot, extDir), + category, + allFiles, + sourceFiles, + allContent, + constants, + schema, + }); +} diff --git a/infra/docs-gen/src/extractor/schema.mjs b/infra/docs-gen/src/extractor/schema.mjs new file mode 100644 index 00000000..ead0d709 --- /dev/null +++ b/infra/docs-gen/src/extractor/schema.mjs @@ -0,0 +1,23 @@ +/** + * English: Resolves extension schema nodes and marks from spec source content. + * + * Русский: Резолвит schema nodes и marks расширения из spec source content. + */ +import {extractAddMark, extractAddNode, extractMarkSpecs, extractNodeSpecs} from './ast.mjs'; +import {resolveAllConstants} from './constants.mjs'; + +/** + * Extracts schema nodes and marks. + */ +export function extractSchema(specContent, constants) { + return { + nodes: resolveAllConstants( + [...extractAddNode(specContent), ...extractNodeSpecs(specContent)], + constants, + ), + marks: resolveAllConstants( + [...extractAddMark(specContent), ...extractMarkSpecs(specContent)], + constants, + ), + }; +} diff --git a/infra/docs-gen/src/extractor/schema.test.mjs b/infra/docs-gen/src/extractor/schema.test.mjs new file mode 100644 index 00000000..d354ad00 --- /dev/null +++ b/infra/docs-gen/src/extractor/schema.test.mjs @@ -0,0 +1,24 @@ +/** + * English: Unit coverage for AST-based schema registration extraction. + * + * Русский: Unit-покрытие AST-based извлечения schema registrations. + */ +import assert from 'node:assert/strict'; +import {test} from 'node:test'; + +import {extractAddMark, extractAddNode, extractMarkSpecs, extractNodeSpecs} from './ast.mjs'; + +test('schema extraction captures legacy and granular builder methods', () => { + const content = [ + "builder.addNode('legacyNode', () => ({}));", + 'builder', + ' .addNodeSpec(NodeName.Granular, () => ({}))', + ' .addMark(MarkName.Legacy, () => ({}))', + " .addMarkSpec('granularMark', () => ({}));", + ].join('\n'); + + assert.deepEqual(extractAddNode(content), ['legacyNode']); + assert.deepEqual(extractNodeSpecs(content), ['NodeName.Granular']); + assert.deepEqual(extractAddMark(content), ['MarkName.Legacy']); + assert.deepEqual(extractMarkSpecs(content), ['granularMark']); +}); diff --git a/infra/docs-gen/src/extractor/source-files.mjs b/infra/docs-gen/src/extractor/source-files.mjs new file mode 100644 index 00000000..c8f5b40d --- /dev/null +++ b/infra/docs-gen/src/extractor/source-files.mjs @@ -0,0 +1,68 @@ +/** + * English: Selects relevant extension source, spec, serializer, and test files. + * + * Русский: Выбирает релевантные source, spec, serializer и test файлы расширения. + */ +import {dirname} from 'node:path'; + +/** + * Checks whether a file path points to a TypeScript test file. + */ +export function isTestFile(path) { + return path.endsWith('.test.ts') || path.endsWith('.test.tsx'); +} + +/** + * Selects production files from all extension files. + */ +export function selectSourceFiles(files) { + return files.filter((file) => !isTestFile(file.path)); +} + +/** + * Joins file contents into one source string. + */ +export function joinContents(files) { + return files.map((file) => file.content).join('\n'); +} + +/** + * Checks whether a source file can contain schema metadata. + */ +export function isSpecSourceFile(file, extDir) { + return ( + file.path.includes('Specs') || + file.path.includes('const') || + file.path.includes('schema') || + file.path.includes('parser') || + (file.path.endsWith('/index.ts') && dirname(file.path) === extDir) + ); +} + +/** + * Selects files that can contain schema registrations. + */ +export function selectSpecFiles(files, extDir) { + return files.filter((file) => isSpecSourceFile(file, extDir)); +} + +/** + * Selects files that can contain serializer hints. + */ +export function selectSerializerFiles(files) { + return files.filter((file) => file.path.includes('serializer') || file.path.includes('Specs')); +} + +/** + * Finds an extension root index file. + */ +export function findRootIndexFile(files, extDir) { + return files.find((file) => file.path.endsWith('/index.ts') && dirname(file.path) === extDir); +} + +/** + * Finds a specs index file. + */ +export function findSpecsIndexFile(files) { + return files.find((file) => file.path.includes('Specs') && file.path.endsWith('/index.ts')); +} diff --git a/infra/docs-gen/src/logger.mjs b/infra/docs-gen/src/logger.mjs new file mode 100644 index 00000000..f24c4be8 --- /dev/null +++ b/infra/docs-gen/src/logger.mjs @@ -0,0 +1,38 @@ +/** + * English: Minimal console logger shared by docs-gen command-line scripts. + * + * Русский: Минимальный console logger для command-line скриптов docs-gen. + */ +/* eslint-disable no-console */ + +export class Logger { + /** + * Writes an informational message. + */ + info(...args) { + console.log(...args); + } + + /** + * Writes a warning message. + */ + warn(...args) { + console.warn('Warning:', ...args); + } + + /** + * Writes an error message. + */ + error(...args) { + console.error('Error:', ...args); + } + + /** + * Writes a successful completion message. + */ + success(...args) { + console.log('Done:', ...args); + } +} + +export const logger = new Logger(); diff --git a/infra/docs-gen/src/utils.mjs b/infra/docs-gen/src/utils.mjs new file mode 100644 index 00000000..d637d6dc --- /dev/null +++ b/infra/docs-gen/src/utils.mjs @@ -0,0 +1,55 @@ +/** + * English: Filesystem helpers shared by docs-gen build and extraction flows. + * + * Русский: Filesystem helper-ы, общие для build и extraction flow docs-gen. + */ +import {existsSync, readFileSync, readdirSync, statSync} from 'node:fs'; +import {join} from 'node:path'; + +/** + * Reads a UTF-8 text file. + */ +export function readText(filePath) { + return readFileSync(filePath, 'utf-8'); +} + +/** + * Lists uppercase-named child directories. + */ +export function listDirs(dir) { + if (!existsSync(dir)) return []; + + return readdirSync(dir) + .filter((name) => { + const full = join(dir, name); + const firstChar = name.charAt(0); + const startsWithUppercaseLetter = + firstChar !== '' && + firstChar === firstChar.toUpperCase() && + firstChar !== firstChar.toLowerCase(); + + return statSync(full).isDirectory() && startsWithUppercaseLetter; + }) + .sort(); +} + +/** + * Finds recursive directory entries matching a predicate. + */ +export function findFiles(dir, predicate) { + if (!existsSync(dir)) return []; + + return readdirSync(dir, {recursive: true}) + .filter((entry) => predicate(entry)) + .map((entry) => join(dir, entry)) + .sort(); +} + +/** + * Reads recursive TypeScript source files. + */ +export function readAllTsFiles(dir) { + return findFiles(dir, (entry) => entry.endsWith('.ts') || entry.endsWith('.tsx')).map( + (path) => ({path, content: readText(path)}), + ); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 410bbfeb..2dc15fbd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -306,6 +306,9 @@ importers: '@diplodoc/cli': specifier: 5.43.0 version: 5.43.0(@types/markdown-it@13.0.9)(@types/node@25.2.1)(react@18.2.0) + typescript: + specifier: catalog:ts + version: 5.9.3 infra/gulp-tasks: dependencies: