From 758c20985005882911354969f4326bf15e268e93 Mon Sep 17 00:00:00 2001 From: makhnatkin Date: Tue, 16 Jun 2026 13:23:24 +0200 Subject: [PATCH 1/2] build(docs-gen): add extension name extractor --- infra/docs-gen/package.json | 7 +- .../docs-gen/src/extract-extension-names.mjs | 267 ++++++++++++++++++ .../src/extract-extension-names.test.mjs | 35 +++ pnpm-lock.yaml | 3 + 4 files changed, 310 insertions(+), 2 deletions(-) create mode 100644 infra/docs-gen/src/extract-extension-names.mjs create mode 100644 infra/docs-gen/src/extract-extension-names.test.mjs diff --git a/infra/docs-gen/package.json b/infra/docs-gen/package.json index de6aa64c..1a5d01e0 100644 --- a/infra/docs-gen/package.json +++ b/infra/docs-gen/package.json @@ -14,9 +14,12 @@ } }, "scripts": { - "build": "node src/generate-docs.mjs && yfm -i ../../tmp/docs-src -o ../../dist/docs" + "build": "node src/generate-docs.mjs && yfm -i ../../tmp/docs-src -o ../../dist/docs", + "extract:names": "node src/extract-extension-names.mjs", + "test": "node --test src/*.test.mjs" }, "dependencies": { - "@diplodoc/cli": "5.43.0" + "@diplodoc/cli": "5.43.0", + "typescript": "catalog:ts" } } diff --git a/infra/docs-gen/src/extract-extension-names.mjs b/infra/docs-gen/src/extract-extension-names.mjs new file mode 100644 index 00000000..1923fbb3 --- /dev/null +++ b/infra/docs-gen/src/extract-extension-names.mjs @@ -0,0 +1,267 @@ +#!/usr/bin/env node +import { + existsSync, + mkdirSync, + readFileSync, + readdirSync, + statSync, + writeFileSync, +} from 'node:fs'; +import {basename, dirname, join, resolve} from 'node:path'; +import {fileURLToPath} from 'node:url'; + +import ts from 'typescript'; + +export const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +export const DOCS_GEN_DIR = join(REPO_ROOT, 'tmp/docs-gen'); + +const EDITOR_PKG_DIR = join(REPO_ROOT, 'packages/editor'); +const PAGE_CONSTRUCTOR_EXTENSION_DIR = join( + REPO_ROOT, + 'packages/page-constructor-extension/src/extension', +); +const EXTENSION_CATEGORIES = ['base', 'behavior', 'markdown', 'yfm', 'additional']; +const EXTENSION_TYPE_NAMES = new Set(['Extension', 'ExtensionAuto', 'ExtensionWithOptions']); + +export const EXTENSION_NAME_BLACKLIST = [ + 'BaseInputRules', + 'BaseKeymap', + 'BaseStyles', + 'ReactRenderer', + 'SharedState', + 'YfmCut', +]; + +const EXTENSION_ENTRY_POINTS = [ + { + id: 'editor', + kind: 'category-dirs', + packageDir: EDITOR_PKG_DIR, + extensionsDir: 'src/extensions', + categories: EXTENSION_CATEGORIES, + }, + { + id: 'page-constructor-extension', + kind: 'single-extension', + extensionDir: PAGE_CONSTRUCTOR_EXTENSION_DIR, + extensionName: 'YfmPageConstructorExtension', + }, +]; + +function parseSource(content, fileName = 'source.tsx') { + return ts.createSourceFile(fileName, content, ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX); +} + +function forEachNode(root, callback) { + const visit = (node) => { + callback(node); + ts.forEachChild(node, visit); + }; + + visit(root); +} + +function unwrapExpression(expression) { + let current = expression; + + while ( + ts.isParenthesizedExpression(current) || + ts.isAsExpression(current) || + ts.isSatisfiesExpression(current) || + ts.isNonNullExpression(current) || + ts.isTypeAssertionExpression(current) + ) { + current = current.expression; + } + + return current; +} + +function getTypeReferenceName(typeName) { + if (ts.isIdentifier(typeName)) return typeName.text; + if (ts.isQualifiedName(typeName)) return typeName.right.text; + + return null; +} + +function isExtensionType(typeNode) { + return ( + typeNode && + ts.isTypeReferenceNode(typeNode) && + EXTENSION_TYPE_NAMES.has(getTypeReferenceName(typeNode.typeName)) + ); +} + +function hasExportModifier(node) { + return ts.getModifiers(node)?.some((modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword); +} + +function isObjectAssignFromKnownExtension(initializer, extensionImplementations) { + if (!initializer) return false; + + const current = unwrapExpression(initializer); + if (!ts.isCallExpression(current) || !ts.isPropertyAccessExpression(current.expression)) { + return false; + } + + const callee = current.expression; + if ( + !ts.isIdentifier(callee.expression) || + callee.expression.text !== 'Object' || + callee.name.text !== 'assign' + ) { + return false; + } + + const firstArg = current.arguments[0]; + return ( + Boolean(firstArg) && + ts.isIdentifier(firstArg) && + extensionImplementations.has(firstArg.text) + ); +} + +function readVariableDeclarations(sourceFile) { + const declarations = []; + + forEachNode(sourceFile, (node) => { + if (!ts.isVariableStatement(node)) return; + + for (const declaration of node.declarationList.declarations) { + if (ts.isIdentifier(declaration.name)) { + declarations.push({statement: node, declaration}); + } + } + }); + + return declarations; +} + +function unique(values) { + return [...new Set(values.filter(Boolean))]; +} + +export function extractExtensionNamesFromSource(content, fileName) { + const sourceFile = parseSource(content, fileName); + const declarations = readVariableDeclarations(sourceFile); + const extensionImplementations = new Set( + declarations + .filter(({declaration}) => isExtensionType(declaration.type)) + .map(({declaration}) => declaration.name.text), + ); + const names = []; + + for (const {statement, declaration} of declarations) { + if (!hasExportModifier(statement)) continue; + + if ( + isExtensionType(declaration.type) || + isObjectAssignFromKnownExtension(declaration.initializer, extensionImplementations) + ) { + names.push(declaration.name.text); + } + } + + return unique(names); +} + +export function filterExtensionNames(names, blacklist = EXTENSION_NAME_BLACKLIST) { + const blockedNames = new Set(blacklist); + + return names.filter((name) => !blockedNames.has(name)); +} + +function startsWithUppercaseLetter(name) { + const firstChar = name.charAt(0); + + return ( + firstChar !== '' && + firstChar === firstChar.toUpperCase() && + firstChar !== firstChar.toLowerCase() + ); +} + +function listExtensionDirs(dir) { + if (!existsSync(dir)) return []; + + return readdirSync(dir) + .filter((name) => { + const fullPath = join(dir, name); + return statSync(fullPath).isDirectory() && startsWithUppercaseLetter(name); + }) + .map((name) => join(dir, name)) + .sort(); +} + +function readSourceFiles(dir) { + if (!existsSync(dir)) return []; + + const files = []; + for (const entry of readdirSync(dir, {withFileTypes: true})) { + const fullPath = join(dir, entry.name); + + if (entry.isDirectory()) { + files.push(...readSourceFiles(fullPath)); + } else if (entry.isFile() && (entry.name.endsWith('.ts') || entry.name.endsWith('.tsx'))) { + files.push({path: fullPath, content: readFileSync(fullPath, 'utf-8')}); + } + } + + return files.sort((left, right) => left.path.localeCompare(right.path)); +} + +function extractExpectedExtensionName(extensionDir, expectedName = basename(extensionDir)) { + const names = unique( + readSourceFiles(extensionDir).flatMap((file) => + extractExtensionNamesFromSource(file.content, file.path), + ), + ); + + return names.includes(expectedName) ? expectedName : null; +} + +function collectCategoryExtensionNames(entryPoint) { + const names = []; + const extensionsRoot = join(entryPoint.packageDir, entryPoint.extensionsDir); + + for (const category of entryPoint.categories) { + const categoryDir = join(extensionsRoot, category); + + for (const extensionDir of listExtensionDirs(categoryDir)) { + names.push(extractExpectedExtensionName(extensionDir)); + } + } + + return names; +} + +function collectSingleExtensionName(entryPoint) { + return [extractExpectedExtensionName(entryPoint.extensionDir, entryPoint.extensionName)]; +} + +export function collectExtensionNames(entryPoints = EXTENSION_ENTRY_POINTS) { + const names = entryPoints.flatMap((entryPoint) => { + if (entryPoint.kind === 'category-dirs') return collectCategoryExtensionNames(entryPoint); + if (entryPoint.kind === 'single-extension') return collectSingleExtensionName(entryPoint); + + return []; + }); + + return filterExtensionNames(unique(names)); +} + +function writeExtensionNames(outDir, names) { + mkdirSync(outDir, {recursive: true}); + writeFileSync( + join(outDir, 'extensions.json'), + `${JSON.stringify({extensions: names}, null, 2)}\n`, + ); +} + +export function main() { + writeExtensionNames(DOCS_GEN_DIR, collectExtensionNames()); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + main(); +} diff --git a/infra/docs-gen/src/extract-extension-names.test.mjs b/infra/docs-gen/src/extract-extension-names.test.mjs new file mode 100644 index 00000000..48f48b25 --- /dev/null +++ b/infra/docs-gen/src/extract-extension-names.test.mjs @@ -0,0 +1,35 @@ +import assert from 'node:assert/strict'; +import {test} from 'node:test'; + +import { + EXTENSION_NAME_BLACKLIST, + extractExtensionNamesFromSource, + filterExtensionNames, +} from './extract-extension-names.mjs'; + +test('extractExtensionNamesFromSource reads exported extension const names from TypeScript AST', () => { + const content = [ + "import type {ExtensionAuto} from '../../../core';", + 'const InternalExtension: ExtensionAuto = (builder) => {', + ' builder.addAction("internal", () => null);', + '};', + 'export const PublicExtension = Object.assign(InternalExtension, {});', + 'export const DirectExtension: ExtensionAuto = (builder) => {', + ' builder.addAction("direct", () => null);', + '};', + 'export const NotAnExtension = "value";', + ].join('\n'); + + assert.deepEqual(extractExtensionNamesFromSource(content), [ + 'PublicExtension', + 'DirectExtension', + ]); +}); + +test('filterExtensionNames removes blacklisted extension names', () => { + assert.deepEqual(filterExtensionNames(['Bold', 'BaseKeymap', 'YfmCut', 'Italic']), [ + 'Bold', + 'Italic', + ]); + assert.equal(EXTENSION_NAME_BLACKLIST.includes('YfmCut'), true); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 410bbfeb..2dc15fbd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -306,6 +306,9 @@ importers: '@diplodoc/cli': specifier: 5.43.0 version: 5.43.0(@types/markdown-it@13.0.9)(@types/node@25.2.1)(react@18.2.0) + typescript: + specifier: catalog:ts + version: 5.9.3 infra/gulp-tasks: dependencies: From b7881f5b763d6bb5c1e59a94c88f79ed2053be6b Mon Sep 17 00:00:00 2001 From: makhnatkin Date: Tue, 16 Jun 2026 13:36:19 +0200 Subject: [PATCH 2/2] refactor(docs-gen): modularize extension extractor --- infra/docs-gen/package.json | 4 +- infra/docs-gen/src/extract-extension-data.mjs | 14 + .../docs-gen/src/extract-extension-names.mjs | 267 ------------------ .../src/extract-extension-names.test.mjs | 35 --- infra/docs-gen/src/extractor/README.md | 18 ++ infra/docs-gen/src/extractor/ast/core.mjs | 38 +++ .../src/extractor/ast/extension-name.mjs | 85 ++++++ infra/docs-gen/src/extractor/blacklist.mjs | 13 + infra/docs-gen/src/extractor/config.mjs | 30 ++ infra/docs-gen/src/extractor/entry-points.mjs | 72 +++++ .../docs-gen/src/extractor/extractor.test.mjs | 55 ++++ infra/docs-gen/src/extractor/field-config.mjs | 22 ++ infra/docs-gen/src/extractor/fields/name.mjs | 12 + infra/docs-gen/src/extractor/index.mjs | 23 ++ infra/docs-gen/src/extractor/output.mjs | 7 + infra/docs-gen/src/extractor/source-files.mjs | 19 ++ 16 files changed, 410 insertions(+), 304 deletions(-) create mode 100644 infra/docs-gen/src/extract-extension-data.mjs delete mode 100644 infra/docs-gen/src/extract-extension-names.mjs delete mode 100644 infra/docs-gen/src/extract-extension-names.test.mjs create mode 100644 infra/docs-gen/src/extractor/README.md create mode 100644 infra/docs-gen/src/extractor/ast/core.mjs create mode 100644 infra/docs-gen/src/extractor/ast/extension-name.mjs create mode 100644 infra/docs-gen/src/extractor/blacklist.mjs create mode 100644 infra/docs-gen/src/extractor/config.mjs create mode 100644 infra/docs-gen/src/extractor/entry-points.mjs create mode 100644 infra/docs-gen/src/extractor/extractor.test.mjs create mode 100644 infra/docs-gen/src/extractor/field-config.mjs create mode 100644 infra/docs-gen/src/extractor/fields/name.mjs create mode 100644 infra/docs-gen/src/extractor/index.mjs create mode 100644 infra/docs-gen/src/extractor/output.mjs create mode 100644 infra/docs-gen/src/extractor/source-files.mjs diff --git a/infra/docs-gen/package.json b/infra/docs-gen/package.json index 1a5d01e0..404f34ac 100644 --- a/infra/docs-gen/package.json +++ b/infra/docs-gen/package.json @@ -15,8 +15,8 @@ }, "scripts": { "build": "node src/generate-docs.mjs && yfm -i ../../tmp/docs-src -o ../../dist/docs", - "extract:names": "node src/extract-extension-names.mjs", - "test": "node --test src/*.test.mjs" + "extract": "node src/extract-extension-data.mjs", + "test": "node --test src/extractor/*.test.mjs" }, "dependencies": { "@diplodoc/cli": "5.43.0", diff --git a/infra/docs-gen/src/extract-extension-data.mjs b/infra/docs-gen/src/extract-extension-data.mjs new file mode 100644 index 00000000..0fe58319 --- /dev/null +++ b/infra/docs-gen/src/extract-extension-data.mjs @@ -0,0 +1,14 @@ +#!/usr/bin/env node +import {fileURLToPath} from 'node:url'; + +import {DOCS_GEN_DIR} from './extractor/config.mjs'; +import {extractExtensionData} from './extractor/index.mjs'; +import {writeExtensionsJson} from './extractor/output.mjs'; + +export function main() { + writeExtensionsJson(DOCS_GEN_DIR, extractExtensionData()); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + main(); +} diff --git a/infra/docs-gen/src/extract-extension-names.mjs b/infra/docs-gen/src/extract-extension-names.mjs deleted file mode 100644 index 1923fbb3..00000000 --- a/infra/docs-gen/src/extract-extension-names.mjs +++ /dev/null @@ -1,267 +0,0 @@ -#!/usr/bin/env node -import { - existsSync, - mkdirSync, - readFileSync, - readdirSync, - statSync, - writeFileSync, -} from 'node:fs'; -import {basename, dirname, join, resolve} from 'node:path'; -import {fileURLToPath} from 'node:url'; - -import ts from 'typescript'; - -export const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); -export const DOCS_GEN_DIR = join(REPO_ROOT, 'tmp/docs-gen'); - -const EDITOR_PKG_DIR = join(REPO_ROOT, 'packages/editor'); -const PAGE_CONSTRUCTOR_EXTENSION_DIR = join( - REPO_ROOT, - 'packages/page-constructor-extension/src/extension', -); -const EXTENSION_CATEGORIES = ['base', 'behavior', 'markdown', 'yfm', 'additional']; -const EXTENSION_TYPE_NAMES = new Set(['Extension', 'ExtensionAuto', 'ExtensionWithOptions']); - -export const EXTENSION_NAME_BLACKLIST = [ - 'BaseInputRules', - 'BaseKeymap', - 'BaseStyles', - 'ReactRenderer', - 'SharedState', - 'YfmCut', -]; - -const EXTENSION_ENTRY_POINTS = [ - { - id: 'editor', - kind: 'category-dirs', - packageDir: EDITOR_PKG_DIR, - extensionsDir: 'src/extensions', - categories: EXTENSION_CATEGORIES, - }, - { - id: 'page-constructor-extension', - kind: 'single-extension', - extensionDir: PAGE_CONSTRUCTOR_EXTENSION_DIR, - extensionName: 'YfmPageConstructorExtension', - }, -]; - -function parseSource(content, fileName = 'source.tsx') { - return ts.createSourceFile(fileName, content, ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX); -} - -function forEachNode(root, callback) { - const visit = (node) => { - callback(node); - ts.forEachChild(node, visit); - }; - - visit(root); -} - -function unwrapExpression(expression) { - let current = expression; - - while ( - ts.isParenthesizedExpression(current) || - ts.isAsExpression(current) || - ts.isSatisfiesExpression(current) || - ts.isNonNullExpression(current) || - ts.isTypeAssertionExpression(current) - ) { - current = current.expression; - } - - return current; -} - -function getTypeReferenceName(typeName) { - if (ts.isIdentifier(typeName)) return typeName.text; - if (ts.isQualifiedName(typeName)) return typeName.right.text; - - return null; -} - -function isExtensionType(typeNode) { - return ( - typeNode && - ts.isTypeReferenceNode(typeNode) && - EXTENSION_TYPE_NAMES.has(getTypeReferenceName(typeNode.typeName)) - ); -} - -function hasExportModifier(node) { - return ts.getModifiers(node)?.some((modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword); -} - -function isObjectAssignFromKnownExtension(initializer, extensionImplementations) { - if (!initializer) return false; - - const current = unwrapExpression(initializer); - if (!ts.isCallExpression(current) || !ts.isPropertyAccessExpression(current.expression)) { - return false; - } - - const callee = current.expression; - if ( - !ts.isIdentifier(callee.expression) || - callee.expression.text !== 'Object' || - callee.name.text !== 'assign' - ) { - return false; - } - - const firstArg = current.arguments[0]; - return ( - Boolean(firstArg) && - ts.isIdentifier(firstArg) && - extensionImplementations.has(firstArg.text) - ); -} - -function readVariableDeclarations(sourceFile) { - const declarations = []; - - forEachNode(sourceFile, (node) => { - if (!ts.isVariableStatement(node)) return; - - for (const declaration of node.declarationList.declarations) { - if (ts.isIdentifier(declaration.name)) { - declarations.push({statement: node, declaration}); - } - } - }); - - return declarations; -} - -function unique(values) { - return [...new Set(values.filter(Boolean))]; -} - -export function extractExtensionNamesFromSource(content, fileName) { - const sourceFile = parseSource(content, fileName); - const declarations = readVariableDeclarations(sourceFile); - const extensionImplementations = new Set( - declarations - .filter(({declaration}) => isExtensionType(declaration.type)) - .map(({declaration}) => declaration.name.text), - ); - const names = []; - - for (const {statement, declaration} of declarations) { - if (!hasExportModifier(statement)) continue; - - if ( - isExtensionType(declaration.type) || - isObjectAssignFromKnownExtension(declaration.initializer, extensionImplementations) - ) { - names.push(declaration.name.text); - } - } - - return unique(names); -} - -export function filterExtensionNames(names, blacklist = EXTENSION_NAME_BLACKLIST) { - const blockedNames = new Set(blacklist); - - return names.filter((name) => !blockedNames.has(name)); -} - -function startsWithUppercaseLetter(name) { - const firstChar = name.charAt(0); - - return ( - firstChar !== '' && - firstChar === firstChar.toUpperCase() && - firstChar !== firstChar.toLowerCase() - ); -} - -function listExtensionDirs(dir) { - if (!existsSync(dir)) return []; - - return readdirSync(dir) - .filter((name) => { - const fullPath = join(dir, name); - return statSync(fullPath).isDirectory() && startsWithUppercaseLetter(name); - }) - .map((name) => join(dir, name)) - .sort(); -} - -function readSourceFiles(dir) { - if (!existsSync(dir)) return []; - - const files = []; - for (const entry of readdirSync(dir, {withFileTypes: true})) { - const fullPath = join(dir, entry.name); - - if (entry.isDirectory()) { - files.push(...readSourceFiles(fullPath)); - } else if (entry.isFile() && (entry.name.endsWith('.ts') || entry.name.endsWith('.tsx'))) { - files.push({path: fullPath, content: readFileSync(fullPath, 'utf-8')}); - } - } - - return files.sort((left, right) => left.path.localeCompare(right.path)); -} - -function extractExpectedExtensionName(extensionDir, expectedName = basename(extensionDir)) { - const names = unique( - readSourceFiles(extensionDir).flatMap((file) => - extractExtensionNamesFromSource(file.content, file.path), - ), - ); - - return names.includes(expectedName) ? expectedName : null; -} - -function collectCategoryExtensionNames(entryPoint) { - const names = []; - const extensionsRoot = join(entryPoint.packageDir, entryPoint.extensionsDir); - - for (const category of entryPoint.categories) { - const categoryDir = join(extensionsRoot, category); - - for (const extensionDir of listExtensionDirs(categoryDir)) { - names.push(extractExpectedExtensionName(extensionDir)); - } - } - - return names; -} - -function collectSingleExtensionName(entryPoint) { - return [extractExpectedExtensionName(entryPoint.extensionDir, entryPoint.extensionName)]; -} - -export function collectExtensionNames(entryPoints = EXTENSION_ENTRY_POINTS) { - const names = entryPoints.flatMap((entryPoint) => { - if (entryPoint.kind === 'category-dirs') return collectCategoryExtensionNames(entryPoint); - if (entryPoint.kind === 'single-extension') return collectSingleExtensionName(entryPoint); - - return []; - }); - - return filterExtensionNames(unique(names)); -} - -function writeExtensionNames(outDir, names) { - mkdirSync(outDir, {recursive: true}); - writeFileSync( - join(outDir, 'extensions.json'), - `${JSON.stringify({extensions: names}, null, 2)}\n`, - ); -} - -export function main() { - writeExtensionNames(DOCS_GEN_DIR, collectExtensionNames()); -} - -if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { - main(); -} diff --git a/infra/docs-gen/src/extract-extension-names.test.mjs b/infra/docs-gen/src/extract-extension-names.test.mjs deleted file mode 100644 index 48f48b25..00000000 --- a/infra/docs-gen/src/extract-extension-names.test.mjs +++ /dev/null @@ -1,35 +0,0 @@ -import assert from 'node:assert/strict'; -import {test} from 'node:test'; - -import { - EXTENSION_NAME_BLACKLIST, - extractExtensionNamesFromSource, - filterExtensionNames, -} from './extract-extension-names.mjs'; - -test('extractExtensionNamesFromSource reads exported extension const names from TypeScript AST', () => { - const content = [ - "import type {ExtensionAuto} from '../../../core';", - 'const InternalExtension: ExtensionAuto = (builder) => {', - ' builder.addAction("internal", () => null);', - '};', - 'export const PublicExtension = Object.assign(InternalExtension, {});', - 'export const DirectExtension: ExtensionAuto = (builder) => {', - ' builder.addAction("direct", () => null);', - '};', - 'export const NotAnExtension = "value";', - ].join('\n'); - - assert.deepEqual(extractExtensionNamesFromSource(content), [ - 'PublicExtension', - 'DirectExtension', - ]); -}); - -test('filterExtensionNames removes blacklisted extension names', () => { - assert.deepEqual(filterExtensionNames(['Bold', 'BaseKeymap', 'YfmCut', 'Italic']), [ - 'Bold', - 'Italic', - ]); - assert.equal(EXTENSION_NAME_BLACKLIST.includes('YfmCut'), true); -}); diff --git a/infra/docs-gen/src/extractor/README.md b/infra/docs-gen/src/extractor/README.md new file mode 100644 index 00000000..e8923464 --- /dev/null +++ b/infra/docs-gen/src/extractor/README.md @@ -0,0 +1,18 @@ +# Extension Extractor Layout + +This extractor is intentionally modular from the first PR, even though it currently writes only +the `name` field. + +- `config.mjs` defines repository paths and extension entry points. +- `blacklist.mjs` owns internal and public extension skip lists. +- `entry-points.mjs` turns configured entry points into extension refs and applies blacklist + filtering. +- `source-files.mjs` reads TypeScript/TSX sources for one extension ref. +- `ast/` contains TypeScript AST primitives and source scanners. +- `fields/` contains one controller per extracted field. +- `field-config.mjs` declares the active output fields and connects each field to its controller. +- `index.mjs` orchestrates ref collection, source reading, field extraction, and record creation. +- `output.mjs` writes extracted records to `tmp/docs-gen/extensions.json`. + +To add another field later, add its controller under `fields/`, add any focused AST scanner under +`ast/`, then register the field in `field-config.mjs`. diff --git a/infra/docs-gen/src/extractor/ast/core.mjs b/infra/docs-gen/src/extractor/ast/core.mjs new file mode 100644 index 00000000..144f1e51 --- /dev/null +++ b/infra/docs-gen/src/extractor/ast/core.mjs @@ -0,0 +1,38 @@ +import ts from 'typescript'; + +export function parseSource(content, fileName = 'source.tsx') { + return ts.createSourceFile(fileName, content, ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX); +} + +export function forEachNode(root, callback) { + const visit = (node) => { + callback(node); + ts.forEachChild(node, visit); + }; + + visit(root); +} + +export function unwrapExpression(expression) { + let current = expression; + + while ( + ts.isParenthesizedExpression(current) || + ts.isAsExpression(current) || + ts.isSatisfiesExpression(current) || + ts.isNonNullExpression(current) || + ts.isTypeAssertionExpression(current) + ) { + current = current.expression; + } + + return current; +} + +export function hasExportModifier(node) { + return ts.getModifiers(node)?.some((modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword); +} + +export function unique(values) { + return [...new Set(values.filter(Boolean))]; +} diff --git a/infra/docs-gen/src/extractor/ast/extension-name.mjs b/infra/docs-gen/src/extractor/ast/extension-name.mjs new file mode 100644 index 00000000..042b138a --- /dev/null +++ b/infra/docs-gen/src/extractor/ast/extension-name.mjs @@ -0,0 +1,85 @@ +import ts from 'typescript'; + +import {forEachNode, hasExportModifier, parseSource, unique, unwrapExpression} from './core.mjs'; + +const EXTENSION_TYPE_NAMES = new Set(['Extension', 'ExtensionAuto', 'ExtensionWithOptions']); + +function getTypeReferenceName(typeName) { + if (ts.isIdentifier(typeName)) return typeName.text; + if (ts.isQualifiedName(typeName)) return typeName.right.text; + + return null; +} + +function isExtensionType(typeNode) { + return ( + typeNode && + ts.isTypeReferenceNode(typeNode) && + EXTENSION_TYPE_NAMES.has(getTypeReferenceName(typeNode.typeName)) + ); +} + +function isObjectAssignFromKnownExtension(initializer, extensionImplementations) { + if (!initializer) return false; + + const current = unwrapExpression(initializer); + if (!ts.isCallExpression(current) || !ts.isPropertyAccessExpression(current.expression)) { + return false; + } + + const callee = current.expression; + if ( + !ts.isIdentifier(callee.expression) || + callee.expression.text !== 'Object' || + callee.name.text !== 'assign' + ) { + return false; + } + + const firstArg = current.arguments[0]; + return ( + Boolean(firstArg) && + ts.isIdentifier(firstArg) && + extensionImplementations.has(firstArg.text) + ); +} + +function readVariableDeclarations(sourceFile) { + const declarations = []; + + forEachNode(sourceFile, (node) => { + if (!ts.isVariableStatement(node)) return; + + for (const declaration of node.declarationList.declarations) { + if (ts.isIdentifier(declaration.name)) { + declarations.push({statement: node, declaration}); + } + } + }); + + return declarations; +} + +export function extractExtensionNamesFromSource(content, fileName) { + const sourceFile = parseSource(content, fileName); + const declarations = readVariableDeclarations(sourceFile); + const extensionImplementations = new Set( + declarations + .filter(({declaration}) => isExtensionType(declaration.type)) + .map(({declaration}) => declaration.name.text), + ); + const names = []; + + for (const {statement, declaration} of declarations) { + if (!hasExportModifier(statement)) continue; + + if ( + isExtensionType(declaration.type) || + isObjectAssignFromKnownExtension(declaration.initializer, extensionImplementations) + ) { + names.push(declaration.name.text); + } + } + + return unique(names); +} diff --git a/infra/docs-gen/src/extractor/blacklist.mjs b/infra/docs-gen/src/extractor/blacklist.mjs new file mode 100644 index 00000000..3864bd0e --- /dev/null +++ b/infra/docs-gen/src/extractor/blacklist.mjs @@ -0,0 +1,13 @@ +export const INTERNAL_EXTENSION_BLACKLIST = [ + 'BaseInputRules', + 'BaseKeymap', + 'BaseStyles', + 'ReactRenderer', + 'SharedState', +]; + +export const EXTENSION_BLACKLIST = ['YfmCut']; + +export function isBlacklistedExtension(name) { + return [...INTERNAL_EXTENSION_BLACKLIST, ...EXTENSION_BLACKLIST].includes(name); +} diff --git a/infra/docs-gen/src/extractor/config.mjs b/infra/docs-gen/src/extractor/config.mjs new file mode 100644 index 00000000..f9d0f574 --- /dev/null +++ b/infra/docs-gen/src/extractor/config.mjs @@ -0,0 +1,30 @@ +import {dirname, join, resolve} from 'node:path'; +import {fileURLToPath} from 'node:url'; + +export const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../..'); +export const DOCS_GEN_DIR = join(REPO_ROOT, 'tmp/docs-gen'); +export const EDITOR_PKG_DIR = join(REPO_ROOT, 'packages/editor'); +export const PAGE_CONSTRUCTOR_EXTENSION_DIR = join( + REPO_ROOT, + 'packages/page-constructor-extension/src/extension', +); + +export const EXTENSION_CATEGORIES = ['base', 'behavior', 'markdown', 'yfm', 'additional']; + +export const EXTENSION_ENTRY_POINTS = [ + { + id: 'editor', + packageName: '@gravity-ui/markdown-editor', + kind: 'category-dirs', + packageDir: EDITOR_PKG_DIR, + extensionsDir: 'src/extensions', + categories: EXTENSION_CATEGORIES, + }, + { + id: 'page-constructor-extension', + packageName: '@gravity-ui/markdown-editor-page-constructor-extension', + kind: 'single-extension', + extensionDir: PAGE_CONSTRUCTOR_EXTENSION_DIR, + extensionName: 'YfmPageConstructorExtension', + }, +]; diff --git a/infra/docs-gen/src/extractor/entry-points.mjs b/infra/docs-gen/src/extractor/entry-points.mjs new file mode 100644 index 00000000..0d2849dd --- /dev/null +++ b/infra/docs-gen/src/extractor/entry-points.mjs @@ -0,0 +1,72 @@ +import {existsSync, readdirSync, statSync} from 'node:fs'; +import {join} from 'node:path'; + +import {isBlacklistedExtension} from './blacklist.mjs'; + +function startsWithUppercaseLetter(name) { + const firstChar = name.charAt(0); + + return ( + firstChar !== '' && + firstChar === firstChar.toUpperCase() && + firstChar !== firstChar.toLowerCase() + ); +} + +function listExtensionDirs(dir) { + if (!existsSync(dir)) return []; + + return readdirSync(dir) + .filter((name) => { + const fullPath = join(dir, name); + return statSync(fullPath).isDirectory() && startsWithUppercaseLetter(name); + }) + .map((name) => ({name, dir: join(dir, name)})) + .sort((left, right) => left.name.localeCompare(right.name)); +} + +function collectCategoryExtensionRefs(entryPoint) { + const extensionsRoot = join(entryPoint.packageDir, entryPoint.extensionsDir); + const refs = []; + + for (const category of entryPoint.categories) { + const categoryDir = join(extensionsRoot, category); + + for (const extensionDir of listExtensionDirs(categoryDir)) { + refs.push({ + name: extensionDir.name, + sourceDir: extensionDir.dir, + category, + entryPoint: entryPoint.id, + packageName: entryPoint.packageName, + }); + } + } + + return refs; +} + +function collectSingleExtensionRef(entryPoint) { + return [ + { + name: entryPoint.extensionName, + sourceDir: entryPoint.extensionDir, + category: 'external', + entryPoint: entryPoint.id, + packageName: entryPoint.packageName, + }, + ]; +} + +export function collectExtensionRefs(entryPoints) { + return entryPoints.flatMap((entryPoint) => { + if (entryPoint.kind === 'category-dirs') return collectCategoryExtensionRefs(entryPoint); + if (entryPoint.kind === 'single-extension') return collectSingleExtensionRef(entryPoint); + + return []; + }); +} + +export function filterExtensionRefs(extensionRefs) { + return extensionRefs.filter((extensionRef) => !isBlacklistedExtension(extensionRef.name)); +} diff --git a/infra/docs-gen/src/extractor/extractor.test.mjs b/infra/docs-gen/src/extractor/extractor.test.mjs new file mode 100644 index 00000000..8936f3f7 --- /dev/null +++ b/infra/docs-gen/src/extractor/extractor.test.mjs @@ -0,0 +1,55 @@ +import assert from 'node:assert/strict'; +import {test} from 'node:test'; + +import {extractExtensionNamesFromSource} from './ast/extension-name.mjs'; +import {EXTENSION_BLACKLIST, INTERNAL_EXTENSION_BLACKLIST} from './blacklist.mjs'; +import {filterExtensionRefs} from './entry-points.mjs'; +import {createExtensionRecord, EXTENSION_FIELD_CONFIG} from './field-config.mjs'; + +function createSourceFile(content) { + return {path: 'extension.ts', content}; +} + +test('extractExtensionNamesFromSource reads exported extension const names from TypeScript AST', () => { + const content = [ + "import type {ExtensionAuto} from '../../../core';", + 'const InternalExtension: ExtensionAuto = (builder) => {', + ' builder.addAction("internal", () => null);', + '};', + 'export const PublicExtension = Object.assign(InternalExtension, {});', + 'export const DirectExtension: ExtensionAuto = (builder) => {', + ' builder.addAction("direct", () => null);', + '};', + 'export const NotAnExtension = "value";', + ].join('\n'); + + assert.deepEqual(extractExtensionNamesFromSource(content), [ + 'PublicExtension', + 'DirectExtension', + ]); +}); + +test('filterExtensionRefs removes internal and public blacklist entries', () => { + assert.deepEqual( + filterExtensionRefs([ + {name: 'Bold'}, + {name: 'BaseKeymap'}, + {name: 'YfmCut'}, + {name: 'Italic'}, + ]).map((extensionRef) => extensionRef.name), + ['Bold', 'Italic'], + ); + assert.equal(INTERNAL_EXTENSION_BLACKLIST.includes('BaseKeymap'), true); + assert.equal(EXTENSION_BLACKLIST.includes('YfmCut'), true); +}); + +test('createExtensionRecord extracts only configured fields', () => { + const content = 'export const Bold: ExtensionAuto = (builder) => builder;'; + const record = createExtensionRecord({ + extensionRef: {name: 'Bold'}, + sourceFiles: [createSourceFile(content)], + }); + + assert.deepEqual(Object.keys(EXTENSION_FIELD_CONFIG), ['name']); + assert.deepEqual(record, {name: 'Bold'}); +}); diff --git a/infra/docs-gen/src/extractor/field-config.mjs b/infra/docs-gen/src/extractor/field-config.mjs new file mode 100644 index 00000000..c5281ae4 --- /dev/null +++ b/infra/docs-gen/src/extractor/field-config.mjs @@ -0,0 +1,22 @@ +import {extractNameField} from './fields/name.mjs'; + +export const EXTENSION_FIELD_CONFIG = { + name: { + from: 'extension source files: exported Extension/ExtensionAuto/ExtensionWithOptions const', + required: true, + extract: extractNameField, + }, +}; + +export function createExtensionRecord(context, fieldConfig = EXTENSION_FIELD_CONFIG) { + const record = {}; + + for (const [fieldName, config] of Object.entries(fieldConfig)) { + const value = config.extract(context); + if (config.required && value === null) return null; + + record[fieldName] = value; + } + + return record; +} diff --git a/infra/docs-gen/src/extractor/fields/name.mjs b/infra/docs-gen/src/extractor/fields/name.mjs new file mode 100644 index 00000000..a30ad491 --- /dev/null +++ b/infra/docs-gen/src/extractor/fields/name.mjs @@ -0,0 +1,12 @@ +import {extractExtensionNamesFromSource} from '../ast/extension-name.mjs'; +import {unique} from '../ast/core.mjs'; + +export function extractNameField({extensionRef, sourceFiles}) { + const names = unique( + sourceFiles.flatMap((sourceFile) => + extractExtensionNamesFromSource(sourceFile.content, sourceFile.path), + ), + ); + + return names.includes(extensionRef.name) ? extensionRef.name : null; +} diff --git a/infra/docs-gen/src/extractor/index.mjs b/infra/docs-gen/src/extractor/index.mjs new file mode 100644 index 00000000..5d83ef87 --- /dev/null +++ b/infra/docs-gen/src/extractor/index.mjs @@ -0,0 +1,23 @@ +import {EXTENSION_ENTRY_POINTS} from './config.mjs'; +import {collectExtensionRefs, filterExtensionRefs} from './entry-points.mjs'; +import {createExtensionRecord, EXTENSION_FIELD_CONFIG} from './field-config.mjs'; +import {readSourceFiles} from './source-files.mjs'; + +function scanExtension(extensionRef, fieldConfig) { + return createExtensionRecord( + { + extensionRef, + sourceFiles: readSourceFiles(extensionRef.sourceDir), + }, + fieldConfig, + ); +} + +export function extractExtensionData({ + entryPoints = EXTENSION_ENTRY_POINTS, + fieldConfig = EXTENSION_FIELD_CONFIG, +} = {}) { + return filterExtensionRefs(collectExtensionRefs(entryPoints)) + .map((extensionRef) => scanExtension(extensionRef, fieldConfig)) + .filter(Boolean); +} diff --git a/infra/docs-gen/src/extractor/output.mjs b/infra/docs-gen/src/extractor/output.mjs new file mode 100644 index 00000000..8799f0dc --- /dev/null +++ b/infra/docs-gen/src/extractor/output.mjs @@ -0,0 +1,7 @@ +import {mkdirSync, writeFileSync} from 'node:fs'; +import {join} from 'node:path'; + +export function writeExtensionsJson(outDir, extensions) { + mkdirSync(outDir, {recursive: true}); + writeFileSync(join(outDir, 'extensions.json'), `${JSON.stringify({extensions}, null, 2)}\n`); +} diff --git a/infra/docs-gen/src/extractor/source-files.mjs b/infra/docs-gen/src/extractor/source-files.mjs new file mode 100644 index 00000000..1a338121 --- /dev/null +++ b/infra/docs-gen/src/extractor/source-files.mjs @@ -0,0 +1,19 @@ +import {existsSync, readFileSync, readdirSync} from 'node:fs'; +import {join} from 'node:path'; + +export function readSourceFiles(dir) { + if (!existsSync(dir)) return []; + + const files = []; + for (const entry of readdirSync(dir, {withFileTypes: true})) { + const fullPath = join(dir, entry.name); + + if (entry.isDirectory()) { + files.push(...readSourceFiles(fullPath)); + } else if (entry.isFile() && (entry.name.endsWith('.ts') || entry.name.endsWith('.tsx'))) { + files.push({path: fullPath, content: readFileSync(fullPath, 'utf-8')}); + } + } + + return files.sort((left, right) => left.path.localeCompare(right.path)); +}