diff --git a/infra/docs-gen/package.json b/infra/docs-gen/package.json index de6aa64c..404f34ac 100644 --- a/infra/docs-gen/package.json +++ b/infra/docs-gen/package.json @@ -14,9 +14,12 @@ } }, "scripts": { - "build": "node src/generate-docs.mjs && yfm -i ../../tmp/docs-src -o ../../dist/docs" + "build": "node src/generate-docs.mjs && yfm -i ../../tmp/docs-src -o ../../dist/docs", + "extract": "node src/extract-extension-data.mjs", + "test": "node --test src/extractor/*.test.mjs" }, "dependencies": { - "@diplodoc/cli": "5.43.0" + "@diplodoc/cli": "5.43.0", + "typescript": "catalog:ts" } } diff --git a/infra/docs-gen/src/extract-extension-data.mjs b/infra/docs-gen/src/extract-extension-data.mjs new file mode 100644 index 00000000..0fe58319 --- /dev/null +++ b/infra/docs-gen/src/extract-extension-data.mjs @@ -0,0 +1,14 @@ +#!/usr/bin/env node +import {fileURLToPath} from 'node:url'; + +import {DOCS_GEN_DIR} from './extractor/config.mjs'; +import {extractExtensionData} from './extractor/index.mjs'; +import {writeExtensionsJson} from './extractor/output.mjs'; + +export function main() { + writeExtensionsJson(DOCS_GEN_DIR, extractExtensionData()); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + main(); +} diff --git a/infra/docs-gen/src/extractor/README.md b/infra/docs-gen/src/extractor/README.md new file mode 100644 index 00000000..e8923464 --- /dev/null +++ b/infra/docs-gen/src/extractor/README.md @@ -0,0 +1,18 @@ +# Extension Extractor Layout + +This extractor is intentionally modular from the first PR, even though it currently writes only +the `name` field. + +- `config.mjs` defines repository paths and extension entry points. +- `blacklist.mjs` owns internal and public extension skip lists. +- `entry-points.mjs` turns configured entry points into extension refs and applies blacklist + filtering. +- `source-files.mjs` reads TypeScript/TSX sources for one extension ref. +- `ast/` contains TypeScript AST primitives and source scanners. +- `fields/` contains one controller per extracted field. +- `field-config.mjs` declares the active output fields and connects each field to its controller. +- `index.mjs` orchestrates ref collection, source reading, field extraction, and record creation. +- `output.mjs` writes extracted records to `tmp/docs-gen/extensions.json`. + +To add another field later, add its controller under `fields/`, add any focused AST scanner under +`ast/`, then register the field in `field-config.mjs`. diff --git a/infra/docs-gen/src/extractor/ast/core.mjs b/infra/docs-gen/src/extractor/ast/core.mjs new file mode 100644 index 00000000..144f1e51 --- /dev/null +++ b/infra/docs-gen/src/extractor/ast/core.mjs @@ -0,0 +1,38 @@ +import ts from 'typescript'; + +export function parseSource(content, fileName = 'source.tsx') { + return ts.createSourceFile(fileName, content, ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX); +} + +export function forEachNode(root, callback) { + const visit = (node) => { + callback(node); + ts.forEachChild(node, visit); + }; + + visit(root); +} + +export function unwrapExpression(expression) { + let current = expression; + + while ( + ts.isParenthesizedExpression(current) || + ts.isAsExpression(current) || + ts.isSatisfiesExpression(current) || + ts.isNonNullExpression(current) || + ts.isTypeAssertionExpression(current) + ) { + current = current.expression; + } + + return current; +} + +export function hasExportModifier(node) { + return ts.getModifiers(node)?.some((modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword); +} + +export function unique(values) { + return [...new Set(values.filter(Boolean))]; +} diff --git a/infra/docs-gen/src/extractor/ast/extension-name.mjs b/infra/docs-gen/src/extractor/ast/extension-name.mjs new file mode 100644 index 00000000..042b138a --- /dev/null +++ b/infra/docs-gen/src/extractor/ast/extension-name.mjs @@ -0,0 +1,85 @@ +import ts from 'typescript'; + +import {forEachNode, hasExportModifier, parseSource, unique, unwrapExpression} from './core.mjs'; + +const EXTENSION_TYPE_NAMES = new Set(['Extension', 'ExtensionAuto', 'ExtensionWithOptions']); + +function getTypeReferenceName(typeName) { + if (ts.isIdentifier(typeName)) return typeName.text; + if (ts.isQualifiedName(typeName)) return typeName.right.text; + + return null; +} + +function isExtensionType(typeNode) { + return ( + typeNode && + ts.isTypeReferenceNode(typeNode) && + EXTENSION_TYPE_NAMES.has(getTypeReferenceName(typeNode.typeName)) + ); +} + +function isObjectAssignFromKnownExtension(initializer, extensionImplementations) { + if (!initializer) return false; + + const current = unwrapExpression(initializer); + if (!ts.isCallExpression(current) || !ts.isPropertyAccessExpression(current.expression)) { + return false; + } + + const callee = current.expression; + if ( + !ts.isIdentifier(callee.expression) || + callee.expression.text !== 'Object' || + callee.name.text !== 'assign' + ) { + return false; + } + + const firstArg = current.arguments[0]; + return ( + Boolean(firstArg) && + ts.isIdentifier(firstArg) && + extensionImplementations.has(firstArg.text) + ); +} + +function readVariableDeclarations(sourceFile) { + const declarations = []; + + forEachNode(sourceFile, (node) => { + if (!ts.isVariableStatement(node)) return; + + for (const declaration of node.declarationList.declarations) { + if (ts.isIdentifier(declaration.name)) { + declarations.push({statement: node, declaration}); + } + } + }); + + return declarations; +} + +export function extractExtensionNamesFromSource(content, fileName) { + const sourceFile = parseSource(content, fileName); + const declarations = readVariableDeclarations(sourceFile); + const extensionImplementations = new Set( + declarations + .filter(({declaration}) => isExtensionType(declaration.type)) + .map(({declaration}) => declaration.name.text), + ); + const names = []; + + for (const {statement, declaration} of declarations) { + if (!hasExportModifier(statement)) continue; + + if ( + isExtensionType(declaration.type) || + isObjectAssignFromKnownExtension(declaration.initializer, extensionImplementations) + ) { + names.push(declaration.name.text); + } + } + + return unique(names); +} diff --git a/infra/docs-gen/src/extractor/blacklist.mjs b/infra/docs-gen/src/extractor/blacklist.mjs new file mode 100644 index 00000000..3864bd0e --- /dev/null +++ b/infra/docs-gen/src/extractor/blacklist.mjs @@ -0,0 +1,13 @@ +export const INTERNAL_EXTENSION_BLACKLIST = [ + 'BaseInputRules', + 'BaseKeymap', + 'BaseStyles', + 'ReactRenderer', + 'SharedState', +]; + +export const EXTENSION_BLACKLIST = ['YfmCut']; + +export function isBlacklistedExtension(name) { + return [...INTERNAL_EXTENSION_BLACKLIST, ...EXTENSION_BLACKLIST].includes(name); +} diff --git a/infra/docs-gen/src/extractor/config.mjs b/infra/docs-gen/src/extractor/config.mjs new file mode 100644 index 00000000..f9d0f574 --- /dev/null +++ b/infra/docs-gen/src/extractor/config.mjs @@ -0,0 +1,30 @@ +import {dirname, join, resolve} from 'node:path'; +import {fileURLToPath} from 'node:url'; + +export const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../../..'); +export const DOCS_GEN_DIR = join(REPO_ROOT, 'tmp/docs-gen'); +export const EDITOR_PKG_DIR = join(REPO_ROOT, 'packages/editor'); +export const PAGE_CONSTRUCTOR_EXTENSION_DIR = join( + REPO_ROOT, + 'packages/page-constructor-extension/src/extension', +); + +export const EXTENSION_CATEGORIES = ['base', 'behavior', 'markdown', 'yfm', 'additional']; + +export const EXTENSION_ENTRY_POINTS = [ + { + id: 'editor', + packageName: '@gravity-ui/markdown-editor', + kind: 'category-dirs', + packageDir: EDITOR_PKG_DIR, + extensionsDir: 'src/extensions', + categories: EXTENSION_CATEGORIES, + }, + { + id: 'page-constructor-extension', + packageName: '@gravity-ui/markdown-editor-page-constructor-extension', + kind: 'single-extension', + extensionDir: PAGE_CONSTRUCTOR_EXTENSION_DIR, + extensionName: 'YfmPageConstructorExtension', + }, +]; diff --git a/infra/docs-gen/src/extractor/entry-points.mjs b/infra/docs-gen/src/extractor/entry-points.mjs new file mode 100644 index 00000000..0d2849dd --- /dev/null +++ b/infra/docs-gen/src/extractor/entry-points.mjs @@ -0,0 +1,72 @@ +import {existsSync, readdirSync, statSync} from 'node:fs'; +import {join} from 'node:path'; + +import {isBlacklistedExtension} from './blacklist.mjs'; + +function startsWithUppercaseLetter(name) { + const firstChar = name.charAt(0); + + return ( + firstChar !== '' && + firstChar === firstChar.toUpperCase() && + firstChar !== firstChar.toLowerCase() + ); +} + +function listExtensionDirs(dir) { + if (!existsSync(dir)) return []; + + return readdirSync(dir) + .filter((name) => { + const fullPath = join(dir, name); + return statSync(fullPath).isDirectory() && startsWithUppercaseLetter(name); + }) + .map((name) => ({name, dir: join(dir, name)})) + .sort((left, right) => left.name.localeCompare(right.name)); +} + +function collectCategoryExtensionRefs(entryPoint) { + const extensionsRoot = join(entryPoint.packageDir, entryPoint.extensionsDir); + const refs = []; + + for (const category of entryPoint.categories) { + const categoryDir = join(extensionsRoot, category); + + for (const extensionDir of listExtensionDirs(categoryDir)) { + refs.push({ + name: extensionDir.name, + sourceDir: extensionDir.dir, + category, + entryPoint: entryPoint.id, + packageName: entryPoint.packageName, + }); + } + } + + return refs; +} + +function collectSingleExtensionRef(entryPoint) { + return [ + { + name: entryPoint.extensionName, + sourceDir: entryPoint.extensionDir, + category: 'external', + entryPoint: entryPoint.id, + packageName: entryPoint.packageName, + }, + ]; +} + +export function collectExtensionRefs(entryPoints) { + return entryPoints.flatMap((entryPoint) => { + if (entryPoint.kind === 'category-dirs') return collectCategoryExtensionRefs(entryPoint); + if (entryPoint.kind === 'single-extension') return collectSingleExtensionRef(entryPoint); + + return []; + }); +} + +export function filterExtensionRefs(extensionRefs) { + return extensionRefs.filter((extensionRef) => !isBlacklistedExtension(extensionRef.name)); +} diff --git a/infra/docs-gen/src/extractor/extractor.test.mjs b/infra/docs-gen/src/extractor/extractor.test.mjs new file mode 100644 index 00000000..8936f3f7 --- /dev/null +++ b/infra/docs-gen/src/extractor/extractor.test.mjs @@ -0,0 +1,55 @@ +import assert from 'node:assert/strict'; +import {test} from 'node:test'; + +import {extractExtensionNamesFromSource} from './ast/extension-name.mjs'; +import {EXTENSION_BLACKLIST, INTERNAL_EXTENSION_BLACKLIST} from './blacklist.mjs'; +import {filterExtensionRefs} from './entry-points.mjs'; +import {createExtensionRecord, EXTENSION_FIELD_CONFIG} from './field-config.mjs'; + +function createSourceFile(content) { + return {path: 'extension.ts', content}; +} + +test('extractExtensionNamesFromSource reads exported extension const names from TypeScript AST', () => { + const content = [ + "import type {ExtensionAuto} from '../../../core';", + 'const InternalExtension: ExtensionAuto = (builder) => {', + ' builder.addAction("internal", () => null);', + '};', + 'export const PublicExtension = Object.assign(InternalExtension, {});', + 'export const DirectExtension: ExtensionAuto = (builder) => {', + ' builder.addAction("direct", () => null);', + '};', + 'export const NotAnExtension = "value";', + ].join('\n'); + + assert.deepEqual(extractExtensionNamesFromSource(content), [ + 'PublicExtension', + 'DirectExtension', + ]); +}); + +test('filterExtensionRefs removes internal and public blacklist entries', () => { + assert.deepEqual( + filterExtensionRefs([ + {name: 'Bold'}, + {name: 'BaseKeymap'}, + {name: 'YfmCut'}, + {name: 'Italic'}, + ]).map((extensionRef) => extensionRef.name), + ['Bold', 'Italic'], + ); + assert.equal(INTERNAL_EXTENSION_BLACKLIST.includes('BaseKeymap'), true); + assert.equal(EXTENSION_BLACKLIST.includes('YfmCut'), true); +}); + +test('createExtensionRecord extracts only configured fields', () => { + const content = 'export const Bold: ExtensionAuto = (builder) => builder;'; + const record = createExtensionRecord({ + extensionRef: {name: 'Bold'}, + sourceFiles: [createSourceFile(content)], + }); + + assert.deepEqual(Object.keys(EXTENSION_FIELD_CONFIG), ['name']); + assert.deepEqual(record, {name: 'Bold'}); +}); diff --git a/infra/docs-gen/src/extractor/field-config.mjs b/infra/docs-gen/src/extractor/field-config.mjs new file mode 100644 index 00000000..c5281ae4 --- /dev/null +++ b/infra/docs-gen/src/extractor/field-config.mjs @@ -0,0 +1,22 @@ +import {extractNameField} from './fields/name.mjs'; + +export const EXTENSION_FIELD_CONFIG = { + name: { + from: 'extension source files: exported Extension/ExtensionAuto/ExtensionWithOptions const', + required: true, + extract: extractNameField, + }, +}; + +export function createExtensionRecord(context, fieldConfig = EXTENSION_FIELD_CONFIG) { + const record = {}; + + for (const [fieldName, config] of Object.entries(fieldConfig)) { + const value = config.extract(context); + if (config.required && value === null) return null; + + record[fieldName] = value; + } + + return record; +} diff --git a/infra/docs-gen/src/extractor/fields/name.mjs b/infra/docs-gen/src/extractor/fields/name.mjs new file mode 100644 index 00000000..a30ad491 --- /dev/null +++ b/infra/docs-gen/src/extractor/fields/name.mjs @@ -0,0 +1,12 @@ +import {extractExtensionNamesFromSource} from '../ast/extension-name.mjs'; +import {unique} from '../ast/core.mjs'; + +export function extractNameField({extensionRef, sourceFiles}) { + const names = unique( + sourceFiles.flatMap((sourceFile) => + extractExtensionNamesFromSource(sourceFile.content, sourceFile.path), + ), + ); + + return names.includes(extensionRef.name) ? extensionRef.name : null; +} diff --git a/infra/docs-gen/src/extractor/index.mjs b/infra/docs-gen/src/extractor/index.mjs new file mode 100644 index 00000000..5d83ef87 --- /dev/null +++ b/infra/docs-gen/src/extractor/index.mjs @@ -0,0 +1,23 @@ +import {EXTENSION_ENTRY_POINTS} from './config.mjs'; +import {collectExtensionRefs, filterExtensionRefs} from './entry-points.mjs'; +import {createExtensionRecord, EXTENSION_FIELD_CONFIG} from './field-config.mjs'; +import {readSourceFiles} from './source-files.mjs'; + +function scanExtension(extensionRef, fieldConfig) { + return createExtensionRecord( + { + extensionRef, + sourceFiles: readSourceFiles(extensionRef.sourceDir), + }, + fieldConfig, + ); +} + +export function extractExtensionData({ + entryPoints = EXTENSION_ENTRY_POINTS, + fieldConfig = EXTENSION_FIELD_CONFIG, +} = {}) { + return filterExtensionRefs(collectExtensionRefs(entryPoints)) + .map((extensionRef) => scanExtension(extensionRef, fieldConfig)) + .filter(Boolean); +} diff --git a/infra/docs-gen/src/extractor/output.mjs b/infra/docs-gen/src/extractor/output.mjs new file mode 100644 index 00000000..8799f0dc --- /dev/null +++ b/infra/docs-gen/src/extractor/output.mjs @@ -0,0 +1,7 @@ +import {mkdirSync, writeFileSync} from 'node:fs'; +import {join} from 'node:path'; + +export function writeExtensionsJson(outDir, extensions) { + mkdirSync(outDir, {recursive: true}); + writeFileSync(join(outDir, 'extensions.json'), `${JSON.stringify({extensions}, null, 2)}\n`); +} diff --git a/infra/docs-gen/src/extractor/source-files.mjs b/infra/docs-gen/src/extractor/source-files.mjs new file mode 100644 index 00000000..1a338121 --- /dev/null +++ b/infra/docs-gen/src/extractor/source-files.mjs @@ -0,0 +1,19 @@ +import {existsSync, readFileSync, readdirSync} from 'node:fs'; +import {join} from 'node:path'; + +export function readSourceFiles(dir) { + if (!existsSync(dir)) return []; + + const files = []; + for (const entry of readdirSync(dir, {withFileTypes: true})) { + const fullPath = join(dir, entry.name); + + if (entry.isDirectory()) { + files.push(...readSourceFiles(fullPath)); + } else if (entry.isFile() && (entry.name.endsWith('.ts') || entry.name.endsWith('.tsx'))) { + files.push({path: fullPath, content: readFileSync(fullPath, 'utf-8')}); + } + } + + return files.sort((left, right) => left.path.localeCompare(right.path)); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 410bbfeb..2dc15fbd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -306,6 +306,9 @@ importers: '@diplodoc/cli': specifier: 5.43.0 version: 5.43.0(@types/markdown-it@13.0.9)(@types/node@25.2.1)(react@18.2.0) + typescript: + specifier: catalog:ts + version: 5.9.3 infra/gulp-tasks: dependencies: