Skip to content

Commit cc593c5

Browse files
ovflowdavivkeller
ovflowd
and
avivkeller
authored
meta: move generators (#616)
* meta: move generators * fixup! --------- Co-authored-by: Aviv Keller <me@aviv.sh>
1 parent 32a824c commit cc593c5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+1500
-1265
lines changed

‎docs/generators.md‎

Lines changed: 253 additions & 148 deletions
Large diffs are not rendered by default.

‎src/generators.mjs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,8 @@ const createGenerator = () => {
5353
return;
5454
}
5555

56-
const { dependsOn, generate, processChunk } =
57-
await allGenerators[generatorName]();
56+
const { dependsOn, generate, hasParallelProcessor } =
57+
allGenerators[generatorName];
5858

5959
// Schedule dependency first
6060
if (dependsOn && !(dependsOn in cachedGenerators)) {
@@ -63,7 +63,7 @@ const createGenerator = () => {
6363

6464
generatorsLogger.debug(`Scheduling "${generatorName}"`, {
6565
dependsOn: dependsOn || 'none',
66-
streaming: Boolean(processChunk),
66+
streaming: hasParallelProcessor,
6767
});
6868

6969
// Schedule the generator
@@ -73,7 +73,7 @@ const createGenerator = () => {
7373
generatorsLogger.debug(`Starting "${generatorName}"`);
7474

7575
// Create parallel worker for streaming generators
76-
const worker = processChunk
76+
const worker = hasParallelProcessor
7777
? createParallelWorker(generatorName, pool, configuration)
7878
: Promise.resolve(null);
7979

‎src/generators/__tests__/index.test.mjs‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,11 @@ import { allGenerators } from '../index.mjs';
77

88
const validDependencies = Object.keys(allGenerators);
99

10-
const allGeneratorsReaolved = await Promise.all(
11-
Object.entries(allGenerators).map(async ([key, loader]) => [
12-
key,
13-
await loader(),
14-
])
15-
);
10+
const allGeneratorsEntries = Object.entries(allGenerators);
1611

1712
describe('All Generators', () => {
18-
it('should have keys matching their name property', async () => {
19-
allGeneratorsReaolved.forEach(([key, generator]) => {
13+
it('should have keys matching their name property', () => {
14+
allGeneratorsEntries.forEach(([key, generator]) => {
2015
assert.equal(
2116
key,
2217
generator.name,
@@ -25,8 +20,8 @@ describe('All Generators', () => {
2520
});
2621
});
2722

28-
it('should have valid semver versions', async () => {
29-
allGeneratorsReaolved.forEach(([key, generator]) => {
23+
it('should have valid semver versions', () => {
24+
allGeneratorsEntries.forEach(([key, generator]) => {
3025
const isValid = semver.valid(generator.version);
3126
assert.ok(
3227
isValid,
@@ -35,8 +30,8 @@ describe('All Generators', () => {
3530
});
3631
});
3732

38-
it('should have valid dependsOn references', async () => {
39-
allGeneratorsReaolved.forEach(([key, generator]) => {
33+
it('should have valid dependsOn references', () => {
34+
allGeneratorsEntries.forEach(([key, generator]) => {
4035
if (generator.dependsOn) {
4136
assert.ok(
4237
validDependencies.includes(generator.dependsOn),
@@ -46,8 +41,8 @@ describe('All Generators', () => {
4641
});
4742
});
4843

49-
it('should have ast generator as a top-level generator with no dependencies', async () => {
50-
const ast = await allGenerators.ast();
44+
it('should have ast generator as a top-level generator with no dependencies', () => {
45+
const ast = allGenerators.ast;
5146
assert.ok(ast, 'ast generator should exist');
5247
assert.equal(
5348
ast.dependsOn,

‎src/generators/addon-verify/generate.mjs‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
'use strict';
2+
3+
import { mkdir, writeFile } from 'node:fs/promises';
4+
import { join } from 'node:path';
5+
6+
import { visit } from 'unist-util-visit';
7+
8+
import { EXTRACT_CODE_FILENAME_COMMENT } from './constants.mjs';
9+
import { generateFileList } from './utils/generateFileList.mjs';
10+
import {
11+
generateSectionFolderName,
12+
isBuildableSection,
13+
normalizeSectionName,
14+
} from './utils/section.mjs';
15+
import getConfig from '../../utils/configuration/index.mjs';
16+
17+
/**
18+
* Generates a file list from code blocks.
19+
*
20+
* @type {import('./types').Generator['generate']}
21+
*/
22+
export async function generate(input) {
23+
const config = getConfig('addon-verify');
24+
25+
const sectionsCodeBlocks = input.reduce((addons, node) => {
26+
const sectionName = node.heading.data.name;
27+
28+
const content = node.content;
29+
30+
visit(content, childNode => {
31+
if (childNode.type === 'code') {
32+
const filename = childNode.value.match(EXTRACT_CODE_FILENAME_COMMENT);
33+
34+
if (filename === null) {
35+
return;
36+
}
37+
38+
if (!addons[sectionName]) {
39+
addons[sectionName] = [];
40+
}
41+
42+
addons[sectionName].push({
43+
name: filename[1],
44+
content: childNode.value,
45+
});
46+
}
47+
});
48+
49+
return addons;
50+
}, {});
51+
52+
const files = await Promise.all(
53+
Object.entries(sectionsCodeBlocks)
54+
.filter(([, codeBlocks]) => isBuildableSection(codeBlocks))
55+
.flatMap(async ([sectionName, codeBlocks], index) => {
56+
const files = generateFileList(codeBlocks);
57+
58+
if (config.output) {
59+
const normalizedSectionName = normalizeSectionName(sectionName);
60+
61+
const folderName = generateSectionFolderName(
62+
normalizedSectionName,
63+
index
64+
);
65+
66+
await mkdir(join(config.output, folderName), { recursive: true });
67+
68+
for (const file of files) {
69+
await writeFile(
70+
join(config.output, folderName, file.name),
71+
file.content
72+
);
73+
}
74+
}
75+
76+
return files;
77+
})
78+
);
79+
80+
return files;
81+
}

‎src/generators/addon-verify/index.mjs‎

Lines changed: 3 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,6 @@
11
'use strict';
22

3-
import { mkdir, writeFile } from 'node:fs/promises';
4-
import { join } from 'node:path';
5-
6-
import { visit } from 'unist-util-visit';
7-
8-
import { EXTRACT_CODE_FILENAME_COMMENT } from './constants.mjs';
9-
import { generateFileList } from './utils/generateFileList.mjs';
10-
import {
11-
generateSectionFolderName,
12-
isBuildableSection,
13-
normalizeSectionName,
14-
} from './utils/section.mjs';
15-
import getConfig from '../../utils/configuration/index.mjs';
3+
import { createLazyGenerator } from '../../utils/generators.mjs';
164

175
/**
186
* This generator generates a file list from code blocks extracted from
@@ -21,7 +9,7 @@ import getConfig from '../../utils/configuration/index.mjs';
219
*
2210
* @type {import('./types').Generator}
2311
*/
24-
export default {
12+
export default await createLazyGenerator({
2513
name: 'addon-verify',
2614

2715
version: '1.0.0',
@@ -30,68 +18,4 @@ export default {
3018
'Generates a file list from code blocks extracted from `doc/api/addons.md` to facilitate C++ compilation and JavaScript runtime validations',
3119

3220
dependsOn: 'metadata',
33-
34-
/**
35-
* Generates a file list from code blocks.
36-
*/
37-
async generate(input) {
38-
const config = getConfig('addon-verify');
39-
40-
const sectionsCodeBlocks = input.reduce((addons, node) => {
41-
const sectionName = node.heading.data.name;
42-
43-
const content = node.content;
44-
45-
visit(content, childNode => {
46-
if (childNode.type === 'code') {
47-
const filename = childNode.value.match(EXTRACT_CODE_FILENAME_COMMENT);
48-
49-
if (filename === null) {
50-
return;
51-
}
52-
53-
if (!addons[sectionName]) {
54-
addons[sectionName] = [];
55-
}
56-
57-
addons[sectionName].push({
58-
name: filename[1],
59-
content: childNode.value,
60-
});
61-
}
62-
});
63-
64-
return addons;
65-
}, {});
66-
67-
const files = await Promise.all(
68-
Object.entries(sectionsCodeBlocks)
69-
.filter(([, codeBlocks]) => isBuildableSection(codeBlocks))
70-
.flatMap(async ([sectionName, codeBlocks], index) => {
71-
const files = generateFileList(codeBlocks);
72-
73-
if (config.output) {
74-
const normalizedSectionName = normalizeSectionName(sectionName);
75-
76-
const folderName = generateSectionFolderName(
77-
normalizedSectionName,
78-
index
79-
);
80-
81-
await mkdir(join(config.output, folderName), { recursive: true });
82-
83-
for (const file of files) {
84-
await writeFile(
85-
join(config.output, folderName, file.name),
86-
file.content
87-
);
88-
}
89-
}
90-
91-
return files;
92-
})
93-
);
94-
95-
return files;
96-
},
97-
};
21+
});

‎src/generators/api-links/__tests__/fixtures.test.mjs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ import { globSync } from 'tinyglobby';
66
import createWorkerPool from '../../../threading/index.mjs';
77
import createParallelWorker from '../../../threading/parallel.mjs';
88
import { setConfig } from '../../../utils/configuration/index.mjs';
9-
import astJs from '../../ast-js/index.mjs';
10-
import apiLinks from '../index.mjs';
9+
import { generate as astJsGenerate } from '../../ast-js/generate.mjs';
10+
import { generate as apiLinksGenerate } from '../generate.mjs';
1111

1212
const relativePath = relative(process.cwd(), import.meta.dirname);
1313

@@ -40,11 +40,11 @@ describe('api links', () => {
4040
// Collect results from the async generator
4141
const astJsResults = [];
4242

43-
for await (const chunk of astJs.generate(undefined, worker)) {
43+
for await (const chunk of astJsGenerate(undefined, worker)) {
4444
astJsResults.push(...chunk);
4545
}
4646

47-
const actualOutput = await apiLinks.generate(astJsResults);
47+
const actualOutput = await apiLinksGenerate(astJsResults);
4848

4949
for (const [k, v] of Object.entries(actualOutput)) {
5050
actualOutput[k] = v.replace(/.*(?=lib\/)/, '');

‎src/generators/api-links/generate.mjs‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
'use strict';
2+
3+
import { writeFile } from 'node:fs/promises';
4+
import { basename, join } from 'node:path';
5+
6+
import { checkIndirectReferences } from './utils/checkIndirectReferences.mjs';
7+
import { extractExports } from './utils/extractExports.mjs';
8+
import { findDefinitions } from './utils/findDefinitions.mjs';
9+
import getConfig from '../../utils/configuration/index.mjs';
10+
import {
11+
GITHUB_BLOB_URL,
12+
populate,
13+
} from '../../utils/configuration/templates.mjs';
14+
15+
/**
16+
* Generates the `apilinks.json` file.
17+
*
18+
* @type {import('./types').Generator['generate']}
19+
*/
20+
export async function generate(input) {
21+
const config = getConfig('api-links');
22+
/**
23+
* @type Record<string, string>
24+
*/
25+
const definitions = {};
26+
27+
input.forEach(program => {
28+
/**
29+
* Mapping of definitions to their line number
30+
*
31+
* @type {Record<string, number>}
32+
* @example { 'someclass.foo': 10 }
33+
*/
34+
const nameToLineNumberMap = {};
35+
36+
// `http.js` -> `http`
37+
const baseName = basename(program.path, '.js');
38+
39+
const exports = extractExports(program, baseName, nameToLineNumberMap);
40+
41+
findDefinitions(program, baseName, nameToLineNumberMap, exports);
42+
43+
checkIndirectReferences(program, exports, nameToLineNumberMap);
44+
45+
const fullGitUrl = `${populate(GITHUB_BLOB_URL, config)}lib/${baseName}.js`;
46+
47+
// Add the exports we found in this program to our output
48+
Object.keys(nameToLineNumberMap).forEach(key => {
49+
const lineNumber = nameToLineNumberMap[key];
50+
51+
definitions[key] = `${fullGitUrl}#L${lineNumber}`;
52+
});
53+
});
54+
55+
if (config.output) {
56+
const out = join(config.output, 'apilinks.json');
57+
58+
await writeFile(
59+
out,
60+
config.minify
61+
? JSON.stringify(definitions)
62+
: JSON.stringify(definitions, null, 2)
63+
);
64+
}
65+
66+
return definitions;
67+
}

0 commit comments

Comments
 (0)