Skip to content

Commit 2849265

Browse files
FridayCandourFridayCandour
committed
feat(compiler): add metadata extraction and page tree generation
- Remove unnecessary blank lines for code cleanup - Standardize quote style from single to double quotes in attribute access - Add DocmachTagMetadata interface to track docmach tag information - Add PageMetadata interface to store compiled page metadata with tags - Add PageTreeNode interface for hierarchical page structure representation - Implement extractDocmachTags function to parse and extract docmach tags from content - Add compileFileWithMetadata function to return compiled content with extracted tags - Update parseFiles to return PageMetadata array and track source/output paths and links - Add buildPageTree function to construct hierarchical page tree from metadata - Improve parser imports to include relative path utility and new compiler exports - Enable metadata tracking throughout compilation pipeline for better page structure analysis
1 parent 36e7030 commit 2849265

2 files changed

Lines changed: 247 additions & 47 deletions

File tree

‎src/compiler.ts‎

Lines changed: 78 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,6 @@ export const templateCache = new LRUCache<
6565
}
6666
>();
6767

68-
69-
7068
// Helper: Parse attributes string into a key/value object.
7169
function parseAttributes(attrString: string): Record<string, string> {
7270
const attrs: Record<string, string> = {};
@@ -116,7 +114,6 @@ function parseParams(
116114
return params;
117115
}
118116

119-
120117
async function processWrapperTags(fileContent: string, filePath: string) {
121118
const replacements: { original: string; replacement: string }[] = [];
122119
const wrapperRegex = /<docmach\b([^>]*?)\s*([^/])>([\s\S]*?)<\/docmach>/g;
@@ -127,8 +124,8 @@ async function processWrapperTags(fileContent: string, filePath: string) {
127124
for (const match of matches) {
128125
const attrStr = match[1].endsWith('"') ? match[1] : match[1] + '"';
129126
const attrs = parseAttributes(attrStr);
130-
if (attrs['type'] === "wrapper" && attrs['file']) {
131-
templatePaths.add(resolve(attrs['file']));
127+
if (attrs["type"] === "wrapper" && attrs["file"]) {
128+
templatePaths.add(resolve(attrs["file"]));
132129
}
133130
}
134131

@@ -164,11 +161,11 @@ async function processWrapperTags(fileContent: string, filePath: string) {
164161
const attrStr = match[1].endsWith('"') ? match[1] : match[1] + '"';
165162
let innerContent = match[3].trim();
166163
const attrs = parseAttributes(attrStr);
167-
if (attrs['type'] === "wrapper" && attrs['file'] && attrs['replacement']) {
168-
const resolvedPath = resolve(attrs['file']);
164+
if (attrs["type"] === "wrapper" && attrs["file"] && attrs["replacement"]) {
165+
const resolvedPath = resolve(attrs["file"]);
169166
let templateContent = loadedTemplates.get(resolvedPath);
170167
if (templateContent) {
171-
const params = attrs['params'] ? parseParams(attrs['params']) : {};
168+
const params = attrs["params"] ? parseParams(attrs["params"]) : {};
172169
if (params) {
173170
Object.entries(params).forEach(([key, value]) => {
174171
templateContent = templateContent!.replace(
@@ -185,7 +182,7 @@ async function processWrapperTags(fileContent: string, filePath: string) {
185182
replacements.push({ original: fullMatch, replacement: replaced });
186183
}
187184
} else {
188-
if (!attrs['replacement']) {
185+
if (!attrs["replacement"]) {
189186
console.error(
190187
"Docmach: a wrapper tag must have a replacement attribute!"
191188
);
@@ -204,8 +201,8 @@ async function processSelfClosingTags(fileContent: string, filePath: string) {
204201

205202
for (const match of matches) {
206203
const attributes = parseAttributes(match[1]);
207-
if (attributes['type'] === "function" && attributes['file']) {
208-
functionPaths.add(resolve(attributes['file']));
204+
if (attributes["type"] === "function" && attributes["file"]) {
205+
functionPaths.add(resolve(attributes["file"]));
209206
}
210207
}
211208

@@ -243,7 +240,9 @@ async function processSelfClosingTags(fileContent: string, filePath: string) {
243240
const { file, type } = attributes;
244241
if (!file || !type) continue;
245242

246-
const params = attributes['params'] ? parseParams(attributes['params']) : {};
243+
const params = attributes["params"]
244+
? parseParams(attributes["params"])
245+
: {};
247246
const resolvedPath = resolve(file);
248247

249248
if (type === "function") {
@@ -288,6 +287,64 @@ async function processSelfClosingTags(fileContent: string, filePath: string) {
288287
return replacements;
289288
}
290289

290+
export interface DocmachTagMetadata {
291+
type: string;
292+
file?: string;
293+
params?: Record<string, string | number | object>;
294+
replacement?: string;
295+
}
296+
297+
export interface PageMetadata {
298+
sourcePath: string;
299+
outputPath: string;
300+
link: string;
301+
docmachTags: DocmachTagMetadata[];
302+
}
303+
304+
export interface PageTreeNode {
305+
name: string;
306+
type: "file" | "directory";
307+
path: string; // Relative to docs-directory
308+
link?: string; // Web URL (files only)
309+
sourcePath?: string; // Source .md file (files only)
310+
outputPath?: string; // Generated .html file (files only)
311+
docmachTags?: DocmachTagMetadata[]; // Files only
312+
children?: PageTreeNode[]; // Directories only
313+
}
314+
315+
function extractDocmachTags(fileContent: string): DocmachTagMetadata[] {
316+
const tags: DocmachTagMetadata[] = [];
317+
318+
// Extract wrapper tags
319+
const wrapperRegex = /<docmach\b([^>]*?)\s*([^/])>([\s\S]*?)<\/docmach>/g;
320+
let match;
321+
while ((match = wrapperRegex.exec(fileContent)) !== null) {
322+
const attrStr = match[1].endsWith('"') ? match[1] : match[1] + '"';
323+
const attrs = parseAttributes(attrStr);
324+
const tag: DocmachTagMetadata = {
325+
type: attrs["type"] || "unknown",
326+
};
327+
if (attrs["file"]) tag.file = attrs["file"];
328+
if (attrs["params"]) tag.params = parseParams(attrs["params"]);
329+
if (attrs["replacement"]) tag.replacement = attrs["replacement"];
330+
tags.push(tag);
331+
}
332+
333+
// Extract self-closing tags
334+
const tagRegex = /<docmach([^>]+)\/?\>/g;
335+
while ((match = tagRegex.exec(fileContent)) !== null) {
336+
const attrs = parseAttributes(match[1]);
337+
const tag: DocmachTagMetadata = {
338+
type: attrs["type"] || "unknown",
339+
};
340+
if (attrs["file"]) tag.file = attrs["file"];
341+
if (attrs["params"]) tag.params = parseParams(attrs["params"]);
342+
tags.push(tag);
343+
}
344+
345+
return tags;
346+
}
347+
291348
export async function compileFile(filePath: string): Promise<string> {
292349
let fileContent = await readFile(normalizePath(filePath), "utf8");
293350
fileContent = md.render(fileContent);
@@ -302,3 +359,12 @@ export async function compileFile(filePath: string): Promise<string> {
302359
});
303360
return fileContent;
304361
}
362+
363+
export async function compileFileWithMetadata(
364+
filePath: string
365+
): Promise<{ content: string; tags: DocmachTagMetadata[] }> {
366+
const rawContent = await readFile(normalizePath(filePath), "utf8");
367+
const tags = extractDocmachTags(rawContent);
368+
const content = await compileFile(filePath);
369+
return { content, tags };
370+
}

‎src/parser.ts‎

Lines changed: 169 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,10 @@ import {
3434
import { pipeline } from "stream/promises";
3535
import { cwd } from "node:process";
3636
import { join } from "path";
37-
import path, { resolve } from "node:path";
37+
import path, { resolve, relative } from "node:path";
3838
//
39-
import { compileFile } from "./compiler.ts";
39+
import { compileFileWithMetadata } from "./compiler.ts";
40+
import type { PageMetadata, PageTreeNode } from "./compiler.ts";
4041

4142
type configType = {
4243
"docs-directory": string;
@@ -98,18 +99,137 @@ function ensureFileSync(filePath: string) {
9899
}
99100
writeFileSync(normalizePath(filePath), "");
100101
}
101-
async function parseFiles(files: string[], config: configType) {
102+
async function parseFiles(
103+
files: string[],
104+
config: configType
105+
): Promise<PageMetadata[]> {
106+
const metadata: PageMetadata[] = [];
107+
102108
await Promise.all(
103109
files.map(async (file) => {
104-
const content = await compileFile(file);
105-
const path = resolve(
110+
const { content, tags } = await compileFileWithMetadata(file);
111+
const outputPath = resolve(
106112
cwd(),
107113
file.replace(config["docs-directory"], config["build-directory"])
108114
).replace(".md", ".html");
109-
ensureFileSync(path);
110-
await writeFile(normalizePath(path), content);
115+
116+
ensureFileSync(outputPath);
117+
await writeFile(normalizePath(outputPath), content);
118+
119+
// Generate relative link from build directory
120+
const link =
121+
"/" +
122+
relative(config["build-directory"], outputPath).replace(/\\/g, "/");
123+
124+
metadata.push({
125+
sourcePath: relative(cwd(), file),
126+
outputPath: relative(cwd(), outputPath),
127+
link,
128+
docmachTags: tags,
129+
});
111130
})
112131
);
132+
133+
return metadata;
134+
}
135+
136+
function buildPageTree(
137+
metadata: PageMetadata[],
138+
config: configType
139+
): PageTreeNode {
140+
const root: PageTreeNode = {
141+
name: "/",
142+
type: "directory",
143+
path: "/",
144+
children: [],
145+
};
146+
147+
// Build tree structure
148+
metadata.forEach((page) => {
149+
// Get path relative to docs-directory
150+
const relPath = relative(config["docs-directory"], page.sourcePath);
151+
const pathParts = relPath.split(path.sep).filter(Boolean);
152+
153+
let currentNode = root;
154+
155+
// Navigate/create directory structure
156+
for (let i = 0; i < pathParts.length - 1; i++) {
157+
const dirName = pathParts[i];
158+
let childDir = currentNode.children?.find(
159+
(child) => child.name === dirName && child.type === "directory"
160+
);
161+
162+
if (!childDir) {
163+
const dirPath = "/" + pathParts.slice(0, i + 1).join("/");
164+
childDir = {
165+
name: dirName,
166+
type: "directory",
167+
path: dirPath,
168+
children: [],
169+
};
170+
currentNode.children!.push(childDir);
171+
}
172+
173+
currentNode = childDir;
174+
}
175+
176+
// Add file node
177+
const fileName = pathParts[pathParts.length - 1].replace(".md", ".html");
178+
const fileNode: PageTreeNode = {
179+
name: fileName,
180+
type: "file",
181+
path: page.link,
182+
link: page.link,
183+
sourcePath: relative(cwd(), page.sourcePath),
184+
outputPath: relative(cwd(), page.outputPath),
185+
docmachTags: page.docmachTags,
186+
};
187+
188+
currentNode.children!.push(fileNode);
189+
});
190+
191+
// Sort children: directories first, then files, both alphabetically
192+
function sortChildren(node: PageTreeNode) {
193+
if (node.children) {
194+
node.children.sort((a, b) => {
195+
if (a.type !== b.type) {
196+
return a.type === "directory" ? -1 : 1;
197+
}
198+
return a.name.localeCompare(b.name);
199+
});
200+
201+
node.children.forEach(sortChildren);
202+
}
203+
}
204+
205+
sortChildren(root);
206+
return root;
207+
}
208+
209+
async function generateManifest(
210+
metadata: PageMetadata[],
211+
config: configType
212+
): Promise<void> {
213+
const manifestPath = join(config["build-directory"], "docmach-manifest.json");
214+
215+
// Build tree structure
216+
const tree = buildPageTree(metadata, config);
217+
218+
const manifest = {
219+
generatedAt: new Date().toISOString(),
220+
docsDirectory: relative(cwd(), config["docs-directory"]),
221+
buildDirectory: relative(cwd(), config["build-directory"]),
222+
totalPages: metadata.length,
223+
tree,
224+
// Keep flat list for backward compatibility
225+
pages: metadata,
226+
};
227+
228+
await writeFile(
229+
normalizePath(manifestPath),
230+
JSON.stringify(manifest, null, 2)
231+
);
232+
console.log(`Generated manifest: ${relative(cwd(), manifestPath)}`);
113233
}
114234

115235
const getList = async (config: configType, file?: string) => {
@@ -179,40 +299,54 @@ export const parseDocmachFIles = async (config: configType, file?: string) => {
179299
// check
180300
if (
181301
config["assets-folder"] &&
182-
file.startsWith(normalizePath(config["assets-folder"])) &&
183-
(await open(normalizePath(file)))
302+
file.startsWith(normalizePath(config["assets-folder"]))
184303
) {
185-
const sourceDir = path.relative(
186-
cwd(),
187-
normalizePath(config["assets-folder"])
188-
);
189-
const destinationDir = path.relative(
190-
cwd(),
191-
normalizePath(config["build-directory"])
192-
);
193-
await copyChangedFiles(sourceDir, destinationDir);
304+
try {
305+
const handle = await open(normalizePath(file));
306+
await handle.close();
307+
const sourceDir = path.relative(
308+
cwd(),
309+
normalizePath(config["assets-folder"])
310+
);
311+
const destinationDir = path.relative(
312+
cwd(),
313+
normalizePath(config["build-directory"])
314+
);
315+
await copyChangedFiles(sourceDir, destinationDir);
316+
} catch (_e) {
317+
// File doesn't exist or can't be opened
318+
}
194319
}
195320
}
196321
return;
197322
}
198-
if (
199-
config["assets-folder"] &&
200-
(await open(normalizePath(config["assets-folder"])))
201-
) {
202-
const sourceDir = path.relative(
203-
cwd(),
204-
normalizePath(config["assets-folder"])
205-
);
206-
const destinationDir = path.relative(
207-
cwd(),
208-
normalizePath(config["build-directory"])
209-
);
210-
await copyChangedFiles(
211-
normalizePath(sourceDir),
212-
normalizePath(destinationDir)
213-
);
323+
if (config["assets-folder"]) {
324+
try {
325+
const handle = await open(normalizePath(config["assets-folder"]));
326+
await handle.close();
327+
const sourceDir = path.relative(
328+
cwd(),
329+
normalizePath(config["assets-folder"])
330+
);
331+
const destinationDir = path.relative(
332+
cwd(),
333+
normalizePath(config["build-directory"])
334+
);
335+
await copyChangedFiles(
336+
normalizePath(sourceDir),
337+
normalizePath(destinationDir)
338+
);
339+
} catch (_e) {
340+
// Assets folder doesn't exist
341+
}
214342
}
215-
await parseFiles(files, config);
343+
const metadata = await parseFiles(files, config);
344+
345+
// Generate manifest only during full builds (not incremental file updates)
346+
if (!file) {
347+
await generateManifest(metadata, config);
348+
}
349+
216350
return files;
217351
};
218352

0 commit comments

Comments
 (0)