rename ts package, rehome at quarto-markdown

Author: cscheid

ts-packages/README.md

@@ -0,0 +1,22 @@
+# TypeScript Packages
+
+This directory contains standalone TypeScript packages associated with the Kyoto Rust workspace.
+
+Following the convention used in Rust monorepos (similar to how `target/` contains build artifacts),
+this `ts-packages/` directory contains TypeScript packages that complement the Rust crates.
+
+## Packages
+
+- **annotated-qmd** (`@quarto/annotated-qmd`): Converts quarto-markdown-pandoc JSON output
+  to AnnotatedParse structures compatible with quarto-cli's YAML validation infrastructure.
+
+## Development
+
+Each package is independent with its own `package.json` and can be developed/tested separately:
+
+```bash
+cd ts-packages/annotated-qmd
+npm install
+npm test
+npm run build
+```

ts-packages/rust-qmd-json/.gitignore ts-packages/annotated-qmd/.gitignorets-packages/rust-qmd-json/.gitignore renamed to ts-packages/annotated-qmd/.gitignore

@@ -1,4 +1,4 @@
-# @quarto/rust-qmd-json
+# @quarto/annotated-qmd
 
 Convert quarto-markdown-pandoc JSON output to AnnotatedParse structures with full source mapping.
 
@@ -11,14 +11,14 @@ infrastructure. It preserves complete source location information through the co
 ## Installation
 
 ```bash
-npm install @quarto/rust-qmd-json
+npm install @quarto/annotated-qmd
 ```
 
 ## Quick Start
 
 ```typescript
-import { parseRustQmdMetadata } from '@quarto/rust-qmd-json';
-import type { RustQmdJson } from '@quarto/rust-qmd-json';
+import { parseRustQmdMetadata } from '@quarto/annotated-qmd';
+import type { RustQmdJson } from '@quarto/annotated-qmd';
 
 // JSON from quarto-markdown-pandoc
 const json: RustQmdJson = {
@@ -58,7 +58,7 @@ Main entry point for converting quarto-markdown-pandoc JSON to AnnotatedParse.
 **Example with error handling:**
 
 ```typescript
-import { parseRustQmdMetadata } from '@quarto/rust-qmd-json';
+import { parseRustQmdMetadata } from '@quarto/annotated-qmd';
 
 const errorHandler = (msg: string, id?: number) => {
   console.error(`SourceInfo error: ${msg}`, id);
@@ -81,15 +81,15 @@ import type {
   SerializableSourceInfo,
   SourceContext,
   SourceInfoErrorHandler
-} from '@quarto/rust-qmd-json';
+} from '@quarto/annotated-qmd';
 ```
 
 ### Advanced Usage
 
 For more control, you can use the underlying classes directly:
 
 ```typescript
-import { SourceInfoReconstructor, MetadataConverter } from '@quarto/rust-qmd-json';
+import { SourceInfoReconstructor, MetadataConverter } from '@quarto/annotated-qmd';
 
 const reconstructor = new SourceInfoReconstructor(
   json.source_pool,
@@ -115,24 +115,3 @@ npm test
 # Clean
 npm run clean
 ```
-
-## Architecture
-
-The conversion happens in two phases:
-
-1. **SourceInfo Reconstruction**: Convert the pooled SourceInfo format from JSON into
-   MappedString objects that track source locations through transformation chains.
-
-2. **Metadata Conversion**: Recursively convert MetaValue variants into AnnotatedParse
-   structures with proper source tracking. MetaInlines/MetaBlocks are treated as leaf
-   nodes with the JSON array structure preserved in the result.
-
-## Design Decisions
-
-- **Direct JSON Value Mapping**: MetaInlines and MetaBlocks are preserved as JSON arrays
-  in the `result` field, avoiding any text reconstruction
-- **Source Tracking**: Every value can be traced back to original file location via SourceInfo
-- **Compatible Types**: Produces AnnotatedParse structures compatible with existing validation code
-
-See repository's `claude-notes/plans/2025-10-23-json-to-annotated-parse-conversion.md` for
-detailed implementation plan.

ts-packages/rust-qmd-json/README.md ts-packages/annotated-qmd/README.mdts-packages/rust-qmd-json/README.md renamed to ts-packages/annotated-qmd/README.md

@@ -1,6 +1,6 @@
 {
-  "name": "@quarto/rust-qmd-json",
-  "version": "0.1.0",
+  "name": "@quarto/annotated-qmd",
+  "version": "0.1.1",
   "description": "Convert quarto-markdown-pandoc JSON output to AnnotatedParse structures",
   "license": "MIT",
   "author": {
@@ -12,8 +12,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/quarto-dev/quarto.git",
-    "directory": "ts-packages/rust-qmd-json"
+    "url": "git+https://github.com/quarto-dev/quarto-markdown.git"
   },
   "type": "module",
   "main": "dist/index.js",

ts-packages/rust-qmd-json/package.json ts-packages/annotated-qmd/package.jsonts-packages/rust-qmd-json/package.json renamed to ts-packages/annotated-qmd/package.json

@@ -1,5 +1,5 @@
 /**
- * @quarto/rust-qmd-json
+ * @quarto/annotated-qmd
  *
  * Converts quarto-markdown-pandoc JSON output to AnnotatedParse structures
  * compatible with quarto-cli's YAML validation infrastructure.
@@ -43,21 +43,22 @@ import type { SourceInfoErrorHandler } from './source-map.js';
  *
  * @example
  * ```typescript
- * import { parseRustQmdMetadata } from '@quarto/rust-qmd-json';
+ * import { parseRustQmdMetadata } from '@quarto/annotated-qmd';
  *
  * const json = {
  *   meta: {
  *     title: { t: 'MetaString', c: 'Hello', s: 0 }
  *   },
  *   blocks: [],
- *   source_pool: [
- *     { r: [11, 16], t: 0, d: 0 }
- *   ],
- *   source_context: {
+ *   astContext: {
+ *     sourceInfoPool: [
+ *       { r: [11, 16], t: 0, d: 0 }
+ *     ],
  *     files: [
- *       { id: 0, path: 'test.qmd', content: '---\ntitle: Hello\n---' }
+ *       { name: 'test.qmd', content: '---\ntitle: Hello\n---' }
  *     ]
- *   }
+ *   },
+ *   'pandoc-api-version': [1, 23, 1]
  * };
  *
  * const metadata = parseRustQmdMetadata(json);
@@ -68,15 +69,27 @@ export function parseRustQmdMetadata(
   json: RustQmdJson,
   errorHandler?: SourceInfoErrorHandler
 ): AnnotatedParse {
+  // Normalize the JSON structure to internal format
+  const sourceContext = {
+    files: json.astContext.files.map((f, idx) => ({
+      id: idx,
+      path: f.name,
+      content: f.content || ''
+    }))
+  };
+
   // 1. Create SourceInfoReconstructor with pool and context
   const sourceReconstructor = new SourceInfoReconstructor(
-    json.source_pool,
-    json.source_context,
+    json.astContext.sourceInfoPool,
+    sourceContext,
     errorHandler
   );
 
-  // 2. Create MetadataConverter
-  const converter = new MetadataConverter(sourceReconstructor);
+  // 2. Create MetadataConverter with metaTopLevelKeySources
+  const converter = new MetadataConverter(
+    sourceReconstructor,
+    json.astContext.metaTopLevelKeySources
+  );
 
   // 3. Convert metadata to AnnotatedParse
   return converter.convertMeta(json.meta);

ts-packages/rust-qmd-json/src/index.ts ts-packages/annotated-qmd/src/index.tsts-packages/rust-qmd-json/src/index.ts renamed to ts-packages/annotated-qmd/src/index.ts

@@ -49,7 +49,10 @@ function isTaggedSpan(obj: unknown): obj is {
  * Converts metadata from quarto-markdown-pandoc JSON to AnnotatedParse
  */
 export class MetadataConverter {
-  constructor(private sourceReconstructor: SourceInfoReconstructor) {}
+  constructor(
+    private sourceReconstructor: SourceInfoReconstructor,
+    private metaTopLevelKeySources?: Record<string, number>
+  ) {}
 
   /**
    * Convert top-level metadata object to AnnotatedParse
@@ -58,7 +61,8 @@ export class MetadataConverter {
     // Create a synthetic MetaMap for the top-level metadata
     const entries: MetaMapEntry[] = Object.entries(jsonMeta).map(([key, value]) => ({
       key,
-      key_source: value.s,  // Use value's source for key (not ideal, but metadata doesn't include key sources)
+      // Use metaTopLevelKeySources if available, otherwise fall back to value's source
+      key_source: this.metaTopLevelKeySources?.[key] ?? value.s,
       value
     }));
 
@@ -83,8 +87,10 @@ export class MetadataConverter {
 
     for (const [key, value] of Object.entries(jsonMeta)) {
       // Create AnnotatedParse for key
-      const [keyStart, keyEnd] = this.sourceReconstructor.getOffsets(value.s);
-      const keySource = this.sourceReconstructor.toMappedString(value.s);
+      // Use metaTopLevelKeySources if available, otherwise fall back to value's source
+      const keySourceId = this.metaTopLevelKeySources?.[key] ?? value.s;
+      const [keyStart, keyEnd] = this.sourceReconstructor.getOffsets(keySourceId);
+      const keySource = this.sourceReconstructor.toMappedString(keySourceId);
 
       const keyAP: AnnotatedParse = {
         result: key,

…ages/rust-qmd-json/src/meta-converter.ts …ages/annotated-qmd/src/meta-converter.tsts-packages/rust-qmd-json/src/meta-converter.ts renamed to ts-packages/annotated-qmd/src/meta-converter.ts ts-packages/rust-qmd-json/src/meta-converter.ts renamed to ts-packages/annotated-qmd/src/meta-converter.ts

@@ -18,13 +18,11 @@ export interface SerializableSourceInfo {
 
 /**
  * Type guard for Concat data structure
+ * Rust serializes Concat data as a plain array: [[source_info_id, offset, length], ...]
  */
-function isConcatData(data: unknown): data is { pieces: [number, number, number][] } {
-  return (
-    typeof data === 'object' &&
-    data !== null &&
-    'pieces' in data &&
-    Array.isArray((data as { pieces: unknown }).pieces)
+function isConcatData(data: unknown): data is [number, number, number][] {
+  return Array.isArray(data) && data.every(
+    item => Array.isArray(item) && item.length === 3
   );
 }
 
@@ -181,16 +179,17 @@ export class SourceInfoReconstructor {
 
   /**
    * Handle Concat SourceInfo type (t=2)
-   * Data format: {pieces: [[source_info_id, offset, length], ...]}
+   * Data format: [[source_info_id, offset, length], ...]
+   * (Rust serializes as plain array, not object with pieces field)
    */
   private handleConcat(id: number, info: SerializableSourceInfo): MappedString {
     // Runtime type check
     if (!isConcatData(info.d)) {
-      this.errorHandler(`Invalid Concat data format (expected {pieces: [...]}), got ${typeof info.d}`, id);
+      this.errorHandler(`Invalid Concat data format (expected array of [id, offset, length]), got ${typeof info.d}`, id);
       return asMappedString('');
     }
 
-    const pieces = info.d.pieces;
+    const pieces = info.d;  // Direct array access
 
     // Build MappedString array from pieces
     const mappedPieces: MappedString[] = [];
@@ -272,7 +271,7 @@ export class SourceInfoReconstructor {
             this.errorHandler(`Invalid Concat data format`, id);
             resolved = { file_id: -1, range: info.r };
           } else {
-            const pieces = info.d.pieces;
+            const pieces = info.d;  // Direct array access
             if (pieces.length === 0) {
               this.errorHandler(`Empty Concat pieces`, id);
               resolved = { file_id: -1, range: info.r };

…packages/rust-qmd-json/src/source-map.ts …packages/annotated-qmd/src/source-map.tsts-packages/rust-qmd-json/src/source-map.ts renamed to ts-packages/annotated-qmd/src/source-map.ts ts-packages/rust-qmd-json/src/source-map.ts renamed to ts-packages/annotated-qmd/src/source-map.ts

@@ -47,18 +47,26 @@ export interface MetaMapEntry {
   value: JsonMetaValue;
 }
 
+/**
+ * File information from Rust JSON output
+ */
+export interface RustFileInfo {
+  name: string;              // File path/name
+  line_breaks?: number[];    // Byte offsets of newlines
+  total_length?: number;     // Total file length in bytes
+  content?: string;          // File content (populated by consumer)
+}
+
 /**
  * Complete JSON output from quarto-markdown-pandoc
  */
 export interface RustQmdJson {
   meta: Record<string, JsonMetaValue>;
   blocks: unknown[];  // Not used in metadata conversion
-  source_pool: SerializableSourceInfo[];
-  source_context: {
-    files: Array<{
-      id: number;
-      path: string;
-      content: string;
-    }>;
+  astContext: {
+    sourceInfoPool: SerializableSourceInfo[];
+    files: RustFileInfo[];
+    metaTopLevelKeySources?: Record<string, number>;  // Maps metadata keys to SourceInfo IDs
   };
+  'pandoc-api-version': [number, number, number];
 }

ts-packages/rust-qmd-json/src/types.ts ts-packages/annotated-qmd/src/types.tsts-packages/rust-qmd-json/src/types.ts renamed to ts-packages/annotated-qmd/src/types.ts

@@ -41,15 +41,16 @@ test('can convert complete JSON to AnnotatedParse', async () => {
       author: { t: 'MetaString', c: 'Alice', s: 1 }
     },
     blocks: [],
-    source_pool: [
-      { r: [11, 22], t: 0, d: 0 },  // "Hello World"
-      { r: [31, 36], t: 0, d: 0 }   // "Alice"
-    ],
-    source_context: {
+    astContext: {
+      sourceInfoPool: [
+        { r: [11, 22], t: 0, d: 0 },  // "Hello World"
+        { r: [31, 36], t: 0, d: 0 }   // "Alice"
+      ],
       files: [
-        { id: 0, path: 'test.qmd', content: '---\ntitle: Hello World\nauthor: Alice\n---' }
+        { name: 'test.qmd', content: '---\ntitle: Hello World\nauthor: Alice\n---' }
       ]
-    }
+    },
+    'pandoc-api-version': [1, 23, 1]
   };
 
   const result = parseRustQmdMetadata(json);
@@ -60,3 +61,34 @@ test('can convert complete JSON to AnnotatedParse', async () => {
   assert.strictEqual((result.result as any).author, 'Alice');
   assert.strictEqual(result.components.length, 4);  // title key, title value, author key, author value
 });
+
+test('can parse math-with-attr.json', async () => {
+  const { parseRustQmdMetadata } = await import('../src/index.js');
+  const fs = await import('fs/promises');
+  const path = await import('path');
+  const { fileURLToPath } = await import('url');
+
+  // Get the directory of this test file
+  const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+  // Load JSON fixture from test/fixtures
+  const jsonPath = path.join(__dirname, 'fixtures', 'math-with-attr.json');
+  const jsonText = await fs.readFile(jsonPath, 'utf-8');
+  const json = JSON.parse(jsonText);
+
+  // Read the QMD file content from test/fixtures
+  const qmdPath = path.join(__dirname, 'fixtures', 'math-with-attr.qmd');
+  const qmdContent = await fs.readFile(qmdPath, 'utf-8');
+
+  // Populate file content (simulating what user would do)
+  for (const file of json.astContext.files) {
+    file.content = qmdContent;
+  }
+
+  const result = parseRustQmdMetadata(json);
+
+  // Basic validation that it didn't throw
+  assert.strictEqual(result.kind, 'mapping');
+  assert.ok(result.result);
+  assert.ok((result.result as any).title);
+});

…ackages/rust-qmd-json/test/basic.test.ts …ackages/annotated-qmd/test/basic.test.tsts-packages/rust-qmd-json/test/basic.test.ts renamed to ts-packages/annotated-qmd/test/basic.test.ts ts-packages/rust-qmd-json/test/basic.test.ts renamed to ts-packages/annotated-qmd/test/basic.test.ts

@@ -0,0 +1 @@
+{"astContext":{"files":[{"line_breaks":[3,35,39,40,94,95,124,125,128,177,195,196,256],"name":"math-with-attr.qmd","total_length":257}],"metaTopLevelKeySources":{"title":47},"sourceInfoPool":[{"d":0,"r":[0,4],"t":0},{"d":0,"r":[4,5],"t":0},{"d":0,"r":[5,9],"t":0},{"d":0,"r":[9,10],"t":0},{"d":0,"r":[12,22],"t":0},{"d":0,"r":[10,24],"t":0},{"d":0,"r":[0,257],"t":0},{"d":6,"r":[4,35],"t":1},{"d":7,"r":[7,31],"t":1},{"d":0,"r":[41,47],"t":0},{"d":0,"r":[47,48],"t":0},{"d":0,"r":[48,52],"t":0},{"d":0,"r":[52,53],"t":0},{"d":0,"r":[53,57],"t":0},{"d":0,"r":[57,58],"t":0},{"d":0,"r":[58,67],"t":0},{"d":0,"r":[67,68],"t":0},{"d":[[15,0,9],[16,9,1]],"r":[0,10],"t":2},{"d":0,"r":[68,69],"t":0},{"d":0,"r":[69,79],"t":0},{"d":0,"r":[0,0],"t":0},{"d":0,"r":[41,95],"t":0},{"d":0,"r":[96,103],"t":0},{"d":0,"r":[103,104],"t":0},{"d":0,"r":[104,108],"t":0},{"d":0,"r":[108,109],"t":0},{"d":0,"r":[109,113],"t":0},{"d":0,"r":[113,114],"t":0},{"d":0,"r":[114,123],"t":0},{"d":0,"r":[123,124],"t":0},{"d":[[28,0,9],[29,9,1]],"r":[0,10],"t":2},{"d":0,"r":[96,125],"t":0},{"d":0,"r":[126,180],"t":0},{"d":0,"r":[0,0],"t":0},{"d":0,"r":[126,196],"t":0},{"d":0,"r":[197,204],"t":0},{"d":0,"r":[204,205],"t":0},{"d":0,"r":[205,211],"t":0},{"d":0,"r":[211,212],"t":0},{"d":0,"r":[212,219],"t":0},{"d":0,"r":[219,220],"t":0},{"d":[[39,0,7],[40,7,1]],"r":[0,8],"t":2},{"d":0,"r":[220,221],"t":0},{"d":0,"r":[221,238],"t":0},{"d":0,"r":[0,0],"t":0},{"d":0,"r":[197,257],"t":0},{"d":6,"r":[4,35],"t":1},{"d":46,"r":[0,5],"t":1}]},"blocks":[{"c":[{"c":"Inline","s":9,"t":"Str"},{"s":10,"t":"Space"},{"c":"math","s":11,"t":"Str"},{"s":12,"t":"Space"},{"c":"with","s":13,"t":"Str"},{"s":14,"t":"Space"},{"c":"attribute:","s":17,"t":"Str"},{"s":18,"t":"Space"},{"c":[["eq-einstein",["quarto-math-with-attribute"],[]],[{"c":[{"t":"InlineMath"},"E = mc^2"],"s":19,"t":"Math"}]],"s":20,"t":"Span"}],"s":21,"t":"Para"},{"c":[{"c":"Display","s":22,"t":"Str"},{"s":23,"t":"Space"},{"c":"math","s":24,"t":"Str"},{"s":25,"t":"Space"},{"c":"with","s":26,"t":"Str"},{"s":27,"t":"Space"},{"c":"attribute:","s":30,"t":"Str"}],"s":31,"t":"Para"},{"c":[{"c":[["eq-gaussian",["quarto-math-with-attribute"],[]],[{"c":[{"t":"DisplayMath"},"\n\\int_0^\\infty e^{-x^2} dx = \\frac{\\sqrt{\\pi}}{2}\n"],"s":32,"t":"Math"}]],"s":33,"t":"Span"}],"s":34,"t":"Para"},{"c":[{"c":"Another","s":35,"t":"Str"},{"s":36,"t":"Space"},{"c":"inline","s":37,"t":"Str"},{"s":38,"t":"Space"},{"c":"example:","s":41,"t":"Str"},{"s":42,"t":"Space"},{"c":[["eq-pythagorean",["quarto-math-with-attribute"],[]],[{"c":[{"t":"InlineMath"},"a^2 + b^2 = c^2"],"s":43,"t":"Math"}]],"s":44,"t":"Span"}],"s":45,"t":"Para"}],"meta":{"title":{"c":[{"c":"math","s":0,"t":"Str"},{"s":1,"t":"Space"},{"c":"with","s":2,"t":"Str"},{"s":3,"t":"Space"},{"c":[{"c":"attributes","s":4,"t":"Str"}],"s":5,"t":"Strong"}],"s":8,"t":"MetaInlines"}},"pandoc-api-version":[1,23,1]}