From c73d0cf7fb25cfc9c33366d324a50ed9f29b3c2c Mon Sep 17 00:00:00 2001 From: Mikers Date: Mon, 23 Mar 2026 10:57:17 -1000 Subject: [PATCH] Issue 62 PR 04: Bounded-cost controls and chain-aware limits --- SPEC.md | 32 ++++- packages/cli/src/index.ts | 114 ++++++++++++++++-- packages/generator/src/index.ts | 2 +- .../src/solidity/generateAppSolidity.ts | 36 +++++- schemas/tokenhost-chain-config.schema.json | 20 +++ test/testCliBuildArtifacts.js | 36 +++++- test/testCliDev.js | 2 + test/testCliVerify.js | 2 + 8 files changed, 226 insertions(+), 18 deletions(-) diff --git a/SPEC.md b/SPEC.md index e772394..3d95cca 100644 --- a/SPEC.md +++ b/SPEC.md @@ -3,7 +3,7 @@ Status: Draft (spec-driven design) Owner: Token Host Domain: `tokenhost.com` -Last updated: 2026-02-05 +Last updated: 2026-03-23 Scope: Production system. All existing repos are legacy prototypes and are not binding. This document defines the intended production design of the Token Host platform: a managed schema-to-dapp builder that generates and deploys smart contracts, generates a hosted UI, optionally provisions indexing, and publishes apps under a dedicated hosted-app origin (default `*.apps.tokenhost.com`, plus optional custom domains). @@ -857,7 +857,7 @@ Implementation note: list methods MUST cap `limit` to a safe maximum (configurab Recommended defaults (non-normative): - If chain config provides `limits.lists.maxLimit` and/or `limits.lists.maxScanSteps`, the generator SHOULD use those values (or stricter). -- Otherwise, a safe default is `maxLimit=50` and `maxScanSteps = min(1000, maxLimit * 20)`. +- Otherwise, a safe default is `maxLimit=50` and `maxScanSteps=1000`. - The UI SHOULD handle “short pages” (fewer than `limit` results) gracefully and SHOULD allow loading more using the next cursor (the smallest returned ID). ### 7.6 Update semantics @@ -931,6 +931,25 @@ For each indexed field `f`: - If a record’s field changes, the record ID MAY appear in multiple buckets historically; readers MUST validate current field value when interpreting results. - Index accessors MUST be paginated (offset + limit) and MUST cap `limit`. +**Tokenized index (secondary index; when `onChainIndexing=true`)** +For each tokenized indexed field `f`: +- `indexC_f: mapping(bytes32 => uint256[])` mapping of normalized token key to append-only list of candidate record IDs. +- Tokenized indexes MUST apply deterministic extraction and normalization rules defined by the configured tokenizer. +- In v1, the only supported tokenizer is `hashtag`: + - source field type MUST be `string`, + - tokens begin with `#`, + - the token body is limited to ASCII letters, digits, and underscore, + - normalized token content is lowercased ASCII without the leading `#`, + - empty markers (e.g. a bare `#`) MUST be ignored, + - duplicate tokens from the same write MUST be de-duplicated before bucket append. +- If a record’s field changes, the record ID MAY appear in multiple token buckets historically; readers MUST validate current field value when interpreting results. +- Token extraction during create/update MUST be bounded. In v1, a safe default is: + - maximum extracted tokens per indexed field: `8` + - maximum normalized token length: `32` bytes + - writes that exceed those bounds MUST revert. +- If chain config provides stricter tokenized-index guidance (for example `limits.indexing.tokenized.maxTokens` and `limits.indexing.tokenized.maxTokenLength`), the generator SHOULD use those values or stricter values. +- Index accessors MUST be paginated (offset + limit) and MUST cap `limit`. + **Reference reverse index (when `onChainIndexing=true`)** For reference fields `ref`: - `refIndexC_ref: mapping(uint256 => uint256[])` mapping of referenced ID to append-only list of record IDs. @@ -939,6 +958,7 @@ For reference fields `ref`: **Index accessor semantics (normative)** - Index accessors MUST return **candidate IDs only**. Callers MUST validate record existence, deletion status, and current field values via `getC`/`existsC` when correctness matters. - For append-only equality/reference indexes, ordering MUST be insertion order (oldest to newest) within the bucket. +- For append-only tokenized indexes, ordering MUST be insertion order (oldest to newest) within the bucket. - For swap-and-pop sets (e.g., owner index sets), ordering MUST NOT be relied upon. - Index accessors MUST be paginated and MUST cap `limit` to avoid RPC timeouts. @@ -957,6 +977,14 @@ This produces a uniform `bytes32` key space. Normalization note: Token Host does not implicitly normalize string values on-chain. If an app requires case-insensitive uniqueness or trimmed matching, the schema/UI MUST enforce canonicalization by storing the canonical form as the field value. +For tokenized hashtag indexes: +- the lookup input is the normalized token body without the leading `#`, +- normalization MUST lowercase ASCII `A-Z` to `a-z`, +- only ASCII letters, digits, and underscore are preserved in the token body, +- the key is `keccak256(bytes(normalizedToken))`. + +This allows generated UIs, self-hosted runtimes, and external indexers to derive the same lookup key without replaying contract internals. + ### 7.9 Event model Events are the primary integration surface for indexers and external analytics. diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 2f3ecf4..88dad93 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -9,7 +9,7 @@ import { createRequire } from 'module'; import { fileURLToPath, pathToFileURL } from 'url'; import { Command } from 'commander'; -import { generateAppSolidity } from '@tokenhost/generator'; +import { generateAppSolidity, type GeneratorLimits } from '@tokenhost/generator'; import { computeSchemaHash, importLegacyContractsJson, @@ -1442,6 +1442,76 @@ function titleFromSlug(slug: string): string { type KnownChainName = 'anvil' | 'sepolia' | 'filecoin_calibration' | 'filecoin_mainnet'; +const DEFAULT_GENERATION_LIMITS: Required = { + listMaxLimit: 50, + listMaxScanSteps: 1000, + multicallMaxCalls: 20, + tokenizedIndexMaxTokens: 8, + tokenizedIndexMaxTokenLength: 32 +}; + +const MAX_GENERATION_LIMITS: Required = { + listMaxLimit: 100, + listMaxScanSteps: 5000, + multicallMaxCalls: 50, + tokenizedIndexMaxTokens: 16, + tokenizedIndexMaxTokenLength: 64 +}; + +function clampGeneratorLimit(value: number | undefined, fallback: number, cap: number): number { + const numeric = Number(value); + if (!Number.isFinite(numeric) || numeric < 1) return fallback; + return Math.min(Math.floor(numeric), cap); +} + +function generationLimitsForChain(chainName?: KnownChainName): Required { + const chainDefaults: Partial>>> = { + filecoin_calibration: { + listMaxLimit: 25, + listMaxScanSteps: 500, + multicallMaxCalls: 12, + tokenizedIndexMaxTokens: 6, + tokenizedIndexMaxTokenLength: 24 + }, + filecoin_mainnet: { + listMaxLimit: 25, + listMaxScanSteps: 500, + multicallMaxCalls: 12, + tokenizedIndexMaxTokens: 6, + tokenizedIndexMaxTokenLength: 24 + } + }; + + const selected = { + ...DEFAULT_GENERATION_LIMITS, + ...(chainName ? chainDefaults[chainName] ?? {} : {}) + }; + + return { + listMaxLimit: clampGeneratorLimit(selected.listMaxLimit, DEFAULT_GENERATION_LIMITS.listMaxLimit, MAX_GENERATION_LIMITS.listMaxLimit), + listMaxScanSteps: clampGeneratorLimit( + selected.listMaxScanSteps, + DEFAULT_GENERATION_LIMITS.listMaxScanSteps, + MAX_GENERATION_LIMITS.listMaxScanSteps + ), + multicallMaxCalls: clampGeneratorLimit( + selected.multicallMaxCalls, + DEFAULT_GENERATION_LIMITS.multicallMaxCalls, + MAX_GENERATION_LIMITS.multicallMaxCalls + ), + tokenizedIndexMaxTokens: clampGeneratorLimit( + selected.tokenizedIndexMaxTokens, + DEFAULT_GENERATION_LIMITS.tokenizedIndexMaxTokens, + MAX_GENERATION_LIMITS.tokenizedIndexMaxTokens + ), + tokenizedIndexMaxTokenLength: clampGeneratorLimit( + selected.tokenizedIndexMaxTokenLength, + DEFAULT_GENERATION_LIMITS.tokenizedIndexMaxTokenLength, + MAX_GENERATION_LIMITS.tokenizedIndexMaxTokenLength + ) + }; +} + function resolveKnownChain(name: string): { chainName: KnownChainName; chain: any } { const n = name.toLowerCase().trim(); if (n === 'anvil') return { chainName: 'anvil', chain: anvil }; @@ -1471,6 +1541,7 @@ function resolveRpcUrl(chainName: KnownChainName, chain: any, override?: string) function buildChainConfigArtifact(args: { chainName: KnownChainName; chain: any; rpcUrl: string }): any { const now = new Date().toISOString(); const isLocal = args.chainName === 'anvil'; + const generationLimits = generationLimitsForChain(args.chainName); const chainConfig: any = { chainConfigVersion: '1.0.0', @@ -1501,6 +1572,21 @@ function buildChainConfigArtifact(args: { chainName: KnownChainName; chain: any; } ] }, + limits: { + lists: { + maxLimit: generationLimits.listMaxLimit, + maxScanSteps: generationLimits.listMaxScanSteps + }, + multicall: { + maxCalls: generationLimits.multicallMaxCalls + }, + indexing: { + tokenized: { + maxTokens: generationLimits.tokenizedIndexMaxTokens, + maxTokenLength: generationLimits.tokenizedIndexMaxTokenLength + } + } + }, issuer: { name: 'Token Host (local)', issuedAt: now @@ -2360,14 +2446,16 @@ function buildFromSchema( txMode?: string; relayBaseUrl?: string; targetChainId?: number; + targetChainName?: KnownChainName; compileProfile?: CompileProfile; } ): { outDir: string; uiBundleDir: string | null; uiSiteDir: string | null } { const resolvedOutDir = path.resolve(outDir); ensureDir(resolvedOutDir); + const generationLimits = generationLimitsForChain(opts.targetChainName); // 1) Generate Solidity source - const appSol = generateAppSolidity(schema); + const appSol = generateAppSolidity(schema, { limits: generationLimits }); ensureDir(path.join(resolvedOutDir, path.dirname(appSol.path))); fs.writeFileSync(path.join(resolvedOutDir, appSol.path), appSol.contents); @@ -2409,7 +2497,9 @@ function buildFromSchema( fs.rmSync(uiBundleDir, { recursive: true, force: true }); ensureDir(uiBundleDir); - const uiWorkDir = fs.mkdtempSync(path.join(os.tmpdir(), 'tokenhost-ui-build-')); + const uiTempRoot = path.join(resolvedOutDir, '.tokenhost-build-tmp'); + ensureDir(uiTempRoot); + const uiWorkDir = fs.mkdtempSync(path.join(uiTempRoot, 'ui-build-')); try { const templateDir = resolveNextExportUiTemplateDir(); copyDir(templateDir, uiWorkDir); @@ -2443,6 +2533,7 @@ function buildFromSchema( copyDir(exportedDir, uiBundleDir); } finally { fs.rmSync(uiWorkDir, { recursive: true, force: true }); + fs.rmSync(uiTempRoot, { recursive: true, force: true }); } uiBundleDigest = computeDirectoryDigest(uiBundleDir); @@ -3257,9 +3348,10 @@ program .argument('', 'Path to THS schema JSON file') .option('--out ', 'Output directory', 'artifacts') .option('--no-ui', 'Do not generate UI output') + .option('--chain ', 'Target chain for generation limits (anvil|sepolia|filecoin_calibration|filecoin_mainnet)') .option('--compiler-profile ', 'Compiler profile (auto|default|large-app)', 'auto') .option('--with-tests', 'Emit generated app test scaffold', false) - .action((schemaPath: string, opts: { out: string; ui: boolean; compilerProfile?: string; withTests: boolean }) => { + .action((schemaPath: string, opts: { out: string; ui: boolean; chain?: string; compilerProfile?: string; withTests: boolean }) => { const input = readJsonFile(schemaPath); const structural = validateThsStructural(input); if (!structural.ok) { @@ -3278,7 +3370,8 @@ program } const outDir = opts.out; - const appSol = generateAppSolidity(schema); + const generationLimits = generationLimitsForChain(opts.chain ? resolveKnownChain(opts.chain).chainName : undefined); + const appSol = generateAppSolidity(schema, { limits: generationLimits }); const contractsDir = path.join(outDir, path.dirname(appSol.path)); ensureDir(contractsDir); fs.writeFileSync(path.join(outDir, appSol.path), appSol.contents); @@ -3348,18 +3441,22 @@ program .argument('', 'Path to THS schema JSON file') .option('--out ', 'Output directory', 'artifacts') .option('--no-ui', 'Do not generate/build UI bundle') + .option('--chain ', 'Target chain for generation limits (anvil|sepolia|filecoin_calibration|filecoin_mainnet)') .option('--compiler-profile ', 'Compiler profile (auto|default|large-app)', 'auto') .option('--tx-mode ', 'Transaction mode (auto|userPays|sponsored)', 'auto') .option('--relay-base-url ', 'Relay base URL for sponsored mode', '/__tokenhost/relay') - .action((schemaPath: string, opts: { out: string; ui: boolean; compilerProfile?: string; txMode?: string; relayBaseUrl?: string }) => { + .action((schemaPath: string, opts: { out: string; ui: boolean; chain?: string; compilerProfile?: string; txMode?: string; relayBaseUrl?: string }) => { try { const schema = loadThsSchemaOrThrow(schemaPath); + const targetChainName = opts.chain ? resolveKnownChain(opts.chain).chainName : undefined; buildFromSchema(schema, opts.out, { ui: opts.ui, schemaPathForHints: schemaPath, compileProfile: normalizeCompileProfile(opts.compilerProfile), txMode: opts.txMode, - relayBaseUrl: opts.relayBaseUrl + relayBaseUrl: opts.relayBaseUrl, + targetChainName, + targetChainId: targetChainName ? resolveKnownChain(targetChainName).chain.id : undefined }); } catch (e: any) { console.error(String(e?.message ?? e)); @@ -3548,7 +3645,8 @@ program compileProfile, txMode: opts.txMode, relayBaseUrl: opts.relayBaseUrl, - targetChainId: chain.id + targetChainId: chain.id, + targetChainName: chainName }); console.log('Build complete.'); diff --git a/packages/generator/src/index.ts b/packages/generator/src/index.ts index 69f3220..54dc831 100644 --- a/packages/generator/src/index.ts +++ b/packages/generator/src/index.ts @@ -1,2 +1,2 @@ export { generateAppSolidity } from './solidity/generateAppSolidity.js'; - +export type { GenerateAppSolidityOptions, GeneratorLimits } from './solidity/generateAppSolidity.js'; diff --git a/packages/generator/src/solidity/generateAppSolidity.ts b/packages/generator/src/solidity/generateAppSolidity.ts index f095989..d560b08 100644 --- a/packages/generator/src/solidity/generateAppSolidity.ts +++ b/packages/generator/src/solidity/generateAppSolidity.ts @@ -1,6 +1,26 @@ import type { Access, Collection, FieldType, QueryIndex, Relation, ThsField, ThsSchema, UniqueIndex } from '@tokenhost/schema'; import { computeSchemaHash } from '@tokenhost/schema'; +export type GeneratorLimits = { + listMaxLimit?: number; + listMaxScanSteps?: number; + multicallMaxCalls?: number; + tokenizedIndexMaxTokens?: number; + tokenizedIndexMaxTokenLength?: number; +}; + +export type GenerateAppSolidityOptions = { + limits?: GeneratorLimits; +}; + +const DEFAULT_GENERATOR_LIMITS = Object.freeze({ + listMaxLimit: 50, + listMaxScanSteps: 1000, + multicallMaxCalls: 20, + tokenizedIndexMaxTokens: 8, + tokenizedIndexMaxTokenLength: 32 +}); + type SolidityType = 'string' | 'uint256' | 'int256' | 'bool' | 'address' | 'bytes32'; function solidityStorageType(t: FieldType): SolidityType { @@ -114,7 +134,7 @@ function recordHashFnName(collectionName: string): string { return `_hashRecord${collectionName}`; } -export function generateAppSolidity(schema: ThsSchema): { path: string; contents: string } { +export function generateAppSolidity(schema: ThsSchema, options: GenerateAppSolidityOptions = {}): { path: string; contents: string } { const w = new W(); const schemaHash = computeSchemaHash(schema); @@ -122,6 +142,10 @@ export function generateAppSolidity(schema: ThsSchema): { path: string; contents const onChainIndexing = schema.app.features?.onChainIndexing ?? true; const anyPaidCreates = schema.collections.some(hasPaidCreates); + const limits = { + ...DEFAULT_GENERATOR_LIMITS, + ...(options.limits ?? {}) + }; w.line('// SPDX-License-Identifier: UNLICENSED'); w.line('pragma solidity ^0.8.24;'); @@ -147,11 +171,11 @@ export function generateAppSolidity(schema: ThsSchema): { path: string; contents w.line(); w.line(`bool public constant ON_CHAIN_INDEXING = ${onChainIndexing ? 'true' : 'false'};`); - w.line('uint256 public constant MAX_LIST_LIMIT = 50;'); - w.line('uint256 public constant MAX_SCAN_STEPS = 1000;'); - w.line('uint256 public constant MAX_MULTICALL_CALLS = 20;'); - w.line('uint256 public constant MAX_TOKENIZED_INDEX_TOKENS = 8;'); - w.line('uint256 public constant MAX_TOKENIZED_INDEX_TOKEN_LENGTH = 32;'); + w.line(`uint256 public constant MAX_LIST_LIMIT = ${limits.listMaxLimit};`); + w.line(`uint256 public constant MAX_SCAN_STEPS = ${limits.listMaxScanSteps};`); + w.line(`uint256 public constant MAX_MULTICALL_CALLS = ${limits.multicallMaxCalls};`); + w.line(`uint256 public constant MAX_TOKENIZED_INDEX_TOKENS = ${limits.tokenizedIndexMaxTokens};`); + w.line(`uint256 public constant MAX_TOKENIZED_INDEX_TOKEN_LENGTH = ${limits.tokenizedIndexMaxTokenLength};`); w.line(); // Errors diff --git a/schemas/tokenhost-chain-config.schema.json b/schemas/tokenhost-chain-config.schema.json index fd41d13..45a5ad3 100644 --- a/schemas/tokenhost-chain-config.schema.json +++ b/schemas/tokenhost-chain-config.schema.json @@ -315,6 +315,26 @@ } } }, + "indexing": { + "type": "object", + "additionalProperties": false, + "properties": { + "tokenized": { + "type": "object", + "additionalProperties": false, + "properties": { + "maxTokens": { + "type": "integer", + "minimum": 1 + }, + "maxTokenLength": { + "type": "integer", + "minimum": 1 + } + } + } + } + }, "realtime": { "type": "object", "additionalProperties": false, diff --git a/test/testCliBuildArtifacts.js b/test/testCliBuildArtifacts.js index 52161f1..56d05f3 100644 --- a/test/testCliBuildArtifacts.js +++ b/test/testCliBuildArtifacts.js @@ -71,5 +71,39 @@ describe('th build (artifacts)', function () { expect(manifest?.artifacts?.compiledContracts?.url).to.match(/^file:\/\//); expect(manifest?.signatures?.[0]?.sig).to.be.a('string'); }); -}); + it('uses stricter chain-targeted generation limits when build is given a target chain', function () { + this.timeout(60000); + + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'th-build-chain-limits-')); + const schemaPath = path.join(dir, 'schema.json'); + const outDir = path.join(dir, 'out'); + writeJson(schemaPath, minimalSchema()); + + const res = runTh(['build', schemaPath, '--out', outDir, '--no-ui', '--chain', 'filecoin_calibration'], process.cwd()); + expect(res.status, res.stderr || res.stdout).to.equal(0); + + const appSol = fs.readFileSync(path.join(outDir, 'contracts', 'App.sol'), 'utf-8'); + expect(appSol).to.include('uint256 public constant MAX_LIST_LIMIT = 25;'); + expect(appSol).to.include('uint256 public constant MAX_SCAN_STEPS = 500;'); + expect(appSol).to.include('uint256 public constant MAX_MULTICALL_CALLS = 12;'); + expect(appSol).to.include('uint256 public constant MAX_TOKENIZED_INDEX_TOKENS = 6;'); + expect(appSol).to.include('uint256 public constant MAX_TOKENIZED_INDEX_TOKEN_LENGTH = 24;'); + }); + + it('cleans its temporary UI build workspace after a successful build', function () { + this.timeout(180000); + + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'th-build-ui-temp-cleanup-')); + const schemaPath = path.join(dir, 'schema.json'); + const outDir = path.join(dir, 'out'); + writeJson(schemaPath, minimalSchema()); + + const res = runTh(['build', schemaPath, '--out', outDir], process.cwd()); + expect(res.status, res.stderr || res.stdout).to.equal(0); + + expect(fs.existsSync(path.join(outDir, '.tokenhost-build-tmp'))).to.equal(false); + expect(fs.existsSync(path.join(outDir, 'ui-bundle', 'index.html'))).to.equal(true); + expect(fs.existsSync(path.join(outDir, 'ui-site', 'index.html'))).to.equal(true); + }); +}); diff --git a/test/testCliDev.js b/test/testCliDev.js index 8a64061..2376130 100644 --- a/test/testCliDev.js +++ b/test/testCliDev.js @@ -41,6 +41,8 @@ function minimalSchema(overrides = {}) { } describe('th up/run/dev', function () { + this.timeout(15000); + it('supports --dry-run (no side effects) via th up', function () { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'th-dev-')); const schemaPath = path.join(dir, 'schema.json'); diff --git a/test/testCliVerify.js b/test/testCliVerify.js index 98fae81..f9c9dbd 100644 --- a/test/testCliVerify.js +++ b/test/testCliVerify.js @@ -76,6 +76,8 @@ function writeVerifyFixtureBuild(outDir, chainId = 11155111) { } describe('th verify', function () { + this.timeout(15000); + it('fails when Etherscan key is missing', function () { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'th-verify-no-key-')); writeVerifyFixtureBuild(dir);