Skip to content

Commit 8d250f0

Browse files
keckec
committed
Add full-set ledger ingest driver and shared sectioning helper
LedgerGeneratorMain (entity/builder) is the persisted-output driver the #869 generator lacked: loads a real protobuf artifact, decompiles it via TaxonomySectioner + SectionEmitter, and writes one compilable Java source file per section plus a delegating aggregator — turning GeneratorEndToEndIT's proven in-memory round trip into real ledger source for a genesis repo's ingested-foundation section (#872). Extracted the section-plus-residual-catch-all-plus-dedup algorithm GeneratorEndToEndIT already proved into TaxonomySectioner.sectionsCoveringFullStore, shared by both the IT and the new driver so they can't silently drift apart. Full 10-class/44-test generator IT suite and the entity javadoc gate stay green. Refs: IKE-Network/ike-issues#872
1 parent 3ea13bd commit 8d250f0

3 files changed

Lines changed: 229 additions & 38 deletions

File tree

‎entity/src/main/java/dev/ikm/tinkar/entity/builder/LedgerGeneratorMain.java‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
/*
2+
* Copyright © 2015 Integrated Knowledge Management (support@ikm.dev)
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package dev.ikm.tinkar.entity.builder;
17+
18+
import dev.ikm.tinkar.common.service.CachingService;
19+
import dev.ikm.tinkar.common.service.PrimitiveData;
20+
import dev.ikm.tinkar.coordinate.Calculators;
21+
import dev.ikm.tinkar.coordinate.language.calculator.LanguageCalculator;
22+
import dev.ikm.tinkar.coordinate.stamp.calculator.StampCalculator;
23+
import dev.ikm.tinkar.entity.builder.generator.SectionEmitter;
24+
import dev.ikm.tinkar.entity.builder.generator.TaxonomySectioner;
25+
import dev.ikm.tinkar.entity.builder.generator.TaxonomySectioner.Section;
26+
import dev.ikm.tinkar.entity.builder.generator.TinkarTermReferenceResolver;
27+
import dev.ikm.tinkar.entity.load.LoadEntitiesFromProtobufFile;
28+
import dev.ikm.tinkar.terms.TinkarTerm;
29+
30+
import java.io.File;
31+
import java.io.IOException;
32+
import java.nio.file.Files;
33+
import java.nio.file.Path;
34+
import java.util.ArrayList;
35+
import java.util.List;
36+
37+
/**
38+
* The full-set ledger-ingest entry point: loads a real protobuf artifact into an
39+
* ephemeral store, decompiles it via the #869 generator ({@link TaxonomySectioner} +
40+
* {@link SectionEmitter}), and writes one compilable Java source file per section —
41+
* turning {@code GeneratorEndToEndIT}'s proven, in-memory round trip into real,
42+
* persisted ledger source for a genesis repo's own ingested-foundation section
43+
* (IKE-Network/ike-issues#872).
44+
* <p>
45+
* Like {@link KbAssembleMain}/{@link BindingsMain}, a reflective seam for build
46+
* tooling: run via {@code exec:java} over a classpath carrying the entity, executor,
47+
* and ephemeral-store providers with their legacy {@code META-INF/services}
48+
* registrations. Unlike those, this is intentionally a one-time-per-target-repo
49+
* operation, not a repeated build step — the ingested-foundation section it produces
50+
* is regeneration-locked once landed (declared identities, not re-derived on every
51+
* build; future upstream updates arrive as change sets via the #847 lift machinery,
52+
* never by re-running this generator).
53+
* <p>
54+
* Arguments: {@code pbZip outputSourceRoot packageName moduleRef authorRef}.
55+
* {@code moduleRef}/{@code authorRef} are fully-qualified Java source expressions
56+
* (e.g. {@code network.ike.foundation.ike.terms.Ike.MODULE}) spliced literally into
57+
* every generated section's stamp — see {@link SectionEmitter#emitSection}.
58+
*/
59+
public final class LedgerGeneratorMain {
60+
61+
private LedgerGeneratorMain() {
62+
}
63+
64+
/**
65+
* Generates one section source file per taxonomy bucket (plus a residual
66+
* catch-all), and a delegating aggregator that composes them all onto a
67+
* caller-supplied {@code KnowledgeSet}.
68+
*
69+
* @param args {@code pbZip outputSourceRoot packageName moduleRef authorRef}
70+
* @throws Exception if the store cannot be opened, the artifact fails to load, a
71+
* section cannot be written, or any component needed hand
72+
* authoring (zero tolerance for a silently-skipped component in
73+
* a real ingest)
74+
*/
75+
public static void main(String[] args) throws Exception {
76+
if (args.length != 5) {
77+
throw new IllegalArgumentException("Usage: LedgerGeneratorMain <pbZip> <outputSourceRoot>"
78+
+ " <packageName> <moduleRef> <authorRef>");
79+
}
80+
File pbZip = new File(args[0]);
81+
Path outputSourceRoot = Path.of(args[1]);
82+
String packageName = args[2];
83+
String moduleRef = args[3];
84+
String authorRef = args[4];
85+
if (!pbZip.isFile()) {
86+
throw new IllegalArgumentException("Not a file: " + pbZip);
87+
}
88+
89+
CachingService.clearAll();
90+
PrimitiveData.selectControllerByName("Load Ephemeral Store");
91+
PrimitiveData.start();
92+
List<String> sectionClassNames = new ArrayList<>();
93+
int distinctMembers;
94+
try {
95+
new LoadEntitiesFromProtobufFile(pbZip).compute();
96+
97+
StampCalculator calculator = Calculators.Stamp.DevelopmentLatestActiveOnly();
98+
TinkarTermReferenceResolver resolver = TinkarTermReferenceResolver.build();
99+
LanguageCalculator languageCalculator =
100+
Calculators.Language.UsEnglishFullyQualifiedName(calculator.stampCoordinate());
101+
TaxonomySectioner sectioner = TaxonomySectioner.fromStatedNavigation(calculator);
102+
103+
List<Section> sections = sectioner.sectionsCoveringFullStore(
104+
TinkarTerm.ROOT_VERTEX.nid(), 60, 4, 50);
105+
distinctMembers = sections.stream().mapToInt(section -> section.members().size()).sum();
106+
107+
List<String> emissionNotes = new ArrayList<>();
108+
int index = 0;
109+
for (Section section : sections) {
110+
index++;
111+
if (section.members().isEmpty()) {
112+
continue;
113+
}
114+
String className = "Section" + index;
115+
sectionClassNames.add(className);
116+
SectionEmitter.EmittedSection emitted = SectionEmitter.emitSection(packageName, className, section,
117+
calculator, languageCalculator, resolver, moduleRef, authorRef);
118+
emissionNotes.addAll(emitted.manifestNotes());
119+
writeSourceFile(outputSourceRoot, packageName, className, emitted.source());
120+
}
121+
if (!emissionNotes.isEmpty()) {
122+
throw new IllegalStateException("Zero tolerance for a silently-skipped component in a real"
123+
+ " ingest — " + emissionNotes.size() + " manifest note(s):\n"
124+
+ String.join("\n", emissionNotes));
125+
}
126+
127+
String aggregatorSource = emitDelegatingAggregator(packageName, sectionClassNames);
128+
writeSourceFile(outputSourceRoot, packageName, "FoundationSet", aggregatorSource);
129+
} finally {
130+
PrimitiveData.stop();
131+
}
132+
System.out.println("Ledger generated: " + sectionClassNames.size() + " sections, "
133+
+ distinctMembers + " distinct components, into " + outputSourceRoot);
134+
}
135+
136+
/**
137+
* Emits the delegating aggregator: unlike {@link SectionEmitter#emitAggregator}
138+
* (which mints its own fresh {@code KnowledgeSet}), this composes every section
139+
* onto a {@code KnowledgeSet} the caller already created — a genesis repo's own
140+
* session-root constant, not a new identity.
141+
*
142+
* @param packageName the target package
143+
* @param sectionClassNames every emitted section class's simple name, in
144+
* composition order
145+
* @return the aggregator class's full compilable source text
146+
*/
147+
private static String emitDelegatingAggregator(String packageName, List<String> sectionClassNames) {
148+
StringBuilder source = new StringBuilder();
149+
source.append("package ").append(packageName).append(";\n\n");
150+
source.append("import dev.ikm.tinkar.entity.builder.KnowledgeSet;\n\n");
151+
source.append("/** Composes every ingested-foundation section onto the caller's KnowledgeSet"
152+
+ " (IKE-Network/ike-issues#872). */\n");
153+
source.append("public final class FoundationSet {\n\n");
154+
source.append(" private FoundationSet() {\n }\n\n");
155+
source.append(" public static void compose(KnowledgeSet set) {\n");
156+
for (String sectionClassName : sectionClassNames) {
157+
source.append(" ").append(sectionClassName).append(".compose(set);\n");
158+
}
159+
source.append(" }\n}\n");
160+
return source.toString();
161+
}
162+
163+
private static void writeSourceFile(Path sourceRoot, String packageName, String className, String source)
164+
throws IOException {
165+
Path packageDir = sourceRoot.resolve(packageName.replace('.', '/'));
166+
Files.createDirectories(packageDir);
167+
Files.writeString(packageDir.resolve(className + ".java"), source);
168+
}
169+
}

‎entity/src/main/java/dev/ikm/tinkar/entity/builder/generator/TaxonomySectioner.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
import dev.ikm.tinkar.common.id.IntIdSet;
1919
import dev.ikm.tinkar.coordinate.stamp.calculator.StampCalculator;
20+
import dev.ikm.tinkar.entity.EntityService;
2021
import dev.ikm.tinkar.terms.TinkarTerm;
2122
import org.eclipse.collections.api.factory.primitive.IntLists;
2223
import org.eclipse.collections.api.list.primitive.MutableIntList;
@@ -162,6 +163,55 @@ public List<Section> sectionsUnder(int taxonomyRootNid, int splitThreshold, int
162163
return sections;
163164
}
164165

166+
/**
167+
* Convenience for callers that need FULL store coverage, not just the
168+
* taxonomy-reachable subset: {@link #sectionsUnder(int, int, int)}, first-section-wins
169+
* deduplicated across sections (this class's own documented "dedup is the caller's
170+
* job" contract, applied once here so every caller doesn't reimplement it), plus a
171+
* residual catch-all covering every concept and pattern NOT reachable via stated
172+
* navigation — meta-schema concepts the taxonomy scan found unanchored, and the
173+
* entirely separate pattern taxonomy, which {@link #sectionsUnder} never walks.
174+
* Residual members are batched by {@code residualBatchSize}: a single
175+
* {@code compose()} method over every residual member can approach the same 64KB
176+
* bytecode-per-method limit that drives {@code splitThreshold}/{@code maxDepth}
177+
* above.
178+
*
179+
* @param taxonomyRootNid forwarded to {@link #sectionsUnder}
180+
* @param splitThreshold forwarded to {@link #sectionsUnder}
181+
* @param maxDepth forwarded to {@link #sectionsUnder}
182+
* @param residualBatchSize the maximum members per residual-catch-all section
183+
* @return every section needed for full-store coverage, deduplicated
184+
*/
185+
public List<Section> sectionsCoveringFullStore(int taxonomyRootNid, int splitThreshold, int maxDepth,
186+
int residualBatchSize) {
187+
List<Section> sections = sectionsUnder(taxonomyRootNid, splitThreshold, maxDepth);
188+
189+
Set<Integer> assigned = new HashSet<>();
190+
List<Section> exclusiveSections = new ArrayList<>();
191+
for (Section section : sections) {
192+
List<Integer> exclusiveMembers = section.members().stream().filter(assigned::add).toList();
193+
exclusiveSections.add(new Section(section.rootNid(), exclusiveMembers));
194+
}
195+
196+
List<Integer> residualMembers = new ArrayList<>();
197+
EntityService.get().forEachConceptEntity(concept -> {
198+
if (assigned.add(concept.nid())) {
199+
residualMembers.add(concept.nid());
200+
}
201+
});
202+
EntityService.get().forEachPatternEntity(pattern -> {
203+
if (assigned.add(pattern.nid())) {
204+
residualMembers.add(pattern.nid());
205+
}
206+
});
207+
for (int start = 0; start < residualMembers.size(); start += residualBatchSize) {
208+
List<Integer> batch = residualMembers.subList(start,
209+
Math.min(start + residualBatchSize, residualMembers.size()));
210+
exclusiveSections.add(new Section(taxonomyRootNid, List.copyOf(batch)));
211+
}
212+
return exclusiveSections;
213+
}
214+
165215
private void splitInto(int nid, int splitThreshold, int maxDepth, int depth, List<Section> sections,
166216
Set<Integer> ancestorPath) {
167217
if (!ancestorPath.add(nid)) {

‎integration/src/test/java/dev/ikm/tinkar/integration/builder/GeneratorEndToEndIT.java‎

Lines changed: 10 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -116,48 +116,20 @@ void generateCompileReplayVerify() throws Exception {
116116
int descriptionPatternVersionsBefore = EntityHandle.get(TinkarTerm.DESCRIPTION_PATTERN.nid()).expectPattern()
117117
.versions().size();
118118

119-
List<Section> sections = sectioner.sectionsUnder(TinkarTerm.ROOT_VERTEX.nid(), 60, 4);
120-
int totalMembers = sections.stream().mapToInt(section -> section.members().size()).sum();
121-
LOG.info("Sectioned into {} files covering {} component slots", sections.size(), totalMembers);
122-
123119
// Sections are not disjoint by design (TaxonomySectioner's own contract) — a
124120
// dual-parented concept is a member of every section whose root reaches it.
125-
// The generator must still declare each component exactly once: first section
126-
// to claim it wins, matching "the caller resolves as primary" in
127-
// TaxonomySectioner's javadoc.
128-
Set<Integer> assigned = new HashSet<>();
129-
List<Section> exclusiveSections = new ArrayList<>();
130-
for (Section section : sections) {
131-
List<Integer> exclusiveMembers = section.members().stream().filter(assigned::add).toList();
132-
exclusiveSections.add(new Section(section.rootNid(), exclusiveMembers));
133-
}
134-
// TaxonomySectioner only walks concepts reachable via stated navigation — the
135-
// ~60 unanchored meta-schema concepts the #873 scan found, and all 28
121+
// TaxonomySectioner only walks concepts reachable via stated navigation, too —
122+
// the ~60 unanchored meta-schema concepts the #873 scan found, and all 28
136123
// patterns (an entirely separate taxonomy), need a residual catch-all so the
137124
// round trip actually covers the full store, not just the navigable subset.
138-
List<Integer> residualMembers = new ArrayList<>();
139-
EntityService.get().forEachConceptEntity(concept -> {
140-
if (assigned.add(concept.nid())) {
141-
residualMembers.add(concept.nid());
142-
}
143-
});
144-
EntityService.get().forEachPatternEntity(pattern -> {
145-
if (assigned.add(pattern.nid())) {
146-
residualMembers.add(pattern.nid());
147-
}
148-
});
149-
// Chunked, not one section: a single compose() method over ~90 components can
150-
// approach the same 64KB bytecode-per-method limit that forced deeper taxonomy
151-
// splitting above.
152-
int batchSize = 50;
153-
for (int start = 0; start < residualMembers.size(); start += batchSize) {
154-
List<Integer> batch = residualMembers.subList(start, Math.min(start + batchSize, residualMembers.size()));
155-
exclusiveSections.add(new Section(TinkarTerm.ROOT_VERTEX.nid(), List.copyOf(batch)));
156-
}
157-
LOG.info("{} unanchored/pattern components in the residual catch-all section", residualMembers.size());
158-
159-
int distinctMembers = assigned.size();
160-
LOG.info("{} distinct components after cross-section dedup", distinctMembers);
125+
// sectionsCoveringFullStore does both (first-section-wins dedup + residual
126+
// catch-all, batched to stay under the JVM's 64KB bytecode-per-method limit) —
127+
// shared with LedgerGeneratorMain's real ingest run so the two can't drift.
128+
List<Section> exclusiveSections = sectioner.sectionsCoveringFullStore(
129+
TinkarTerm.ROOT_VERTEX.nid(), 60, 4, 50);
130+
int distinctMembers = exclusiveSections.stream().mapToInt(section -> section.members().size()).sum();
131+
LOG.info("{} sections covering {} distinct components after cross-section dedup + residual catch-all",
132+
exclusiveSections.size(), distinctMembers);
161133

162134
Path sourceDir = Files.createTempDirectory("ledger-generator-it-src");
163135
Path classesDir = Files.createTempDirectory("ledger-generator-it-classes");

0 commit comments

Comments
 (0)