diff --git a/CLAUDE.md b/CLAUDE.md index 52e7a1e..6e64eba 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -53,7 +53,8 @@ extraction, Meilisearch indexing with scheduled refresh, and four MCP tools │ ├── ContentIndexer.java # orchestrates discover→diff→fetch→upsert/delete │ ├── IndexReport.java # per-pass counters │ ├── ContentIndexStore.java # index read/write seam (StoredDocument) -│ └── MeilisearchContentIndexStore.java # ContentIndexStore backed by MeilisearchClient +│ ├── MeilisearchContentIndexStore.java # ContentIndexStore backed by MeilisearchClient +│ └── ContentBootstrapStep.java # SearchIndexBootstrapStep: startup full reindex ├── src/main/resources/application.yaml # datasource, JPA, OAuth2, MCP, Meilisearch, content config ├── src/test/java/com/openelements/content/ # behavior tests (context, MCP enabled/disabled, search-down, jsoup) └── docs/ @@ -105,7 +106,10 @@ items via the strategy → batch-upsert, and delete documents that 404 or vanish `MeilisearchContentIndexStore` implements the store over `MeilisearchClient` (paged `multiSearch` to read state, `addDocuments`+`waitForTask` to upsert, `deleteDocument`), since the library's `BatchWriter` is package-private. The indexer is the reusable engine for the bootstrap step (009) and -refresh scheduler (010). +refresh scheduler (010). `ContentBootstrapStep` implements the library's `SearchIndexBootstrapStep`: +at startup the library's `MeilisearchBootstrapRunner` discovers it, consumes its lazy `documents()` +stream (over all enabled sources, via `ContentIndexer.streamAllDocuments`) in batches into Meilisearch, +and toggles `SearchReadinessState`. > **Key gotcha:** the library ships no Spring Boot auto-configuration and couples MCP to a JPA > datasource, so `@Import({ McpConfiguration, SearchConfig })` alone does **not** boot. See diff --git a/docs/specs/009-content-bootstrap-step/steps.md b/docs/specs/009-content-bootstrap-step/steps.md new file mode 100644 index 0000000..52a9750 --- /dev/null +++ b/docs/specs/009-content-bootstrap-step/steps.md @@ -0,0 +1,39 @@ +# Implementation Steps: Content Bootstrap Step + +## Step 1: `ContentBootstrapStep` + +- [x] `@Component` (gated on `meilisearch.enabled`) implementing `SearchIndexBootstrapStep` +- [x] `indexUid()` → `resolveIndex("content")` +- [x] `documents()` → lazy `flatMap` over enabled sources via `ContentIndexer.streamAllDocuments` +- [x] Per-source fault isolation: a source that fails to stream is logged and skipped, others still contribute + +**Acceptance criteria:** +- [x] Project builds; behaviors verified by tests +- [x] Discovered by the library `MeilisearchBootstrapRunner` (auto-injected `List`) + +**Related behaviors:** indexUid targets content index; all enabled sources contribute; stream is lazy; failing source does not block others + +--- + +## Step 2: Tests + +- [x] `ContentBootstrapStepTest` (real `ContentIndexer` + stub strategy): indexUid, enabled-only sources, laziness, failing-source isolation + +**Acceptance criteria:** +- [x] All tests pass (`mvn test`); build green + +--- + +## Behavior Coverage + +| Scenario | Layer | Covered in Step | +|----------|-------|-----------------| +| Step is discovered by the runner | Backend (integration) | Delegated to the library `MeilisearchBootstrapRunner` (auto-injects `SearchIndexBootstrapStep` beans); `@Component` registration verified by the full app-context build. Live end-to-end run needs a running Meilisearch. | +| indexUid targets the content index | Backend | Step 2 (`indexUidTargetsContentIndex`) | +| All enabled sources contribute documents | Backend | Step 2 (`onlyEnabledSourcesContribute`) | +| Stream is lazy | Backend | Step 2 (`streamIsLazy`) | +| Readiness flips after bootstrap | Backend (integration) | Owned by the library runner (`markBootstrappingStarted/Finished` on `SearchReadinessState`); needs a live Meilisearch. | +| Search short-circuits during bootstrap | Backend | Deferred to spec 011 (search layer observes `SearchReadinessState.isBootstrapping()`). | +| A failing source does not block others | Backend | Step 2 (`failingSourceDoesNotBlockOthers`) | + +All scenarios are backend; there is no frontend in this spec. The readiness/runner scenarios are the library's responsibility and require a live Meilisearch, consistent with the project's established live-search test boundary. diff --git a/docs/specs/INDEX.md b/docs/specs/INDEX.md index e732467..984ade4 100644 --- a/docs/specs/INDEX.md +++ b/docs/specs/INDEX.md @@ -14,7 +14,7 @@ roadmap step, to be implemented sequentially (each builds on the previous). | 006 | 006-content-extractor | Content extractor | backend, crawler | `ContentExtractor` — jsoup content + metadata extraction | #11 | done | | 007 | 007-source-strategy | Source strategy | backend, architecture | `ContentSourceStrategy` interface + `WebsiteSourceStrategy` | #13 | done | | 008 | 008-content-indexer | Content indexer | backend, search, crawler | `ContentIndexer` — orchestration, diff, upsert/delete | #15 | done | -| 009 | 009-content-bootstrap-step | Content bootstrap step | backend, search | `ContentBootstrapStep` — initial reindex via `SearchIndexBootstrapStep` | — | open | +| 009 | 009-content-bootstrap-step | Content bootstrap step | backend, search | `ContentBootstrapStep` — initial reindex via `SearchIndexBootstrapStep` | #17 | done | | 010 | 010-content-refresh-scheduler | Content refresh scheduler | backend, scheduling | `ContentRefreshScheduler` — `@Scheduled` incremental re-crawl | — | open | | 011 | 011-content-search-service | Content search service | backend, search | `ContentSearchService` — `multiSearch` + highlighting facade | — | open | | 012 | 012-content-mcp-tools | Content MCP tools | backend, mcp, api | `ContentMcpToolProvider` — the 4 MCP tools | — | open | diff --git a/src/main/java/com/openelements/content/ContentBootstrapStep.java b/src/main/java/com/openelements/content/ContentBootstrapStep.java new file mode 100644 index 0000000..9d413da --- /dev/null +++ b/src/main/java/com/openelements/content/ContentBootstrapStep.java @@ -0,0 +1,65 @@ +package com.openelements.content; + +import com.openelements.spring.base.services.search.MeilisearchProperties; +import com.openelements.spring.base.services.search.SearchIndexBootstrapStep; +import java.util.Map; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +/** + * Populates the content index at application startup by implementing the library's + * {@link SearchIndexBootstrapStep}. + * + *

The step is intentionally thin: it declares the target index and provides a lazy document + * stream over all enabled sources (via {@link ContentIndexer#streamAllDocuments(ContentSource)}). The + * library's {@code MeilisearchBootstrapRunner} discovers the step, consumes the stream in batches, + * and toggles {@code SearchReadinessState} around the run. Reusing {@link ContentIndexer} keeps + * bootstrap and the refresh scheduler (spec 010) from diverging. + * + *

Only created when the Meilisearch stack is enabled. + */ +@Component +@ConditionalOnProperty(prefix = "openelements.meilisearch", name = "enabled", havingValue = "true") +public class ContentBootstrapStep implements SearchIndexBootstrapStep { + + private static final Logger log = LoggerFactory.getLogger(ContentBootstrapStep.class); + + private final ContentSourceProperties properties; + private final ContentIndexer indexer; + private final MeilisearchProperties meilisearchProperties; + + public ContentBootstrapStep(ContentSourceProperties properties, ContentIndexer indexer, + MeilisearchProperties meilisearchProperties) { + this.properties = properties; + this.indexer = indexer; + this.meilisearchProperties = meilisearchProperties; + } + + @Override + public String indexUid() { + return meilisearchProperties.resolveIndex("content"); + } + + @Override + public Stream> documents() { + return properties.sources().stream() + .filter(ContentSource::enabled) + .flatMap(this::streamSourceSafely); + } + + /** + * Streams one source's documents, isolating a source-level failure so the remaining sources still + * contribute (per-item fetch/extract errors are already contained as {@code SKIP} by the strategy). + */ + private Stream> streamSourceSafely(ContentSource source) { + try { + return indexer.streamAllDocuments(source); + } catch (Exception e) { + log.warn("Skipping source {} during bootstrap: {}", source.id(), e.toString()); + return Stream.empty(); + } + } +} diff --git a/src/test/java/com/openelements/content/ContentBootstrapStepTest.java b/src/test/java/com/openelements/content/ContentBootstrapStepTest.java new file mode 100644 index 0000000..175d4d8 --- /dev/null +++ b/src/test/java/com/openelements/content/ContentBootstrapStepTest.java @@ -0,0 +1,135 @@ +package com.openelements.content; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.openelements.spring.base.services.search.MeilisearchProperties; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.util.unit.DataSize; + +/** + * Unit tests for {@link ContentBootstrapStep} using a real {@link ContentIndexer} driven by a + * test-double strategy, so document streaming is exercised without a running Meilisearch. + * + *

Readiness toggling and batching are the library {@code MeilisearchBootstrapRunner}'s + * responsibility and require a live instance; they are not reproduced here. + */ +@DisplayName("Content bootstrap step") +class ContentBootstrapStepTest { + + private final MeilisearchProperties meilisearch = + new MeilisearchProperties("http://localhost:7700", "", "content_", Duration.ofSeconds(10)); + private final StubStrategy strategy = new StubStrategy(); + private final ContentIndexer indexer = + new ContentIndexer(new SourceStrategyRegistry(List.of(strategy)), new NoOpStore()); + + private static ContentSource source(String id, boolean enabled) { + return new ContentSource( + id, SourceType.WEBSITE, "https://ex.com", List.of(), List.of("/**"), List.of(), "article", List.of(), enabled); + } + + private static ContentSourceProperties properties(ContentSource... sources) { + return new ContentSourceProperties( + true, null, "UA", 2.0, Duration.ofSeconds(10), DataSize.ofMegabytes(5), List.of(sources)); + } + + private ContentBootstrapStep step(ContentSourceProperties properties) { + return new ContentBootstrapStep(properties, indexer, meilisearch); + } + + @Test + @DisplayName("indexUid targets the prefixed content index") + void indexUidTargetsContentIndex() { + assertThat(step(properties()).indexUid()).isEqualTo("content_content"); + } + + @Test + @DisplayName("only enabled sources contribute documents") + void onlyEnabledSourcesContribute() { + ContentBootstrapStep step = step(properties(source("a", true), source("b", true), source("c", false))); + + List sources = step.documents().map(doc -> (String) doc.get("source")).toList(); + + assertThat(sources).containsExactlyInAnyOrder("a", "b"); + } + + @Test + @DisplayName("the document stream is lazy — nothing is fetched until it is consumed") + void streamIsLazy() { + strategy.itemsPerSource = 3; + ContentBootstrapStep step = step(properties(source("a", true))); + + Stream> documents = step.documents(); + assertThat(strategy.fetchCount.get()).isZero(); // building the stream fetches nothing + + List> materialized = documents.toList(); + assertThat(materialized).hasSize(3); + assertThat(strategy.fetchCount.get()).isEqualTo(3); + } + + @Test + @DisplayName("a source that fails while streaming does not block the others") + void failingSourceDoesNotBlockOthers() { + strategy.throwingSourceIds.add("bad"); + ContentBootstrapStep step = step(properties(source("a", true), source("bad", true))); + + List sources = step.documents().map(doc -> (String) doc.get("source")).toList(); + + assertThat(sources).containsExactly("a"); + } + + /** Test-double strategy: one INDEX document per discovered item; can throw for chosen sources. */ + private static final class StubStrategy implements ContentSourceStrategy { + private final List throwingSourceIds = new ArrayList<>(); + private final AtomicInteger fetchCount = new AtomicInteger(); + private int itemsPerSource = 1; + + @Override + public SourceType type() { + return SourceType.WEBSITE; + } + + @Override + public List discover(ContentSource source) { + if (throwingSourceIds.contains(source.id())) { + throw new IllegalStateException("boom " + source.id()); + } + return IntStream.range(0, itemsPerSource) + .mapToObj(i -> new DiscoveredItem("https://ex.com/" + source.id() + "/" + i, "L")) + .toList(); + } + + @Override + public FetchOutcome fetch(ContentSource source, DiscoveredItem item) { + fetchCount.incrementAndGet(); + ContentDocument document = new ContentDocument( + ContentDocument.id(source.id(), item.url()), source.id(), "en", item.url(), + "T", "E", "body", "a", List.of(), "2026-01-01", item.lastmod(), null); + return FetchOutcome.index(document); + } + } + + /** streamAllDocuments never touches the store, so a no-op suffices. */ + private static final class NoOpStore implements ContentIndexStore { + @Override + public Map loadState(String source) { + return Map.of(); + } + + @Override + public int upsert(List> documents) { + return documents.size(); + } + + @Override + public void delete(String id) { + } + } +}