-**📋 Document Owner:** CEO | **📄 Version:** 6.6 | **📅 Last Updated:** 2026-04-25 (UTC)
+**📋 Document Owner:** CEO | **📄 Version:** 6.7 | **📅 Last Updated:** 2026-04-27 (UTC)
**🔄 Review Cycle:** Quarterly | **⏰ Next Review:** 2026-07-21
**🏢 Owner:** Hack23 AB (Org.nr 5595347807) | **🏷️ Classification:** Public
@@ -87,11 +87,13 @@ Scripts run the download. Example:
```bash
npx tsx scripts/download-parliamentary-data.ts \
--date ${ARTICLE_DATE} \
- --scope ${DOC_TYPE} \
- --out analysis/daily/${ARTICLE_DATE}/${DOC_TYPE}/data/
+ --doc-type ${DOC_TYPE} \
+ --auto-full-text-top-n 2
```
-**Write `data-download-manifest.md`** using the [manifest template](../templates/data-download-manifest.md). It records what arrived, from which MCP tools, with what data-depth distribution (FULL-TEXT / SUMMARY / METADATA-ONLY).
+**`--auto-full-text-top-n 2`** (recommended for L2/L3 runs): after the bulk download, the script calls `get_dokument_innehall` with `include_full_text=true` for the top-2 documents (by order in the downloaded batch) and persists the retrieved content to `analysis/daily/${ARTICLE_DATE}/${DOC_TYPE}/full-text/{dok_id}.md`. Accept the extra 30–60 s as a documented quality investment. The manifest's `## Full-Text Fetch Outcomes` table records `full_text_available` per `dok_id`; the analysis gate (check 10) enforces that ≥ 2 succeed or a `` annotation is present.
+
+**Write `data-download-manifest.md`** using the [manifest template](../templates/data-download-manifest.md). It records what arrived, from which MCP tools, with what data-depth distribution (FULL-TEXT / SUMMARY / METADATA-ONLY) and — when `--auto-full-text-top-n` is used — the `## Full-Text Fetch Outcomes` table.
After `download-parliamentary-data.ts` completes for `committeeReports`, also run the voting-records script to capture party-level vote counts and defector detection for each betänkande:
diff --git a/scripts/download-parliamentary-data.ts b/scripts/download-parliamentary-data.ts
index b023681a0c..e5863a9b90 100644
--- a/scripts/download-parliamentary-data.ts
+++ b/scripts/download-parliamentary-data.ts
@@ -34,8 +34,9 @@ import {
flattenDocuments,
subtractBusinessDays,
MAX_LOOKBACK_BUSINESS_DAYS,
+ fetchFullTextForTopN,
} from './parliamentary-data/data-downloader.js';
-import type { DocumentTypeKey } from './parliamentary-data/data-downloader.js';
+import type { DocumentTypeKey, FullTextFetchOutcome } from './parliamentary-data/data-downloader.js';
import { persistDownloadedData, sanitizeDokId } from './parliamentary-data/data-persistence.js';
@@ -148,10 +149,11 @@ export function parseArgs(argv: string[]): {
})
: [];
- // --auto-full-text-top-n: Override the per-type full-text enrichment limit.
- // When set, only the top N documents per type receive fetchDocumentDetails
- // (full-text) enrichment, enabling more targeted significance-scoring input.
- // Defaults to MAX_ENRICHMENT_PER_TYPE when omitted (null → caller uses default).
+ // --auto-full-text-top-n: Override the per-type full-text enrichment limit and
+ // persist full text outcomes for the first N documents in the current filtered
+ // array order. Defaults to null when omitted so downloadAllDocuments uses
+ // MAX_ENRICHMENT_PER_TYPE; explicit 0 disables per-type enrichment and
+ // persisted full-text fetching. No DIW significance ranking is applied here.
const autoFullTextTopNArg = get('--auto-full-text-top-n');
let autoFullTextTopN: number | null = null;
if (autoFullTextTopNArg !== null) {
@@ -235,6 +237,7 @@ function serializeDataManifest(
docCounts: Record' + 'X'.repeat(FULL_TEXT_MIN_LENGTH + 50) + '
'; + const docs = [ + makeDoc({ dok_id: 'HD01FiU48' }), + makeDoc({ dok_id: 'HD01CU25' }), + ]; + const client = createMockClient(async (_dokId, _) => ({ text: longContent, snippet: 'Test snippet' })); + + const outcomes = await fetchFullTextForTopN(client, docs, 2, tmpDir); + + expect(outcomes).toHaveLength(2); + expect(outcomes[0]!.dokId).toBe('HD01FiU48'); + expect(outcomes[0]!.success).toBe(true); + expect(outcomes[0]!.chars).toBeGreaterThan(FULL_TEXT_MIN_LENGTH); + + expect(outcomes[1]!.dokId).toBe('HD01CU25'); + expect(outcomes[1]!.success).toBe(true); + + // Verify files were written to full-text/ subdirectory + const fullTextDir = path.join(tmpDir, 'full-text'); + expect(fs.existsSync(path.join(fullTextDir, 'HD01FiU48.md'))).toBe(true); + expect(fs.existsSync(path.join(fullTextDir, 'HD01CU25.md'))).toBe(true); + }); + + it('only fetches top-N documents even when more docs provided', async () => { + const longContent = 'A'.repeat(FULL_TEXT_MIN_LENGTH + 10); + const docs = [ + makeDoc({ dok_id: 'DOC1' }), + makeDoc({ dok_id: 'DOC2' }), + makeDoc({ dok_id: 'DOC3' }), + ]; + const fetchDetails = vi.fn().mockResolvedValue({ text: longContent }); + const client = createMockClient(fetchDetails); + + await fetchFullTextForTopN(client, docs, 2, tmpDir); + + expect(fetchDetails).toHaveBeenCalledTimes(2); + expect(fetchDetails).toHaveBeenCalledWith('DOC1', true); + expect(fetchDetails).toHaveBeenCalledWith('DOC2', true); + expect(fetchDetails).not.toHaveBeenCalledWith('DOC3', true); + }); + + it('creates full-text directory automatically', async () => { + const longContent = 'B'.repeat(FULL_TEXT_MIN_LENGTH + 1); + const client = createMockClient(async () => ({ text: longContent })); + const nestedDir = path.join(tmpDir, 'deeply', 'nested'); + + await fetchFullTextForTopN(client, [makeDoc({ dok_id: 'X1' })], 1, nestedDir); + + expect(fs.existsSync(path.join(nestedDir, 'full-text', 'X1.md'))).toBe(true); + }); + + describe('graceful degradation — metadata-only response', () => { + it('records success=false when content is below FULL_TEXT_MIN_LENGTH', async () => { + const shortContent = 'short'; + const client = createMockClient(async () => ({ text: shortContent, snippet: 'snippet' })); + + const outcomes = await fetchFullTextForTopN( + client, [makeDoc({ dok_id: 'HD03104' })], 1, tmpDir, + ); + + expect(outcomes).toHaveLength(1); + expect(outcomes[0]!.success).toBe(false); + expect(outcomes[0]!.chars).toBe(0); + expect(outcomes[0]!.reason).toMatch(/metadata-only/); + + // No file should be written for failed fetches + const fullTextDir = path.join(tmpDir, 'full-text'); + expect(fs.existsSync(path.join(fullTextDir, 'HD03104.md'))).toBe(false); + }); + + it('falls back from text to fullText field when text is too short', async () => { + const longFullText = 'C'.repeat(FULL_TEXT_MIN_LENGTH + 20); + const client = createMockClient(async () => ({ + text: 'short', + fullText: longFullText, + })); + + const outcomes = await fetchFullTextForTopN( + client, [makeDoc({ dok_id: 'DOC_FT' })], 1, tmpDir, + ); + + expect(outcomes[0]!.success).toBe(true); + expect(outcomes[0]!.chars).toBe(longFullText.length); + }); + + it('falls back to html field when text and fullText are too short', async () => { + const longHtml = '' + 'D'.repeat(FULL_TEXT_MIN_LENGTH + 20) + ''; + const client = createMockClient(async () => ({ + text: 'x', + fullText: 'y', + html: longHtml, + })); + + const outcomes = await fetchFullTextForTopN( + client, [makeDoc({ dok_id: 'DOC_HTML' })], 1, tmpDir, + ); + + expect(outcomes[0]!.success).toBe(true); + expect(outcomes[0]!.chars).toBe(longHtml.length); + }); + }); + + describe('graceful degradation — fetchDocumentDetails throws', () => { + it('records success=false with reason when MCP call rejects', async () => { + const client = createMockClient(async () => { + throw new Error('MCP connection timeout'); + }); + + const outcomes = await fetchFullTextForTopN( + client, [makeDoc({ dok_id: 'HD01CU24' })], 1, tmpDir, + ); + + expect(outcomes).toHaveLength(1); + expect(outcomes[0]!.success).toBe(false); + expect(outcomes[0]!.chars).toBe(0); + expect(outcomes[0]!.reason).toMatch(/fetchDocumentDetails failed/); + expect(outcomes[0]!.reason).toMatch(/MCP connection timeout/); + }); + + it('continues to next document after one fails', async () => { + const longContent = 'E'.repeat(FULL_TEXT_MIN_LENGTH + 5); + const client = createMockClient(async (dokId) => { + if (dokId === 'FAIL_DOC') throw new Error('timeout'); + return { text: longContent }; + }); + + const docs = [ + makeDoc({ dok_id: 'FAIL_DOC' }), + makeDoc({ dok_id: 'OK_DOC' }), + ]; + + const outcomes = await fetchFullTextForTopN(client, docs, 2, tmpDir); + + expect(outcomes).toHaveLength(2); + expect(outcomes[0]!.success).toBe(false); + expect(outcomes[1]!.success).toBe(true); + expect(outcomes[1]!.chars).toBeGreaterThan(0); + }); + }); + + describe('dokId resolution', () => { + it('skips documents with no resolvable dok_id', async () => { + const fetchDetails = vi.fn().mockResolvedValue({}); + const client = createMockClient(fetchDetails); + const docNoId = makeDoc({ + dok_id: undefined, + dokument_id: undefined, + rel_dok_id: undefined, + id: undefined, + dokumentnamn: undefined, + }); + + const outcomes = await fetchFullTextForTopN(client, [docNoId], 1, tmpDir); + + expect(outcomes).toHaveLength(0); + expect(fetchDetails).not.toHaveBeenCalled(); + }); + + it('resolves dok_id from dokument_id when dok_id is absent', async () => { + const longContent = 'F'.repeat(FULL_TEXT_MIN_LENGTH + 5); + const fetchDetails = vi.fn().mockResolvedValue({ text: longContent }); + const client = createMockClient(fetchDetails); + const doc = makeDoc({ + dok_id: undefined, + dokument_id: 'DOKU1', + }); + + const outcomes = await fetchFullTextForTopN(client, [doc], 1, tmpDir); + + expect(fetchDetails).toHaveBeenCalledWith('DOKU1', true); + expect(outcomes[0]!.dokId).toBe('DOKU1'); + }); + }); + + describe('file content', () => { + it('writes markdown file with header and content', async () => { + const content = 'G'.repeat(FULL_TEXT_MIN_LENGTH + 5); + const client = createMockClient(async () => ({ + text: content, + snippet: 'A short summary', + })); + + await fetchFullTextForTopN(client, [makeDoc({ dok_id: 'DOC99' })], 1, tmpDir); + + const written = fs.readFileSync(path.join(tmpDir, 'full-text', 'DOC99.md'), 'utf8'); + expect(written).toContain('# Full Text — DOC99'); + expect(written).toContain('A short summary'); + expect(written).toContain(content); + // Blank lines must be preserved so the horizontal rule renders correctly + expect(written).toContain('\n\n---\n'); + }); + + it('filePath in outcome is relative to outputDir (not CWD)', async () => { + const content = 'Z'.repeat(FULL_TEXT_MIN_LENGTH + 5); + const client = createMockClient(async () => ({ text: content })); + + const outcomes = await fetchFullTextForTopN( + client, [makeDoc({ dok_id: 'RELPATH' })], 1, tmpDir, + ); + + expect(outcomes[0]!.success).toBe(true); + // filePath should be relative to outputDir, not an absolute path or CWD-relative + expect(outcomes[0]!.filePath).toBe(path.join('full-text', 'RELPATH.md')); + }); + + it('sanitizes MP profile text (isPersonProfileText filter)', async () => { + const profileText = 'Tjänstgörande riksdagsledamot ' + 'A'.repeat(200); + const client = createMockClient(async () => ({ + text: 'short', + fullText: profileText, + })); + + const outcomes = await fetchFullTextForTopN( + client, [makeDoc({ dok_id: 'PROF_DOC' })], 1, tmpDir, + ); + + // Profile text is sanitized, falls back to short text which is below threshold + expect(outcomes[0]!.success).toBe(false); + }); + }); +}); + +// --------------------------------------------------------------------------- +// serializeDataManifest — full-text outcomes section +// --------------------------------------------------------------------------- + +// We import buildWeeklySummaryMarkdown as a smoke-test that the module loads +// without errors after the changes to download-parliamentary-data.ts. +import { buildWeeklySummaryMarkdown } from '../scripts/download-parliamentary-data.js'; + +describe('serializeDataManifest (via buildWeeklySummaryMarkdown sanity)', () => { + it('buildWeeklySummaryMarkdown still works after refactor', () => { + const md = buildWeeklySummaryMarkdown({ + weekLabel: '2026-W17', + generatedAt: '2026-04-26 06:00 UTC', + documentsDownloaded: 42, + daysIncluded: 5, + dayList: ['2026-04-22', '2026-04-23', '2026-04-24'], + }); + expect(md).toContain('2026-W17'); + expect(md).toContain('42'); + }); +}); + +// We test the manifest full-text section indirectly by checking parseArgs +// correctly exposes autoFullTextTopN so the caller can pass outcomes to +// serializeDataManifest. Direct testing of the private serialize function +// is done via the integration path in the pipeline. +describe('manifest full-text outcomes integration contract', () => { + it('parseArgs exposes autoFullTextTopN=2 when flag is set', () => { + const args = parseArgs(['node', 'script.ts', '--auto-full-text-top-n', '2']); + expect(args.autoFullTextTopN).toBe(2); + }); + + it('fetchFullTextForTopN returns outcome with filePath for successful fetch', async () => { + const longContent = 'H'.repeat(FULL_TEXT_MIN_LENGTH + 10); + const client = createMockClient(async () => ({ text: longContent })); + + const outcomes = await fetchFullTextForTopN( + client, + [makeDoc({ dok_id: 'HD01FiU48' }), makeDoc({ dok_id: 'HD01CU25' })], + 2, + tmpDir, + ); + + for (const o of outcomes) { + if (o.success) { + expect(o.filePath).toBeDefined(); + expect(o.chars).toBeGreaterThan(FULL_TEXT_MIN_LENGTH); + } + } + }); + + it('analysis-gate can determine if top-2 full texts are available from outcomes', async () => { + const longContent = 'I'.repeat(FULL_TEXT_MIN_LENGTH + 1); + const client = createMockClient(async () => ({ text: longContent })); + + const docs = [ + makeDoc({ dok_id: 'TOP1' }), + makeDoc({ dok_id: 'TOP2' }), + ]; + + const outcomes = await fetchFullTextForTopN(client, docs, 2, tmpDir); + + const successCount = outcomes.filter(o => o.success).length; + // Gate can check: successCount >= 2 OR fallback annotation present + expect(successCount).toBe(2); + }); +}); diff --git a/tests/pir-status-contract.test.ts b/tests/pir-status-contract.test.ts index df453f6156..929f38327e 100644 --- a/tests/pir-status-contract.test.ts +++ b/tests/pir-status-contract.test.ts @@ -850,7 +850,10 @@ describe('analysis-gate pir-status.json contract', () => { }); it('05-analysis-gate.md keeps PIR and supplementary checks sequential', () => { expect(gate).toContain('# Check 9 — PIR status sidecar'); - expect(gate).toContain('# Check 10 — supplementary artifacts'); + // Check 10 is now "top-2 full-text availability" (added by --auto-full-text-top-n); + // supplementary artifacts shifted to Check 11. + expect(gate).toContain('# Check 10 — top-2 full-text availability'); + expect(gate).toContain('# Check 11 — supplementary artifacts'); }); it('ai-driven-analysis-guide.md references pir-status.json', () => { expect(guide).toContain('pir-status.json'); diff --git a/tests/statskontoret-feasibility-contract.test.ts b/tests/statskontoret-feasibility-contract.test.ts index 617b9a4042..a5f099917f 100644 --- a/tests/statskontoret-feasibility-contract.test.ts +++ b/tests/statskontoret-feasibility-contract.test.ts @@ -125,8 +125,7 @@ describe('Statskontoret inventory → implementation-feasibility coverage contra expect(inv.datasets['myndighetsforteckning'].admiralty).toBe('A1'); }); - it('at least one implementation-feasibility.md file in the analysis tree ' + - 'mentions a known Swedish agency (otherwise the per-file coverage test is vacuous)', () => { + it('at least one implementation-feasibility.md file in the analysis tree mentions a known Swedish agency (otherwise the per-file coverage test is vacuous)', () => { const filesWithMentions = feasibilityFiles.filter((filePath) => { const content = fs.readFileSync(filePath, 'utf-8'); return extractAgencyMentions(content, KNOWN_AGENCIES).length > 0; @@ -138,8 +137,7 @@ describe('Statskontoret inventory → implementation-feasibility coverage contra ).toBeGreaterThan(0); }); - it('every implementation-feasibility.md mentioning a known agency resolves to ' + - 'a Statskontoret dataset that covers it via myndighetsforteckning', () => { + it('every implementation-feasibility.md mentioning a known agency resolves to a Statskontoret dataset that covers it via myndighetsforteckning', () => { // Since myndighetsforteckning covers ALL Swedish government bodies by // definition, one dataset entry suffices for all named agencies. This // test enforces the contract per-file: every file mentioning an agency