diff --git a/.gitignore b/.gitignore index fffc41e5d..8d64bc517 100644 --- a/.gitignore +++ b/.gitignore @@ -211,3 +211,9 @@ docs/superpowers/ # Superpowers brainstorming scratch files .superpowers/ + +# Demo corpus-import zips (large binaries, staged locally from ~/Code/EDGARx2) +demo/import_zips/ + +# Local-only governance-graph demo (kept on disk, excluded from PRs) +/demo/ diff --git a/changelog.d/corpus-reference-enrichment.added.md b/changelog.d/corpus-reference-enrichment.added.md new file mode 100644 index 000000000..db978a724 --- /dev/null +++ b/changelog.d/corpus-reference-enrichment.added.md @@ -0,0 +1,68 @@ +- **Corpus reference enrichment agent.** A corpus-scoped agent tool pair that + crawls a corpus, inventories explicit references, and persists them — proven + on a real 55-document S-1 corpus (348 references created). + - New deterministic engine in `opencontractserver/enrichment/` + (`extractor.py`, `resolver.py`, `writer.py`, `services/`): regex/grammar + extraction of law citations (`Section 145 of the Delaware General + Corporation Law` → canonical key `dgcl:145`), document/exhibit references, + internal section references, and defined-term definition sites + (`(the "Company")` / `"Change of Control" means …` → `term:company` / + `term:change-of-control`, opt-in and capped); resolution to in-corpus target + documents and OC_SECTION annotations. + - New `CorpusReference` model (`opencontractserver/annotations/models.py`, + migration `annotations/0078_corpusreference`): a first-class + cross-document / cross-corpus / external-law connection with an indexed + `canonical_key` join key (`target_corpus` anticipates future cross-corpus + linking). Within-document section links continue to use `Relationship`; + every reference also gets a mention `Annotation` (law payloads stored in + `Annotation.data`, resolved document links in `Annotation.link_url` as + in-app site-relative paths). + - Two agent tools (`opencontractserver/llms/tools/core_tools/corpus_references.py`, + registered in `tool_registry.py`): `scan_corpus_references` (read-only + inventory) and `apply_corpus_reference_enrichment` + (`requires_approval=True`, `requires_write_permission=True`) — the + CAML-style scan → approval-gated apply pattern. Enrichment is idempotent. + - Read-only GraphQL `corpusReferences(corpusId)` query + (`config/graphql/annotation_types.py`, `annotation_queries.py`), visibility + scoped to corpus READ via `CorpusReferenceService`. + - Resolved document/exhibit references additionally roll up to + `DocumentRelationship` rows (`enrichment/writer.py`), feeding the corpus + document graph. Mention dedup is by (document, label, span start) so a + growing alias registry cannot duplicate mentions. +- **Authority corpora + cross-corpus law linking.** + - `opencontractserver/enrichment/authorities.py`: `AuthorityCorpusBootstrapper` + materialises statute sections as keyed text documents + (`Document.custom_meta.canonical_key`, e.g. `dgcl:145`), idempotently + (skip / version-up on amendment / self-healing restamp after concurrent + pipeline saves). `find_authority_target` resolves citations with + subsection→section fallback (`dgcl:122(17)` → `dgcl:122`), visibility- and + current-version-aware. + - `EnrichmentService.link_external_references` upgrades EXTERNAL law + citations to RESOLVED cross-corpus links (`target_corpus`/`target_document` + + in-app `link_url` on the mention); runs automatically inside `apply()` + and is re-runnable as new authority corpora appear. + - Data-driven authority alias registry: `authority_alias_registry(user)` + merges static defaults with `custom_meta.authority_aliases` declared by + authority corpora — adding a body of law is a bootstrap call, zero code. +- **Citation grammar coverage** (`enrichment/extractor.py`): suffix form + ("Section 145 of the DGCL"), prefix form ("Securities Act Section 4(a)(5)"), + statute-internal relative form ("§ 251 of this title", keyed via the + document's own authority context), and bare SEC rules ("Rule 506(b)", + "Rule 144A", "Rule 10b-5" → `sec-rule:*`). +- **Analyzer framework: corpus-scoped analyzers.** + - New `@corpus_analyzer_task` decorator (`shared/decorators.py`) — the + corpus-scoped sibling of `@doc_analyzer_task`: runs once per Analysis, + owns its own writes, wrapper manages the Analysis lifecycle + (RUNNING/COMPLETED/FAILED, timestamps, result/error messages). + - `get_corpus_analyzer_task_by_name` / `get_analyzer_task_by_name` + (`utils/celery_tasks.py`); `run_task_name_analyzer` dispatches + corpus-scoped analyzers as a single task (no per-doc fan-out); analyzer + auto-sync and the `analyzer.W001` check cover both flavours. + - Enrichment adapter task + (`opencontractserver/tasks/corpus_analysis_tasks.py`) registers the engine + as a real task-based Analyzer (dispatchable via `CorpusAction`), attaching + to the framework-created Analysis instead of creating its own. +- **Demo pipeline** (`demo/`, untracked zips excluded): authority seed JSONs + with real statutory text (DGCL, Securities Act, Exchange Act, IRC, ICA, + SEC Rules — 17 C.F.R.), bootstrap/import/export scripts, and a standalone + D3 governance-graph visualization (`governance_graph.html`). diff --git a/changelog.d/corpusvote-index-state-reconcile.fixed.md b/changelog.d/corpusvote-index-state-reconcile.fixed.md new file mode 100644 index 000000000..eaa878941 --- /dev/null +++ b/changelog.d/corpusvote-index-state-reconcile.fixed.md @@ -0,0 +1,8 @@ +- **Reconciled `CorpusVote` migration state with the model** (migration + `corpuses/0057`): migration `0049` created the three `CorpusVote` indexes + with explicit names (`corpuses_co_corpus__vote_idx`, …) while the model + declares them unnamed (Django auto-hashed names), and the `backend_lock` + field's `db_index=True` was missing from migration state. The drift made + `makemigrations --check` fail on every branch. Operations are three + Postgres `ALTER INDEX … RENAME` calls plus one index-state alter — no data + changes. diff --git a/changelog.d/enrichment-review-fixes.fixed.md b/changelog.d/enrichment-review-fixes.fixed.md new file mode 100644 index 000000000..48c31de88 --- /dev/null +++ b/changelog.d/enrichment-review-fixes.fixed.md @@ -0,0 +1,18 @@ +- **Enrichment mention links no longer 404** — the writer and + `link_external_references` wrote `link_url` as `/corpus/{id}/document/{id}`, + a shape no frontend route serves (it falls into the `*` catch-all → 404). + They now emit the canonical slug path + `/d/{corpus.creator.slug}/{corpus.slug}/{document.slug}` via the new + `opencontractserver/utils/frontend_paths.py::document_in_corpus_path` + helper (mirrors the frontend's `buildCanonicalPath`); cross-corpus law + links point into the authority corpus. Links with missing slugs are + skipped rather than written broken. +- **`@corpus_analyzer_task` no longer marks a retrying Analysis FAILED** + (`opencontractserver/shared/decorators.py`) — `celery.exceptions.Retry` + extends `Exception`, so the wrapper's failure branch stamped + `Analysis.status=FAILED` before the retry ran. `Retry` is now re-raised + untouched, mirroring `doc_analyzer_task`; regression test pins the + RUNNING status surviving a retry. +- **Enrichment perf**: OC_SECTION lookups batched to one query per corpus + (was one per document), and `link_external_references` collapses O(N) + row-by-row saves into two `bulk_update` calls (refs + mentions). diff --git a/changelog.d/governance-graph-query.added.md b/changelog.d/governance-graph-query.added.md new file mode 100644 index 000000000..d8ebbde87 --- /dev/null +++ b/changelog.d/governance-graph-query.added.md @@ -0,0 +1,19 @@ +- **`governanceGraph` GraphQL query** — the corpus-scoped reference web in + node-link form, the in-app successor to `demo/export_governance_graph.py` + (first integration step from #1976's callout). Nodes are documents (filing + primaries, exhibits, statute sections) plus "ghost" nodes for law citations + with no visible target document; edges are mention-weighted `LAW` (resolved, + possibly cross-corpus), `LAW_EXTERNAL` (rolled up to the citation's section + root), and `DOCUMENT` (`DocumentRelationship` rollups). + - Graph assembly lives in + `opencontractserver/enrichment/services/governance_graph_service.py`; + the resolver (`config/graphql/annotation_queries.py`) only encodes relay + ids — no inline Tier-0 (E001 green). + - Visibility: corpus READ gates the query (invisible corpus → empty graph); + every surfaced document is independently READ-checked — invisible source + documents drop their edges, invisible target documents degrade to external + ghost nodes so titles never leak, and only READ-visible target corpora are + listed (pinned by `opencontractserver/tests/test_governance_graph.py`). + - Node lists are degree-capped at `GOVERNANCE_GRAPH_MAX_NODES` (200, + `opencontractserver/constants/stats.py`) with full-graph counts + + `truncated` flag, mirroring `corpusDocumentGraph`'s contract. diff --git a/config/graphql/annotation_queries.py b/config/graphql/annotation_queries.py index b76dd124f..093fbcc3f 100644 --- a/config/graphql/annotation_queries.py +++ b/config/graphql/annotation_queries.py @@ -13,12 +13,17 @@ from graphene_django.filter import DjangoFilterConnectionField from graphql import GraphQLError from graphql_jwt.decorators import login_required -from graphql_relay import from_global_id +from graphql_relay import from_global_id, to_global_id from config.graphql.filters import LabelFilter, LabelsetFilter, RelationshipFilter from config.graphql.graphene_types import ( AnnotationLabelType, AnnotationType, + CorpusReferenceType, + GovernanceGraphCorpusType, + GovernanceGraphEdgeType, + GovernanceGraphNodeType, + GovernanceGraphType, LabelSetType, NoteType, PageAwareAnnotationType, @@ -37,7 +42,9 @@ DOCUMENT_ANNOTATION_INDEX_LIMIT, MANUAL_ANNOTATION_SENTINEL, ) +from opencontractserver.constants.stats import GOVERNANCE_GRAPH_MAX_NODES from opencontractserver.documents.models import Document +from opencontractserver.enrichment import constants as enrichment_constants from opencontractserver.shared.services.base import BaseService from opencontractserver.types.enums import LabelType @@ -47,6 +54,153 @@ class AnnotationQueryMixin: """Query fields and resolvers for annotation, relationship, label, labelset, and note queries.""" + # CORPUS REFERENCE RESOLVERS ############################### + corpus_references = DjangoConnectionField( + CorpusReferenceType, + corpus_id=graphene.ID(required=True), + reference_type=graphene.String(), + canonical_key=graphene.String(), + ) + + def resolve_corpus_references(self, info, corpus_id, **kwargs) -> Any: + """List enrichment cross-references for a corpus the user can read. + + Visibility is enforced by ``CorpusReferenceService`` (corpus-derived); + no inline Tier-0 permission fusion here. + """ + from opencontractserver.annotations.models import CorpusReference + from opencontractserver.enrichment.services import CorpusReferenceService + + pk_str = from_global_id(corpus_id)[1] + if not str(pk_str).isdigit(): + return CorpusReference.objects.none() + pk = int(pk_str) + qs = CorpusReferenceService.for_corpus(info.context.user, pk) + if kwargs.get("reference_type"): + qs = qs.filter(reference_type=kwargs["reference_type"]) + if kwargs.get("canonical_key"): + qs = qs.filter(canonical_key=kwargs["canonical_key"]) + # Pull the FK targets the type resolves in one pass — without this each + # CorpusReferenceType row fires a separate query per FK (N+1). + return qs.select_related( + "source_annotation", + "corpus", + "target_document", + "target_annotation", + "target_corpus", + ) + + governance_graph = graphene.Field( + GovernanceGraphType, + corpus_id=graphene.ID(required=True), + limit=graphene.Int(required=False), + description=( + "The corpus-scoped reference web in node-link form: documents, " + "statute sections, and external-citation ghost nodes, with " + "mention-weighted LAW / LAW_EXTERNAL / DOCUMENT edges. Powers the " + "Governance Graph panel on the Corpus Intelligence home." + ), + ) + + @graphql_ratelimit_dynamic(get_rate=get_user_tier_rate("READ_MEDIUM")) + def resolve_governance_graph(self, info, corpus_id, limit=None) -> Any: + """Build the governance graph through ``GovernanceGraphService``. + + All visibility decisions (corpus READ gate, per-document READ checks, + ghost degradation for invisible targets) live in the service; this + resolver only translates raw PKs / canonical keys into relay ids. + """ + # Function-local service import (services import GraphQL types + # transitively — module-level import would cycle). + from opencontractserver.enrichment.services import GovernanceGraphService + + empty = GovernanceGraphType( + corpora=[], + nodes=[], + edges=[], + document_count=0, + external_key_count=0, + edge_count=0, + mention_count=0, + truncated=False, + ) + + corpus_pk = from_global_id(corpus_id)[1] + # Malformed/empty global ids decode to a non-numeric pk; treat as + # not-found (empty graph) rather than erroring. + if not str(corpus_pk).isdigit(): + return empty + + node_cap = GOVERNANCE_GRAPH_MAX_NODES + if limit is not None and 0 < limit < node_cap: + node_cap = limit + + data = GovernanceGraphService.build( + info.context.user, int(corpus_pk), node_cap, request=info.context + ) + if data is None: + return empty + + def _node_id(endpoint) -> str: + kind, val = endpoint + if kind == "doc": + return to_global_id("DocumentType", val) + return f"key:{val}" + + nodes = [ + GovernanceGraphNodeType( + id=to_global_id("DocumentType", n["doc_pk"]), + document_id=to_global_id("DocumentType", n["doc_pk"]), + title=n["title"], + kind=n["kind"], + corpus_id=( + to_global_id("CorpusType", n["corpus_pk"]) + if n["corpus_pk"] + else None + ), + authority=n["authority"], + degree=n["degree"], + ) + for n in data["doc_nodes"] + ] + [ + GovernanceGraphNodeType( + id=f"key:{g['key']}", + document_id=None, + title=g["key"], + kind=enrichment_constants.GRAPH_NODE_EXTERNAL, + corpus_id=None, + authority=g["authority"], + degree=g["degree"], + ) + for g in data["ghost_nodes"] + ] + + return GovernanceGraphType( + corpora=[ + GovernanceGraphCorpusType( + id=to_global_id("CorpusType", c["corpus_pk"]), + title=c["title"], + kind=c["kind"], + ) + for c in data["corpora"] + ], + nodes=nodes, + edges=[ + GovernanceGraphEdgeType( + source=_node_id(e["source"]), + target=_node_id(e["target"]), + edge_type=e["edge_type"], + weight=e["weight"], + ) + for e in data["edges"] + ], + document_count=data["document_count"], + external_key_count=data["external_key_count"], + edge_count=data["edge_count"], + mention_count=data["mention_count"], + truncated=data["truncated"], + ) + # ANNOTATION RESOLVERS ##################################### annotations = DjangoConnectionField( AnnotationType, diff --git a/config/graphql/annotation_types.py b/config/graphql/annotation_types.py index 727b89ec6..456a0dc80 100644 --- a/config/graphql/annotation_types.py +++ b/config/graphql/annotation_types.py @@ -19,6 +19,7 @@ from opencontractserver.annotations.models import ( Annotation, AnnotationLabel, + CorpusReference, LabelSet, Note, NoteRevision, @@ -46,6 +47,104 @@ class Meta: connection_class = CountableConnection +class CorpusReferenceType(DjangoObjectType): + """Read-only view of an enrichment cross-reference. + + No ``AnnotatePermissionsForReadMixin``: ``CorpusReference`` has no guardian + permission tables — visibility derives from the parent corpus and is + enforced by ``CorpusReferenceService`` in the resolver. + """ + + normalized_data = GenericScalar() # noqa + + class Meta: + model = CorpusReference + interfaces = [relay.Node] + connection_class = CountableConnection + + +class GovernanceGraphCorpusType(graphene.ObjectType): + """A corpus participating in the governance graph (filing or authority).""" + + id = graphene.ID(required=True, description="Global CorpusType id.") + title = graphene.String() + kind = graphene.String( + required=True, description='"filing" or "authority" (cited body of law).' + ) + + +class GovernanceGraphNodeType(graphene.ObjectType): + """One governance-graph node: a document or an external-citation ghost.""" + + id = graphene.String( + required=True, + description=( + "Node id: the global DocumentType id for document nodes, or " + '"key:" for external ghost nodes.' + ), + ) + document_id = graphene.ID( + description="Global DocumentType id (null for external ghost nodes)." + ) + title = graphene.String( + description="Document title, or the canonical key for ghost nodes." + ) + kind = graphene.String( + required=True, description='"primary", "exhibit", "statute" or "external".' + ) + corpus_id = graphene.ID( + description="Global CorpusType id of the node's corpus (null for ghosts)." + ) + authority = graphene.String( + description='Body-of-law key prefix (e.g. "dgcl") for statute/ghost nodes.' + ) + degree = graphene.Int( + required=True, description="Summed mention weight of edges touching the node." + ) + + +class GovernanceGraphEdgeType(graphene.ObjectType): + """One weighted reference edge between two governance-graph nodes.""" + + source = graphene.String(required=True, description="Source node id.") + target = graphene.String(required=True, description="Target node id.") + edge_type = graphene.String( + required=True, description='"LAW", "LAW_EXTERNAL" or "DOCUMENT".' + ) + weight = graphene.Int(required=True, description="Mention count.") + + +class GovernanceGraphType(graphene.ObjectType): + """The corpus-scoped reference web in node-link form. + + Built by ``GovernanceGraphService`` from corpus-as-gate ``CorpusReference`` + rows + permission-filtered ``DocumentRelationship`` rows, with every + surfaced document independently READ-checked (invisible targets degrade to + external ghost nodes). Counts describe the full visible graph; the + node/edge lists may be degree-capped (``truncated``). + """ + + corpora = graphene.List(graphene.NonNull(GovernanceGraphCorpusType), required=True) + nodes = graphene.List(graphene.NonNull(GovernanceGraphNodeType), required=True) + edges = graphene.List(graphene.NonNull(GovernanceGraphEdgeType), required=True) + document_count = graphene.Int( + required=True, description="Distinct visible document nodes (pre-cap)." + ) + external_key_count = graphene.Int( + required=True, description="Distinct external ghost nodes (pre-cap)." + ) + edge_count = graphene.Int( + required=True, description="Distinct edges in the full graph (pre-cap)." + ) + mention_count = graphene.Int( + required=True, description="Total reference mentions across all edges." + ) + truncated = graphene.Boolean( + required=True, + description="True when nodes/edges were dropped to honor the node cap.", + ) + + class RelationInputType(AnnotatePermissionsForReadMixin, graphene.InputObjectType): id = graphene.String() source_ids = graphene.List(graphene.String) diff --git a/config/graphql/graphene_types.py b/config/graphql/graphene_types.py index b2cece36b..518d13b04 100644 --- a/config/graphql/graphene_types.py +++ b/config/graphql/graphene_types.py @@ -19,6 +19,11 @@ AnnotationInputType, AnnotationLabelType, AnnotationType, + CorpusReferenceType, + GovernanceGraphCorpusType, + GovernanceGraphEdgeType, + GovernanceGraphNodeType, + GovernanceGraphType, LabelSetType, NoteRevisionType, NoteType, diff --git a/docs/test_scripts/corpus_reference_enrichment.md b/docs/test_scripts/corpus_reference_enrichment.md new file mode 100644 index 000000000..321cda52c --- /dev/null +++ b/docs/test_scripts/corpus_reference_enrichment.md @@ -0,0 +1,110 @@ +# Test: Corpus Reference Enrichment on a real S-1 corpus + +## Purpose +Prove the corpus reference enrichment engine (`opencontractserver/enrichment/`) +detects, normalizes, and persists the explicit references in a *real* S-1 +filing corpus: external law citations (as cross-corpus-trackable stubs), +document/exhibit references (as in-app links), and internal section references. + +## Prerequisites +- Migration `annotations/0078_corpusreference` applied. +- Local dev stack up (`docker compose -f local.yml`). +- An S-1 corpus loaded. NOTE: the local dev DB "thrashes" — corpus IDs get + reassigned out from under you mid-session. Root cause (diagnosed 2026-06-09): + this is NOT a periodic/celery-beat task and NOT this feature's code; it is + *external manual activity* against the shared local Postgres — the + `docs/test_scripts/smoke_reingest_remap.py` smoke script ("Smoke Source …" + corpuses) and EDGAR scrape imports run from the `~/Code/EDGARx2/` workspace + (an additional working dir) create/delete corpuses. The isolated **test + database is unaffected**, so the pytest suite (`test_enrichment_*.py`) is the + reproducible proof; this live run is a snapshot. Find the current S-1 corpus + by title rather than assuming an ID. If none is loaded, import one of the v2 + zips under `~/Code/EDGARx2/` (e.g. `spacex_s1_oc_corpus_v3.zip`). + + For defined-term coverage, pass + `types=list(__import__('opencontractserver.enrichment.constants', fromlist=['ALL_REFERENCE_TYPES']).ALL_REFERENCE_TYPES)` + (defined terms are opt-in and excluded from the default scan/apply set). + +## Steps + +1. Find the current S-1 corpus and a user who can read it. + ```bash + docker compose -f local.yml run --rm django python manage.py shell -c " + import warnings; warnings.simplefilter('ignore') + from opencontractserver.corpuses.models import Corpus + for c in Corpus.objects.all(): + n = c._get_active_documents().count() + if n > 10: + print(c.id, n, repr(c.title), 'creator', c.creator_id) + " + ``` + +2. Scan (read-only inventory — writes nothing). Replace ``/``. + ```bash + docker compose -f local.yml run --rm django python manage.py shell -c " + import warnings; warnings.simplefilter('ignore') + from opencontractserver.enrichment.services import EnrichmentService + out = EnrichmentService().scan(corpus_id=, creator_id=, sample_n=10) + print('scanned', out['documents_scanned'], 'candidates', out['total_candidates']) + print('by_type', out['counts_by_type']) + print('by_status', out['counts_by_status']) + " + ``` + +3. Apply (approval-gated at the agent layer; direct here for the proof). + ```bash + docker compose -f local.yml run --rm django python manage.py shell -c " + import warnings; warnings.simplefilter('ignore') + from opencontractserver.enrichment.services import EnrichmentService + from opencontractserver.annotations.models import Annotation, CorpusReference + out = EnrichmentService().apply(corpus_id=, creator_id=) + print(out) + refs = CorpusReference.objects.filter(corpus_id=) + print('LAW keys:', sorted(set(refs.filter(reference_type='LAW').values_list('canonical_key', flat=True)))[:15]) + print('DOC resolved:', refs.filter(reference_type='DOCUMENT', target_document__isnull=False).count()) + print('link_url anns:', Annotation.objects.filter(corpus_id=).exclude(link_url__isnull=True).exclude(link_url='').count()) + # Idempotency: + out2 = EnrichmentService().apply(corpus_id=, creator_id=) + print('re-apply created:', out2['annotations_created'], out2['references_created']) + " + ``` + +## Expected Results / Observed (corpus 25 "Select 2026 IPO S-1 Filings (SpaceX, Fervo Energy)", 55 docs, 2026-06-09) + +- **Scan:** 55 documents scanned, **348 candidates** — + `{'LAW': 59, 'DOCUMENT': 63, 'SECTION': 226}`; + statuses `{'EXTERNAL': 59, 'RESOLVED': 242, 'UNRESOLVED': 47}`. +- **Apply:** 348 mention annotations + 348 `CorpusReference` rows created. + - LAW canonical keys (cross-corpus stubs) included: + `dgcl:116, dgcl:122(17), dgcl:151, dgcl:202, dgcl:203, dgcl:212, dgcl:224, + dgcl:228, dgcl:242(b)(2), exchange-act:10(b), exchange-act:12, + exchange-act:13(d), irc:451, securities-act:4(a)(2), + securities-act:7(a)(2)(b), securities-act:8(a)`. + - **16** DOCUMENT references resolved to a target document, each mention + carrying a site-relative `link_url` like `/corpus/25/document/774` that the + frontend routes in-app. + - SECTION references resolved via the heading-text fallback (this corpus's + docs lack a dense OC_SECTION index); where an OC_SECTION annotation exists + and matches, an `OC_REFERENCES` Relationship is created instead (proven in + `test_enrichment_writer.py::...creates_relationship`). +- **Idempotent re-run:** `annotations_created == 0`, `references_created == 0`. +- **Defined terms (opt-in, `types=ALL_REFERENCE_TYPES`):** 599 distinct terms + across 925 definition sites — top cross-corpus stubs `term:company` (16 docs), + `term:board`, `term:common-stock`, `term:change-in-control`, + `term:class-a-common-stock`, `term:affiliate`. Each is a `DEFINED_TERM` + `CorpusReference` with `resolution_status='RESOLVED'` and `canonical_key` + `term:` (usage→definition linking is a future increment — deferred to + avoid the volume explosion of high-frequency terms like "Company"). + +## Cleanup +Delete the run's rows (the `Analysis` groups them): +```bash +docker compose -f local.yml run --rm django python manage.py shell -c " +import warnings; warnings.simplefilter('ignore') +from opencontractserver.annotations.models import CorpusReference, Annotation +from opencontractserver.enrichment import constants as C +CorpusReference.objects.filter(corpus_id=).delete() +Annotation.objects.filter(corpus_id=, annotation_label__text__in=[ + C.LABEL_REF_LAW, C.LABEL_REF_DOC, C.LABEL_REF_SECTION, C.LABEL_REF_TERM]).delete() +" +``` diff --git a/frontend/tests/DocumentGraphLive.ct.tsx b/frontend/tests/DocumentGraphLive.ct.tsx new file mode 100644 index 000000000..c344d79e6 --- /dev/null +++ b/frontend/tests/DocumentGraphLive.ct.tsx @@ -0,0 +1,131 @@ +/** + * Component tests for DocumentGraphLive — the Apollo-wired thin shell that + * fetches GET_CORPUS_DOCUMENT_GRAPH and feeds the presentational + * DocumentGraphGlimpse. Mounts under MockedProvider so the query resolves + * synchronously in the test environment. + * + * NOTE: each JSX-component import is kept in its own ``import`` statement, + * separate from all other imports, per the Playwright CT split-import rule. + * Mixing a component reference with helpers in a single import statement + * causes Playwright CT's babel transform to leave the component unrewritten + * and ``mount()`` throws. + */ +import { test, expect } from "./utils/coverage"; +import { MockedProvider } from "@apollo/client/testing"; +import { DocumentGraphLive } from "../src/components/corpuses/CorpusHome/intelligence/DocumentGraphLive"; +import { docScreenshot } from "./utils/docScreenshot"; +import { GET_CORPUS_DOCUMENT_GRAPH } from "../src/graphql/queries"; + +const CORPUS_ID = "Q29ycHVzVHlwZTox"; + +// Two graph mock instances so Apollo does not exhaust the mock on a potential +// refetch (MockedProvider removes each mock after one use by default). +const makeGraphMock = () => ({ + request: { + query: GET_CORPUS_DOCUMENT_GRAPH, + // DocumentGraphLive builds variables as { corpusId } — no limit field. + variables: { corpusId: CORPUS_ID }, + }, + result: { + data: { + corpusDocumentGraph: { + nodes: [ + { + id: "Doc:1", + title: "Alpha", + fileType: "application/pdf", + degree: 2, + }, + { + id: "Doc:2", + title: "Beta", + fileType: "application/pdf", + degree: 1, + }, + { + id: "Doc:3", + title: "Gamma", + fileType: "application/pdf", + degree: 1, + }, + ], + edges: [ + { + id: "e1", + source: "Doc:1", + target: "Doc:2", + label: "Cites", + relationshipType: "RELATIONSHIP", + }, + ], + totalNodeCount: 3, + totalEdgeCount: 1, + truncated: false, + }, + }, + }, +}); + +test.describe("DocumentGraphLive", () => { + test("renders the graph glimpse with live data from the Apollo mock", async ({ + mount, + page, + }) => { + const component = await mount( + + + + ); + + // Wait for the SVG to appear — the skeleton disappears once data resolves. + await expect( + page.locator('[data-testid="document-graph-glimpse-svg"]') + ).toBeVisible({ timeout: 20000 }); + + // Three nodes rendered — one per document in the mock. + await expect( + page.locator('[data-testid="document-graph-glimpse-node"]') + ).toHaveCount(3, { timeout: 20000 }); + + // One edge line rendered. + await expect( + page.locator('[data-testid="document-graph-glimpse-edges"] line') + ).toHaveCount(1); + + // Meta summary reflects the returned counts. + await expect( + page.locator('[data-testid="document-graph-glimpse-meta"]') + ).toContainText("3 linked documents"); + + await docScreenshot(page, "corpus--document-graph-live--with-data"); + + await component.unmount(); + }); + + test("shows the loading skeleton before the query resolves", async ({ + mount, + page, + }) => { + // Use a delayed mock so we can catch the loading state. + const delayedMock = { + ...makeGraphMock(), + delay: 2000, + }; + + const component = await mount( + + + + ); + + // While the query is in-flight the skeleton must be visible. + await expect( + page.locator('[data-testid="document-graph-glimpse-skeleton"]') + ).toBeVisible({ timeout: 10000 }); + + await component.unmount(); + }); +}); diff --git a/frontend/tests/SuggestedQuestions.ct.tsx b/frontend/tests/SuggestedQuestions.ct.tsx new file mode 100644 index 000000000..31a06cad0 --- /dev/null +++ b/frontend/tests/SuggestedQuestions.ct.tsx @@ -0,0 +1,68 @@ +/** + * Component tests for SuggestedQuestions — the cross-document question chip + * card. This is a purely presentational component: no GraphQL, no providers + * needed. It takes a single required prop ``onAskQuestion`` and renders one + * clickable chip per entry in ``SUGGESTED_QUESTIONS``. + * + * NOTE: the JSX component import and the constant import are kept in SEPARATE + * statements per the Playwright CT split-import rule (mixing a component with + * helpers in one statement leaves the component unrewritten by Playwright CT's + * babel transform, causing ``mount()`` to throw). + */ +import { test, expect } from "./utils/coverage"; +import { docScreenshot } from "./utils/docScreenshot"; +import { SuggestedQuestions } from "../src/components/corpuses/CorpusHome/intelligence/SuggestedQuestions"; +import { SUGGESTED_QUESTIONS } from "../src/components/corpuses/CorpusHome/intelligence/SuggestedQuestions"; + +test.describe("SuggestedQuestions", () => { + test("renders all chips and invokes the callback when one is clicked", async ({ + mount, + page, + }) => { + const submitted: string[] = []; + + const component = await mount( + { + submitted.push(q); + }} + /> + ); + + // At least the first question must be visible as a clickable chip. + const firstQuestion = SUGGESTED_QUESTIONS[0]; + const chips = page.locator('[data-testid="ask-across-docs-suggestion"]'); + await expect(chips.first()).toBeVisible({ timeout: 10000 }); + + // All chips are rendered — one per entry in SUGGESTED_QUESTIONS. + await expect(chips).toHaveCount(SUGGESTED_QUESTIONS.length); + + // The chip text matches the exported constant. + await expect(chips.first()).toContainText(firstQuestion); + + // Click the first chip and verify the callback fires with the right question. + await chips.first().click(); + await expect.poll(() => submitted.length).toBeGreaterThan(0); + expect(submitted[0]).toBe(firstQuestion); + + await docScreenshot(page, "corpus--suggested-questions--default"); + + await component.unmount(); + }); + + test("does not render the card when the callback is a no-op stub", async ({ + mount, + page, + }) => { + // Smoke-test: the container renders even if the handler does nothing. + const component = await mount( + {}} /> + ); + + await expect( + page.locator('[data-testid="ask-across-docs-suggestions"]') + ).toBeVisible({ timeout: 10000 }); + + await component.unmount(); + }); +}); diff --git a/opencontractserver/analyzer/checks.py b/opencontractserver/analyzer/checks.py index b12a23cd2..dd44e18bd 100644 --- a/opencontractserver/analyzer/checks.py +++ b/opencontractserver/analyzer/checks.py @@ -6,8 +6,8 @@ @register() def check_unsynced_analyzers(app_configs: Any, **kwargs: Any) -> list[Warning]: """ - Check if there are doc_analyzer_task decorated functions - that haven't been synced to the database. + Check if there are doc_analyzer_task / corpus_analyzer_task decorated + functions that haven't been synced to the database. """ warnings: list[Warning] = [] @@ -15,13 +15,13 @@ def check_unsynced_analyzers(app_configs: Any, **kwargs: Any) -> list[Warning]: from opencontractserver.analyzer.models import Analyzer from opencontractserver.utils.celery_tasks import ( celery_app, - get_doc_analyzer_task_by_name, + get_analyzer_task_by_name, ) unsynced = [] for task_name in celery_app.tasks.keys(): - analyzer_task = get_doc_analyzer_task_by_name(task_name) + analyzer_task = get_analyzer_task_by_name(task_name) if analyzer_task is None: continue diff --git a/opencontractserver/analyzer/utils.py b/opencontractserver/analyzer/utils.py index 3a236fb49..dd1ad11d4 100644 --- a/opencontractserver/analyzer/utils.py +++ b/opencontractserver/analyzer/utils.py @@ -15,11 +15,11 @@ # Attempt to import Celery logic in a typical run from opencontractserver.utils.celery_tasks import ( celery_app, - get_doc_analyzer_task_by_name, + get_analyzer_task_by_name, ) except ImportError: # If Celery or tasks aren't available in this context, set placeholders - get_doc_analyzer_task_by_name = None # type: ignore[assignment] + get_analyzer_task_by_name = None # type: ignore[assignment] celery_app = None logger = logging.getLogger(__name__) @@ -78,9 +78,9 @@ def auto_create_doc_analyzers( :param update_existing: If True, update existing analyzers with missing/outdated fields. """ - if celery_app is None or get_doc_analyzer_task_by_name is None: + if celery_app is None or get_analyzer_task_by_name is None: logger.warning( - "auto_create_doc_analyzers: Celery or doc_analyzer_task accessor not available." + "auto_create_doc_analyzers: Celery or analyzer task accessor not available." ) return @@ -101,9 +101,9 @@ def auto_create_doc_analyzers( model_fields = [f.name for f in AnalyzerModel._meta.get_fields()] has_input_schema = "input_schema" in model_fields - # Iterate over tasks in Celery registry + # Iterate over tasks in Celery registry (doc- and corpus-scoped analyzers) for task_name in celery_app.tasks.keys(): - analyzer_task = get_doc_analyzer_task_by_name(task_name) + analyzer_task = get_analyzer_task_by_name(task_name) if analyzer_task is None: continue @@ -114,7 +114,9 @@ def auto_create_doc_analyzers( trimmed_desc = Truncator(docstring).chars(max_docstring_length) default_desc = "Auto-created from @doc_analyzer_task-decorated Celery task." description = trimmed_desc if trimmed_desc else default_desc - schema = getattr(analyzer_task, "_oc_doc_analyzer_input_schema", None) + schema = getattr( + analyzer_task, "_oc_doc_analyzer_input_schema", None + ) or getattr(analyzer_task, "_oc_corpus_analyzer_input_schema", None) # Check for existing analyzer existing_analyzer = None diff --git a/opencontractserver/annotations/migrations/0078_corpusreference.py b/opencontractserver/annotations/migrations/0078_corpusreference.py new file mode 100644 index 000000000..a3020ffba --- /dev/null +++ b/opencontractserver/annotations/migrations/0078_corpusreference.py @@ -0,0 +1,165 @@ +# Generated by Django 5.2.14 on 2026-06-09 15:39 + +import django.db.models.deletion +import opencontractserver.shared.user_can_mixin +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("analyzer", "0022_alter_analysis_status"), + ("annotations", "0077_annotation_structural_modified_index"), + ("corpuses", "0056_corpus_auto_branding_enabled"), + ("documents", "0042_pendingcorpusimport"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="CorpusReference", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("backend_lock", models.BooleanField(db_index=True, default=False)), + ("is_public", models.BooleanField(default=False)), + ("created", models.DateTimeField(auto_now_add=True)), + ("modified", models.DateTimeField(auto_now=True)), + ( + "reference_type", + models.CharField( + choices=[ + ("LAW", "Law citation"), + ("DOCUMENT", "Document reference"), + ("SECTION", "Internal section reference"), + ("DEFINED_TERM", "Defined term"), + ], + db_index=True, + max_length=16, + ), + ), + ( + "canonical_key", + models.CharField( + blank=True, db_index=True, max_length=255, null=True + ), + ), + ("normalized_data", models.JSONField(blank=True, null=True)), + ("confidence", models.FloatField(default=1.0)), + ( + "resolution_status", + models.CharField( + choices=[ + ("RESOLVED", "Resolved"), + ("UNRESOLVED", "Unresolved"), + ("EXTERNAL", "External (no internal target)"), + ], + default="RESOLVED", + max_length=16, + ), + ), + ( + "corpus", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="references", + to="corpuses.corpus", + ), + ), + ( + "created_by_analysis", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="created_references", + to="analyzer.analysis", + ), + ), + ( + "creator", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "source_annotation", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="outbound_references", + to="annotations.annotation", + ), + ), + ( + "target_annotation", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="inbound_references", + to="annotations.annotation", + ), + ), + ( + "target_corpus", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="inbound_references", + to="corpuses.corpus", + ), + ), + ( + "target_document", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="inbound_references", + to="documents.document", + ), + ), + ( + "user_lock", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="locked_%(class)s_objects", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "indexes": [ + models.Index( + fields=["corpus", "reference_type"], + name="annotations_corpus__a5d62c_idx", + ), + models.Index( + fields=["canonical_key"], name="annotations_canonic_92d0e7_idx" + ), + ], + "constraints": [ + models.UniqueConstraint( + fields=("source_annotation", "reference_type", "canonical_key"), + name="uniq_corpusref_source_type_key", + ) + ], + }, + bases=( + opencontractserver.shared.user_can_mixin.InstanceUserCanMixin, + models.Model, + ), + ), + ] diff --git a/opencontractserver/annotations/models.py b/opencontractserver/annotations/models.py index 369338d1c..8a51b2522 100644 --- a/opencontractserver/annotations/models.py +++ b/opencontractserver/annotations/models.py @@ -30,6 +30,7 @@ from pgvector.django import HnswIndex, VectorField from opencontractserver.constants.search import HNSW_EF_CONSTRUCTION, HNSW_M +from opencontractserver.enrichment import constants as enrichment_constants from opencontractserver.shared.defaults import ( jsonfield_default_value, ) @@ -1876,3 +1877,102 @@ class Meta: def __str__(self) -> str: return f"NoteRevision(note_id={self.note_id}, v={self.version})" + + +# --------------------------------------------------------------------------- # +# Cross-document / cross-corpus reference substrate (enrichment) # +# --------------------------------------------------------------------------- # +# Discriminator values are single-sourced from the enrichment engine's +# constants (`enrichment/constants.py` is pure — no model imports — so this +# cannot cycle); only the human-readable labels live here. +REFERENCE_TYPE_CHOICES = [ + (enrichment_constants.REF_LAW, "Law citation"), + (enrichment_constants.REF_DOCUMENT, "Document reference"), + (enrichment_constants.REF_SECTION, "Internal section reference"), + (enrichment_constants.REF_DEFINED_TERM, "Defined term"), +] +RESOLUTION_STATUS_CHOICES = [ + (enrichment_constants.STATUS_RESOLVED, "Resolved"), + (enrichment_constants.STATUS_UNRESOLVED, "Unresolved"), + (enrichment_constants.STATUS_EXTERNAL, "External (no internal target)"), +] + + +class CorpusReference(BaseOCModel): + """First-class connection between a reference *mention* and its target. + + Within-document links continue to use :class:`Relationship`; + ``CorpusReference`` carries the cross-document, cross-corpus, and + external-law (statute) links that ``Relationship`` cannot express (a + relationship is pinned to a single document). ``canonical_key`` is the + cross-corpus join key (e.g. ``dgcl:145``) — cross-corpus *linking* is future + work, but the shape anticipates it via ``target_corpus`` + ``canonical_key``. + """ + + corpus = django.db.models.ForeignKey( + "corpuses.Corpus", + on_delete=django.db.models.CASCADE, + related_name="references", + db_index=True, + ) + reference_type = django.db.models.CharField( + max_length=16, choices=REFERENCE_TYPE_CHOICES, db_index=True + ) + source_annotation = django.db.models.ForeignKey( + "annotations.Annotation", + on_delete=django.db.models.CASCADE, + related_name="outbound_references", + ) + target_annotation = django.db.models.ForeignKey( + "annotations.Annotation", + on_delete=django.db.models.SET_NULL, + null=True, + blank=True, + related_name="inbound_references", + ) + target_document = django.db.models.ForeignKey( + "documents.Document", + on_delete=django.db.models.SET_NULL, + null=True, + blank=True, + related_name="inbound_references", + ) + target_corpus = django.db.models.ForeignKey( + "corpuses.Corpus", + on_delete=django.db.models.SET_NULL, + null=True, + blank=True, + related_name="inbound_references", + ) + canonical_key = django.db.models.CharField( + max_length=255, null=True, blank=True, db_index=True + ) + normalized_data = django.db.models.JSONField(null=True, blank=True) + confidence = django.db.models.FloatField(default=1.0) + resolution_status = django.db.models.CharField( + max_length=16, choices=RESOLUTION_STATUS_CHOICES, default="RESOLVED" + ) + created_by_analysis = django.db.models.ForeignKey( + "analyzer.Analysis", + on_delete=django.db.models.SET_NULL, + null=True, + blank=True, + related_name="created_references", + ) + + class Meta: + indexes = [ + django.db.models.Index(fields=["corpus", "reference_type"]), + django.db.models.Index(fields=["canonical_key"]), + ] + constraints = [ + # Idempotency guard: one reference per (source span, type, key). + django.db.models.UniqueConstraint( + fields=["source_annotation", "reference_type", "canonical_key"], + name="uniq_corpusref_source_type_key", + ), + ] + + def __str__(self) -> str: + target = self.canonical_key or self.target_document_id + return f"CorpusReference({self.reference_type} -> {target})" diff --git a/opencontractserver/constants/stats.py b/opencontractserver/constants/stats.py index f6e6af95f..6ff794105 100644 --- a/opencontractserver/constants/stats.py +++ b/opencontractserver/constants/stats.py @@ -28,3 +28,10 @@ # How many distinct annotation labels to surface in the IntelligencePanel's # label-distribution mini-chart before collapsing the long tail. CORPUS_INTELLIGENCE_LABEL_DISTRIBUTION_TOP_N = 8 + +# Node cap for the corpus-scoped governance graph (reference web: documents, +# statute sections, and external-citation ghosts). Larger than the +# relationship-glimpse cap because the governance panel is a dedicated +# full-bleed visualization, not a landing-page card; nodes are ranked by +# degree (summed mention weight) and the rest summarised via ``truncated``. +GOVERNANCE_GRAPH_MAX_NODES = 200 diff --git a/opencontractserver/corpuses/migrations/0057_rename_corpuses_co_corpus__vote_idx_corpuses_co_corpus__64f0b6_idx_and_more.py b/opencontractserver/corpuses/migrations/0057_rename_corpuses_co_corpus__vote_idx_corpuses_co_corpus__64f0b6_idx_and_more.py new file mode 100644 index 000000000..9a8f0bc73 --- /dev/null +++ b/opencontractserver/corpuses/migrations/0057_rename_corpuses_co_corpus__vote_idx_corpuses_co_corpus__64f0b6_idx_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 5.2.14 on 2026-06-10 05:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("corpuses", "0056_corpus_auto_branding_enabled"), + ] + + operations = [ + migrations.RenameIndex( + model_name="corpusvote", + new_name="corpuses_co_corpus__64f0b6_idx", + old_name="corpuses_co_corpus__vote_idx", + ), + migrations.RenameIndex( + model_name="corpusvote", + new_name="corpuses_co_creator_912c61_idx", + old_name="corpuses_co_creator_idx", + ), + migrations.RenameIndex( + model_name="corpusvote", + new_name="corpuses_co_session_4e4eaf_idx", + old_name="corpuses_co_session_idx", + ), + migrations.AlterField( + model_name="corpusvote", + name="backend_lock", + field=models.BooleanField(db_index=True, default=False), + ), + ] diff --git a/opencontractserver/enrichment/__init__.py b/opencontractserver/enrichment/__init__.py new file mode 100644 index 000000000..1b6432f67 --- /dev/null +++ b/opencontractserver/enrichment/__init__.py @@ -0,0 +1,6 @@ +"""Corpus reference enrichment engine. + +Deterministic extraction → resolution → persistence of explicit references +(law citations, document/exhibit references, internal section references, and +defined terms) across an OpenContracts corpus. +""" diff --git a/opencontractserver/enrichment/authorities.py b/opencontractserver/enrichment/authorities.py new file mode 100644 index 000000000..91b4a0f41 --- /dev/null +++ b/opencontractserver/enrichment/authorities.py @@ -0,0 +1,232 @@ +"""Bootstrap "authority corpora" — statute / regulation reference targets. + +An authority corpus holds one text document per statute section (e.g. DGCL +§ 145). Each document carries its canonical key in ``Document.custom_meta`` +(``{"canonical_key": "dgcl:145", "authority": "dgcl"}``) so the cross-corpus +resolution pass can upgrade EXTERNAL law references (emitted by the +enrichment engine with the same canonical keys) into RESOLVED links that +point at a concrete document in another corpus. + +Document creation is delegated to the existing text-import tool +(``create_or_update_text_document`` -> ``Corpus.import_content``), so +authority documents get real versioning, paths, and permissions like any +other corpus document. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass + +from django.contrib.auth import get_user_model + +from opencontractserver.corpuses.models import Corpus +from opencontractserver.corpuses.services.corpus_documents import CorpusDocumentService +from opencontractserver.documents.models import Document +from opencontractserver.enrichment import constants as C +from opencontractserver.utils.files import read_field_file_text + +logger = logging.getLogger(__name__) +User = get_user_model() + +# Subsection canonical keys fall back to their root section: the authority +# corpus stores one document per *section* (dgcl:122), while citations may be +# subsection-precise (dgcl:122(17)). Root = authority prefix + section number +# (with optional trailing letter), i.e. everything before the first "(". +_ROOT_KEY_RE = re.compile(r"^(?P[a-z0-9-]+:\d+[a-z]?)", re.IGNORECASE) + + +@dataclass +class AuthoritySection: + """One statute section destined to become an authority document.""" + + key: str # canonical key, e.g. "dgcl:145" + heading: str # document title, e.g. "DGCL § 145 — Indemnification" + text: str # full section text + source_url: str | None = None + + +def candidate_keys(canonical_key: str) -> list[str]: + """Keys to try when resolving a citation: exact first, then section root.""" + keys = [canonical_key] + m = _ROOT_KEY_RE.match(canonical_key) + if m and m.group("root") != canonical_key: + keys.append(m.group("root")) + return keys + + +def authority_alias_registry(user=None) -> dict[str, str]: + """Build the authority alias -> canonical-prefix map for extraction. + + Static defaults (``constants.AUTHORITY_PREFIX``) merged with aliases + declared by authority corpora: the bootstrapper stamps each section + document with ``custom_meta.authority_aliases``, so adding a body of law + is a bootstrap call, not a code change. DB-declared aliases override + static entries on collision. When ``user`` is given, only aliases on + documents visible to that user are included. + """ + mapping: dict[str, str] = dict(C.AUTHORITY_PREFIX) + # Fail closed: without a user we contribute only the static defaults rather + # than aliases from EVERY document (including private corpora). A future + # caller that forgets to thread ``creator`` gets no DB-declared aliases, not + # a cross-tenant metadata leak. + qs = ( + Document.objects.none() + if user is None + else Document.objects.visible_to_user(user) + ) + rows = qs.filter(custom_meta__has_key="authority_aliases").values_list( + "custom_meta", flat=True + ) + for meta in rows: + meta = meta or {} + prefix = meta.get("authority") + if not prefix: + continue + for alias in meta.get("authority_aliases") or []: + if isinstance(alias, str) and alias.strip(): + mapping[alias.strip().lower()] = prefix + return mapping + + +def find_authority_target(canonical_key: str, user) -> Document | None: + """Find the authority document for a canonical key, visible to ``user``. + + Falls back from subsection keys (``dgcl:122(17)``) to the root section + (``dgcl:122``). Returns ``None`` when no visible authority document + carries the key. + """ + for key in candidate_keys(canonical_key): + doc = ( + Document.objects.visible_to_user(user) + .filter( + custom_meta__canonical_key=key, + # Version-ups create a new Document row; only the current + # (non-superseded) version has a live path record. + path_records__is_current=True, + path_records__is_deleted=False, + ) + .order_by("id") + .first() + ) + if doc is not None: + return doc + return None + + +class AuthorityCorpusBootstrapper: + """Create or refresh an authority corpus from a section spec.""" + + def bootstrap( + self, + *, + creator_id: int, + corpus_title: str, + sections: list[AuthoritySection], + corpus_id: int | None = None, + aliases: list[str] | None = None, + ) -> dict: + """Idempotently materialise ``sections`` as keyed documents. + + Unchanged sections are skipped; changed text version-ups the existing + document at the same path (title-derived). Returns a summary dict. + """ + # Local import: the tool module imports services that import models — + # keep the engine importable without dragging the whole tool package. + from opencontractserver.llms.tools.core_tools.text_document_import import ( + create_or_update_text_document, + ) + + user = User.objects.get(pk=creator_id) + if corpus_id is not None: + # Visibility-scoped: invisible and nonexistent corpora raise the + # same ``Corpus.DoesNotExist`` (no existence oracle). + corpus = Corpus.objects.visible_to_user(user).get(pk=corpus_id) + corpus_created = False + else: + corpus, corpus_created = Corpus.objects.get_or_create( + title=corpus_title, creator=user + ) + + created = updated = skipped = restamped = 0 + for sec in sections: + # Match by key, falling back to title: a concurrent full-row save + # from the document-processing pipeline can clobber a freshly + # stamped custom_meta (lost update), so a re-run must recognise + # the document by title and restamp rather than re-import it. + existing = self._find_by_key(user, corpus, sec.key) or self._find_by_title( + user, corpus, sec.heading + ) + if existing is not None: + try: + current = read_field_file_text(existing.txt_extract_file) + except (OSError, ValueError, AttributeError): + # Unreadable (missing file / no file associated / decode + # error) -> treat as needs-rewrite below. Narrow on purpose + # so genuine bugs surface instead of being swallowed. + current = None + if current == sec.text: + meta = existing.custom_meta or {} + if meta.get("canonical_key") != sec.key or ( + aliases and meta.get("authority_aliases") != aliases + ): + self._stamp_key(existing.id, sec, aliases) + restamped += 1 + else: + skipped += 1 + continue + + out = create_or_update_text_document( + corpus_id=corpus.id, + title=sec.heading, + content=sec.text, + author_id=creator_id, + description=f"Authority text for {sec.key}" + + (f" (source: {sec.source_url})" if sec.source_url else ""), + ) + self._stamp_key(out["document_id"], sec, aliases) + if out["status"] == "created": + created += 1 + else: + updated += 1 + + return { + "corpus_id": corpus.id, + "corpus_created": corpus_created, + "documents_created": created, + "documents_updated": updated, + "documents_skipped": skipped, + "documents_restamped": restamped, + } + + @staticmethod + def _find_by_key(user, corpus, key: str) -> Document | None: + return ( + CorpusDocumentService.get_corpus_documents(user, corpus) + .filter(custom_meta__canonical_key=key) + .first() + ) + + @staticmethod + def _find_by_title(user, corpus, title: str) -> Document | None: + return ( + CorpusDocumentService.get_corpus_documents(user, corpus) + .filter(title=title) + .first() + ) + + @staticmethod + def _stamp_key( + document_id: int, sec: AuthoritySection, aliases: list[str] | None = None + ) -> None: + doc = Document.objects.get(pk=document_id) + meta = dict(doc.custom_meta or {}) + meta["canonical_key"] = sec.key + meta["authority"] = sec.key.split(":", 1)[0] + if sec.source_url: + meta["source_url"] = sec.source_url + if aliases: + meta["authority_aliases"] = aliases + doc.custom_meta = meta + doc.save(update_fields=["custom_meta"]) diff --git a/opencontractserver/enrichment/constants.py b/opencontractserver/enrichment/constants.py new file mode 100644 index 000000000..88d9b8df7 --- /dev/null +++ b/opencontractserver/enrichment/constants.py @@ -0,0 +1,85 @@ +"""Constants for the corpus reference enrichment engine. + +No magic numbers / strings in the engine modules — they import from here. +""" + +# Mention annotation labels (one per reference type). +LABEL_REF_LAW = "OC_REF_LAW" +LABEL_REF_DOC = "OC_REF_DOC" +LABEL_REF_SECTION = "OC_REF_SECTION" +LABEL_REF_TERM = "OC_REF_TERM" + +# Relationship label used for within-document reference links (section->section, +# definition->usage). One label keeps the relationship graph legible. +LABEL_RELATIONSHIP = "OC_REFERENCES" + +# Reference type discriminators — must match CorpusReference.REFERENCE_TYPE_CHOICES. +REF_LAW = "LAW" +REF_DOCUMENT = "DOCUMENT" +REF_SECTION = "SECTION" +REF_DEFINED_TERM = "DEFINED_TERM" + +# Resolution statuses — must match CorpusReference.RESOLUTION_STATUS_CHOICES. +STATUS_RESOLVED = "RESOLVED" +STATUS_UNRESOLVED = "UNRESOLVED" +STATUS_EXTERNAL = "EXTERNAL" + +LABEL_FOR_TYPE = { + REF_LAW: LABEL_REF_LAW, + REF_DOCUMENT: LABEL_REF_DOC, + REF_SECTION: LABEL_REF_SECTION, + REF_DEFINED_TERM: LABEL_REF_TERM, +} + +# Authority name (as it appears in text, lowercased) -> canonical_key prefix. +AUTHORITY_PREFIX = { + "delaware general corporation law": "dgcl", + "dgcl": "dgcl", + "securities act": "securities-act", + "securities exchange act": "exchange-act", + "exchange act": "exchange-act", + "internal revenue code": "irc", + "investment company act": "ica", + "investment advisers act": "iaa", +} + +# Canonical-key prefix for bare SEC rule citations ("Rule 506(b)") — these are +# 17 CFR rules cited without a named authority. +SEC_RULE_PREFIX = "sec-rule" + +# Document-level relationship type for graph rollups — must match +# DocumentRelationship.RELATIONSHIP_TYPE_CHOICES. +DOC_REL_RELATIONSHIP = "RELATIONSHIP" + +# Provenance analyzer identity for enrichment runs. The task name is the +# REAL registered Celery task (the @corpus_analyzer_task adapter in +# opencontractserver/tasks/corpus_analysis_tasks.py) so the analyzer row the +# service creates is dispatchable via run_task_name_analyzer / CorpusAction. +ENRICHMENT_ANALYZER_TASK = ( + "opencontractserver.tasks.corpus_analysis_tasks.corpus_reference_enrichment" +) +ENRICHMENT_ANALYZER_ID = "corpus-reference-enrichment" +ENRICHMENT_ANALYZER_TITLE = "Corpus Reference Enrichment" + +# Governance-graph vocabulary (node kinds / edge types / corpus roles) — the +# contract between GovernanceGraphService, the GraphQL types, and the frontend +# panel. Mirrors demo/export_governance_graph.py. +GRAPH_EDGE_LAW = "LAW" # resolved law citation -> statute document +GRAPH_EDGE_LAW_EXTERNAL = "LAW_EXTERNAL" # citation with no visible target doc +GRAPH_EDGE_DOCUMENT = "DOCUMENT" # DocumentRelationship rollup (doc -> doc) +GRAPH_NODE_PRIMARY = "primary" +GRAPH_NODE_EXHIBIT = "exhibit" +GRAPH_NODE_STATUTE = "statute" +GRAPH_NODE_EXTERNAL = "external" # ghost node for an unresolved canonical key +GRAPH_CORPUS_FILING = "filing" +GRAPH_CORPUS_AUTHORITY = "authority" + +# Defaults / thresholds. +DEFAULT_SAMPLE_N = 10 +MAX_DEFINED_TERMS = 50 # cap to control precision/volume in v1 +# Punctuation stripped from the tail of a captured defined term +# (e.g. (the "Notes," ...) -> "Notes"). +TRAILING_PUNCT = ",.;:" +ALL_REFERENCE_TYPES = (REF_LAW, REF_DOCUMENT, REF_SECTION, REF_DEFINED_TERM) +# Defined-terms are opt-in (precision risk); not scanned/applied by default. +DEFAULT_REFERENCE_TYPES = (REF_LAW, REF_DOCUMENT, REF_SECTION) diff --git a/opencontractserver/enrichment/extractor.py b/opencontractserver/enrichment/extractor.py new file mode 100644 index 000000000..3f7035103 --- /dev/null +++ b/opencontractserver/enrichment/extractor.py @@ -0,0 +1,245 @@ +"""Deterministic reference extraction from document text (no DB access). + +The extractor recognises *explicit* references with high-precision grammars and +emits :class:`Candidate` spans. Target resolution (which document/section/corpus +a candidate points at) is the resolver's job — the extractor only normalises +what the text itself states (e.g. a law citation's canonical key). +""" + +from __future__ import annotations + +import re +from collections.abc import Iterator +from dataclasses import dataclass, field + +from opencontractserver.enrichment import constants as C + + +@dataclass +class Candidate: + """A single detected reference span within one document's text.""" + + reference_type: str + start: int + end: int + raw_text: str + canonical_key: str | None = None + normalized_data: dict = field(default_factory=dict) + + +# "Section 145", "Section 4(a)(2)", "Section 7(a)(2)(B)" followed by authority. +# The authority alternation is compiled per extractor instance from an alias +# registry (static defaults + DB-declared aliases on authority corpora), so +# adding a body of law never requires a code change. Aliases are matched +# longest-first so "Securities Exchange Act" wins over "Exchange Act". +def _compile_law_re(aliases) -> re.Pattern: + alt = "|".join(re.escape(a) for a in sorted(aliases, key=len, reverse=True)) + return re.compile( + r"Section\s+(?P\d+[A-Za-z]?(?:\([0-9a-zA-Z]+\))*)\s+of\s+the\s+" + r"(?P" + alt + r")", + re.IGNORECASE, + ) + + +def _compile_prefix_law_re(aliases) -> re.Pattern: + """Form-style citations with the authority BEFORE the section: + "Securities Act Section 4(a)(5)", "Investment Company Act Section 3(c)".""" + alt = "|".join(re.escape(a) for a in sorted(aliases, key=len, reverse=True)) + return re.compile( + r"(?P" + alt + r")\s+Section\s+" + r"(?P\d+[A-Za-z]?(?:\([0-9a-zA-Z]+\))*)", + re.IGNORECASE, + ) + + +# Statute-internal relative idiom: "§ 251 of this title", "Section 141(b) of +# this chapter". Only keyable when the caller supplies the document's own +# authority (from the authority corpus's custom_meta) as default_authority. +_LAW_RELATIVE_RE = re.compile( + r"(?:§|Section)\s+(?P\d+[A-Za-z]?(?:\([0-9a-zA-Z]+\))*)\s+of\s+this\s+" + r"(?:title|chapter)", + re.IGNORECASE, +) +# Bare SEC rule citations: "Rule 506(b)", "Rule 504(b)(1)", "Rule 144A", +# "Rule 10b-5". No authority name in text — keyed under SEC_RULE_PREFIX. +_RULE_RE = re.compile( + r"Rule\s+(?P\d+[A-Za-z]?(?:-\d+)?(?:\([0-9a-zA-Z]+\))*)", +) +_EXHIBIT_RE = re.compile(r"Exhibit\s+(?P\d+\.\d+[A-Za-z()]*)", re.IGNORECASE) +# see / under ... "Heading" (curly or straight quotes) +_SECTION_RE = re.compile( + r"(?:see|under)\b[^\"“]{0,40}?[\"“]" r"(?P[A-Z][^\"”]{2,60})[\"”]", + re.IGNORECASE, +) +# Defined-term DEFINITION sites — high precision only: +# 1. parenthetical: (the "Company") ("Notes") (collectively, the "Shares") +# 2. copular: "Change of Control" means | shall mean | refers to +_TERM_PAREN_RE = re.compile( + r"\((?:the\s+|collectively,?\s+the\s+|each\s+an?\s+|together(?:,)?\s+the\s+)?" + r"[\"“](?P[A-Z][^\"”]{1,60}?)[\"”]", +) +_TERM_MEANS_RE = re.compile( + r"[\"“](?P[A-Z][^\"”]{1,60}?)[\"”]\s+" + r"(?:means\b|shall\s+mean\b|refers\s+to\b|has\s+the\s+meaning\b)", +) + + +def _slugify_term(term: str) -> str: + """Normalize a defined term to a stable cross-corpus key fragment.""" + return re.sub(r"[^a-z0-9]+", "-", term.strip().lower()).strip("-") + + +class ReferenceExtractor: + """Stateless-per-text extractor: ``extract(text) -> list[Candidate]``. + + ``authority_aliases`` maps authority alias (as it appears in text) -> + canonical key prefix; defaults to the static ``AUTHORITY_PREFIX`` set. + Callers with corpus context should pass the merged registry from + :func:`opencontractserver.enrichment.authorities.authority_alias_registry`. + + ``default_authority`` keys relative citations ("§ 251 of this title") + when extracting from a statute document that knows its own authority; + without it, relative citations are skipped (they cannot be keyed). + """ + + def __init__(self, authority_aliases: dict[str, str] | None = None) -> None: + aliases = authority_aliases or C.AUTHORITY_PREFIX + self._alias_to_prefix = {k.lower(): v for k, v in aliases.items()} + self._law_re = _compile_law_re(self._alias_to_prefix) + self._prefix_law_re = _compile_prefix_law_re(self._alias_to_prefix) + + def extract( + self, text: str, default_authority: str | None = None + ) -> list[Candidate]: + out: list[Candidate] = [] + out.extend(self._laws(text)) + out.extend(self._prefix_laws(text)) + out.extend(self._rules(text)) + if default_authority: + out.extend(self._relative_laws(text, default_authority)) + out.extend(self._exhibits(text)) + out.extend(self._sections(text)) + out.extend(self._terms(text)) + return out + + def _law_candidate(self, m: re.Match) -> Candidate: + sec = m.group("sec") + auth_text = m.group("auth") + prefix = self._alias_to_prefix[auth_text.lower()] + return Candidate( + reference_type=C.REF_LAW, + start=m.start(), + end=m.end(), + raw_text=m.group(0), + canonical_key=f"{prefix}:{sec.lower()}", + normalized_data={ + "authority": prefix, + "section": sec, + "authority_text": auth_text, + }, + ) + + def _laws(self, text: str) -> Iterator[Candidate]: + for m in self._law_re.finditer(text): + yield self._law_candidate(m) + + def _prefix_laws(self, text: str) -> Iterator[Candidate]: + for m in self._prefix_law_re.finditer(text): + yield self._law_candidate(m) + + def _rules(self, text: str) -> Iterator[Candidate]: + for m in _RULE_RE.finditer(text): + rule = m.group("rule") + yield Candidate( + reference_type=C.REF_LAW, + start=m.start(), + end=m.end(), + raw_text=m.group(0), + canonical_key=f"{C.SEC_RULE_PREFIX}:{rule.lower()}", + normalized_data={ + "authority": C.SEC_RULE_PREFIX, + "section": rule, + "rule": True, + }, + ) + + def _relative_laws(self, text: str, authority: str) -> Iterator[Candidate]: + for m in _LAW_RELATIVE_RE.finditer(text): + sec = m.group("sec") + yield Candidate( + reference_type=C.REF_LAW, + start=m.start(), + end=m.end(), + raw_text=m.group(0), + canonical_key=f"{authority}:{sec.lower()}", + normalized_data={ + "authority": authority, + "section": sec, + "relative": True, + }, + ) + + def _exhibits(self, text: str) -> Iterator[Candidate]: + for m in _EXHIBIT_RE.finditer(text): + yield Candidate( + reference_type=C.REF_DOCUMENT, + start=m.start(), + end=m.end(), + raw_text=m.group(0), + normalized_data={"exhibit_number": m.group("num")}, + ) + + def _sections(self, text: str) -> Iterator[Candidate]: + for m in _SECTION_RE.finditer(text): + head = m.group("head").strip() + # Span covers the quoted heading (including the surrounding quotes). + yield Candidate( + reference_type=C.REF_SECTION, + start=m.start("head") - 1, + end=m.end("head") + 1, + raw_text=m.group(0), + normalized_data={"heading": head}, + ) + + def _terms(self, text: str) -> Iterator[Candidate]: + """Defined-term DEFINITION sites, deduped by slug, capped per document. + + Opt-in (not in the default reference-type set) and capped at + ``MAX_DEFINED_TERMS`` to keep precision/volume in check. The cap is a + TOTAL per document across both grammar forms: matches are merged in + document order before capping, so a document heavy in parenthetical + definitions cannot starve the "means" form — the first N definition + sites win regardless of form. Each definition becomes a + cross-corpus-trackable stub keyed ``term:`` — the same + philosophy as law citations. + """ + matches = [ + (m, kind) + for regex, kind in ( + (_TERM_PAREN_RE, "parenthetical"), + (_TERM_MEANS_RE, "means"), + ) + for m in regex.finditer(text) + ] + matches.sort(key=lambda pair: pair[0].start()) + seen: set[str] = set() + emitted = 0 + for m, kind in matches: + if emitted >= C.MAX_DEFINED_TERMS: + return + # Trim trailing punctuation captured inside the quotes + # (e.g. (the "Notes," ...) -> "Notes"). + term = m.group("term").strip().rstrip(C.TRAILING_PUNCT) + slug = _slugify_term(term) + if not slug or slug in seen: + continue + seen.add(slug) + emitted += 1 + yield Candidate( + reference_type=C.REF_DEFINED_TERM, + start=m.start("term") - 1, + end=m.end("term") + 1, + raw_text=term, + canonical_key=f"term:{slug}", + normalized_data={"term": term, "slug": slug, "kind": kind}, + ) diff --git a/opencontractserver/enrichment/resolver.py b/opencontractserver/enrichment/resolver.py new file mode 100644 index 000000000..87386b796 --- /dev/null +++ b/opencontractserver/enrichment/resolver.py @@ -0,0 +1,160 @@ +"""Resolve extracted reference candidates to concrete targets in a corpus. + +The resolver is fed the corpus's documents (already permission-scoped by the +caller via ``CorpusDocumentService``) so it performs no Tier-0 permission +fusions itself. It maps: + +* law citations -> EXTERNAL (no internal target; canonical stub only) +* document/exhibit refs -> a target Document (by exhibit number in its title) +* internal section refs -> an OC_SECTION annotation, or a heading-text offset +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +from opencontractserver.enrichment import constants as C +from opencontractserver.enrichment.extractor import Candidate + +# Exhibit number as it appears in EDGAR document titles, e.g. +# "Cerebras Systems Inc. S-1 (2024-09-30) - Exhibit 10.12(A): EX-10.12(A)". +_TITLE_EXHIBIT_RE = re.compile(r"Exhibit\s+(?P\d+\.\d+[A-Za-z()]*)", re.IGNORECASE) + + +@dataclass +class Resolution: + """A candidate paired with its resolved target (if any).""" + + candidate: Candidate + source_document_id: int + resolution_status: str + target_document_id: int | None = None + target_annotation_id: int | None = None + target_offset: int | None = None + canonical_key: str | None = None + confidence: float = 1.0 + normalized_data: dict = field(default_factory=dict) + + @property + def reference_type(self) -> str: + return self.candidate.reference_type + + +@dataclass +class SectionAnno: + """Minimal view of an OC_SECTION annotation for matching.""" + + id: int + raw_text: str + + +class ReferenceResolver: + """Resolve candidates against a fixed set of corpus documents.""" + + def __init__(self, documents) -> None: + # Build {exhibit_number -> document_id} from document titles. + self._exhibit_index: dict[str, int] = {} + for doc in documents: + for m in _TITLE_EXHIBIT_RE.finditer(doc.title or ""): + self._exhibit_index.setdefault(m.group("num").lower(), doc.id) + + # -- per-type ---------------------------------------------------------- # + + def resolve_law(self, cand: Candidate) -> Resolution: + return Resolution( + candidate=cand, + source_document_id=0, # filled by caller via _stamp_source + resolution_status=C.STATUS_EXTERNAL, + canonical_key=cand.canonical_key, + normalized_data=dict(cand.normalized_data), + ) + + def resolve_document(self, cand: Candidate, source_doc_id: int) -> Resolution: + num = (cand.normalized_data.get("exhibit_number") or "").lower() + target = self._exhibit_index.get(num) + if target is not None and target != source_doc_id: + status = C.STATUS_RESOLVED + else: + target = None + status = C.STATUS_UNRESOLVED + return Resolution( + candidate=cand, + source_document_id=source_doc_id, + resolution_status=status, + target_document_id=target, + canonical_key=cand.canonical_key, + normalized_data=dict(cand.normalized_data), + ) + + def resolve_section( + self, + cand: Candidate, + source_doc_id: int, + doc_text: str, + sections: list[SectionAnno] | None = None, + ) -> Resolution: + heading = (cand.normalized_data.get("heading") or "").strip() + res = Resolution( + candidate=cand, + source_document_id=source_doc_id, + resolution_status=C.STATUS_UNRESOLVED, + normalized_data=dict(cand.normalized_data), + ) + # 1. Prefer an OC_SECTION annotation whose text matches the heading. + for sec in sections or []: + if (sec.raw_text or "").strip().lower() == heading.lower(): + res.target_annotation_id = sec.id + res.resolution_status = C.STATUS_RESOLVED + return res + # 2. Fallback: locate the heading text within the document. + if heading: + idx = doc_text.lower().find(heading.lower(), cand.end) + if idx == -1: + # Backward reference (e.g. "as defined in 'Heading' above"): + # match the nearest *preceding* occurrence, not the first in + # the document — a duplicate heading in an earlier exhibit + # would otherwise mis-resolve to an unrelated section. + idx = doc_text.lower().rfind(heading.lower(), 0, cand.start) + if idx != -1: + res.target_offset = idx + res.resolution_status = C.STATUS_RESOLVED + res.confidence = 0.6 # heading-text match, not a real section anno + return res + + # -- dispatcher -------------------------------------------------------- # + + def resolve( + self, + cand: Candidate, + source_doc_id: int, + doc_text: str, + sections: list[SectionAnno] | None = None, + ) -> Resolution: + if cand.reference_type == C.REF_LAW: + res = self.resolve_law(cand) + res.source_document_id = source_doc_id + return res + if cand.reference_type == C.REF_DOCUMENT: + return self.resolve_document(cand, source_doc_id) + if cand.reference_type == C.REF_SECTION: + return self.resolve_section(cand, source_doc_id, doc_text, sections) + if cand.reference_type == C.REF_DEFINED_TERM: + # A definition site is self-contained: the mention IS the canonical + # target for ``term:``. Usage->definition linking is a future + # increment (volume/precision control needed for terms like "Company"). + return Resolution( + candidate=cand, + source_document_id=source_doc_id, + resolution_status=C.STATUS_RESOLVED, + canonical_key=cand.canonical_key, + normalized_data=dict(cand.normalized_data), + ) + # Any future types: carry through unresolved. + return Resolution( + candidate=cand, + source_document_id=source_doc_id, + resolution_status=C.STATUS_UNRESOLVED, + canonical_key=cand.canonical_key, + normalized_data=dict(cand.normalized_data), + ) diff --git a/opencontractserver/enrichment/services/__init__.py b/opencontractserver/enrichment/services/__init__.py new file mode 100644 index 000000000..484ea3f64 --- /dev/null +++ b/opencontractserver/enrichment/services/__init__.py @@ -0,0 +1,16 @@ +"""Service layer for corpus reference enrichment. + +Follows the repo-wide ``opencontractserver//services/`` convention: +user-context callers (GraphQL resolvers, agent tools, Celery adapters) reach +enrichment data through these services, never via inline Tier-0 ORM fusions. +""" + +from opencontractserver.enrichment.services.corpus_reference_service import ( + CorpusReferenceService, +) +from opencontractserver.enrichment.services.enrichment_service import EnrichmentService +from opencontractserver.enrichment.services.governance_graph_service import ( + GovernanceGraphService, +) + +__all__ = ["CorpusReferenceService", "EnrichmentService", "GovernanceGraphService"] diff --git a/opencontractserver/enrichment/services/corpus_reference_service.py b/opencontractserver/enrichment/services/corpus_reference_service.py new file mode 100644 index 000000000..ff869e802 --- /dev/null +++ b/opencontractserver/enrichment/services/corpus_reference_service.py @@ -0,0 +1,25 @@ +"""Read surface for ``CorpusReference`` rows. + +Visibility derives from corpus visibility — ``CorpusReference`` carries no +per-object guardian rows in v1. +""" + +from __future__ import annotations + +from opencontractserver.annotations.models import CorpusReference +from opencontractserver.corpuses.models import Corpus +from opencontractserver.shared.services.base import BaseService + + +class CorpusReferenceService(BaseService): + """Read surface for CorpusReference rows.""" + + @staticmethod + def visible_to_user(user): + return CorpusReference.objects.filter( + corpus__in=Corpus.objects.visible_to_user(user) + ) + + @classmethod + def for_corpus(cls, user, corpus_id: int): + return cls.visible_to_user(user).filter(corpus_id=corpus_id) diff --git a/opencontractserver/enrichment/services/enrichment_service.py b/opencontractserver/enrichment/services/enrichment_service.py new file mode 100644 index 000000000..e01ad950f --- /dev/null +++ b/opencontractserver/enrichment/services/enrichment_service.py @@ -0,0 +1,307 @@ +"""Orchestration service for corpus reference enrichment. + +``EnrichmentService`` is the single entry point the agent tools call: + +* ``scan`` — extract + resolve across the corpus, return an inventory, NO writes. +* ``apply`` — scan, then persist under an ``Analysis`` (approval-gated at the + tool layer). + +The read surface lives in +:mod:`opencontractserver.enrichment.services.corpus_reference_service`. +""" + +from __future__ import annotations + +import logging +from collections import Counter + +from django.contrib.auth import get_user_model +from django.utils import timezone + +from opencontractserver.analyzer.models import Analysis, Analyzer +from opencontractserver.annotations.models import Annotation, CorpusReference +from opencontractserver.constants.annotations import OC_SECTION_LABEL +from opencontractserver.corpuses.models import Corpus +from opencontractserver.corpuses.services.corpus_documents import CorpusDocumentService +from opencontractserver.enrichment import constants as C +from opencontractserver.enrichment.extractor import ReferenceExtractor +from opencontractserver.enrichment.resolver import ( + ReferenceResolver, + Resolution, + SectionAnno, +) +from opencontractserver.enrichment.writer import EnrichmentWriter +from opencontractserver.types.enums import JobStatus +from opencontractserver.utils.files import read_field_file_text +from opencontractserver.utils.frontend_paths import document_in_corpus_path + +logger = logging.getLogger(__name__) +User = get_user_model() + + +class EnrichmentService: + """Scan and apply reference enrichment for a corpus.""" + + # -- shared internals -------------------------------------------------- # + + def _load(self, corpus_id: int, creator_id: int): + user = User.objects.get(pk=creator_id) + # Visibility-scoped fetch: invisible and nonexistent corpora raise the + # same ``Corpus.DoesNotExist`` (no existence oracle for callers that + # pass arbitrary PKs). + corpus = Corpus.objects.visible_to_user(user).get(pk=corpus_id) + documents = list( + CorpusDocumentService.get_corpus_documents(user, corpus, include_caml=False) + ) + return user, corpus, documents + + def _sections_by_doc(self, documents) -> dict[int, list[SectionAnno]]: + """OC_SECTION annotations grouped by document — one query per corpus.""" + sections: dict[int, list[SectionAnno]] = {} + rows = Annotation.objects.filter( + document_id__in=[d.id for d in documents], + annotation_label__text=OC_SECTION_LABEL, + ).values_list("id", "raw_text", "document_id") + for pk, txt, doc_id in rows: + sections.setdefault(doc_id, []).append( + SectionAnno(id=pk, raw_text=txt or "") + ) + return sections + + def _resolutions(self, corpus, documents, types, user) -> list[Resolution]: + from opencontractserver.enrichment.authorities import authority_alias_registry + + wanted = set(types or C.DEFAULT_REFERENCE_TYPES) + resolver = ReferenceResolver(documents) + # The alias registry is corpus-data-driven (authority corpora declare + # their own aliases) and visibility-scoped to the run user. + extractor = ReferenceExtractor(authority_aliases=authority_alias_registry(user)) + sections_by_doc = self._sections_by_doc(documents) + resolutions: list[Resolution] = [] + for doc in documents: + try: + text = read_field_file_text(doc.txt_extract_file) + except Exception as exc: # isolate per-document failures + logger.warning( + "Enrichment: skip doc %s (text read failed: %s)", doc.id, exc + ) + continue + if not text: + continue + sections = sections_by_doc.get(doc.id, []) + # Authority documents know their own body of law — that context + # keys relative citations ("§ 251 of this title") in statute text. + meta = doc.custom_meta if isinstance(doc.custom_meta, dict) else {} + for cand in extractor.extract( + text, default_authority=meta.get("authority") + ): + if cand.reference_type not in wanted: + continue + resolutions.append(resolver.resolve(cand, doc.id, text, sections)) + return resolutions + + # -- public API -------------------------------------------------------- # + + def scan( + self, + *, + corpus_id: int, + creator_id: int, + types: list[str] | None = None, + sample_n: int = C.DEFAULT_SAMPLE_N, + ) -> dict: + user, corpus, documents = self._load(corpus_id, creator_id) + resolutions = self._resolutions(corpus, documents, types, user) + + by_type = Counter(r.reference_type for r in resolutions) + by_status = Counter(r.resolution_status for r in resolutions) + samples = [ + { + "reference_type": r.reference_type, + "raw_text": r.candidate.raw_text[:120], + "canonical_key": r.canonical_key, + "resolution_status": r.resolution_status, + "target_document_id": r.target_document_id, + "source_document_id": r.source_document_id, + } + for r in resolutions[:sample_n] + ] + unresolved = [ + { + "reference_type": r.reference_type, + "raw_text": r.candidate.raw_text[:120], + "source_document_id": r.source_document_id, + } + for r in resolutions + if r.resolution_status == C.STATUS_UNRESOLVED + ][:sample_n] + + return { + "corpus_id": corpus_id, + "documents_scanned": len(documents), + "total_candidates": len(resolutions), + "counts_by_type": dict(by_type), + "counts_by_status": dict(by_status), + "samples": samples, + "unresolved_samples": unresolved, + } + + def _get_analysis(self, corpus, creator_id: int) -> Analysis: + # task_name is unique and the startup sync may have created the row + # under id == task_name already — converge on it before creating. + analyzer = Analyzer.objects.filter(task_name=C.ENRICHMENT_ANALYZER_TASK).first() + if analyzer is None: + analyzer, _ = Analyzer.objects.get_or_create( + id=C.ENRICHMENT_ANALYZER_ID, + defaults={ + "task_name": C.ENRICHMENT_ANALYZER_TASK, + "description": C.ENRICHMENT_ANALYZER_TITLE, + "creator_id": creator_id, + }, + ) + # Two provenance paths: when ``apply`` runs via the analyzer framework + # (Celery), the @corpus_analyzer_task wrapper owns the Analysis and + # drives RUNNING -> COMPLETED/FAILED. This branch serves the direct + # agent-tool / service call, which runs synchronously inside ``apply`` + # — the Analysis starts RUNNING and is set to COMPLETED on success or + # FAILED on exception by ``apply()``. + return Analysis.objects.create( + analyzer=analyzer, + analyzed_corpus=corpus, + creator_id=creator_id, + status=JobStatus.RUNNING.value, + ) + + def apply( + self, + *, + corpus_id: int, + creator_id: int, + types: list[str] | None = None, + analysis: Analysis | None = None, + ) -> dict: + """Persist the corpus's reference web. + + ``analysis`` lets the analyzer-framework adapter attach the run to the + framework-created ``Analysis``; when omitted (agent tool / direct + service call) a provenance ``Analysis`` is created here. + """ + user, corpus, documents = self._load(corpus_id, creator_id) + resolutions = self._resolutions(corpus, documents, types, user) + if analysis is None: + analysis = self._get_analysis(corpus, creator_id) + writer = EnrichmentWriter(corpus, creator_id, analysis=analysis) + try: + res = writer.write(resolutions) + except Exception: + analysis.status = JobStatus.FAILED.value + analysis.save(update_fields=["status"]) + raise + analysis.status = JobStatus.COMPLETED.value + analysis.save(update_fields=["status"]) + link = self._link_external(user, corpus) + return { + "corpus_id": corpus_id, + "analysis_id": analysis.id, + "documents_scanned": len(documents), + "total_candidates": len(resolutions), + "annotations_created": res.annotations_created, + "relationships_created": res.relationships_created, + "references_created": res.references_created, + "document_relationships_created": res.document_relationships_created, + "law_references_linked": link["law_references_linked"], + } + + # -- cross-corpus linking ----------------------------------------------- # + + def link_external_references(self, *, corpus_id: int, creator_id: int) -> dict: + """Upgrade EXTERNAL law references to RESOLVED cross-corpus links. + + Re-runnable: as new authority corpora are bootstrapped, another pass + links any still-external citations whose canonical keys now have a + visible authority document. + """ + user = User.objects.get(pk=creator_id) + # Same visibility-scoped semantics as ``_load`` (uniform DoesNotExist + # for invisible vs nonexistent). + corpus = Corpus.objects.visible_to_user(user).get(pk=corpus_id) + return self._link_external(user, corpus) + + def _link_external(self, user, corpus) -> dict: + from opencontractserver.documents.models import Document, DocumentPath + from opencontractserver.enrichment.authorities import find_authority_target + + refs = ( + CorpusReference.objects.filter( + corpus=corpus, + reference_type=C.REF_LAW, + target_document__isnull=True, + ) + .exclude(canonical_key=None) + .select_related("source_annotation") + ) + target_cache: dict[str, Document | None] = {} + corpus_cache: dict[int, Corpus] = {} + now = timezone.now() + updated_refs: list[CorpusReference] = [] + updated_mentions: list[Annotation] = [] + # First pass: build target_cache (deduped by canonical key). + for ref in refs: + key = ref.canonical_key + if not key: + continue + if key not in target_cache: + target_cache[key] = find_authority_target(key, user) + # Batch-fetch corpus membership for all resolved targets in one query + # instead of one per target (avoids N+1 on large corpora). + resolved_target_ids = {t.id for t in target_cache.values() if t is not None} + path_corpus_cache: dict[int, int | None] = dict( + DocumentPath.objects.filter( + document_id__in=resolved_target_ids, + is_current=True, + is_deleted=False, + ).values_list("document_id", "corpus_id") + ) + for ref in refs: + key = ref.canonical_key + if not key: # queryset excludes None; guard for type-narrowing + continue + target = target_cache.get(key) + if target is None: + continue + target_corpus_id = path_corpus_cache.get(target.id) + ref.target_document = target + ref.target_corpus_id = target_corpus_id + ref.resolution_status = C.STATUS_RESOLVED + # bulk_update bypasses auto_now — stamp ``modified`` explicitly. + ref.modified = now + updated_refs.append(ref) + if target_corpus_id is not None: + if target_corpus_id not in corpus_cache: + corpus_cache[target_corpus_id] = Corpus.objects.select_related( + "creator" + ).get(pk=target_corpus_id) + target_corpus = corpus_cache[target_corpus_id] + # Canonical slug path into the authority corpus — the only + # shape the frontend router serves (anything else 404s). + link_url = document_in_corpus_path( + corpus_creator_slug=target_corpus.creator.slug, + corpus_slug=target_corpus.slug, + document_slug=target.slug, + ) + if link_url: + mention = ref.source_annotation + mention.link_url = link_url + mention.modified = now + updated_mentions.append(mention) + + # Two queries instead of O(N) row-by-row saves (corpora carry + # hundreds-to-thousands of law references at demo scale). + if updated_refs: + CorpusReference.objects.bulk_update( + updated_refs, + ["target_document", "target_corpus", "resolution_status", "modified"], + ) + if updated_mentions: + Annotation.objects.bulk_update(updated_mentions, ["link_url", "modified"]) + return {"corpus_id": corpus.id, "law_references_linked": len(updated_refs)} diff --git a/opencontractserver/enrichment/services/governance_graph_service.py b/opencontractserver/enrichment/services/governance_graph_service.py new file mode 100644 index 000000000..97680896b --- /dev/null +++ b/opencontractserver/enrichment/services/governance_graph_service.py @@ -0,0 +1,248 @@ +"""Build the corpus-scoped governance graph (the in-app reference web). + +Mirrors ``demo/export_governance_graph.py``, restricted to one source corpus +and enforced through the permission model: + +* **nodes** — documents (filing primaries, exhibits, statute sections) plus + "ghost" nodes for law citations with no *visible* target document. +* **edges** — resolved LAW links (possibly cross-corpus), EXTERNAL law + citations (rolled up to their section root), and ``DocumentRelationship`` + rows — weighted by mention count. + +Visibility rules: + +* The source corpus must be READ-visible or the build returns ``None``. +* Reference rows are corpus-as-gate (``CorpusReferenceService``), but every + *document* surfaced as a node must itself be READ-visible: invisible source + documents drop their edges entirely; invisible target documents degrade to + external ghost nodes so titles never leak. +* Only READ-visible target corpora are listed in ``corpora``. + +The service returns plain data keyed by raw PKs / canonical keys; the GraphQL +resolver owns the relay global-id encoding. Node endpoints are ``("doc", pk)`` +or ``("key", canonical_key)`` tuples. +""" + +from __future__ import annotations + +from collections import Counter +from typing import Any + +from opencontractserver.corpuses.models import Corpus +from opencontractserver.documents.models import Document +from opencontractserver.documents.services import DocumentRelationshipService +from opencontractserver.enrichment import constants as C +from opencontractserver.enrichment.services.corpus_reference_service import ( + CorpusReferenceService, +) +from opencontractserver.shared.services.base import BaseService + +Endpoint = tuple[str, Any] # ("doc", pk) | ("key", canonical_key) + + +class GovernanceGraphService: + """Assemble the node-link governance graph for one corpus.""" + + @classmethod + def build( + cls, + user, + corpus_pk: int, + node_cap: int, + *, + request: Any = None, + ) -> dict | None: + """Return the graph as plain data, or ``None`` for an invisible corpus. + + Counts (``document_count`` / ``external_key_count`` / ``edge_count`` / + ``mention_count``) describe the FULL visible graph; ``truncated`` + signals that the node/edge lists were capped to ``node_cap`` by + degree rank. + """ + from opencontractserver.enrichment.authorities import candidate_keys + + corpus = ( + BaseService.filter_visible(Corpus, user, request=request) + .filter(id=corpus_pk) + .first() + ) + if corpus is None: + return None + + ref_rows = list( + CorpusReferenceService.for_corpus(user, corpus_pk) + .filter(reference_type=C.REF_LAW) + .exclude(canonical_key=None) + .values_list( + "source_annotation__document_id", + "target_document_id", + "target_corpus_id", + "canonical_key", + ) + ) + rel_rows = list( + DocumentRelationshipService.get_visible_relationships( + user, corpus_id=corpus_pk, request=request + ).values_list("source_document_id", "target_document_id") + ) + + # Every document surfaced as a node must itself be READ-visible. + # (Relationship endpoints already are — the service enforces it.) + candidate_doc_ids = {src for src, _t, _c, _k in ref_rows if src} | { + tgt for _s, tgt, _c, _k in ref_rows if tgt + } + visible_doc_ids = set( + BaseService.filter_visible(Document, user, request=request) + .filter(id__in=candidate_doc_ids) + .values_list("id", flat=True) + ) + + edge_weight: Counter = Counter() # (src_endpoint, tgt_endpoint, type) -> w + doc_corpus: dict[int, int | None] = {} + target_corpus_ids: set[int] = set() + + for src, tgt, tgt_corpus, key in ref_rows: + if not src or src not in visible_doc_ids: + continue # invisible source: no edge, no title leak + doc_corpus.setdefault(src, corpus_pk) + if tgt == src: + continue # self-citation ("this section") — no edge to draw + if tgt and tgt in visible_doc_ids: + edge_weight[(("doc", src), ("doc", tgt), C.GRAPH_EDGE_LAW)] += 1 + doc_corpus.setdefault(tgt, tgt_corpus) + if tgt_corpus: + target_corpus_ids.add(tgt_corpus) + else: + # Unresolved — or resolved but invisible to this user: degrade + # to a ghost keyed by the citation's section root so subsection + # variants share one node. + root = candidate_keys(key)[-1] + edge_weight[ + (("doc", src), ("key", root), C.GRAPH_EDGE_LAW_EXTERNAL) + ] += 1 + + for src, tgt in rel_rows: + edge_weight[(("doc", src), ("doc", tgt), C.GRAPH_EDGE_DOCUMENT)] += 1 + doc_corpus.setdefault(src, corpus_pk) + doc_corpus.setdefault(tgt, corpus_pk) + + degree: Counter = Counter() + for (src_ep, tgt_ep, _t), w in edge_weight.items(): + degree[src_ep] += w + degree[tgt_ep] += w + + node_doc_ids = {val for kind, val in degree if kind == "doc"} + ghost_keys = {val for kind, val in degree if kind == "key"} + + # These node objects are read only for ``title``/``custom_meta``, so + # drop the visibility manager's default JOINs/prefetches before + # narrowing with ``.only()``. ``select_related(None)`` is required: + # the manager select_relates ``parent`` (among others), and Django + # forbids a field being both select_related and deferred by ``.only()``. + docs = { + d.id: d + for d in BaseService.filter_visible(Document, user, request=request) + .filter(id__in=node_doc_ids) + .select_related(None) + .prefetch_related(None) + .only("id", "title", "custom_meta") + } + + # Corpora the graph reaches: the queried corpus plus READ-visible + # resolved-target corpora. An invisible target corpus is never listed + # (its documents degraded to ghosts above, so it has no nodes either). + listed_corpora = {corpus.id: corpus} + if target_corpus_ids - {corpus.id}: + for c in BaseService.filter_visible(Corpus, user, request=request).filter( + id__in=target_corpus_ids - {corpus.id} + ): + listed_corpora[c.id] = c + + corpora = [ + { + "corpus_pk": c.id, + "title": c.title, + # A corpus cited by the graph's own references is an authority + # (statutes citing statutes classify the queried corpus too). + "kind": ( + C.GRAPH_CORPUS_AUTHORITY + if c.id in target_corpus_ids + else C.GRAPH_CORPUS_FILING + ), + } + for c in listed_corpora.values() + ] + + doc_nodes = [] + for doc_pk in node_doc_ids: + doc = docs.get(doc_pk) + if doc is None: + continue + meta = doc.custom_meta if isinstance(doc.custom_meta, dict) else {} + if meta.get("canonical_key"): + kind = C.GRAPH_NODE_STATUTE + elif "exhibit" in (doc.title or "").lower(): + kind = C.GRAPH_NODE_EXHIBIT + else: + kind = C.GRAPH_NODE_PRIMARY + node_corpus = doc_corpus.get(doc_pk) + doc_nodes.append( + { + "doc_pk": doc_pk, + "title": doc.title, + "kind": kind, + "corpus_pk": node_corpus if node_corpus in listed_corpora else None, + "authority": meta.get("authority"), + "degree": degree[("doc", doc_pk)], + } + ) + ghost_nodes = [ + { + "key": key, + "authority": key.split(":", 1)[0], + "degree": degree[("key", key)], + } + for key in sorted(ghost_keys) + ] + + # Full-graph stats, then degree-ranked truncation for the payload. + document_count = len(doc_nodes) + external_key_count = len(ghost_nodes) + edge_count = len(edge_weight) + mention_count = sum(edge_weight.values()) + + truncated = (document_count + external_key_count) > node_cap + if truncated: + ranked: list[Endpoint] = [ep for ep, _w in degree.most_common()][:node_cap] + kept = set(ranked) + doc_nodes = [n for n in doc_nodes if ("doc", n["doc_pk"]) in kept] + ghost_nodes = [n for n in ghost_nodes if ("key", n["key"]) in kept] + edge_weight = Counter( + { + (s, t, ty): w + for (s, t, ty), w in edge_weight.items() + if s in kept and t in kept + } + ) + + edges = [ + {"source": src_ep, "target": tgt_ep, "edge_type": etype, "weight": w} + for (src_ep, tgt_ep, etype), w in sorted( + edge_weight.items(), + # Endpoints mix int and str payloads — stringify for a stable, + # type-safe ordering. + key=lambda kv: (str(kv[0][0]), str(kv[0][1]), kv[0][2]), + ) + ] + + return { + "corpora": corpora, + "doc_nodes": doc_nodes, + "ghost_nodes": ghost_nodes, + "edges": edges, + "document_count": document_count, + "external_key_count": external_key_count, + "edge_count": edge_count, + "mention_count": mention_count, + "truncated": truncated, + } diff --git a/opencontractserver/enrichment/writer.py b/opencontractserver/enrichment/writer.py new file mode 100644 index 000000000..25094ccb4 --- /dev/null +++ b/opencontractserver/enrichment/writer.py @@ -0,0 +1,204 @@ +"""Persist resolved references as annotations / relationships / CorpusReferences. + +All writes for one run happen inside a single transaction under one ``Analysis`` +(provenance). Creation is idempotent: re-running enriches only newly-found +references (mentions are deduped by (document, span, label); ``CorpusReference`` +rows by their unique (source_annotation, reference_type, canonical_key) guard). +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field + +from django.db import transaction + +from opencontractserver.annotations.models import ( + RELATIONSHIP_LABEL, + SPAN_LABEL, + Annotation, + CorpusReference, + Relationship, +) +from opencontractserver.documents.models import Document, DocumentRelationship +from opencontractserver.enrichment import constants as C +from opencontractserver.enrichment.resolver import Resolution +from opencontractserver.utils.frontend_paths import document_in_corpus_path + +logger = logging.getLogger(__name__) + + +@dataclass +class WriteResult: + annotations_created: int = 0 + relationships_created: int = 0 + references_created: int = 0 + document_relationships_created: int = 0 + annotation_ids: list[int] = field(default_factory=list) + reference_ids: list[int] = field(default_factory=list) + + +class EnrichmentWriter: + """Turn :class:`Resolution` objects into durable rows.""" + + def __init__(self, corpus, creator_id: int, analysis=None) -> None: + self.corpus = corpus + self.creator_id = creator_id + self.analysis = analysis + self._label_cache: dict[tuple[str, str], object] = {} + # Target-document slugs, prefetched once per write() so mention links + # can carry the canonical /d/ path without a per-mention query. + self._doc_slugs: dict[int, str | None] = {} + + def _label(self, text: str, label_type: str): + key = (text, label_type) + if key not in self._label_cache: + self._label_cache[key] = self.corpus.ensure_label_and_labelset( + label_text=text, creator_id=self.creator_id, label_type=label_type + ) + return self._label_cache[key] + + def _get_or_create_mention(self, res: Resolution) -> tuple[Annotation, bool]: + cand = res.candidate + label = self._label(C.LABEL_FOR_TYPE[res.reference_type], SPAN_LABEL) + json_pos = {"start": cand.start, "end": cand.end} + # Dedup by (document, label, span START): the span END can legitimately + # move when the alias registry grows (a longer authority alias wins + # longest-first matching), and that must not duplicate the mention. + existing = Annotation.objects.filter( + document_id=res.source_document_id, + corpus=self.corpus, + annotation_label=label, + json__start=cand.start, + ).first() + if existing is not None: + return existing, False + + data = dict(cand.normalized_data) + if res.canonical_key: + data["canonical_key"] = res.canonical_key + link_url = None + if res.target_document_id: + # Canonical slug path — the only shape the frontend router serves + # (any other form falls into the catch-all 404). + link_url = document_in_corpus_path( + corpus_creator_slug=self.corpus.creator.slug, + corpus_slug=self.corpus.slug, + document_slug=self._doc_slugs.get(res.target_document_id), + ) + + ann = Annotation( + raw_text=cand.raw_text, + page=1, + json=json_pos, + annotation_label=label, + document_id=res.source_document_id, + corpus=self.corpus, + creator_id=self.creator_id, + annotation_type=SPAN_LABEL, + structural=False, + data=data or None, + link_url=link_url, + analysis=self.analysis, + ) + ann.save() + return ann, True + + def write(self, resolutions: list[Resolution]) -> WriteResult: + result = WriteResult() + rel_label = self._label(C.LABEL_RELATIONSHIP, RELATIONSHIP_LABEL) + + target_ids = {r.target_document_id for r in resolutions if r.target_document_id} + self._doc_slugs = dict( + Document.objects.filter(id__in=target_ids).values_list("id", "slug") + ) + + with transaction.atomic(): + for res in resolutions: + mention, created = self._get_or_create_mention(res) + if created: + result.annotations_created += 1 + result.annotation_ids.append(mention.pk) + + # Within-document section link -> Relationship. + if ( + res.reference_type == C.REF_SECTION + and res.target_annotation_id is not None + ): + self._ensure_section_relationship(mention, res, rel_label, result) + continue + + # Everything else with a cross-boundary/external nature -> + # CorpusReference. (Offset-only section refs also land here so + # the resolved target offset is recorded.) + self._ensure_corpus_reference(mention, res, result) + + # Resolved doc->doc refs additionally roll up to a document- + # level edge: DocumentRelationship is what the corpus document + # graph renders (documents = nodes, relationships = edges). + if ( + res.reference_type == C.REF_DOCUMENT + and res.target_document_id is not None + ): + self._ensure_document_relationship(res, rel_label, result) + + return result + + def _ensure_document_relationship(self, res, rel_label, result): + _, created = DocumentRelationship.objects.get_or_create( + source_document_id=res.source_document_id, + target_document_id=res.target_document_id, + annotation_label=rel_label, + relationship_type=C.DOC_REL_RELATIONSHIP, + defaults={ + "corpus": self.corpus, + "creator_id": self.creator_id, + "data": {"analysis_id": self.analysis.id if self.analysis else None}, + }, + ) + if created: + result.document_relationships_created += 1 + + def _ensure_section_relationship(self, mention, res, rel_label, result): + already = Relationship.objects.filter( + relationship_label=rel_label, + document_id=res.source_document_id, + corpus=self.corpus, + source_annotations=mention, + target_annotations__id=res.target_annotation_id, + ).exists() + if already: + return + rel = Relationship.objects.create( + relationship_label=rel_label, + document_id=res.source_document_id, + corpus=self.corpus, + creator_id=self.creator_id, + analysis=self.analysis, + ) + rel.source_annotations.set([mention]) + rel.target_annotations.set([res.target_annotation_id]) + result.relationships_created += 1 + + def _ensure_corpus_reference(self, mention, res, result): + normalized = dict(res.normalized_data) + if res.target_offset is not None: + normalized["target_offset"] = res.target_offset + ref, created = CorpusReference.objects.get_or_create( + source_annotation=mention, + reference_type=res.reference_type, + canonical_key=res.canonical_key, + defaults={ + "corpus": self.corpus, + "target_annotation_id": res.target_annotation_id, + "target_document_id": res.target_document_id, + "resolution_status": res.resolution_status, + "confidence": res.confidence, + "normalized_data": normalized or None, + "created_by_analysis": self.analysis, + "creator_id": self.creator_id, + }, + ) + if created: + result.references_created += 1 + result.reference_ids.append(ref.pk) diff --git a/opencontractserver/llms/tools/core_tools/__init__.py b/opencontractserver/llms/tools/core_tools/__init__.py index 1d74064a0..327a5ffca 100644 --- a/opencontractserver/llms/tools/core_tools/__init__.py +++ b/opencontractserver/llms/tools/core_tools/__init__.py @@ -37,6 +37,12 @@ aread_corpus_caml_article, ) from .corpus_branding import aregenerate_corpus_icon # noqa: F401 +from .corpus_references import ( # noqa: F401 + aapply_corpus_reference_enrichment, + apply_corpus_reference_enrichment, + ascan_corpus_references, + scan_corpus_references, +) from .descriptions import ( # noqa: F401 aget_corpus_description, aget_document_description, @@ -191,6 +197,10 @@ "AnnotationItem", "aadd_annotations_from_exact_strings", "add_annotations_from_exact_strings", + "ascan_corpus_references", + "scan_corpus_references", + "aapply_corpus_reference_enrichment", + "apply_corpus_reference_enrichment", "aduplicate_annotations_with_label", "duplicate_annotations_with_label", # CAML article review (Readme.CAML) diff --git a/opencontractserver/llms/tools/core_tools/corpus_references.py b/opencontractserver/llms/tools/core_tools/corpus_references.py new file mode 100644 index 000000000..6413cdc0d --- /dev/null +++ b/opencontractserver/llms/tools/core_tools/corpus_references.py @@ -0,0 +1,74 @@ +"""Agent tools for corpus reference enrichment. + +Two-phase, CAML-style: ``scan`` (read-only inventory) then +``apply`` (approval-gated write). ``corpus_id`` and ``creator_id`` are injected +from the agent context and hidden from the LLM. +""" + +from __future__ import annotations + +from ._helpers import _db_sync_to_async + + +def scan_corpus_references( + *, + corpus_id: int, + creator_id: int, + types: list[str] | None = None, + sample_n: int = 10, +) -> dict: + """Inventory explicit references in a corpus WITHOUT writing anything. + + Detects law citations (e.g. "Section 145 of the Delaware General Corporation + Law"), document/exhibit references, and internal section references, then + reports counts by type and sample resolved/unresolved candidates so the user + can review before applying enrichment. + """ + from opencontractserver.enrichment.services import EnrichmentService + + return EnrichmentService().scan( + corpus_id=corpus_id, creator_id=creator_id, types=types, sample_n=sample_n + ) + + +def apply_corpus_reference_enrichment( + *, + corpus_id: int, + creator_id: int, + types: list[str] | None = None, +) -> dict: + """Create reference annotations, relationships, and cross-references. + + Annotates every detected reference, links internal section references via + relationships, resolves document/exhibit references to in-app links, and + records law citations as cross-corpus-trackable stubs. Idempotent: + re-running enriches only newly-found references. + """ + from opencontractserver.enrichment.services import EnrichmentService + + return EnrichmentService().apply( + corpus_id=corpus_id, creator_id=creator_id, types=types + ) + + +async def ascan_corpus_references( + *, + corpus_id: int, + creator_id: int, + types: list[str] | None = None, + sample_n: int = 10, +) -> dict: + return await _db_sync_to_async(scan_corpus_references)( + corpus_id=corpus_id, creator_id=creator_id, types=types, sample_n=sample_n + ) + + +async def aapply_corpus_reference_enrichment( + *, + corpus_id: int, + creator_id: int, + types: list[str] | None = None, +) -> dict: + return await _db_sync_to_async(apply_corpus_reference_enrichment)( + corpus_id=corpus_id, creator_id=creator_id, types=types + ) diff --git a/opencontractserver/llms/tools/tool_registry.py b/opencontractserver/llms/tools/tool_registry.py index 7c2bbd276..20b2d07d1 100644 --- a/opencontractserver/llms/tools/tool_registry.py +++ b/opencontractserver/llms/tools/tool_registry.py @@ -366,6 +366,36 @@ def to_dict(self) -> dict: # ------------------------------------------------------------------------- # CORPUS TOOLS # ------------------------------------------------------------------------- + ToolDefinition( + name="scan_corpus_references", + description=( + "Inventory explicit references across the corpus (law citations such " + "as 'Section 145 of the Delaware General Corporation Law', " + "document/exhibit references, and internal section references) " + "WITHOUT writing anything. Returns counts by type and sample " + "resolved/unresolved candidates. Use before applying enrichment." + ), + category=ToolCategory.CORPUS, + requires_corpus=True, + parameters=( + ("types", "Optional list of reference types to scan", False), + ("sample_n", "Number of sample candidates to return", False), + ), + ), + ToolDefinition( + name="apply_corpus_reference_enrichment", + description=( + "Crawl the corpus and create reference annotations, relationships, " + "and cross-references for every explicit reference found (law " + "citations, document/exhibit links, internal section links). " + "Idempotent: re-running enriches only newly-found references." + ), + category=ToolCategory.CORPUS, + requires_corpus=True, + requires_approval=True, + requires_write_permission=True, + parameters=(("types", "Optional list of reference types to enrich", False),), + ), ToolDefinition( name="get_corpus_description", description="Retrieve the latest markdown description for this corpus.", @@ -1313,6 +1343,7 @@ def _populate(self) -> None: aadd_annotations_from_exact_strings, aadd_document_note, aapply_caml_article_edit, + aapply_corpus_reference_enrichment, acreate_document_index, acreate_markdown_link, acreate_or_update_text_document, @@ -1339,6 +1370,7 @@ def _populate(self) -> None: aregenerate_corpus_icon, arename_document, ascan_and_annotate_pii, + ascan_corpus_references, asearch_corpus_documents, asearch_document_notes, asearch_exact_text_as_sources, @@ -1410,6 +1442,11 @@ def _populate(self) -> None: (), ), "scan_and_annotate_pii": (ascan_and_annotate_pii, ()), + "scan_corpus_references": (ascan_corpus_references, ()), + "apply_corpus_reference_enrichment": ( + aapply_corpus_reference_enrichment, + (), + ), "create_document_index": (acreate_document_index, ()), # Corpus tools "get_corpus_description": (aget_corpus_description, ()), diff --git a/opencontractserver/shared/decorators.py b/opencontractserver/shared/decorators.py index 97b55c4c0..c9572b27f 100644 --- a/opencontractserver/shared/decorators.py +++ b/opencontractserver/shared/decorators.py @@ -11,6 +11,7 @@ from celery.exceptions import Retry from django.core.exceptions import ObjectDoesNotExist from django.db import close_old_connections, transaction +from django.utils import timezone from plasmapdf.models.PdfDataLayer import build_translation_layer from opencontractserver.analyzer.models import Analysis @@ -18,7 +19,7 @@ from opencontractserver.corpuses.models import Corpus from opencontractserver.documents.models import Document, DocumentAnalysisRow from opencontractserver.types.dicts import TextSpan -from opencontractserver.types.enums import LabelType +from opencontractserver.types.enums import JobStatus, LabelType from opencontractserver.utils.compact_pawls import expand_pawls_pages from opencontractserver.utils.etl import is_dict_instance_of_typed_dict @@ -359,6 +360,120 @@ def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: return decorator +def corpus_analyzer_task( + max_retries: Union[int, None] = None, + input_schema: dict[str, Any] | None = None, +) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + """ + Decorator for Celery tasks that analyze a whole corpus in ONE run. + + The corpus-scoped sibling of :func:`doc_analyzer_task`. Where the doc + framework fans out per document and persists span annotations from the + return value, a corpus analyzer needs corpus-wide context (cross-document + indexes, cross-corpus resolution) and owns its own writes — annotations, + relationships, ``CorpusReference`` rows, whatever the job produces. The + framework provides identity, lifecycle, and surfacing: the task syncs to + an ``Analyzer`` row at startup, runs via ``run_task_name_analyzer`` / + ``CorpusAction`` like any task-based analyzer, and this wrapper manages + the ``Analysis`` lifecycle (RUNNING/COMPLETED/FAILED, timestamps, + ``result_message`` / ``error_message``). + + The wrapped function is called with keyword arguments ``corpus_id`` and + ``analysis_id`` (plus any ``analysis_input_data`` passthrough kwargs) and + must return a JSON-serializable ``dict`` of result counts/details, which + is stored on ``Analysis.result_message``. + + :param max_retries: Optional maximum number of retries for the Celery task. + :param input_schema: Optional dictionary defining input schema for the task. + :return: The decorator function. + """ + + def decorator(func: Callable[..., Any]) -> Callable[..., Any]: + @shared_task(bind=True, max_retries=max_retries) + @wraps(func) + def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any: + analysis_id = kwargs.get("analysis_id") + if not analysis_id: + raise ValueError("analysis_id is required for corpus_analyzer_task") + + try: + analysis = Analysis.objects.get(id=analysis_id) + except ObjectDoesNotExist: + raise ValueError(f"Analysis with id {analysis_id} does not exist") + + corpus_id = kwargs.get("corpus_id") or analysis.analyzed_corpus_id + if not corpus_id: + raise ValueError( + "corpus_analyzer_task requires a corpus: pass corpus_id or " + "link the Analysis to a corpus" + ) + # Defensive depth: the corpus must be visible to the analysis + # creator, not merely exist — a task must not operate on a corpus + # the analysis owner couldn't see. Same error either way (no + # existence oracle). + from opencontractserver.shared.services.base import BaseService + + corpus_visible = ( + BaseService.filter_visible(Corpus, analysis.creator) + .filter(id=corpus_id) + .exists() + ) + if not corpus_visible: + raise ValueError(f"Corpus with id {corpus_id} does not exist") + kwargs["corpus_id"] = corpus_id + + analysis.status = JobStatus.RUNNING.value + analysis.analysis_started = timezone.now() + analysis.save() + + try: + result = func(*args, **kwargs) + + if not isinstance(result, dict): + raise ValueError( + "corpus_analyzer_task functions must return a dict of " + "result counts/details" + ) + + analysis.status = JobStatus.COMPLETED.value + analysis.analysis_completed = timezone.now() + analysis.error_message = None + analysis.result_message = json.dumps(result, default=str) + analysis.save() + return result + + except Retry: + # ``Retry`` extends ``Exception`` — re-raise untouched + # (mirroring doc_analyzer_task) so a transient retry does not + # permanently stamp the Analysis FAILED before the retry runs. + logger.info( + f"Retry in corpus_analyzer_task {func.__name__} for " + f"analysis {analysis_id}" + ) + raise + + except Exception as e: + logger.error( + f"corpus_analyzer_task {func.__name__} failed for analysis " + f"{analysis_id}: {e}\n{traceback.format_exc()}" + ) + analysis.status = JobStatus.FAILED.value + analysis.analysis_completed = timezone.now() + analysis.error_message = str(e) + analysis.result_message = None + analysis.save() + raise + + # Identify tasks decorated by this wrapper + wrapper.is_corpus_analyzer_task = True + # Attach the input schema for runtime retrieval + wrapper._oc_corpus_analyzer_input_schema = input_schema + + return wrapper + + return decorator + + def async_doc_analyzer_task( max_retries: Union[int, None] = None, input_schema: dict[str, Any] | None = None, diff --git a/opencontractserver/tasks/__init__.py b/opencontractserver/tasks/__init__.py index 98bad0d65..a1f407f21 100644 --- a/opencontractserver/tasks/__init__.py +++ b/opencontractserver/tasks/__init__.py @@ -4,6 +4,7 @@ cleanup_orphaned_document_blobs_task, delete_analysis_and_annotations_task, ) +from .corpus_analysis_tasks import corpus_reference_enrichment from .corpus_tasks import * # noqa: F403, F401 from .data_extract_tasks import * # noqa: F403, F401 from .doc_analysis_tasks import * # noqa: F403, F401 @@ -33,6 +34,7 @@ # by default search tasks in package.tasks __all__ = [ + "corpus_reference_enrichment", "run_extract", "package_annotated_docs", "burn_doc_annotations", diff --git a/opencontractserver/tasks/corpus_analysis_tasks.py b/opencontractserver/tasks/corpus_analysis_tasks.py new file mode 100644 index 000000000..f80dbd911 --- /dev/null +++ b/opencontractserver/tasks/corpus_analysis_tasks.py @@ -0,0 +1,63 @@ +"""Corpus-scoped analyzer tasks (@corpus_analyzer_task). + +These run ONCE per Analysis with the whole corpus in scope, own their writes, +and are surfaced through the analyzer framework (auto-synced Analyzer rows, +run_task_name_analyzer dispatch, CorpusAction triggers). +""" + +import logging + +from opencontractserver.shared.decorators import corpus_analyzer_task + +logger = logging.getLogger(__name__) + + +@corpus_analyzer_task( + input_schema={ + "type": "object", + "properties": { + "types": { + "type": "array", + "items": { + "type": "string", + "enum": ["LAW", "DOCUMENT", "SECTION", "DEFINED_TERM"], + }, + "description": ( + "Reference types to enrich. Defaults to LAW, DOCUMENT and " + "SECTION; DEFINED_TERM is opt-in (precision/volume)." + ), + } + }, + } +) +def corpus_reference_enrichment( + *args, + corpus_id: int, + analysis_id: int, + types: list[str] | None = None, + **kwargs, +) -> dict: + """Crawl the corpus and persist its reference web. + + Detects explicit references (law citations, document/exhibit references, + internal section references; defined terms opt-in), annotates every + mention, links section references via relationships, rolls resolved + document references up to document-graph edges, records law citations as + canonical-key cross-references, and resolves them against visible + authority corpora. Idempotent: re-running enriches only newly-found + references. + """ + from opencontractserver.analyzer.models import Analysis + from opencontractserver.enrichment.services import EnrichmentService + + analysis = Analysis.objects.get(id=analysis_id) + if analysis.creator_id is None: + raise ValueError( + f"Analysis {analysis_id} has no creator; cannot run enrichment" + ) + return EnrichmentService().apply( + corpus_id=corpus_id, + creator_id=analysis.creator_id, + types=types, + analysis=analysis, + ) diff --git a/opencontractserver/tasks/corpus_tasks.py b/opencontractserver/tasks/corpus_tasks.py index e8030ae21..08eab1352 100644 --- a/opencontractserver/tasks/corpus_tasks.py +++ b/opencontractserver/tasks/corpus_tasks.py @@ -35,6 +35,7 @@ from opencontractserver.types.enums import PermissionTypes from opencontractserver.utils.analysis import create_and_setup_analysis from opencontractserver.utils.celery_tasks import ( + get_corpus_analyzer_task_by_name, get_doc_analyzer_task_by_name, get_task_by_name, ) @@ -58,6 +59,31 @@ def run_task_name_analyzer( analyzer = analysis.analyzer task_name = analyzer.task_name + + # Corpus-scoped analyzers run ONCE per analysis (no per-doc fan-out); + # the @corpus_analyzer_task wrapper owns the Analysis lifecycle. + corpus_task_func = get_corpus_analyzer_task_by_name(task_name) + if corpus_task_func is not None: + if analysis.analyzed_corpus is None: + raise ValueError( + f"Corpus analyzer {task_name} requires the Analysis to be " + "linked to a corpus" + ) + if document_ids: + logger.warning( + f"Corpus analyzer {task_name} operates corpus-wide; ignoring " + f"document_ids subset ({len(document_ids)} ids)" + ) + corpus_id = analysis.analyzed_corpus.id + transaction.on_commit( + lambda: corpus_task_func.si( # type: ignore[attr-defined] + analysis_id=analysis.id, + corpus_id=corpus_id, + **(analysis_input_data if analysis_input_data else {}), + ).apply_async() + ) + return + task_func = get_doc_analyzer_task_by_name(task_name) if task_func is None: diff --git a/opencontractserver/tests/test_analyzer_app_coverage.py b/opencontractserver/tests/test_analyzer_app_coverage.py index aa5e6a788..923a4487a 100644 --- a/opencontractserver/tests/test_analyzer_app_coverage.py +++ b/opencontractserver/tests/test_analyzer_app_coverage.py @@ -224,12 +224,18 @@ def test_swallows_runtime_errors_silently(self) -> None: self.assertEqual(warnings, []) def test_skips_non_doc_analyzer_tasks(self) -> None: + # A plain task carries neither analyzer marker. Set both markers + # explicitly: bare MagicMock attributes are truthy, which would + # misclassify the task as a (corpus) analyzer. + regular_task = MagicMock( + is_doc_analyzer_task=False, is_corpus_analyzer_task=False + ) with patch( "opencontractserver.utils.celery_tasks.celery_app" ) as mock_celery, patch( "opencontractserver.utils.celery_tasks.get_doc_analyzer_task_by_name" ) as mock_get: - mock_celery.tasks = {"some.regular.task": MagicMock()} + mock_celery.tasks = {"some.regular.task": regular_task} # Not a doc_analyzer_task → helper returns None mock_get.return_value = None @@ -412,7 +418,7 @@ def test_returns_none_for_unknown_gremlin_id(self) -> None: class AutoCreateDocAnalyzersEdgeCases(TestCase): def test_no_users_returns_early(self) -> None: - # Patch celery_app and the get_doc_analyzer_task_by_name so the + # Patch celery_app and the get_analyzer_task_by_name so the # iteration body is exercised; UserModel.objects.first() returns # None to trigger the early-return path. fake_task = MagicMock() @@ -423,7 +429,7 @@ def test_no_users_returns_early(self) -> None: with patch( "opencontractserver.analyzer.utils.celery_app" ) as mock_celery, patch( - "opencontractserver.analyzer.utils.get_doc_analyzer_task_by_name", + "opencontractserver.analyzer.utils.get_analyzer_task_by_name", return_value=fake_task, ), self.assertLogs( "opencontractserver.analyzer.utils", level="WARNING" @@ -447,7 +453,7 @@ def test_celery_app_unavailable_short_circuits(self) -> None: auto_create_doc_analyzers( AnalyzerModel=MagicMock(), UserModel=MagicMock() ) - self.assertTrue(any("Celery or doc_analyzer_task" in m for m in cm.output)) + self.assertTrue(any("Celery or analyzer task" in m for m in cm.output)) def test_update_existing_false_skips_update(self) -> None: fake_task = MagicMock() @@ -458,7 +464,7 @@ def test_update_existing_false_skips_update(self) -> None: with patch( "opencontractserver.analyzer.utils.celery_app" ) as mock_celery, patch( - "opencontractserver.analyzer.utils.get_doc_analyzer_task_by_name", + "opencontractserver.analyzer.utils.get_analyzer_task_by_name", return_value=fake_task, ): mock_celery.tasks = {"some.task": fake_task} diff --git a/opencontractserver/tests/test_enrichment_analyzer_integration.py b/opencontractserver/tests/test_enrichment_analyzer_integration.py new file mode 100644 index 000000000..18a6eb6b7 --- /dev/null +++ b/opencontractserver/tests/test_enrichment_analyzer_integration.py @@ -0,0 +1,157 @@ +"""Tests for the @corpus_analyzer_task decorator and the enrichment adapter. + +Covers: marker/lookup helpers, Analysis lifecycle management by the wrapper, +auto-sync of the adapter into an Analyzer row, and the run_task_name_analyzer +corpus-scoped dispatch branch. +""" + +from unittest.mock import patch + +from django.contrib.auth import get_user_model +from django.core.files.base import ContentFile +from django.test import TestCase + +from opencontractserver.analyzer.models import Analysis, Analyzer +from opencontractserver.analyzer.utils import auto_create_doc_analyzers +from opencontractserver.annotations.models import CorpusReference +from opencontractserver.corpuses.models import Corpus +from opencontractserver.documents.models import Document +from opencontractserver.enrichment import constants as C +from opencontractserver.tasks.corpus_analysis_tasks import corpus_reference_enrichment +from opencontractserver.types.enums import JobStatus +from opencontractserver.utils.celery_tasks import ( + get_analyzer_task_by_name, + get_corpus_analyzer_task_by_name, + get_doc_analyzer_task_by_name, +) + +User = get_user_model() + +S1_TEXT = ( + "We are governed by Section 203 of the Delaware General Corporation Law. " + "The underwriting agreement is filed as Exhibit 1.1 hereto." +) + + +def _make_corpus(user): + corpus = Corpus.objects.create(title="S-1 Corpus", creator=user) + doc = Document.objects.create(title="Acme S-1 primary", creator=user) + doc.txt_extract_file.save("s1.txt", ContentFile(S1_TEXT.encode("utf-8"))) + corpus.add_document(document=doc, user=user) + return corpus + + +def _make_analysis(user, corpus): + analyzer, _ = Analyzer.objects.get_or_create( + id=C.ENRICHMENT_ANALYZER_ID, + defaults={ + "task_name": C.ENRICHMENT_ANALYZER_TASK, + "description": C.ENRICHMENT_ANALYZER_TITLE, + "creator": user, + }, + ) + return Analysis.objects.create( + analyzer=analyzer, analyzed_corpus=corpus, creator=user + ) + + +class CorpusAnalyzerTaskTests(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="owner", password="p") + self.corpus = _make_corpus(self.user) + + def test_adapter_is_registered_as_corpus_analyzer(self): + task = get_corpus_analyzer_task_by_name(C.ENRICHMENT_ANALYZER_TASK) + assert task is not None + assert getattr(task, "is_corpus_analyzer_task", False) is True + # ... and is NOT a doc analyzer (dispatch must take the corpus branch). + assert get_doc_analyzer_task_by_name(C.ENRICHMENT_ANALYZER_TASK) is None + assert get_analyzer_task_by_name(C.ENRICHMENT_ANALYZER_TASK) is task + + def test_adapter_runs_enrichment_and_completes_analysis(self): + analysis = _make_analysis(self.user, self.corpus) + result = corpus_reference_enrichment( + corpus_id=self.corpus.id, analysis_id=analysis.id + ) + + assert result["references_created"] > 0 + assert CorpusReference.objects.filter(corpus=self.corpus).exists() + # The run attached to the framework Analysis, not a second one. + ref = CorpusReference.objects.filter(corpus=self.corpus).first() + assert ref is not None + assert ref.created_by_analysis_id == analysis.id + assert Analysis.objects.count() == 1 + + analysis.refresh_from_db() + assert analysis.status == JobStatus.COMPLETED.value + assert analysis.analysis_started is not None + assert analysis.analysis_completed is not None + assert "references_created" in (analysis.result_message or "") + + def test_failure_marks_analysis_failed(self): + analysis = _make_analysis(self.user, self.corpus) + bad_corpus_id = self.corpus.id + 9999 + with self.assertRaises(ValueError): + corpus_reference_enrichment( + corpus_id=bad_corpus_id, analysis_id=analysis.id + ) + analysis.refresh_from_db() + # Corpus existence fails before RUNNING; analysis stays untouched. + assert analysis.status != JobStatus.COMPLETED.value + + def test_failure_inside_function_records_error(self): + analysis = _make_analysis(self.user, self.corpus) + with patch( + "opencontractserver.enrichment.services.enrichment_service.EnrichmentService.apply", + side_effect=RuntimeError("boom"), + ): + with self.assertRaises(RuntimeError): + corpus_reference_enrichment( + corpus_id=self.corpus.id, analysis_id=analysis.id + ) + + analysis.refresh_from_db() + assert analysis.status == JobStatus.FAILED.value + assert "boom" in (analysis.error_message or "") + + def test_celery_retry_does_not_mark_analysis_failed(self): + # ``Retry`` extends ``Exception`` — the wrapper must re-raise it + # untouched (mirroring doc_analyzer_task) so a transient retry does + # not permanently stamp the Analysis FAILED before the retry runs. + from celery.exceptions import Retry + + analysis = _make_analysis(self.user, self.corpus) + with patch( + "opencontractserver.enrichment.services.enrichment_service.EnrichmentService.apply", + side_effect=Retry("requeued"), + ): + with self.assertRaises(Retry): + corpus_reference_enrichment( + corpus_id=self.corpus.id, analysis_id=analysis.id + ) + + analysis.refresh_from_db() + assert analysis.status == JobStatus.RUNNING.value + assert analysis.error_message is None + + def test_auto_sync_recognizes_adapter_without_duplicating(self): + # Service-created row exists (friendly id, real task_name)... + _make_analysis(self.user, self.corpus) + before = Analyzer.objects.filter(task_name=C.ENRICHMENT_ANALYZER_TASK).count() + assert before == 1 + + auto_create_doc_analyzers(AnalyzerModel=Analyzer, UserModel=User) + + # ...and sync does not create a duplicate for the same task. + assert ( + Analyzer.objects.filter(task_name=C.ENRICHMENT_ANALYZER_TASK).count() == 1 + ) + + def test_auto_sync_creates_analyzer_row_when_absent(self): + assert not Analyzer.objects.filter( + task_name=C.ENRICHMENT_ANALYZER_TASK + ).exists() + auto_create_doc_analyzers(AnalyzerModel=Analyzer, UserModel=User) + row = Analyzer.objects.get(task_name=C.ENRICHMENT_ANALYZER_TASK) + assert row.id == C.ENRICHMENT_ANALYZER_TASK + assert "reference web" in (row.description or "").lower() diff --git a/opencontractserver/tests/test_enrichment_authorities.py b/opencontractserver/tests/test_enrichment_authorities.py new file mode 100644 index 000000000..96a01afbc --- /dev/null +++ b/opencontractserver/tests/test_enrichment_authorities.py @@ -0,0 +1,211 @@ +"""Tests for the authority corpus bootstrapper (statute reference targets).""" + +from django.contrib.auth import get_user_model +from django.core.files.base import ContentFile +from django.test import TestCase + +from opencontractserver.corpuses.models import Corpus +from opencontractserver.documents.models import Document +from opencontractserver.enrichment.authorities import ( + AuthorityCorpusBootstrapper, + AuthoritySection, + authority_alias_registry, + candidate_keys, + find_authority_target, +) +from opencontractserver.utils.files import read_field_file_text + +User = get_user_model() + +DGCL_145_TEXT = ( + "(a) A corporation shall have power to indemnify any person who was or is " + "a party to any action by reason of the fact that the person is or was a " + "director, officer, employee or agent of the corporation." +) +DGCL_122_TEXT = ( + "Every corporation created under this chapter shall have power to ... " + "(17) renounce, in its certificate of incorporation or by action of its " + "board of directors, any interest in specified business opportunities." +) + + +def _bootstrap(user, sections): + return AuthorityCorpusBootstrapper().bootstrap( + creator_id=user.id, + corpus_title="Delaware General Corporation Law", + sections=sections, + ) + + +class AuthorityBootstrapTests(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="owner", password="p") + self.sections = [ + AuthoritySection( + key="dgcl:145", + heading="DGCL § 145 — Indemnification of officers and directors", + text=DGCL_145_TEXT, + source_url="https://delcode.delaware.gov/title8/c001/sc04/#145", + ), + AuthoritySection( + key="dgcl:122", + heading="DGCL § 122 — Specific powers", + text=DGCL_122_TEXT, + ), + ] + + def test_bootstrap_creates_corpus_and_keyed_documents(self): + out = _bootstrap(self.user, self.sections) + assert out["corpus_created"] is True + assert out["documents_created"] == 2 + + doc = Document.objects.get(custom_meta__canonical_key="dgcl:145") + assert doc.custom_meta["authority"] == "dgcl" + assert doc.custom_meta["source_url"].endswith("#145") + assert read_field_file_text(doc.txt_extract_file) == DGCL_145_TEXT + corpus = Corpus.objects.get(pk=out["corpus_id"]) + assert corpus.title == "Delaware General Corporation Law" + + def test_bootstrap_is_idempotent(self): + _bootstrap(self.user, self.sections) + doc_count = Document.objects.count() + + out = _bootstrap(self.user, self.sections) + assert out["corpus_created"] is False + assert out["documents_created"] == 0 + assert out["documents_updated"] == 0 + assert out["documents_skipped"] == 2 + assert Document.objects.count() == doc_count + + def test_bootstrap_restamps_clobbered_custom_meta(self): + """A concurrent pipeline save can wipe custom_meta; re-run must heal it. + + The document is still recognised by title, restamped in place — no new + document or version is created. + """ + _bootstrap(self.user, self.sections) + doc = Document.objects.get(custom_meta__canonical_key="dgcl:145") + doc.custom_meta = {} + doc.save(update_fields=["custom_meta"]) + doc_count = Document.objects.count() + + out = _bootstrap(self.user, self.sections) + assert out["documents_restamped"] == 1 + assert out["documents_created"] == 0 + assert out["documents_updated"] == 0 + assert Document.objects.count() == doc_count + doc.refresh_from_db() + assert doc.custom_meta["canonical_key"] == "dgcl:145" + + def test_bootstrap_versions_up_changed_text(self): + _bootstrap(self.user, self.sections) + amended = DGCL_145_TEXT + " [As amended.]" + out = _bootstrap( + self.user, + [ + AuthoritySection( + key="dgcl:145", + heading="DGCL § 145 — Indemnification of officers and directors", + text=amended, + ) + ], + ) + assert out["documents_updated"] == 1 + assert out["documents_created"] == 0 + + user_target = find_authority_target("dgcl:145", self.user) + assert user_target is not None + assert read_field_file_text(user_target.txt_extract_file) == amended + + +class AuthorityAliasRegistryTests(TestCase): + """Adding a body of law = bootstrapping a corpus with aliases — no code.""" + + def setUp(self): + self.user = User.objects.create_user(username="owner", password="p") + AuthorityCorpusBootstrapper().bootstrap( + creator_id=self.user.id, + corpus_title="New York Business Corporation Law", + aliases=["New York Business Corporation Law", "NYBCL"], + sections=[ + AuthoritySection( + key="nybcl:912", + heading="NYBCL § 912 — Requirements relating to certain business combinations", + text="(a) Definitions... business combination with an interested shareholder.", + ) + ], + ) + + def test_registry_merges_db_aliases_with_static_defaults(self): + registry = authority_alias_registry(self.user) + assert registry["nybcl"] == "nybcl" + assert registry["new york business corporation law"] == "nybcl" + # Static defaults still present. + assert registry["dgcl"] == "dgcl" + + def test_registry_respects_visibility(self): + stranger = User.objects.create_user(username="stranger", password="p") + registry = authority_alias_registry(stranger) + assert "nybcl" not in registry + + def test_new_authority_extracted_and_linked_end_to_end(self): + """A filing citing the DB-declared authority resolves with zero code.""" + from opencontractserver.annotations.models import CorpusReference + from opencontractserver.enrichment.services import EnrichmentService + + filing_corpus = Corpus.objects.create(title="NY Filings", creator=self.user) + doc = Document.objects.create(title="Acme merger proxy", creator=self.user) + doc.txt_extract_file.save( + "proxy.txt", + ContentFile( + b"We are subject to Section 912 of the New York Business " + b"Corporation Law as an interested shareholder." + ), + ) + filing_corpus.add_document(document=doc, user=self.user) + + out = EnrichmentService().apply( + corpus_id=filing_corpus.id, creator_id=self.user.id + ) + assert out["law_references_linked"] == 1 + + ref = CorpusReference.objects.get( + corpus=filing_corpus, canonical_key="nybcl:912" + ) + assert ref.target_document is not None + assert (ref.target_document.custom_meta or {})["canonical_key"] == "nybcl:912" + + +class FindAuthorityTargetTests(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="owner", password="p") + _bootstrap( + self.user, + [ + AuthoritySection( + key="dgcl:122", heading="DGCL § 122", text=DGCL_122_TEXT + ) + ], + ) + + def test_candidate_keys_fall_back_to_section_root(self): + assert candidate_keys("dgcl:122(17)") == ["dgcl:122(17)", "dgcl:122"] + assert candidate_keys("securities-act:7(a)(2)(b)") == [ + "securities-act:7(a)(2)(b)", + "securities-act:7", + ] + assert candidate_keys("dgcl:145") == ["dgcl:145"] + + def test_exact_and_subsection_keys_resolve(self): + exact = find_authority_target("dgcl:122", self.user) + sub = find_authority_target("dgcl:122(17)", self.user) + assert exact is not None + assert sub is not None + assert exact.id == sub.id + + def test_unknown_key_returns_none(self): + assert find_authority_target("dgcl:999", self.user) is None + + def test_visibility_respected(self): + stranger = User.objects.create_user(username="stranger", password="p") + assert find_authority_target("dgcl:122", stranger) is None diff --git a/opencontractserver/tests/test_enrichment_extractor.py b/opencontractserver/tests/test_enrichment_extractor.py new file mode 100644 index 000000000..ceb2de5f8 --- /dev/null +++ b/opencontractserver/tests/test_enrichment_extractor.py @@ -0,0 +1,184 @@ +"""Unit tests for the deterministic ReferenceExtractor (no DB).""" + +from django.test import SimpleTestCase + +from opencontractserver.enrichment import constants as C +from opencontractserver.enrichment.extractor import ReferenceExtractor + + +class ReferenceExtractorTests(SimpleTestCase): + def setUp(self) -> None: + self.extractor = ReferenceExtractor() + + def test_law_dgcl_section(self): + text = ( + "...indemnification under Section 145 of the Delaware General " + "Corporation Law, our amended and restated certificate..." + ) + cands = self.extractor.extract(text) + law = [c for c in cands if c.reference_type == C.REF_LAW] + assert any(c.canonical_key == "dgcl:145" for c in law), [ + c.canonical_key for c in law + ] + hit = next(c for c in law if c.canonical_key == "dgcl:145") + assert text[hit.start : hit.end].lower().startswith("section 145") + assert hit.normalized_data["authority"] == "dgcl" + assert hit.normalized_data["section"] == "145" + + def test_law_securities_act_subsection(self): + text = "exempt pursuant to Section 4(a)(2) of the Securities Act and Rule 506." + cands = self.extractor.extract(text) + assert any(c.canonical_key == "securities-act:4(a)(2)" for c in cands), [ + c.canonical_key for c in cands + ] + + def test_law_section_203_dgcl(self): + text = "We are subject to Section 203 of the Delaware General Corporation Law." + cands = self.extractor.extract(text) + assert any(c.canonical_key == "dgcl:203" for c in cands) + + def test_prefix_form_law_citation(self): + """Form D style: authority name BEFORE the section number.""" + text = ( + "Securities Act Section 4(a)(5) and Investment Company Act " + "Section 3(c)(1) apply to this offering." + ) + cands = self.extractor.extract(text) + keys = {c.canonical_key for c in cands if c.reference_type == C.REF_LAW} + assert "securities-act:4(a)(5)" in keys, keys + assert "ica:3(c)(1)" in keys, keys + + def test_prefix_and_suffix_forms_do_not_double_count(self): + text = "Section 145 of the Delaware General Corporation Law applies." + cands = self.extractor.extract(text) + law = [c for c in cands if c.reference_type == C.REF_LAW] + assert len(law) == 1 + + def test_sec_rule_citation(self): + """Bare SEC rule citations: Rule 506(b), Rule 504(b)(1), Rule 10b-5.""" + text = ( + "The offering relies on Rule 506(b) of Regulation D; resales may " + "use Rule 144A, and liability arises under Rule 10b-5." + ) + cands = self.extractor.extract(text) + keys = {c.canonical_key for c in cands if c.reference_type == C.REF_LAW} + assert "sec-rule:506(b)" in keys, keys + assert "sec-rule:144a" in keys, keys + assert "sec-rule:10b-5" in keys, keys + + def test_relative_law_citation_with_authority_context(self): + """Statute-internal idiom: '§ 251 of this title' keyed via the + document's own authority (passed by the caller from custom_meta).""" + text = ( + "Any corporation organized under § 251 of this title may merge, " + "and Section 141(b) of this title governs the board." + ) + cands = self.extractor.extract(text, default_authority="dgcl") + keys = {c.canonical_key for c in cands if c.reference_type == C.REF_LAW} + assert "dgcl:251" in keys, keys + assert "dgcl:141(b)" in keys, keys + hit = next(c for c in cands if c.canonical_key == "dgcl:251") + assert hit.normalized_data["relative"] is True + assert hit.normalized_data["authority"] == "dgcl" + + def test_relative_law_citation_skipped_without_authority(self): + """No authority context -> relative citations cannot be keyed; skip.""" + text = "Any corporation organized under § 251 of this title may merge." + cands = self.extractor.extract(text) + assert not [c for c in cands if c.reference_type == C.REF_LAW] + + def test_named_citation_not_duplicated_by_relative_grammar(self): + text = "Section 145 of the Delaware General Corporation Law applies." + cands = self.extractor.extract(text, default_authority="dgcl") + law = [c for c in cands if c.reference_type == C.REF_LAW] + assert len(law) == 1 + assert law[0].canonical_key == "dgcl:145" + assert "relative" not in law[0].normalized_data + + def test_exhibit_reference_candidate(self): + text = "The form of underwriting agreement is filed as Exhibit 1.1 hereto." + cands = self.extractor.extract(text) + docs = [c for c in cands if c.reference_type == C.REF_DOCUMENT] + assert docs and docs[0].normalized_data["exhibit_number"] == "1.1" + + def test_internal_section_reference_candidate(self): + text = 'For more information, see "Risk Factors" beginning on page 20.' + cands = self.extractor.extract(text) + secs = [c for c in cands if c.reference_type == C.REF_SECTION] + assert secs and secs[0].normalized_data["heading"] == "Risk Factors" + hit = secs[0] + assert "Risk Factors" in text[hit.start : hit.end] + + def test_defined_term_parenthetical(self): + text = 'Fervo Energy, Inc. (the "Company") is a Delaware corporation.' + cands = self.extractor.extract(text) + terms = [c for c in cands if c.reference_type == C.REF_DEFINED_TERM] + assert terms and terms[0].canonical_key == "term:company" + assert terms[0].normalized_data["term"] == "Company" + assert "Company" in text[terms[0].start : terms[0].end] + + def test_defined_term_means_clause(self): + text = '"Change of Control" means any transaction in which control passes.' + cands = self.extractor.extract(text) + terms = [c for c in cands if c.reference_type == C.REF_DEFINED_TERM] + assert any(c.canonical_key == "term:change-of-control" for c in terms) + + def test_defined_term_strips_trailing_comma_inside_quotes(self): + text = 'the senior notes (collectively, the "Notes," and together...)' + cands = self.extractor.extract(text) + terms = [c for c in cands if c.reference_type == C.REF_DEFINED_TERM] + assert any(c.canonical_key == "term:notes" for c in terms) + + def test_defined_terms_deduped_by_slug(self): + text = ( + 'Acme Inc. (the "Company"). The Company grows. ' + '"Company" means Acme Inc. as defined.' + ) + cands = self.extractor.extract(text) + company = [ + c + for c in cands + if c.reference_type == C.REF_DEFINED_TERM + and c.canonical_key == "term:company" + ] + assert len(company) == 1 # deduped across both patterns + + def test_defined_term_cap_is_total_in_document_order(self): + # The MAX_DEFINED_TERMS cap is a TOTAL per document, applied after + # merging both grammar forms in document order — a wall of + # parenthetical definitions must not starve an earlier "means"-form + # definition (review finding: the old per-regex loop returned before + # the means regex ever ran). + means_first = '"Change of Control" means any transfer of control. ' + " ".join( + f'(the "Term{chr(65 + i // 26)}{chr(65 + i % 26)} Item")' + for i in range(C.MAX_DEFINED_TERMS) + ) + cands = self.extractor.extract(means_first) + terms = [c for c in cands if c.reference_type == C.REF_DEFINED_TERM] + assert len(terms) == C.MAX_DEFINED_TERMS # cap holds + assert any(c.canonical_key == "term:change-of-control" for c in terms) + + def test_defined_term_cap_drops_matches_beyond_position_cap(self): + # Conversely, a "means"-form definition AFTER the cap-filling + # parentheticals is dropped: first N definition sites by position win. + means_last = ( + " ".join( + f'(the "Term{chr(65 + i // 26)}{chr(65 + i % 26)} Item")' + for i in range(C.MAX_DEFINED_TERMS) + ) + + ' "Change of Control" means any transfer of control.' + ) + cands = self.extractor.extract(means_last) + terms = [c for c in cands if c.reference_type == C.REF_DEFINED_TERM] + assert len(terms) == C.MAX_DEFINED_TERMS + assert not any(c.canonical_key == "term:change-of-control" for c in terms) + + def test_defined_terms_not_in_default_reference_types(self): + # Opt-in: DEFINED_TERM is detected by the extractor but excluded from the + # default scan/apply set. + assert C.REF_DEFINED_TERM not in C.DEFAULT_REFERENCE_TYPES + assert C.REF_DEFINED_TERM in C.ALL_REFERENCE_TYPES + + def test_no_false_positive_on_plain_text(self): + text = "The company sells software to enterprise customers worldwide." + assert self.extractor.extract(text) == [] diff --git a/opencontractserver/tests/test_enrichment_linking.py b/opencontractserver/tests/test_enrichment_linking.py new file mode 100644 index 000000000..e587e4427 --- /dev/null +++ b/opencontractserver/tests/test_enrichment_linking.py @@ -0,0 +1,165 @@ +"""Tests for the cross-corpus resolution pass (EXTERNAL law refs -> RESOLVED).""" + +from django.contrib.auth import get_user_model +from django.core.files.base import ContentFile +from django.test import TestCase + +from opencontractserver.annotations.models import CorpusReference +from opencontractserver.corpuses.models import Corpus +from opencontractserver.documents.models import Document +from opencontractserver.enrichment import constants as C +from opencontractserver.enrichment.authorities import ( + AuthorityCorpusBootstrapper, + AuthoritySection, +) +from opencontractserver.enrichment.services import EnrichmentService + +User = get_user_model() + +S1_TEXT = ( + "We are governed by Section 203 of the Delaware General Corporation Law. " + "Indemnification is provided per Section 145 of the Delaware General " + "Corporation Law. The offering relies on Section 4(a)(2) of the " + "Securities Act." +) + + +class CrossCorpusLinkingTests(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="owner", password="p") + self.corpus = Corpus.objects.create(title="S-1 Corpus", creator=self.user) + doc = Document.objects.create(title="Acme S-1 primary", creator=self.user) + doc.txt_extract_file.save("s1.txt", ContentFile(S1_TEXT.encode("utf-8"))) + self.corpus.add_document(document=doc, user=self.user) + + def _bootstrap_dgcl(self, user=None): + return AuthorityCorpusBootstrapper().bootstrap( + creator_id=(user or self.user).id, + corpus_title="Delaware General Corporation Law", + sections=[ + AuthoritySection(key="dgcl:145", heading="DGCL § 145", text="..145.."), + AuthoritySection(key="dgcl:203", heading="DGCL § 203", text="..203.."), + ], + ) + + def test_link_upgrades_external_law_refs(self): + EnrichmentService().apply(corpus_id=self.corpus.id, creator_id=self.user.id) + auth = self._bootstrap_dgcl() + + out = EnrichmentService().link_external_references( + corpus_id=self.corpus.id, creator_id=self.user.id + ) + assert out["law_references_linked"] == 2 + + ref = CorpusReference.objects.get(corpus=self.corpus, canonical_key="dgcl:145") + assert ref.resolution_status == C.STATUS_RESOLVED + assert ref.target_corpus_id == auth["corpus_id"] + assert ref.target_document is not None + # Canonical in-app document path INTO THE AUTHORITY CORPUS (the slug + # shape the frontend router serves: /d/:userIdent/:corpusIdent/:docIdent). + auth_corpus = Corpus.objects.select_related("creator").get(pk=auth["corpus_id"]) + assert ref.source_annotation.link_url == ( + f"/d/{auth_corpus.creator.slug}/{auth_corpus.slug}" + f"/{ref.target_document.slug}" + ) + # No DGCL doc for the Securities Act citation -> still external. + sa = CorpusReference.objects.get( + corpus=self.corpus, canonical_key="securities-act:4(a)(2)" + ) + assert sa.resolution_status == C.STATUS_EXTERNAL + assert sa.target_document is None + + def test_apply_links_automatically_when_authority_exists(self): + self._bootstrap_dgcl() + out = EnrichmentService().apply( + corpus_id=self.corpus.id, creator_id=self.user.id + ) + assert out["law_references_linked"] == 2 + + def test_link_is_idempotent(self): + EnrichmentService().apply(corpus_id=self.corpus.id, creator_id=self.user.id) + self._bootstrap_dgcl() + EnrichmentService().link_external_references( + corpus_id=self.corpus.id, creator_id=self.user.id + ) + again = EnrichmentService().link_external_references( + corpus_id=self.corpus.id, creator_id=self.user.id + ) + assert again["law_references_linked"] == 0 + + def test_self_referential_statute_corpus_links_internally(self): + """Enrichment ON an authority corpus: '§ X of this title' citations in + statute text resolve to sibling sections of the same corpus.""" + auth = AuthorityCorpusBootstrapper().bootstrap( + creator_id=self.user.id, + corpus_title="Delaware General Corporation Law", + sections=[ + AuthoritySection( + key="dgcl:251", + heading="DGCL § 251", + text=( + "Any 2 or more corporations may merge into a single " + "corporation. Notice shall be given as provided in " + "§ 222 of this title." + ), + ), + AuthoritySection( + key="dgcl:222", + heading="DGCL § 222", + text="Whenever stockholders are required to take action...", + ), + ], + ) + + out = EnrichmentService().apply( + corpus_id=auth["corpus_id"], creator_id=self.user.id + ) + assert out["law_references_linked"] == 1 + + ref = CorpusReference.objects.get( + corpus_id=auth["corpus_id"], canonical_key="dgcl:222" + ) + assert ref.resolution_status == C.STATUS_RESOLVED + assert ref.target_corpus_id == auth["corpus_id"] + assert ref.target_document is not None + assert (ref.target_document.custom_meta or {})["canonical_key"] == "dgcl:222" + assert (ref.normalized_data or {})["relative"] is True + + def test_link_raises_doesnotexist_for_missing_and_invisible_corpus(self): + # Visibility-scoped corpus fetch: a nonexistent corpus and a corpus + # the caller cannot see raise the SAME ``Corpus.DoesNotExist`` — no + # existence oracle for callers passing arbitrary PKs. + stranger = User.objects.create_user(username="stranger", password="p") + + with self.assertRaises(Corpus.DoesNotExist): + EnrichmentService().link_external_references( + corpus_id=999_999, creator_id=self.user.id + ) + with self.assertRaises(Corpus.DoesNotExist): + EnrichmentService().link_external_references( + corpus_id=self.corpus.id, creator_id=stranger.id + ) + + def test_bootstrap_raises_doesnotexist_for_invisible_corpus_id(self): + # Same visibility-scoped semantics for the bootstrapper's explicit + # ``corpus_id`` path. + stranger = User.objects.create_user(username="stranger2", password="p") + with self.assertRaises(Corpus.DoesNotExist): + AuthorityCorpusBootstrapper().bootstrap( + creator_id=stranger.id, + corpus_title="ignored", + sections=[], + corpus_id=self.corpus.id, + ) + + def test_link_respects_authority_visibility(self): + EnrichmentService().apply(corpus_id=self.corpus.id, creator_id=self.user.id) + other = User.objects.create_user(username="other", password="p") + self._bootstrap_dgcl(user=other) # private to `other` + + out = EnrichmentService().link_external_references( + corpus_id=self.corpus.id, creator_id=self.user.id + ) + assert out["law_references_linked"] == 0 + ref = CorpusReference.objects.get(corpus=self.corpus, canonical_key="dgcl:145") + assert ref.resolution_status == C.STATUS_EXTERNAL diff --git a/opencontractserver/tests/test_enrichment_resolver.py b/opencontractserver/tests/test_enrichment_resolver.py new file mode 100644 index 000000000..c44adef27 --- /dev/null +++ b/opencontractserver/tests/test_enrichment_resolver.py @@ -0,0 +1,90 @@ +"""Unit tests for ReferenceResolver (document-title + heading resolution).""" + +from django.contrib.auth import get_user_model +from django.test import TestCase + +from opencontractserver.documents.models import Document +from opencontractserver.enrichment import constants as C +from opencontractserver.enrichment.extractor import Candidate +from opencontractserver.enrichment.resolver import ReferenceResolver, SectionAnno + +User = get_user_model() + + +class ResolverTests(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="u", password="p") + self.primary = Document.objects.create(title="Primary S-1", creator=self.user) + self.exhibit = Document.objects.create( + title="Acme S-1 (2024-09-30) - Exhibit 1.1: EX-1.1", creator=self.user + ) + self.resolver = ReferenceResolver([self.primary, self.exhibit]) + + def test_resolves_exhibit_number_to_target_document(self): + cand = Candidate( + reference_type=C.REF_DOCUMENT, + start=0, + end=11, + raw_text="Exhibit 1.1", + normalized_data={"exhibit_number": "1.1"}, + ) + r = self.resolver.resolve_document(cand, source_doc_id=self.primary.id) + assert r.resolution_status == C.STATUS_RESOLVED + assert r.target_document_id == self.exhibit.id + + def test_unknown_exhibit_is_unresolved(self): + cand = Candidate( + reference_type=C.REF_DOCUMENT, + start=0, + end=12, + raw_text="Exhibit 99.9", + normalized_data={"exhibit_number": "99.9"}, + ) + r = self.resolver.resolve_document(cand, source_doc_id=self.primary.id) + assert r.resolution_status == C.STATUS_UNRESOLVED + assert r.target_document_id is None + + def test_law_candidate_is_external(self): + cand = Candidate( + reference_type=C.REF_LAW, + start=0, + end=5, + raw_text="x", + canonical_key="dgcl:145", + normalized_data={"authority": "dgcl", "section": "145"}, + ) + r = self.resolver.resolve(cand, source_doc_id=self.primary.id, doc_text="x") + assert r.resolution_status == C.STATUS_EXTERNAL + assert r.canonical_key == "dgcl:145" + assert r.source_document_id == self.primary.id + + def test_section_matches_oc_section_annotation(self): + cand = Candidate( + reference_type=C.REF_SECTION, + start=20, + end=34, + raw_text='see "Risk Factors"', + normalized_data={"heading": "Risk Factors"}, + ) + sections = [SectionAnno(id=42, raw_text="Risk Factors")] + r = self.resolver.resolve_section( + cand, self.primary.id, doc_text="...", sections=sections + ) + assert r.resolution_status == C.STATUS_RESOLVED + assert r.target_annotation_id == 42 + + def test_section_falls_back_to_heading_text_offset(self): + text = "intro... Risk Factors\nThe following risks..." + cand = Candidate( + reference_type=C.REF_SECTION, + start=0, + end=5, + raw_text='see "Risk Factors"', + normalized_data={"heading": "Risk Factors"}, + ) + r = self.resolver.resolve_section( + cand, self.primary.id, doc_text=text, sections=[] + ) + assert r.resolution_status == C.STATUS_RESOLVED + assert r.target_annotation_id is None + assert r.target_offset == text.find("Risk Factors") diff --git a/opencontractserver/tests/test_enrichment_tools.py b/opencontractserver/tests/test_enrichment_tools.py new file mode 100644 index 000000000..c902959fc --- /dev/null +++ b/opencontractserver/tests/test_enrichment_tools.py @@ -0,0 +1,179 @@ +"""Tool-registry + tool-function tests for corpus reference enrichment.""" + +from django.contrib.auth import get_user_model +from django.core.files.base import ContentFile +from django.test import TestCase + +from opencontractserver.annotations.models import Annotation, CorpusReference +from opencontractserver.corpuses.models import Corpus +from opencontractserver.documents.models import Document +from opencontractserver.llms.tools.core_tools import ( + apply_corpus_reference_enrichment, + scan_corpus_references, +) +from opencontractserver.llms.tools.tool_registry import AVAILABLE_TOOLS + +User = get_user_model() + +TEXT = ( + "Indemnification under Section 145 of the Delaware General Corporation Law. " + "Issued per Section 4(a)(2) of the Securities Act." +) + + +class EnrichmentToolRegistryTests(TestCase): + def test_both_tools_are_registered(self): + names = {t.name for t in AVAILABLE_TOOLS} + assert "scan_corpus_references" in names + assert "apply_corpus_reference_enrichment" in names + + def test_apply_tool_requires_approval_and_write(self): + td = next( + t for t in AVAILABLE_TOOLS if t.name == "apply_corpus_reference_enrichment" + ) + assert td.requires_corpus is True + assert td.requires_approval is True + assert td.requires_write_permission is True + + +class EnrichmentToolFunctionTests(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="owner", password="p") + self.corpus = Corpus.objects.create(title="C", creator=self.user) + doc = Document.objects.create(title="S-1 primary document", creator=self.user) + doc.txt_extract_file.save("d.txt", ContentFile(TEXT.encode("utf-8"))) + self.corpus.add_document(document=doc, user=self.user) + + def test_scan_writes_nothing_and_reports(self): + out = scan_corpus_references(corpus_id=self.corpus.id, creator_id=self.user.id) + assert out["total_candidates"] >= 2 + assert Annotation.objects.filter(corpus=self.corpus).count() == 0 + + def test_apply_creates_references(self): + out = apply_corpus_reference_enrichment( + corpus_id=self.corpus.id, creator_id=self.user.id + ) + assert out["references_created"] >= 2 + keys = set( + CorpusReference.objects.filter(corpus=self.corpus).values_list( + "canonical_key", flat=True + ) + ) + assert "dgcl:145" in keys + assert "securities-act:4(a)(2)" in keys + + +class _Ctx: + def __init__(self, user): + self.user = user + + +class EnrichmentGraphQLTests(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="owner", password="p") + self.corpus = Corpus.objects.create(title="C", creator=self.user) + doc = Document.objects.create(title="S-1 primary document", creator=self.user) + doc.txt_extract_file.save("d.txt", ContentFile(TEXT.encode("utf-8"))) + self.corpus.add_document(document=doc, user=self.user) + apply_corpus_reference_enrichment( + corpus_id=self.corpus.id, creator_id=self.user.id + ) + + def _run(self, user): + from graphene.test import Client + from graphql_relay import to_global_id + + from config.graphql.schema import schema + + gid = to_global_id("CorpusType", self.corpus.id) + query = """ + query ($cid: ID!) { + corpusReferences(corpusId: $cid, referenceType: "LAW") { + edges { node { canonicalKey resolutionStatus } } + } + } + """ + return Client(schema, context_value=_Ctx(user)).execute( + query, variables={"cid": gid} + ) + + def test_owner_sees_law_references(self): + result = self._run(self.user) + edges = result["data"]["corpusReferences"]["edges"] + keys = {e["node"]["canonicalKey"] for e in edges} + assert "dgcl:145" in keys + assert all(e["node"]["resolutionStatus"] == "EXTERNAL" for e in edges) + + def test_stranger_sees_nothing(self): + stranger = User.objects.create_user(username="stranger", password="p") + result = self._run(stranger) + assert result["data"]["corpusReferences"]["edges"] == [] + + +class CorpusReferenceTraversalVisibilityTests(TestCase): + """Nested FK traversal on ``CorpusReferenceType`` must not leak documents. + + ``corpusReferences`` is corpus-as-gate (corpus READ unlocks the rows), but + ``targetDocument`` resolution goes through graphene-django's FK converter → + ``DocumentType.get_node`` → ``DocumentType.get_queryset`` (visibility + filtered). This pins that a reader of a public corpus gets ``null`` for a + target document they cannot READ, while the owner still resolves it. + """ + + def setUp(self): + self.owner = User.objects.create_user(username="owner", password="p") + self.reader = User.objects.create_user(username="reader", password="p") + self.corpus = Corpus.objects.create( + title="Public C", creator=self.owner, is_public=True + ) + doc = Document.objects.create(title="S-1 primary document", creator=self.owner) + doc.txt_extract_file.save("d.txt", ContentFile(TEXT.encode("utf-8"))) + self.corpus.add_document(document=doc, user=self.owner) + apply_corpus_reference_enrichment( + corpus_id=self.corpus.id, creator_id=self.owner.id + ) + # Point one reference at a PRIVATE document (owner-only, no public + # corpus path) — the cross-corpus law-link shape. + self.private_doc = Document.objects.create( + title="Private statute section", creator=self.owner, is_public=False + ) + ref = CorpusReference.objects.filter(corpus=self.corpus).first() + assert ref is not None + ref.target_document = self.private_doc + ref.save(update_fields=["target_document", "modified"]) + self.ref = ref + + def _run(self, user): + from graphene.test import Client + from graphql_relay import to_global_id + + from config.graphql.schema import schema + + gid = to_global_id("CorpusType", self.corpus.id) + query = """ + query ($cid: ID!) { + corpusReferences(corpusId: $cid) { + edges { node { canonicalKey targetDocument { id title } } } + } + } + """ + return Client(schema, context_value=_Ctx(user)).execute( + query, variables={"cid": gid} + ) + + def test_owner_resolves_target_document(self): + result = self._run(self.owner) + nodes = [e["node"] for e in result["data"]["corpusReferences"]["edges"]] + assert any( + n["targetDocument"] + and n["targetDocument"]["title"] == "Private statute section" + for n in nodes + ) + + def test_corpus_reader_gets_null_for_invisible_target_document(self): + result = self._run(self.reader) + edges = result["data"]["corpusReferences"]["edges"] + # Corpus-as-gate: the reference rows themselves ARE visible... + assert edges + # ...but the private target document nulls out for the reader. + assert all(e["node"]["targetDocument"] is None for e in edges) diff --git a/opencontractserver/tests/test_enrichment_writer.py b/opencontractserver/tests/test_enrichment_writer.py new file mode 100644 index 000000000..5e377b6af --- /dev/null +++ b/opencontractserver/tests/test_enrichment_writer.py @@ -0,0 +1,316 @@ +"""Integration tests for EnrichmentWriter + EnrichmentService (DB writes).""" + +from django.contrib.auth import get_user_model +from django.core.files.base import ContentFile +from django.test import TestCase +from graphene.test import Client + +from config.graphql.schema import schema +from opencontractserver.annotations.models import ( + SPAN_LABEL, + Annotation, + CorpusReference, + Relationship, +) +from opencontractserver.corpuses.models import Corpus +from opencontractserver.documents.models import Document, DocumentRelationship +from opencontractserver.enrichment import constants as C +from opencontractserver.enrichment.services import ( + CorpusReferenceService, + EnrichmentService, +) + +User = get_user_model() + + +class _GQLContext: + """Minimal info.context stand-in for graphene.test.Client.""" + + def __init__(self, user): + self.user = user + + +PRIMARY_TEXT = ( + "We are subject to Section 203 of the Delaware General Corporation Law. " + "Indemnification is governed by Section 145 of the Delaware General " + "Corporation Law. Shares were issued pursuant to Section 4(a)(2) of the " + 'Securities Act. For details, see "Risk Factors" beginning on page 20. ' + "Risk Factors\nThe following risks are material to our business. " + "The underwriting agreement is filed as Exhibit 1.1 hereto." +) + + +class EnrichmentWriterTests(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="owner", password="p") + self.corpus = Corpus.objects.create(title="S-1 Corpus", creator=self.user) + self.primary = Document.objects.create( + title="Acme S-1 (2024-09-30) - primary document", creator=self.user + ) + self.primary.txt_extract_file.save( + "primary.txt", ContentFile(PRIMARY_TEXT.encode("utf-8")) + ) + self.exhibit = Document.objects.create( + title="Acme S-1 (2024-09-30) - Exhibit 1.1: EX-1.1", creator=self.user + ) + self.exhibit.txt_extract_file.save( + "ex.txt", ContentFile(b"Underwriting agreement body.") + ) + # add_document creates corpus-isolated copies; enrichment operates on + # (and resolves to) those copies, not the originals. + self.primary_in_corpus, _, _ = self.corpus.add_document( + document=self.primary, user=self.user + ) + self.exhibit_in_corpus, _, _ = self.corpus.add_document( + document=self.exhibit, user=self.user + ) + + def test_apply_creates_law_annotation_and_external_reference(self): + out = EnrichmentService().apply( + corpus_id=self.corpus.id, creator_id=self.user.id + ) + assert out["annotations_created"] > 0 + + law_anns = Annotation.objects.filter( + corpus=self.corpus, annotation_label__text=C.LABEL_REF_LAW + ) + keys = {a.data.get("canonical_key") for a in law_anns if a.data} + assert "dgcl:145" in keys + assert "dgcl:203" in keys + assert "securities-act:4(a)(2)" in keys + + ext = CorpusReference.objects.filter( + corpus=self.corpus, reference_type=C.REF_LAW, canonical_key="dgcl:145" + ).first() + assert ext is not None + assert ext.resolution_status == C.STATUS_EXTERNAL + + def test_apply_links_exhibit_reference_to_target_document(self): + EnrichmentService().apply(corpus_id=self.corpus.id, creator_id=self.user.id) + doc_ref = CorpusReference.objects.filter( + corpus=self.corpus, reference_type=C.REF_DOCUMENT + ).first() + assert doc_ref is not None + assert doc_ref.target_document_id == self.exhibit_in_corpus.id + # The mention annotation carries the canonical in-app document path + # (the slug shape the frontend router actually serves — see + # frontend/src/App.tsx /d/:userIdent/:corpusIdent/:docIdent). + self.corpus.refresh_from_db() + self.exhibit_in_corpus.refresh_from_db() + assert doc_ref.source_annotation.link_url == ( + f"/d/{self.corpus.creator.slug}/{self.corpus.slug}" + f"/{self.exhibit_in_corpus.slug}" + ) + + def test_section_reference_creates_relationship_or_external_ref(self): + EnrichmentService().apply(corpus_id=self.corpus.id, creator_id=self.user.id) + # No OC_SECTION annotations exist, so the section ref resolves via the + # heading-text fallback and is stored as a SECTION CorpusReference. + sec_ref = CorpusReference.objects.filter( + corpus=self.corpus, reference_type=C.REF_SECTION + ).first() + assert sec_ref is not None + assert sec_ref.resolution_status == C.STATUS_RESOLVED + + def test_section_reference_with_oc_section_creates_relationship(self): + # Seed an OC_SECTION annotation on the in-corpus primary doc whose text + # matches the `see "Risk Factors"` heading, so the resolver links to it. + oc_label = self.corpus.ensure_label_and_labelset( + label_text="OC_SECTION", creator_id=self.user.id, label_type=SPAN_LABEL + ) + section = Annotation.objects.create( + raw_text="Risk Factors", + page=1, + json={"start": 0, "end": 12}, + annotation_label=oc_label, + document_id=self.primary_in_corpus.id, + corpus=self.corpus, + creator=self.user, + annotation_type=SPAN_LABEL, + ) + + EnrichmentService().apply(corpus_id=self.corpus.id, creator_id=self.user.id) + + rel = Relationship.objects.filter( + corpus=self.corpus, + relationship_label__text=C.LABEL_RELATIONSHIP, + target_annotations=section, + ).first() + assert rel is not None + source = rel.source_annotations.first() + assert source is not None + label = source.annotation_label + assert label is not None + assert label.text == C.LABEL_REF_SECTION + + def test_document_reference_creates_document_relationship(self): + """Resolved exhibit refs roll up to a document-level edge. + + DocumentRelationship rows are what the corpus document graph renders + (documents = nodes, relationships = edges), so enrichment must emit + them alongside the annotation-level CorpusReference. + """ + EnrichmentService().apply(corpus_id=self.corpus.id, creator_id=self.user.id) + + rel = DocumentRelationship.objects.filter( + corpus=self.corpus, + source_document_id=self.primary_in_corpus.id, + target_document_id=self.exhibit_in_corpus.id, + annotation_label__text=C.LABEL_RELATIONSHIP, + ).first() + assert rel is not None + assert rel.relationship_type == C.DOC_REL_RELATIONSHIP + + def test_document_relationship_rollup_is_idempotent(self): + EnrichmentService().apply(corpus_id=self.corpus.id, creator_id=self.user.id) + count = DocumentRelationship.objects.filter(corpus=self.corpus).count() + assert count == 1 + + out = EnrichmentService().apply( + corpus_id=self.corpus.id, creator_id=self.user.id + ) + assert DocumentRelationship.objects.filter(corpus=self.corpus).count() == count + assert out["document_relationships_created"] == 0 + + def test_defined_terms_opt_in_only(self): + # Default apply excludes DEFINED_TERM. + EnrichmentService().apply(corpus_id=self.corpus.id, creator_id=self.user.id) + assert ( + CorpusReference.objects.filter( + corpus=self.corpus, reference_type=C.REF_DEFINED_TERM + ).count() + == 0 + ) + + def test_defined_terms_when_requested(self): + # PRIMARY_TEXT has no parenthetical definition; add a doc that does. + doc = Document.objects.create(title="Defs", creator=self.user) + doc.txt_extract_file.save( + "defs.txt", + ContentFile( + b'Fervo Energy, Inc. (the "Company"). "Change of Control" means a sale.' + ), + ) + self.corpus.add_document(document=doc, user=self.user) + + EnrichmentService().apply( + corpus_id=self.corpus.id, + creator_id=self.user.id, + types=list(C.ALL_REFERENCE_TYPES), + ) + keys = set( + CorpusReference.objects.filter( + corpus=self.corpus, reference_type=C.REF_DEFINED_TERM + ).values_list("canonical_key", flat=True) + ) + assert "term:company" in keys + assert "term:change-of-control" in keys + ref = CorpusReference.objects.filter( + corpus=self.corpus, canonical_key="term:company" + ).first() + assert ref is not None + assert ref.resolution_status == C.STATUS_RESOLVED + + def test_longer_alias_match_does_not_duplicate_mention(self): + """Growing the alias registry lengthens law-citation spans; the + mention dedup must match on span start, not the exact span.""" + from opencontractserver.enrichment.authorities import ( + AuthorityCorpusBootstrapper, + AuthoritySection, + ) + + EnrichmentService().apply(corpus_id=self.corpus.id, creator_id=self.user.id) + law_count = Annotation.objects.filter( + corpus=self.corpus, annotation_label__text=C.LABEL_REF_LAW + ).count() + + # Declare a longer alias that now wins longest-first matching for the + # same citations ("...Delaware General Corporation Law" is unchanged, + # but make the point with an alias extension). + AuthorityCorpusBootstrapper().bootstrap( + creator_id=self.user.id, + corpus_title="Delaware General Corporation Law", + aliases=["Delaware General Corporation Law, as amended"], + sections=[ + AuthoritySection(key="dgcl:145", heading="DGCL § 145", text="...") + ], + ) + + out = EnrichmentService().apply( + corpus_id=self.corpus.id, creator_id=self.user.id + ) + assert out["annotations_created"] == 0 + assert ( + Annotation.objects.filter( + corpus=self.corpus, annotation_label__text=C.LABEL_REF_LAW + ).count() + == law_count + ) + + def test_apply_is_idempotent(self): + EnrichmentService().apply(corpus_id=self.corpus.id, creator_id=self.user.id) + ann_count = Annotation.objects.filter(corpus=self.corpus).count() + ref_count = CorpusReference.objects.filter(corpus=self.corpus).count() + + EnrichmentService().apply(corpus_id=self.corpus.id, creator_id=self.user.id) + assert Annotation.objects.filter(corpus=self.corpus).count() == ann_count + assert CorpusReference.objects.filter(corpus=self.corpus).count() == ref_count + + def test_scan_writes_nothing(self): + before = CorpusReference.objects.count() + out = EnrichmentService().scan( + corpus_id=self.corpus.id, creator_id=self.user.id + ) + assert out["total_candidates"] > 0 + assert CorpusReference.objects.count() == before + assert Annotation.objects.filter(corpus=self.corpus).count() == 0 + + def test_reference_visibility_scoped_to_corpus(self): + EnrichmentService().apply(corpus_id=self.corpus.id, creator_id=self.user.id) + other = User.objects.create_user(username="stranger", password="p") + visible = CorpusReferenceService.for_corpus(other, self.corpus.id) + assert visible.count() == 0 + mine = CorpusReferenceService.for_corpus(self.user, self.corpus.id) + assert mine.count() > 0 + + +class CorpusReferencesResolverGuardTests(TestCase): + """Fix 1: malformed relay-ID guard in resolve_corpus_references. + + A non-relay / non-numeric corpus_id must return an empty result set + (not raise a 500 / ValueError). + """ + + def setUp(self): + self.user = User.objects.create_user(username="gql-guard", password="p") + + def _execute(self, corpus_id_value: str): + client = Client(schema) + query = """ + query CorpusRefs($corpusId: ID!) { + corpusReferences(corpusId: $corpusId) { + edges { node { id } } + } + } + """ + return client.execute( + query, + variable_values={"corpusId": corpus_id_value}, + context_value=_GQLContext(self.user), + ) + + def test_malformed_relay_id_returns_empty_no_error(self): + """A non-relay string must not raise a 500 — should return empty edges.""" + result = self._execute("not-a-real-id") + self.assertNotIn("errors", result) + self.assertEqual(result["data"]["corpusReferences"]["edges"], []) + + def test_non_numeric_decoded_pk_returns_empty_no_error(self): + """A base64-encoded type:pk where pk is non-numeric must return empty.""" + import base64 + + # Encodes to CorpusType:abc — decodes fine but pk "abc" is not a digit. + relay_id = base64.b64encode(b"CorpusType:abc").decode() + result = self._execute(relay_id) + self.assertNotIn("errors", result) + self.assertEqual(result["data"]["corpusReferences"]["edges"], []) diff --git a/opencontractserver/tests/test_governance_graph.py b/opencontractserver/tests/test_governance_graph.py new file mode 100644 index 000000000..8234ea560 --- /dev/null +++ b/opencontractserver/tests/test_governance_graph.py @@ -0,0 +1,210 @@ +"""Tests for the corpus-scoped governanceGraph GraphQL query. + +The governance graph is the in-app surface of the reference web: nodes are +documents (filing primaries, exhibits, statute sections) plus "ghost" nodes +for still-EXTERNAL law citations; edges are resolved LAW links (possibly +cross-corpus), EXTERNAL law citations, and ``DocumentRelationship`` rows — +weighted by mention count. Mirrors ``demo/export_governance_graph.py``, +visibility-enforced through the service layer. +""" + +from django.contrib.auth import get_user_model +from django.core.files.base import ContentFile +from django.test import TestCase +from graphql_relay import to_global_id + +from opencontractserver.corpuses.models import Corpus +from opencontractserver.documents.models import Document +from opencontractserver.enrichment.authorities import ( + AuthorityCorpusBootstrapper, + AuthoritySection, +) +from opencontractserver.enrichment.services import EnrichmentService + +User = get_user_model() + +S1_TEXT = ( + "Indemnification is provided under Section 145 of the Delaware General " + "Corporation Law. As permitted by Section 145 of the Delaware General " + "Corporation Law, our charter limits liability. We are also governed by " + "Section 203 of the Delaware General Corporation Law. The form of " + "underwriting agreement is filed as Exhibit 1.1 hereto." +) + +GRAPH_QUERY = """ + query ($cid: ID!) { + governanceGraph(corpusId: $cid) { + corpora { id title kind } + nodes { id documentId title kind corpusId authority degree } + edges { source target edgeType weight } + documentCount + externalKeyCount + edgeCount + mentionCount + truncated + } + } +""" + + +class _Ctx: + def __init__(self, user): + self.user = user + + +def _run_graph(user, corpus_pk): + from graphene.test import Client + + from config.graphql.schema import schema + + return Client(schema, context_value=_Ctx(user)).execute( + GRAPH_QUERY, variables={"cid": to_global_id("CorpusType", corpus_pk)} + ) + + +class GovernanceGraphTests(TestCase): + """Happy-path graph composition for a filing corpus with a linked authority.""" + + def setUp(self): + self.user = User.objects.create_user(username="owner", password="p") + self.corpus = Corpus.objects.create(title="S-1 Corpus", creator=self.user) + self.primary = Document.objects.create( + title="Acme Inc. S-1 (2026-01-01)", creator=self.user + ) + self.primary.txt_extract_file.save( + "s1.txt", ContentFile(S1_TEXT.encode("utf-8")) + ) + self.exhibit = Document.objects.create( + title="Acme Inc. S-1 (2026-01-01) - Exhibit 1.1: EX-1.1", + creator=self.user, + ) + self.exhibit.txt_extract_file.save("ex.txt", ContentFile(b"Underwriting.")) + self.corpus.add_document(document=self.primary, user=self.user) + self.corpus.add_document(document=self.exhibit, user=self.user) + # `add_document` creates corpus-local copies — enrichment resolves to + # those, so the graph must be asserted against the in-corpus ids. + self.primary_id, self.exhibit_id = ( + self.corpus.document_paths.filter( + is_current=True, document__title=t + ).values_list("document_id", flat=True)[0] + for t in (self.primary.title, self.exhibit.title) + ) + + EnrichmentService().apply(corpus_id=self.corpus.id, creator_id=self.user.id) + # Authority covers § 145 only — § 203 stays an EXTERNAL ghost. + auth = AuthorityCorpusBootstrapper().bootstrap( + creator_id=self.user.id, + corpus_title="Delaware General Corporation Law", + sections=[ + AuthoritySection( + key="dgcl:145", heading="DGCL § 145", text="Indemnification." + ), + ], + ) + self.auth_corpus_id = auth["corpus_id"] + EnrichmentService().link_external_references( + corpus_id=self.corpus.id, creator_id=self.user.id + ) + self.statute_id = ( + Corpus.objects.get(pk=self.auth_corpus_id) + .document_paths.filter(is_current=True) + .values_list("document_id", flat=True)[0] + ) + + def _graph(self): + result = _run_graph(self.user, self.corpus.id) + assert result.get("errors") is None, result.get("errors") + return result["data"]["governanceGraph"] + + def _node_by_id(self, graph, node_id): + return next(n for n in graph["nodes"] if n["id"] == node_id) + + def test_nodes_cover_documents_and_external_ghosts(self): + graph = self._graph() + primary_gid = to_global_id("DocumentType", self.primary_id) + exhibit_gid = to_global_id("DocumentType", self.exhibit_id) + statute_gid = to_global_id("DocumentType", self.statute_id) + + primary = self._node_by_id(graph, primary_gid) + assert primary["kind"] == "primary" + assert primary["documentId"] == primary_gid + assert primary["corpusId"] == to_global_id("CorpusType", self.corpus.id) + + assert self._node_by_id(graph, exhibit_gid)["kind"] == "exhibit" + + statute = self._node_by_id(graph, statute_gid) + assert statute["kind"] == "statute" + assert statute["authority"] == "dgcl" + assert statute["corpusId"] == to_global_id("CorpusType", self.auth_corpus_id) + + ghost = self._node_by_id(graph, "key:dgcl:203") + assert ghost["kind"] == "external" + assert ghost["documentId"] is None + assert ghost["authority"] == "dgcl" + assert ghost["title"] == "dgcl:203" + + def test_edges_weighted_by_mention_count(self): + graph = self._graph() + primary_gid = to_global_id("DocumentType", self.primary_id) + statute_gid = to_global_id("DocumentType", self.statute_id) + exhibit_gid = to_global_id("DocumentType", self.exhibit_id) + + by_key = {(e["source"], e["target"], e["edgeType"]): e for e in graph["edges"]} + law = by_key[(primary_gid, statute_gid, "LAW")] + assert law["weight"] == 2 # § 145 cited twice + external = by_key[(primary_gid, "key:dgcl:203", "LAW_EXTERNAL")] + assert external["weight"] == 1 + assert (primary_gid, exhibit_gid, "DOCUMENT") in by_key + + # Degree = sum of weights touching the node. + primary = self._node_by_id(graph, primary_gid) + doc_weight = by_key[(primary_gid, exhibit_gid, "DOCUMENT")]["weight"] + assert primary["degree"] == 2 + 1 + doc_weight + + def test_corpora_classified_filing_vs_authority(self): + graph = self._graph() + kinds = {c["id"]: c["kind"] for c in graph["corpora"]} + assert kinds[to_global_id("CorpusType", self.corpus.id)] == "filing" + assert kinds[to_global_id("CorpusType", self.auth_corpus_id)] == "authority" + + def test_stats(self): + graph = self._graph() + assert graph["documentCount"] == 3 # primary, exhibit, statute + assert graph["externalKeyCount"] == 1 # dgcl:203 + assert graph["edgeCount"] == len(graph["edges"]) + assert graph["mentionCount"] == sum(e["weight"] for e in graph["edges"]) + assert graph["truncated"] is False + + def test_invisible_corpus_returns_empty_graph(self): + stranger = User.objects.create_user(username="stranger", password="p") + result = _run_graph(stranger, self.corpus.id) + assert result.get("errors") is None, result.get("errors") + graph = result["data"]["governanceGraph"] + assert graph["nodes"] == [] + assert graph["edges"] == [] + assert graph["documentCount"] == 0 + + def test_invisible_authority_target_degrades_to_external_ghost(self): + # A reader of the (public) filing corpus who cannot see the private + # authority corpus must get a ghost node — not the statute document. + reader = User.objects.create_user(username="reader", password="p") + self.corpus.is_public = True + self.corpus.save() + Document.objects.filter(id__in=[self.primary_id, self.exhibit_id]).update( + is_public=True + ) + + result = _run_graph(reader, self.corpus.id) + assert result.get("errors") is None, result.get("errors") + graph = result["data"]["governanceGraph"] + + statute_gid = to_global_id("DocumentType", self.statute_id) + node_ids = {n["id"] for n in graph["nodes"]} + assert statute_gid not in node_ids + assert "key:dgcl:145" in node_ids # degraded to ghost, rolled to root + edge_types = {(e["source"], e["target"]): e["edgeType"] for e in graph["edges"]} + primary_gid = to_global_id("DocumentType", self.primary_id) + assert edge_types[(primary_gid, "key:dgcl:145")] == "LAW_EXTERNAL" + # The private authority corpus must not be listed. + corpus_ids = {c["id"] for c in graph["corpora"]} + assert to_global_id("CorpusType", self.auth_corpus_id) not in corpus_ids diff --git a/opencontractserver/utils/celery_tasks.py b/opencontractserver/utils/celery_tasks.py index db2abcc8e..f135cbd21 100644 --- a/opencontractserver/utils/celery_tasks.py +++ b/opencontractserver/utils/celery_tasks.py @@ -24,3 +24,27 @@ def get_doc_analyzer_task_by_name(task_name) -> Optional[Callable]: return None except Exception: return None + + +def get_corpus_analyzer_task_by_name(task_name) -> Optional[Callable]: + """ + Get celery task function Callable by name, only for tasks decorated with corpus_analyzer_task + """ + try: + task = celery_app.tasks.get(task_name) + if task and getattr(task, "is_corpus_analyzer_task", False): + return task + return None + except Exception: + return None + + +def get_analyzer_task_by_name(task_name) -> Optional[Callable]: + """ + Get celery task function Callable by name for either analyzer flavour + (doc-scoped or corpus-scoped). Used by registration sync and system + checks, which treat both identically; dispatch distinguishes them. + """ + return get_doc_analyzer_task_by_name(task_name) or get_corpus_analyzer_task_by_name( + task_name + ) diff --git a/opencontractserver/utils/frontend_paths.py b/opencontractserver/utils/frontend_paths.py new file mode 100644 index 000000000..b46afb87d --- /dev/null +++ b/opencontractserver/utils/frontend_paths.py @@ -0,0 +1,28 @@ +"""Canonical site-relative frontend paths for backend-written links. + +The frontend serves slug-shaped routes only (``frontend/src/App.tsx``): +``/d/:userIdent/:corpusIdent/:docIdent`` for a document in a corpus, with the +first segment being the CORPUS creator's slug (mirrors the frontend's +``buildCanonicalPath`` in ``navigationUtils.ts`` — corpus slugs are unique per +creator and the document slug resolves within that corpus). Any other shape +falls through to the ``*`` catch-all and renders the 404 page, so backend +writers must emit this canonical form. +""" + +from __future__ import annotations + + +def document_in_corpus_path( + *, + corpus_creator_slug: str | None, + corpus_slug: str | None, + document_slug: str | None, +) -> str | None: + """Return ``/d/{corpus_creator_slug}/{corpus_slug}/{document_slug}``. + + Returns ``None`` when any slug is missing — callers should skip the link + rather than write a path that 404s. + """ + if not (corpus_creator_slug and corpus_slug and document_slug): + return None + return f"/d/{corpus_creator_slug}/{corpus_slug}/{document_slug}"