From 951a7839b9c4e03e41509a8b352a24eb917fb584 Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Thu, 23 Apr 2026 17:14:34 -0600 Subject: [PATCH 01/32] Add USJ book parsing pipeline and interlinearizer display WebView - Add usjBookExtractor and bookTokenizer parsers that transform a Platform.Bible USJ book into the interlinear Book/Segment/Token model - Replace placeholder WebView with SegmentView showing tokenized word and punctuation chips - Add BookChapterControl verse picker wired to the scroll-group scrRef - Add unit tests for both parsers and the full WebView component (100% coverage); fix mock signatures for useLocalizedStrings (tuple) and useRecentScriptureRefs (object) - Add Jest mocks for lucide-react and platform-bible-react to unblock ESM imports in test environment --- __mocks__/lucide-react.tsx | 12 ++ __mocks__/papi-frontend-react.ts | 12 +- __mocks__/platform-bible-react.tsx | 52 +++++ __mocks__/platform-bible-utils.ts | 9 +- cspell.json | 3 + jest.config.ts | 3 + .../interlinearizer.web-view.test.tsx | 182 +++++++++++++++-- .../parsers/papi/bookTokenizer.test.ts | 119 +++++++++++ .../parsers/papi/usjBookExtractor.test.ts | 187 ++++++++++++++++++ .../{ => pt9}/interlinearXmlParser.test.ts | 12 +- src/interlinearizer.web-view.tsx | 121 ++++++++++-- src/parsers/papi/bookTokenizer.ts | 76 +++++++ src/parsers/papi/usjBookExtractor.ts | 128 ++++++++++++ src/parsers/{ => pt9}/interlinearXmlParser.ts | 0 src/parsers/{ => pt9}/pt9-xml.md | 0 15 files changed, 870 insertions(+), 46 deletions(-) create mode 100644 __mocks__/lucide-react.tsx create mode 100644 __mocks__/platform-bible-react.tsx create mode 100644 src/__tests__/parsers/papi/bookTokenizer.test.ts create mode 100644 src/__tests__/parsers/papi/usjBookExtractor.test.ts rename src/__tests__/parsers/{ => pt9}/interlinearXmlParser.test.ts (99%) create mode 100644 src/parsers/papi/bookTokenizer.ts create mode 100644 src/parsers/papi/usjBookExtractor.ts rename src/parsers/{ => pt9}/interlinearXmlParser.ts (100%) rename src/parsers/{ => pt9}/pt9-xml.md (100%) diff --git a/__mocks__/lucide-react.tsx b/__mocks__/lucide-react.tsx new file mode 100644 index 00000000..8a426d67 --- /dev/null +++ b/__mocks__/lucide-react.tsx @@ -0,0 +1,12 @@ +/** + * @file Jest mock for lucide-react. The real package ships ESM which Jest cannot parse without + * extra transform configuration. Each icon is stubbed as a no-op component so tests that render + * components importing lucide-react don't fail on the import. + */ + +import type { SVGProps } from 'react'; + +const Icon = (_props: SVGProps) => null; + +export const ChevronLeft = Icon; +export const ChevronRight = Icon; \ No newline at end of file diff --git a/__mocks__/papi-frontend-react.ts b/__mocks__/papi-frontend-react.ts index a2ba000f..f995456f 100644 --- a/__mocks__/papi-frontend-react.ts +++ b/__mocks__/papi-frontend-react.ts @@ -31,12 +31,16 @@ const useProjectSetting = jest ]); const useDialogCallback = jest.fn().mockReturnValue(jest.fn()); const useDataProviderMulti = jest.fn().mockReturnValue([]); -/** Returns a map of localization key -> key (so tests get a string for each key). */ -const useLocalizedStrings = jest.fn().mockImplementation((keys: string[]) => +/** Returns [record, isLoading] tuple; maps each key to itself so tests get a predictable string. */ +const useLocalizedStrings = jest.fn().mockImplementation((keys: string[]) => [ Array.isArray(keys) ? keys.reduce>((acc, k) => ({ ...acc, [k]: k }), {}) : {}, -); + false, +]); const useWebViewController = jest.fn().mockReturnValue(undefined); -const useRecentScriptureRefs = jest.fn().mockReturnValue([]); +/** Returns { recentScriptureRefs, addRecentScriptureRef } matching the real hook signature. */ +const useRecentScriptureRefs = jest + .fn() + .mockReturnValue({ recentScriptureRefs: [], addRecentScriptureRef: jest.fn() }); module.exports = { __esModule: true, diff --git a/__mocks__/platform-bible-react.tsx b/__mocks__/platform-bible-react.tsx new file mode 100644 index 00000000..5baea566 --- /dev/null +++ b/__mocks__/platform-bible-react.tsx @@ -0,0 +1,52 @@ +/** + * @file Jest mock for platform-bible-react. The real package ships ESM which Jest cannot parse + * without extra transform configuration. This stub provides the subset used by extension + * components: `cn`, `Button`, `BookChapterControl`, and `BOOK_CHAPTER_CONTROL_STRING_KEYS`. + */ + +import type { ButtonHTMLAttributes, ReactNode } from 'react'; + +export const cn = (...args: unknown[]): string => + args + .flat() + .filter((v) => typeof v === 'string' && v.length > 0) + .join(' '); + +interface ButtonProps extends ButtonHTMLAttributes { + variant?: string; + size?: string; + children?: ReactNode; + asChild?: boolean; +} + +export function Button({ children, variant: _v, size: _s, asChild: _a, ...rest }: ButtonProps) { + return ; +} + +interface ScriptureRef { + book: string; + chapterNum: number; + verseNum: number; +} + +export const BOOK_CHAPTER_CONTROL_STRING_KEYS: string[] = []; + +export function BookChapterControl({ + scrRef, + handleSubmit, +}: { + scrRef: ScriptureRef; + handleSubmit: (ref: ScriptureRef) => void; + localizedStrings?: Record; + recentSearches?: ScriptureRef[]; + onAddRecentSearch?: (ref: ScriptureRef) => void; +}) { + return ( +
+ {scrRef.book} {scrRef.chapterNum}:{scrRef.verseNum} + +
+ ); +} diff --git a/__mocks__/platform-bible-utils.ts b/__mocks__/platform-bible-utils.ts index 35990083..02e0e9f1 100644 --- a/__mocks__/platform-bible-utils.ts +++ b/__mocks__/platform-bible-utils.ts @@ -53,13 +53,16 @@ class UnsubscriberAsyncList { } } -/** Minimal PlatformError shape matching the real platform-bible-utils type. */ +/** + * Minimal PlatformError shape matching the real platform-bible-utils type. Uses `platformErrorVersion` + * as the discriminant — the same field the real `isPlatformError` checks. + */ interface PlatformError { message: string; - isPlatformError: true; + platformErrorVersion: number; } const isPlatformError = (value: unknown): value is PlatformError => - typeof value === 'object' && value !== null && (value as PlatformError).isPlatformError === true; + typeof value === 'object' && value !== null && 'platformErrorVersion' in (value as object); export { UnsubscriberAsyncList, isPlatformError }; diff --git a/cspell.json b/cspell.json index efd45568..edacb5b8 100644 --- a/cspell.json +++ b/cspell.json @@ -22,6 +22,7 @@ "eflomal", "electronmon", "endregion", + "eten", "finalizer", "Fragmenter", "guids", @@ -44,6 +45,8 @@ "pdps", "plusplus", "proxied", + "Punct", + "recalc", "reinitializing", "reserialized", "scriptio", diff --git a/jest.config.ts b/jest.config.ts index d4dd6155..b12ee344 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -92,6 +92,9 @@ const config: Config = { '^@papi/frontend/react$': '/__mocks__/papi-frontend-react.ts', /** Mock so test-helpers get UnsubscriberAsyncList without loading ESM deps. */ '^platform-bible-utils$': '/__mocks__/platform-bible-utils.ts', + /** Mock platform-bible-react and lucide-react — both ship ESM that Jest cannot parse. */ + '^platform-bible-react$': '/__mocks__/platform-bible-react.tsx', + '^lucide-react$': '/__mocks__/lucide-react.tsx', /** Resolve webpack ?inline imports. */ '^(.+)\\.web-view\\?inline$': '/__mocks__/web-view-inline.ts', /** Resolve webpack ?inline imports: SCSS content. */ diff --git a/src/__tests__/interlinearizer.web-view.test.tsx b/src/__tests__/interlinearizer.web-view.test.tsx index bab23ce8..2ed327a7 100644 --- a/src/__tests__/interlinearizer.web-view.test.tsx +++ b/src/__tests__/interlinearizer.web-view.test.tsx @@ -4,8 +4,19 @@ import type { WebViewProps } from '@papi/core'; import type { SerializedVerseRef } from '@sillsdev/scripture'; -import { render, screen } from '@testing-library/react'; -import { useProjectData } from '@papi/frontend/react'; +import { render, screen, fireEvent } from '@testing-library/react'; +import { + useProjectData, + useProjectSetting, + useLocalizedStrings, + useRecentScriptureRefs, +} from '@papi/frontend/react'; +import type { Book } from 'interlinearizer'; +import { extractBookFromUsj } from 'parsers/papi/usjBookExtractor'; +import { tokenizeBook } from 'parsers/papi/bookTokenizer'; + +jest.mock('parsers/papi/usjBookExtractor'); +jest.mock('parsers/papi/bookTokenizer'); /** * Load the WebView module; it assigns the component to globalThis.webViewComponent. This pattern is @@ -24,8 +35,65 @@ const defaultScrRef: SerializedVerseRef = { book: 'GEN', chapterNum: 1, verseNum const testProjectId = 'test-project-id'; +/** Pre-built Book with one GEN 1:1 segment — used by tests that need the strip to render. */ +const GEN_1_1_BOOK: Book = { + id: 'GEN', + bookRef: 'GEN', + textVersion: 'v1', + segments: [ + { + id: 'GEN 1:1', + startRef: { book: 'GEN', chapter: 1, verse: 1 }, + endRef: { book: 'GEN', chapter: 1, verse: 1 }, + baselineText: 'In the beginning.', + tokens: [ + { + id: 'GEN 1:1:0', + surfaceText: 'In', + writingSystem: 'en', + type: 'word', + charStart: 0, + charEnd: 2, + }, + ], + }, + ], +}; + +/** Pre-built Book with no segments — used by the no-verse-data test. */ +const EMPTY_BOOK: Book = { id: 'GEN', bookRef: 'GEN', textVersion: 'v1', segments: [] }; + +/** Book with a non-word (punctuation) token — exercises the non-word chip branch. */ +const GEN_1_1_PUNCTUATION_BOOK: Book = { + id: 'GEN', + bookRef: 'GEN', + textVersion: 'v1', + segments: [ + { + id: 'GEN 1:1', + startRef: { book: 'GEN', chapter: 1, verse: 1 }, + endRef: { book: 'GEN', chapter: 1, verse: 1 }, + baselineText: '.', + tokens: [ + { + id: 'GEN 1:1:0', + surfaceText: '.', + writingSystem: 'en', + type: 'punctuation', + charStart: 0, + charEnd: 1, + }, + ], + }, + ], +}; + /** Builds a minimal WebViewProps for tests. */ -function makeProps(projectId?: string): WebViewProps { +function makeProps( + projectId?: string, + scrRef: SerializedVerseRef = defaultScrRef, + setScrRef: (r: SerializedVerseRef) => void = () => {}, +): WebViewProps { return { id: 'test-id', webViewType: 'interlinearizer.mainWebView', @@ -40,7 +108,7 @@ function makeProps(projectId?: string): WebViewProps { (r: SerializedVerseRef) => void, number | undefined, (id: number | undefined) => void, - ] => [defaultScrRef, () => {}, undefined, () => {}], + ] => [scrRef, setScrRef, undefined, () => {}], updateWebViewDefinition: () => true, }; } @@ -52,9 +120,29 @@ function mockBookData(value: unknown, isLoading = false): void { })); } +/** Configures useProjectSetting to return the given writing system tag. */ +function mockWritingSystem( + tag: string | { platformErrorVersion: number; message: string } = 'en', +): void { + jest.mocked(useProjectSetting).mockReturnValue([tag, jest.fn(), jest.fn(), false]); +} + describe('InterlinearizerWebView', () => { beforeEach(() => { mockBookData(undefined); + mockWritingSystem(); + jest.mocked(useLocalizedStrings).mockReturnValue([{}, false]); + jest.mocked(useRecentScriptureRefs).mockReturnValue({ + recentScriptureRefs: [], + addRecentScriptureRef: jest.fn(), + }); + jest.mocked(extractBookFromUsj).mockReturnValue({ + bookCode: 'GEN', + writingSystem: 'en', + contentHash: 'abc', + verses: [], + }); + jest.mocked(tokenizeBook).mockReturnValue(GEN_1_1_BOOK); }); it('renders the heading "Interlinearizer"', () => { @@ -69,12 +157,12 @@ describe('InterlinearizerWebView', () => { expect(screen.getByText(/open this webview from a paratext project/i)).toBeInTheDocument(); }); - it('shows the book and projectId when a project is linked', () => { - mockBookData({ type: 'USJ', version: '3.1', content: [] }); + it('shows the book chapter control and renders a segment when a project is linked', () => { + mockBookData({}); render(); - expect(screen.getByText(new RegExp(testProjectId))).toBeInTheDocument(); - expect(screen.getByText(/GEN · project/)).toBeInTheDocument(); + expect(screen.getByTestId('book-chapter-control')).toBeInTheDocument(); + expect(screen.getByText('In')).toBeInTheDocument(); }); it('shows Loading when projectId is set but book data has not arrived', () => { @@ -92,22 +180,84 @@ describe('InterlinearizerWebView', () => { expect(screen.getByText(/no usj book available for gen in project/i)).toBeInTheDocument(); }); - it('shows the raw USFM when book data arrives', () => { - mockBookData({ - type: 'USJ', - version: '3.1', - content: [{ type: 'book', marker: 'id', code: 'EXO' }], - }); + it('renders token chips when the tokenized book has a segment for the current reference', () => { + mockBookData({}); render(); - expect(screen.getByText(/"code": "EXO"/)).toBeInTheDocument(); + expect(screen.getByText('In')).toBeInTheDocument(); + }); + + it('shows a no-verse message when the tokenized book has no segments at all', () => { + mockBookData({}); + jest.mocked(tokenizeBook).mockReturnValue(EMPTY_BOOK); + render(); + + expect(screen.getByText(/no verse data for gen 1:1/i)).toBeInTheDocument(); + }); + + it('falls back to the first segment when the current reference has no exact match (e.g. verse 0)', () => { + mockBookData({}); + const titleRef: SerializedVerseRef = { book: 'GEN', chapterNum: 1, verseNum: 0 }; + render(); + + expect(screen.getByText('In')).toBeInTheDocument(); }); it('shows an error heading and message when book data is a PlatformError', () => { - mockBookData({ isPlatformError: true, message: 'Project not found' }); + mockBookData({ platformErrorVersion: 1, message: 'Project not found' }); render(); expect(screen.getByRole('heading', { name: /error loading book/i })).toBeInTheDocument(); expect(screen.getByText(/project not found/i)).toBeInTheDocument(); }); + + it('falls back to "und" writing system when useProjectSetting returns a PlatformError', () => { + mockBookData({}); + mockWritingSystem({ platformErrorVersion: 1, message: 'Setting unavailable' }); + render(); + + expect(screen.getByText('In')).toBeInTheDocument(); + }); + + it('falls back to "und" writing system when useProjectSetting returns an empty string', () => { + mockBookData({}); + mockWritingSystem(''); + render(); + + expect(screen.getByText('In')).toBeInTheDocument(); + }); + + it('shows a no-verse message when tokenization throws', () => { + mockBookData({}); + jest.mocked(tokenizeBook).mockImplementation(() => { + throw new Error('parse failure'); + }); + render(); + + expect(screen.getByText(/no verse data for gen 1:1/i)).toBeInTheDocument(); + }); + + it('renders non-word tokens as muted chips', () => { + mockBookData({}); + jest.mocked(tokenizeBook).mockReturnValue(GEN_1_1_PUNCTUATION_BOOK); + render(); + + expect(screen.getByText('.')).toBeInTheDocument(); + }); + + it('calls setScrRef and addRecentScriptureRef when the verse picker submits', () => { + mockBookData({}); + const mockSetScrRef = jest.fn(); + const mockAddRecentRef = jest.fn(); + jest.mocked(useRecentScriptureRefs).mockReturnValue({ + recentScriptureRefs: [], + addRecentScriptureRef: mockAddRecentRef, + }); + render(); + + fireEvent.click(screen.getByRole('button', { name: /submit reference/i })); + + expect(mockSetScrRef).toHaveBeenCalledWith(defaultScrRef); + expect(mockAddRecentRef).toHaveBeenCalledWith(defaultScrRef); + }); }); diff --git a/src/__tests__/parsers/papi/bookTokenizer.test.ts b/src/__tests__/parsers/papi/bookTokenizer.test.ts new file mode 100644 index 00000000..a5905bcf --- /dev/null +++ b/src/__tests__/parsers/papi/bookTokenizer.test.ts @@ -0,0 +1,119 @@ +/** @file Unit tests for {@link tokenizeBook}. */ +/// + +import { tokenizeBook } from 'parsers/papi/bookTokenizer'; +import type { RawBook } from 'parsers/papi/usjBookExtractor'; + +function makeRawBook(verses: { sid: string; text: string }[]): RawBook { + return { bookCode: 'GEN', writingSystem: 'en', contentHash: 'abc123', verses }; +} + +describe('tokenizeBook', () => { + it('maps bookCode, contentHash, and writingSystem onto the Book', () => { + const raw = makeRawBook([]); + const book = tokenizeBook(raw); + expect(book.bookRef).toBe('GEN'); + expect(book.id).toBe('GEN'); + expect(book.textVersion).toBe('abc123'); + }); + + it('produces no segments when there are no verses', () => { + expect(tokenizeBook(makeRawBook([])).segments).toEqual([]); + }); + + it('produces one segment per verse in order', () => { + const raw = makeRawBook([ + { sid: 'GEN 1:1', text: 'First.' }, + { sid: 'GEN 1:2', text: 'Second.' }, + ]); + const { segments } = tokenizeBook(raw); + expect(segments).toHaveLength(2); + expect(segments[0].id).toBe('GEN 1:1'); + expect(segments[1].id).toBe('GEN 1:2'); + }); + + it('sets baselineText to the raw verse text', () => { + const text = 'In the beginning God created the heavens and the earth.'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].baselineText).toBe(text); + }); + + it('sets startRef and endRef from the verse SID', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'Hello.' }])); + expect(segments[0].startRef).toEqual({ book: 'GEN', chapter: 1, verse: 1 }); + expect(segments[0].endRef).toEqual({ book: 'GEN', chapter: 1, verse: 1 }); + }); + + it('upholds the charStart/charEnd invariant for every token', () => { + const text = 'In the beginning, God created.'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + segments[0].tokens.forEach((token) => + expect(text.slice(token.charStart, token.charEnd)).toBe(token.surfaceText), + ); + }); + + it('labels word tokens as word', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'Hello world' }])); + expect(segments[0].tokens.every((t) => t.type === 'word')).toBe(true); + }); + + it('labels punctuation tokens as punctuation', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: '., ;!' }])); + expect(segments[0].tokens.every((t) => t.type === 'punctuation')).toBe(true); + }); + + it('produces mixed word and punctuation tokens in the correct order', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'Hello, world.' }])); + const types = segments[0].tokens.map((t) => t.type); + expect(types).toEqual(['word', 'punctuation', 'word', 'punctuation']); + }); + + it('does not produce tokens for whitespace', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: ' ' }])); + expect(segments[0].tokens).toEqual([]); + }); + + it('assigns unique IDs within a segment', () => { + const text = 'A B C.'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + const ids = segments[0].tokens.map((t) => t.id); + expect(new Set(ids).size).toBe(ids.length); + }); + + it('assigns unique IDs across segments', () => { + const raw = makeRawBook([ + { sid: 'GEN 1:1', text: 'Word.' }, + { sid: 'GEN 1:2', text: 'Word.' }, + ]); + const allIds = tokenizeBook(raw).segments.flatMap((s) => s.tokens.map((t) => t.id)); + expect(new Set(allIds).size).toBe(allIds.length); + }); + + it('assigns writingSystem to every token', () => { + const raw: RawBook = { + ...makeRawBook([{ sid: 'GEN 1:1', text: 'Hello.' }]), + writingSystem: 'kmr', + }; + const { segments } = tokenizeBook(raw); + expect(segments[0].tokens.every((t) => t.writingSystem === 'kmr')).toBe(true); + }); + + it('produces an empty token list for an empty verse', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: '' }])); + expect(segments[0].tokens).toEqual([]); + }); + + it('handles Unicode letters (non-ASCII word characters)', () => { + const text = 'Ελληνικά.'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + const wordTokens = segments[0].tokens.filter((t) => t.type === 'word'); + expect(wordTokens).toHaveLength(1); + expect(wordTokens[0].surfaceText).toBe('Ελληνικά'); + }); + + it('throws on an invalid verse SID', () => { + expect(() => tokenizeBook(makeRawBook([{ sid: 'not-a-ref', text: 'text' }]))).toThrow( + 'Invalid verse SID', + ); + }); +}); diff --git a/src/__tests__/parsers/papi/usjBookExtractor.test.ts b/src/__tests__/parsers/papi/usjBookExtractor.test.ts new file mode 100644 index 00000000..84be0981 --- /dev/null +++ b/src/__tests__/parsers/papi/usjBookExtractor.test.ts @@ -0,0 +1,187 @@ +/** @file Unit tests for {@link extractBookFromUsj}. */ +/// + +import { extractBookFromUsj, type UsjDocument } from 'parsers/papi/usjBookExtractor'; + +const WS = 'en'; + +describe('extractBookFromUsj', () => { + it('extracts bookCode from the book marker', () => { + const usj: UsjDocument = { + content: [{ type: 'book', code: 'GEN', content: [] }], + }; + expect(extractBookFromUsj(usj, WS).bookCode).toBe('GEN'); + }); + + it('sets writingSystem from the parameter', () => { + const usj: UsjDocument = { + content: [{ type: 'book', code: 'GEN', content: [] }], + }; + expect(extractBookFromUsj(usj, 'kmr').writingSystem).toBe('kmr'); + }); + + it('produces a stable contentHash for identical content', () => { + const usj: UsjDocument = { content: [{ type: 'book', code: 'GEN', content: [] }] }; + expect(extractBookFromUsj(usj, WS).contentHash).toBe(extractBookFromUsj(usj, WS).contentHash); + }); + + it('produces different contentHashes for different content', () => { + const a: UsjDocument = { content: [{ type: 'book', code: 'GEN', content: [] }] }; + const b: UsjDocument = { content: [{ type: 'book', code: 'MAT', content: [] }] }; + expect(extractBookFromUsj(a, WS).contentHash).not.toBe(extractBookFromUsj(b, WS).contentHash); + }); + + it('returns empty verses when there are no verse markers', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'GEN', content: [] }, + { type: 'chapter', number: '1', sid: 'GEN 1' }, + ], + }; + expect(extractBookFromUsj(usj, WS).verses).toEqual([]); + }); + + it('extracts a single verse with its text', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'GEN', content: [] }, + { + type: 'para', + marker: 'p', + content: [ + { type: 'verse', sid: 'GEN 1:1' }, + 'In the beginning God created the heavens and the earth.', + ], + }, + ], + }; + const result = extractBookFromUsj(usj, WS); + expect(result.verses).toHaveLength(1); + expect(result.verses[0]).toEqual({ + sid: 'GEN 1:1', + text: 'In the beginning God created the heavens and the earth.', + }); + }); + + it('extracts multiple verses in document order', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'GEN', content: [] }, + { + type: 'para', + marker: 'p', + content: [ + { type: 'verse', sid: 'GEN 1:1' }, + 'First verse text.', + { type: 'verse', sid: 'GEN 1:2' }, + 'Second verse text.', + ], + }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses).toHaveLength(2); + expect(verses[0]).toEqual({ sid: 'GEN 1:1', text: 'First verse text.' }); + expect(verses[1]).toEqual({ sid: 'GEN 1:2', text: 'Second verse text.' }); + }); + + it('accumulates text across multiple paragraphs within a verse', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'PSA', content: [] }, + { + type: 'para', + marker: 'q1', + content: [{ type: 'verse', sid: 'PSA 1:1' }, 'Blessed is the man'], + }, + { + type: 'para', + marker: 'q2', + content: ['who walks not in the counsel of the wicked.'], + }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses).toHaveLength(1); + expect(verses[0].text).toBe( + 'Blessed is the man' + 'who walks not in the counsel of the wicked.', + ); + }); + + it('includes text inside inline char nodes', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'JHN', content: [] }, + { + type: 'para', + marker: 'p', + content: [ + { type: 'verse', sid: 'JHN 1:1' }, + 'In the beginning was the ', + { type: 'char', marker: 'nd', content: ['Word'] }, + '.', + ], + }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses[0].text).toBe('In the beginning was the Word.'); + }); + + it('excludes note content from verse text', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'MAT', content: [] }, + { + type: 'para', + marker: 'p', + content: [ + { type: 'verse', sid: 'MAT 1:1' }, + 'The book of the genealogy', + { type: 'note', marker: 'f', content: ['Some footnote text.'] }, + ' of Jesus Christ.', + ], + }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses[0].text).toBe('The book of the genealogy of Jesus Christ.'); + }); + + it('produces an empty-text RawVerse when a verse marker has no following text', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'GEN', content: [] }, + { + type: 'para', + marker: 'p', + content: [ + { type: 'verse', sid: 'GEN 1:1' }, + // no text before the next verse + { type: 'verse', sid: 'GEN 1:2' }, + 'Some text.', + ], + }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses).toHaveLength(2); + expect(verses[0]).toEqual({ sid: 'GEN 1:1', text: '' }); + expect(verses[1]).toEqual({ sid: 'GEN 1:2', text: 'Some text.' }); + }); + + it('throws when a verse marker is missing its sid attribute', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'GEN', content: [] }, + { type: 'para', marker: 'p', content: [{ type: 'verse' }] }, + ], + }; + expect(() => extractBookFromUsj(usj, WS)).toThrow('verse marker missing required sid attribute'); + }); + + it('throws when no book marker with a code attribute is found', () => { + const usj: UsjDocument = { content: [{ type: 'para', content: ['Some text.'] }] }; + expect(() => extractBookFromUsj(usj, WS)).toThrow('no book marker'); + }); +}); diff --git a/src/__tests__/parsers/interlinearXmlParser.test.ts b/src/__tests__/parsers/pt9/interlinearXmlParser.test.ts similarity index 99% rename from src/__tests__/parsers/interlinearXmlParser.test.ts rename to src/__tests__/parsers/pt9/interlinearXmlParser.test.ts index 875789f1..aaf1d48a 100644 --- a/src/__tests__/parsers/interlinearXmlParser.test.ts +++ b/src/__tests__/parsers/pt9/interlinearXmlParser.test.ts @@ -4,7 +4,7 @@ import * as fs from 'fs'; import * as path from 'path'; -import { InterlinearXmlParser } from 'parsers/interlinearXmlParser'; +import { InterlinearXmlParser } from 'parsers/pt9/interlinearXmlParser'; describe('InterlinearXmlParser', () => { let parser: InterlinearXmlParser; @@ -578,7 +578,15 @@ describe('InterlinearXmlParser', () => { }); it('parses real test-data file without throwing', () => { - const xmlPath = path.join(__dirname, '..', '..', '..', 'test-data', 'Interlinear_en_MAT.xml'); + const xmlPath = path.join( + __dirname, + '..', + '..', + '..', + '..', + 'test-data', + 'Interlinear_en_MAT.xml', + ); const xml = fs.readFileSync(xmlPath, 'utf-8'); const result = parser.parse(xml); diff --git a/src/interlinearizer.web-view.tsx b/src/interlinearizer.web-view.tsx index 7eb9c8e6..c8492204 100644 --- a/src/interlinearizer.web-view.tsx +++ b/src/interlinearizer.web-view.tsx @@ -1,40 +1,115 @@ import type { WebViewProps } from '@papi/core'; -import { useProjectData } from '@papi/frontend/react'; +import { + useProjectData, + useProjectSetting, + useLocalizedStrings, + useRecentScriptureRefs, +} from '@papi/frontend/react'; import { isPlatformError } from 'platform-bible-utils'; +import { useMemo } from 'react'; +import { BookChapterControl, BOOK_CHAPTER_CONTROL_STRING_KEYS } from 'platform-bible-react'; +import { extractBookFromUsj } from 'parsers/papi/usjBookExtractor'; +import { tokenizeBook } from 'parsers/papi/bookTokenizer'; +import type { Segment } from 'interlinearizer'; + +/** Renders the tokens of a single segment as inline chips. */ +function SegmentView({ segment }: { segment: Segment }) { + return ( +
+

+ {segment.id} +

+
+ {segment.tokens.map((token) => + token.type === 'word' ? ( + + {token.surfaceText} + + ) : ( + + {token.surfaceText} + + ), + )} +
+
+ ); +} /** - * Fetches and displays the USJ book data for the given project and scripture reference. Shows a - * loading indicator while data is in flight, an error message if the fetch fails or returns no - * data, and the raw JSON of the book otherwise. + * Fetches the USJ book for the given project, tokenizes it, finds the segment matching `scrRef`, + * and renders a {@link SegmentView} for that verse. Shows loading / error states while data is in + * flight or unavailable. */ function ProjectBookFetcher({ projectId, scrRef, + setScrRef, }: { projectId: string; scrRef: ReturnType[0]; + setScrRef: ReturnType[1]; }) { const [bookResult, , isLoading] = useProjectData('platformScripture.USJ_Book', projectId).BookUSJ( scrRef, undefined, ); - let bookUsj: typeof bookResult | undefined; - let bookError: string | undefined; + const [writingSystem] = useProjectSetting(projectId, 'platform.languageTag', ''); + + const [localizedStrings] = useLocalizedStrings( + useMemo(() => [...BOOK_CHAPTER_CONTROL_STRING_KEYS], []), + ); + const { recentScriptureRefs: recentRefs, addRecentScriptureRef: onAddRecentRef } = + useRecentScriptureRefs(); + const book = useMemo(() => { + if (!bookResult || isPlatformError(bookResult)) return undefined; + try { + const ws = isPlatformError(writingSystem) ? 'und' : writingSystem || 'und'; + return tokenizeBook(extractBookFromUsj(bookResult, ws)); + } catch { + return undefined; + } + }, [bookResult, writingSystem]); + + const currentSegment = useMemo(() => { + if (!book) return undefined; + return ( + book.segments.find( + (seg) => + seg.startRef.book === scrRef.book && + seg.startRef.chapter === scrRef.chapterNum && + seg.startRef.verse === scrRef.verseNum, + ) ?? book.segments[0] + ); + }, [book, scrRef]); + + let bookError: string | undefined; if (isPlatformError(bookResult)) { bookError = bookResult.message; } else if (!isLoading && bookResult === undefined) { bookError = `No USJ book available for ${scrRef.book} in project ${projectId}`; - } else { - bookUsj = bookResult; } return ( - <> -

- {scrRef.book} · project {projectId} -

+
+ { + setScrRef(ref); + onAddRecentRef(ref); + }} + localizedStrings={localizedStrings} + recentSearches={recentRefs} + onAddRecentSearch={onAddRecentRef} + /> {bookError && (
@@ -45,12 +120,16 @@ function ProjectBookFetcher({
)} - {!bookError && ( - 
-          {isLoading ? 'Loading…' : JSON.stringify(bookUsj, undefined, 2)}
-
+ {!bookError && isLoading &&

Loading…

} + + {!bookError && !isLoading && !currentSegment && ( +

+ No verse data for {scrRef.book} {scrRef.chapterNum}:{scrRef.verseNum}. +

)} - + + {!bookError && !isLoading && currentSegment && } +
); } @@ -63,14 +142,14 @@ globalThis.webViewComponent = function InterlinearizerWebView({ projectId, useWebViewScrollGroupScrRef, }: WebViewProps) { - const [scrRef] = useWebViewScrollGroupScrRef(); + const [scrRef, setScrRef] = useWebViewScrollGroupScrRef(); return ( -
-

Interlinearizer

+
+

Interlinearizer

{projectId ? ( - + ) : (

Open this WebView from a Paratext project to load its source book. diff --git a/src/parsers/papi/bookTokenizer.ts b/src/parsers/papi/bookTokenizer.ts new file mode 100644 index 00000000..2e2201cc --- /dev/null +++ b/src/parsers/papi/bookTokenizer.ts @@ -0,0 +1,76 @@ +/** @file Tokenizes a {@link RawBook} into the interlinear model's `Book → Segment → Token` chain. */ + +import { VerseRef } from '@sillsdev/scripture'; +import type { Book, ScriptureRef, Segment, Token, TokenType } from 'interlinearizer'; + +import type { RawBook } from './usjBookExtractor'; + +// Matches word tokens (Unicode letters / digits / combining marks) and punctuation tokens +// (any single non-word, non-whitespace character). Whitespace is not tokenized. +const TOKEN_RE = /[\p{L}\p{N}\p{M}]+|[^\p{L}\p{N}\p{M}\s]/gu; +const WORD_START_RE = /[\p{L}\p{N}\p{M}]/u; + +/** + * Parses a USJ verse SID (e.g. `"GEN 1:1"`) into a {@link ScriptureRef}. + * + * @throws {Error} If `sid` is not a valid scripture reference string. + */ +function parseSid(sid: string): ScriptureRef { + const { success, verseRef } = VerseRef.tryParse(sid); + if (!success) throw new Error(`Invalid verse SID: "${sid}"`); + return { book: verseRef.book, chapter: verseRef.chapterNum, verse: verseRef.verseNum }; +} + +/** + * Splits a verse's plain text into an ordered array of {@link Token}s. + * + * Word tokens (`\p{L}\p{N}\p{M}` runs) and punctuation tokens (any single non-word, non-whitespace + * character) are emitted in document order. Whitespace is not tokenized. Character offsets are + * zero-based relative to `text`; `charEnd` is exclusive. + * + * @param text - The verse's `baselineText` string. + * @param sid - The verse SID used as the token ID prefix (e.g. `"GEN 1:1"`). + * @param writingSystem - BCP 47 tag assigned to every token's `writingSystem` field. + */ +function tokenizeVerse(text: string, sid: string, writingSystem: string): Token[] { + return Array.from(text.matchAll(TOKEN_RE)).map((match) => { + const surfaceText = match[0]; + const charStart = match.index; + const charEnd = charStart + surfaceText.length; + const type: TokenType = WORD_START_RE.test(surfaceText[0]) ? 'word' : 'punctuation'; + return { id: `${sid}:${charStart}`, surfaceText, writingSystem, type, charStart, charEnd }; + }); +} + +/** + * Tokenizes a {@link RawBook} into the interlinear model's `Book` (text layer only — no analysis). + * + * Each `RawVerse` becomes one `Segment`. The verse SID is parsed into `startRef` / `endRef` (both + * equal — verse-level granularity). The verse text is split into `Token`s using Unicode-aware + * word/punctuation splitting; character offsets are relative to `Segment.baselineText`. + * + * Invariant upheld for every token: `segment.baselineText.slice(token.charStart, token.charEnd) === + * token.surfaceText`. + * + * @param rawBook - Extracted book data from {@link extractBookFromUsj}. + * @throws {Error} If any `RawVerse.sid` cannot be parsed as a valid scripture reference. + */ +export function tokenizeBook(rawBook: RawBook): Book { + const segments: Segment[] = rawBook.verses.map(({ sid, text }) => { + const ref = parseSid(sid); + return { + id: sid, + startRef: ref, + endRef: ref, + baselineText: text, + tokens: tokenizeVerse(text, sid, rawBook.writingSystem), + }; + }); + + return { + id: rawBook.bookCode, + bookRef: rawBook.bookCode, + textVersion: rawBook.contentHash, + segments, + }; +} diff --git a/src/parsers/papi/usjBookExtractor.ts b/src/parsers/papi/usjBookExtractor.ts new file mode 100644 index 00000000..027b8643 --- /dev/null +++ b/src/parsers/papi/usjBookExtractor.ts @@ -0,0 +1,128 @@ +/** @file Extracts {@link RawBook} from a papi USJ book response. */ + +/** Plain text of a single verse extracted from a USJ document, ready to be tokenized. */ +export interface RawVerse { + /** SID from the USJ verse marker, e.g. `"GEN 1:1"`. Parsed into `Segment.startRef` / `endRef`. */ + sid: string; + /** + * Accumulated plain-text content of the verse. Note and footnote content is excluded. Becomes + * `Segment.baselineText`; token `charStart` / `charEnd` are expressed relative to this string. + */ + text: string; +} + +/** + * Raw book data captured from a papi USJ response. Self-contained — everything the tokenizer needs + * to produce `Book → Segment → Token`. + */ +export interface RawBook { + /** 3-letter book code, e.g. `"GEN"`. */ + bookCode: string; + /** BCP 47 writing system tag for the baseline text, from `platform.languageTag`. */ + writingSystem: string; + /** FNV-1a hash of the serialized USJ content. Becomes `Book.textVersion`. */ + contentHash: string; + /** Verse entries in document order, one per USJ `verse` marker. */ + verses: RawVerse[]; +} + +// --------------------------------------------------------------------------- +// Minimal local types for USJ traversal. +// @eten-tech-foundation/scripture-utilities is not a direct dependency of this +// extension, so we define the subset we need here. +// --------------------------------------------------------------------------- + +/** A USJ content item: either a plain text string or a marker node. */ +type MarkerContent = string | UsjNode; + +/** A USJ marker node. Only the fields used during extraction are declared. */ +interface UsjNode { + type: string; + marker?: string; + number?: string; + code?: string; + sid?: string; + content?: MarkerContent[]; +} + +/** Minimal shape of a USJ document as returned by the papi `platformScripture.USJ_Book` provider. */ +export interface UsjDocument { + content: MarkerContent[]; +} + +// --------------------------------------------------------------------------- +// Implementation +// --------------------------------------------------------------------------- + +/** Mutable state threaded through the recursive USJ traversal. */ +interface TraversalState { + bookCode: string; + currentVerse: { sid: string; text: string } | undefined; + verses: RawVerse[]; +} + +/** + * Recursively walks a USJ content array, accumulating verse text into `state`. + * + * @param nodes - Content items to walk (`string` or {@link UsjNode}). + * @param state - Shared mutable state updated in place during traversal. + * @throws {Error} If a `verse` node is missing its `sid` attribute. + */ +function traverse(nodes: MarkerContent[], state: TraversalState): void { + nodes.forEach((node) => { + if (typeof node === 'string') { + if (state.currentVerse !== undefined) state.currentVerse.text += node; + } else if (node.type === 'book') { + if (node.code) state.bookCode = node.code; + if (node.content) traverse(node.content, state); + } else if (node.type === 'verse') { + if (state.currentVerse !== undefined) state.verses.push(state.currentVerse); + if (!node.sid) throw new Error('Invalid USJ: verse marker missing required sid attribute'); + state.currentVerse = { sid: node.sid, text: '' }; + } else if (node.type === 'note') { + // Skip note and footnote content — not part of the baseline text. + } else if (node.content) { + traverse(node.content, state); + } + }); +} + +/** FNV-1a 32-bit hash — sufficient for one-way internal content versioning. */ +function fnv1a32(s: string): string { + let h = 2166136261; + for (let i = 0; i < s.length; i++) { + // eslint-disable-next-line no-bitwise + h = Math.imul(h ^ s.charCodeAt(i), 16777619); + } + // eslint-disable-next-line no-bitwise + return (h >>> 0).toString(16); +} + +/** + * Extracts a {@link RawBook} from a papi USJ book response. + * + * Each `verse` marker in the USJ document becomes one {@link RawVerse}. Text strings within the + * verse scope are accumulated into `RawVerse.text`; `note` nodes are skipped entirely. Verse + * markers with no following text produce an empty `RawVerse` (`text: ""`). + * + * @param usj - USJ document returned by `useProjectData('platformScripture.USJ_Book', ...)`. + * @param writingSystem - BCP 47 tag for the baseline, from `platform.languageTag`. + * @throws {Error} If no `book` marker with a `code` attribute is found in the document. + */ +export function extractBookFromUsj(usj: UsjDocument, writingSystem: string): RawBook { + const contentHash = fnv1a32(JSON.stringify(usj.content)); + const state: TraversalState = { bookCode: '', currentVerse: undefined, verses: [] }; + + traverse(usj.content, state); + + if (state.currentVerse !== undefined) state.verses.push(state.currentVerse); + + if (!state.bookCode) throw new Error('Invalid USJ: no book marker with a code attribute found'); + + return { + bookCode: state.bookCode, + writingSystem, + contentHash, + verses: state.verses, + }; +} diff --git a/src/parsers/interlinearXmlParser.ts b/src/parsers/pt9/interlinearXmlParser.ts similarity index 100% rename from src/parsers/interlinearXmlParser.ts rename to src/parsers/pt9/interlinearXmlParser.ts diff --git a/src/parsers/pt9-xml.md b/src/parsers/pt9/pt9-xml.md similarity index 100% rename from src/parsers/pt9-xml.md rename to src/parsers/pt9/pt9-xml.md From 2392764c6850cf263d990afc1b653ca497df5664 Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Fri, 24 Apr 2026 10:58:25 -0600 Subject: [PATCH 02/32] Fix lint error --- src/__tests__/parsers/papi/usjBookExtractor.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/__tests__/parsers/papi/usjBookExtractor.test.ts b/src/__tests__/parsers/papi/usjBookExtractor.test.ts index 84be0981..4b879c10 100644 --- a/src/__tests__/parsers/papi/usjBookExtractor.test.ts +++ b/src/__tests__/parsers/papi/usjBookExtractor.test.ts @@ -103,9 +103,7 @@ describe('extractBookFromUsj', () => { }; const { verses } = extractBookFromUsj(usj, WS); expect(verses).toHaveLength(1); - expect(verses[0].text).toBe( - 'Blessed is the man' + 'who walks not in the counsel of the wicked.', - ); + expect(verses[0].text).toBe('Blessed is the manwho walks not in the counsel of the wicked.'); }); it('includes text inside inline char nodes', () => { @@ -177,7 +175,9 @@ describe('extractBookFromUsj', () => { { type: 'para', marker: 'p', content: [{ type: 'verse' }] }, ], }; - expect(() => extractBookFromUsj(usj, WS)).toThrow('verse marker missing required sid attribute'); + expect(() => extractBookFromUsj(usj, WS)).toThrow( + 'verse marker missing required sid attribute', + ); }); it('throws when no book marker with a code attribute is found', () => { From a2e7552ab36b6f1ba0d4d54b5b19256e560ad5f2 Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Fri, 24 Apr 2026 11:51:41 -0600 Subject: [PATCH 03/32] Add div for BookChapterControl, display entire chapters at once --- .../interlinearizer.web-view.test.tsx | 97 ++++++++++++- src/interlinearizer.web-view.tsx | 135 +++++++++++------- 2 files changed, 176 insertions(+), 56 deletions(-) diff --git a/src/__tests__/interlinearizer.web-view.test.tsx b/src/__tests__/interlinearizer.web-view.test.tsx index 2ed327a7..5eaf2704 100644 --- a/src/__tests__/interlinearizer.web-view.test.tsx +++ b/src/__tests__/interlinearizer.web-view.test.tsx @@ -63,6 +63,47 @@ const GEN_1_1_BOOK: Book = { /** Pre-built Book with no segments — used by the no-verse-data test. */ const EMPTY_BOOK: Book = { id: 'GEN', bookRef: 'GEN', textVersion: 'v1', segments: [] }; +/** Book with two segments in GEN 1 — used by chapter-display tests. */ +const GEN_1_MULTI_BOOK: Book = { + id: 'GEN', + bookRef: 'GEN', + textVersion: 'v1', + segments: [ + { + id: 'GEN 1:1', + startRef: { book: 'GEN', chapter: 1, verse: 1 }, + endRef: { book: 'GEN', chapter: 1, verse: 1 }, + baselineText: 'In the beginning.', + tokens: [ + { + id: 'GEN 1:1:0', + surfaceText: 'In', + writingSystem: 'en', + type: 'word', + charStart: 0, + charEnd: 2, + }, + ], + }, + { + id: 'GEN 1:2', + startRef: { book: 'GEN', chapter: 1, verse: 2 }, + endRef: { book: 'GEN', chapter: 1, verse: 2 }, + baselineText: 'And the earth.', + tokens: [ + { + id: 'GEN 1:2:0', + surfaceText: 'And', + writingSystem: 'en', + type: 'word', + charStart: 0, + charEnd: 3, + }, + ], + }, + ], +}; + /** Book with a non-word (punctuation) token — exercises the non-word chip branch. */ const GEN_1_1_PUNCTUATION_BOOK: Book = { id: 'GEN', @@ -145,10 +186,10 @@ describe('InterlinearizerWebView', () => { jest.mocked(tokenizeBook).mockReturnValue(GEN_1_1_BOOK); }); - it('renders the heading "Interlinearizer"', () => { + it('shows the book chapter control regardless of whether a project is linked', () => { render(); - expect(screen.getByRole('heading', { name: /interlinearizer/i })).toBeInTheDocument(); + expect(screen.getByTestId('book-chapter-control')).toBeInTheDocument(); }); it('shows a prompt to open from a project when no projectId is provided', () => { @@ -192,15 +233,38 @@ describe('InterlinearizerWebView', () => { jest.mocked(tokenizeBook).mockReturnValue(EMPTY_BOOK); render(); - expect(screen.getByText(/no verse data for gen 1:1/i)).toBeInTheDocument(); + expect(screen.getByText(/no verse data for gen 1\./i)).toBeInTheDocument(); + }); + + it('renders all segments in the current chapter', () => { + mockBookData({}); + jest.mocked(tokenizeBook).mockReturnValue(GEN_1_MULTI_BOOK); + render(); + + expect(screen.getByText('In')).toBeInTheDocument(); + expect(screen.getByText('And')).toBeInTheDocument(); + }); + + it('highlights only the segment matching the current verse', () => { + mockBookData({}); + jest.mocked(tokenizeBook).mockReturnValue(GEN_1_MULTI_BOOK); + // defaultScrRef is GEN 1:1, so verse 1 is active + const { container } = render(); + + const activeSegments = Array.from(container.querySelectorAll('button')).filter((el) => + el.className.includes('tw-bg-muted/50'), + ); + expect(activeSegments).toHaveLength(1); }); - it('falls back to the first segment when the current reference has no exact match (e.g. verse 0)', () => { + it('shows all chapter segments when navigating to a title reference (verse 0)', () => { mockBookData({}); + jest.mocked(tokenizeBook).mockReturnValue(GEN_1_MULTI_BOOK); const titleRef: SerializedVerseRef = { book: 'GEN', chapterNum: 1, verseNum: 0 }; render(); expect(screen.getByText('In')).toBeInTheDocument(); + expect(screen.getByText('And')).toBeInTheDocument(); }); it('shows an error heading and message when book data is a PlatformError', () => { @@ -234,7 +298,7 @@ describe('InterlinearizerWebView', () => { }); render(); - expect(screen.getByText(/no verse data for gen 1:1/i)).toBeInTheDocument(); + expect(screen.getByText(/no verse data for gen 1\./i)).toBeInTheDocument(); }); it('renders non-word tokens as muted chips', () => { @@ -260,4 +324,27 @@ describe('InterlinearizerWebView', () => { expect(mockSetScrRef).toHaveBeenCalledWith(defaultScrRef); expect(mockAddRecentRef).toHaveBeenCalledWith(defaultScrRef); }); + + it('calls setScrRef with the segment ref when a verse box is clicked', () => { + mockBookData({}); + jest.mocked(tokenizeBook).mockReturnValue(GEN_1_MULTI_BOOK); + const mockSetScrRef = jest.fn(); + // Start at verse 1; click verse 2's token to select it + render(); + + fireEvent.click(screen.getByText('And')); + + expect(mockSetScrRef).toHaveBeenCalledWith({ book: 'GEN', chapterNum: 1, verseNum: 2 }); + }); + + it('passes a book-stable ref to BookUSJ so verse changes do not re-fetch the book', () => { + const mockBookUSJ = jest.fn().mockReturnValue([{}, jest.fn(), false]); + jest.mocked(useProjectData).mockImplementation(() => ({ BookUSJ: mockBookUSJ })); + render(); + + expect(mockBookUSJ).toHaveBeenCalledWith( + { book: 'GEN', chapterNum: 1, verseNum: 1 }, + undefined, + ); + }); }); diff --git a/src/interlinearizer.web-view.tsx b/src/interlinearizer.web-view.tsx index c8492204..3226f05e 100644 --- a/src/interlinearizer.web-view.tsx +++ b/src/interlinearizer.web-view.tsx @@ -13,11 +13,27 @@ import { tokenizeBook } from 'parsers/papi/bookTokenizer'; import type { Segment } from 'interlinearizer'; /** Renders the tokens of a single segment as inline chips. */ -function SegmentView({ segment }: { segment: Segment }) { +function SegmentView({ + segment, + isActive, + onClick, +}: { + segment: Segment; + isActive?: boolean; + onClick?: () => void; +}) { return ( -

+
+ ); } /** - * Fetches the USJ book for the given project, tokenizes it, finds the segment matching `scrRef`, - * and renders a {@link SegmentView} for that verse. Shows loading / error states while data is in - * flight or unavailable. + * Fetches the USJ book for the given project, tokenizes it, and renders all segments in the current + * chapter. Shows loading / error states while data is in flight or unavailable. */ function ProjectBookFetcher({ projectId, @@ -56,19 +71,17 @@ function ProjectBookFetcher({ scrRef: ReturnType[0]; setScrRef: ReturnType[1]; }) { + const bookScrRef = useMemo( + () => ({ book: scrRef.book, chapterNum: 1, verseNum: 1 }), + [scrRef.book], + ); const [bookResult, , isLoading] = useProjectData('platformScripture.USJ_Book', projectId).BookUSJ( - scrRef, + bookScrRef, undefined, ); const [writingSystem] = useProjectSetting(projectId, 'platform.languageTag', ''); - const [localizedStrings] = useLocalizedStrings( - useMemo(() => [...BOOK_CHAPTER_CONTROL_STRING_KEYS], []), - ); - const { recentScriptureRefs: recentRefs, addRecentScriptureRef: onAddRecentRef } = - useRecentScriptureRefs(); - const book = useMemo(() => { if (!bookResult || isPlatformError(bookResult)) return undefined; try { @@ -79,17 +92,13 @@ function ProjectBookFetcher({ } }, [bookResult, writingSystem]); - const currentSegment = useMemo(() => { - if (!book) return undefined; - return ( - book.segments.find( - (seg) => - seg.startRef.book === scrRef.book && - seg.startRef.chapter === scrRef.chapterNum && - seg.startRef.verse === scrRef.verseNum, - ) ?? book.segments[0] - ); - }, [book, scrRef]); + const chapterSegments = useMemo( + () => + book?.segments.filter( + (seg) => seg.startRef.book === scrRef.book && seg.startRef.chapter === scrRef.chapterNum, + ) ?? [], + [book, scrRef], + ); let bookError: string | undefined; if (isPlatformError(bookResult)) { @@ -100,17 +109,6 @@ function ProjectBookFetcher({ return (
- { - setScrRef(ref); - onAddRecentRef(ref); - }} - localizedStrings={localizedStrings} - recentSearches={recentRefs} - onAddRecentSearch={onAddRecentRef} - /> - {bookError && (

Error loading book

@@ -122,21 +120,37 @@ function ProjectBookFetcher({ {!bookError && isLoading &&

Loading…

} - {!bookError && !isLoading && !currentSegment && ( + {!bookError && !isLoading && chapterSegments.length === 0 && (

- No verse data for {scrRef.book} {scrRef.chapterNum}:{scrRef.verseNum}. + No verse data for {scrRef.book} {scrRef.chapterNum}.

)} - {!bookError && !isLoading && currentSegment && } + {!bookError && !isLoading && chapterSegments.length > 0 && ( +
+ {chapterSegments.map((seg) => ( + + setScrRef({ + book: seg.startRef.book, + chapterNum: seg.startRef.chapter, + verseNum: seg.startRef.verse, + }) + } + /> + ))} +
+ )}
); } /** - * Root WebView component for the Interlinearizer. Reads the scroll-group scripture reference and - * delegates book fetching to {@link ProjectBookFetcher}. Shows a placeholder when no projectId is - * provided (i.e. the WebView was opened without a project). + * Root WebView component for the Interlinearizer. Renders a sticky reference picker at the top and + * delegates book fetching to {@link ProjectBookFetcher} in the scrollable content area below. */ globalThis.webViewComponent = function InterlinearizerWebView({ projectId, @@ -144,17 +158,36 @@ globalThis.webViewComponent = function InterlinearizerWebView({ }: WebViewProps) { const [scrRef, setScrRef] = useWebViewScrollGroupScrRef(); + const [localizedStrings] = useLocalizedStrings( + useMemo(() => [...BOOK_CHAPTER_CONTROL_STRING_KEYS], []), + ); + const { recentScriptureRefs: recentRefs, addRecentScriptureRef: onAddRecentRef } = + useRecentScriptureRefs(); + return ( -
-

Interlinearizer

+
+
+ { + setScrRef(ref); + onAddRecentRef(ref); + }} + localizedStrings={localizedStrings} + recentSearches={recentRefs} + onAddRecentSearch={onAddRecentRef} + /> +
- {projectId ? ( - - ) : ( -

- Open this WebView from a Paratext project to load its source book. -

- )} +
+ {projectId ? ( + + ) : ( +

+ Open this WebView from a Paratext project to load its source book. +

+ )} +
); }; From c58467fd21c1956fbab8009483061b0894164d41 Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Fri, 24 Apr 2026 12:45:39 -0600 Subject: [PATCH 04/32] Fix USJ chapter/verse traversal, surrogate-pair tokenization, and memo deps - Flush open verse at chapter boundaries and traverse chapter/verse inline content - Add inter-paragraph spacing in usjBookExtractor - Use stableStringify for deterministic content hashing - Fix bookTokenizer to test full surfaceText (not first code unit) so astral-plane letters tokenize as words - Narrow scrRef memo deps to primitives to avoid stale re-renders - Add aria-current to active segment button - Upgrade cn mock to handle object-form class arguments --- __mocks__/platform-bible-react.tsx | 17 +++-- .../interlinearizer.web-view.test.tsx | 17 ++--- .../parsers/papi/bookTokenizer.test.ts | 11 ++++ .../parsers/papi/usjBookExtractor.test.ts | 64 ++++++++++++++++++- src/interlinearizer.web-view.tsx | 3 +- src/parsers/papi/bookTokenizer.ts | 2 +- src/parsers/papi/usjBookExtractor.ts | 23 ++++++- 7 files changed, 119 insertions(+), 18 deletions(-) diff --git a/__mocks__/platform-bible-react.tsx b/__mocks__/platform-bible-react.tsx index 5baea566..40ea4d89 100644 --- a/__mocks__/platform-bible-react.tsx +++ b/__mocks__/platform-bible-react.tsx @@ -6,11 +6,17 @@ import type { ButtonHTMLAttributes, ReactNode } from 'react'; -export const cn = (...args: unknown[]): string => - args - .flat() - .filter((v) => typeof v === 'string' && v.length > 0) - .join(' '); +function flattenCn(arg: unknown): string[] { + if (typeof arg === 'string') return arg.length > 0 ? [arg] : []; + if (Array.isArray(arg)) return arg.flatMap(flattenCn); + if (arg !== null && typeof arg === 'object') + return Object.entries(arg as Record) + .filter(([, v]) => Boolean(v)) + .map(([k]) => k); + return []; +} + +export const cn = (...args: unknown[]): string => args.flatMap(flattenCn).join(' '); interface ButtonProps extends ButtonHTMLAttributes { variant?: string; @@ -39,7 +45,6 @@ export function BookChapterControl({ handleSubmit: (ref: ScriptureRef) => void; localizedStrings?: Record; recentSearches?: ScriptureRef[]; - onAddRecentSearch?: (ref: ScriptureRef) => void; }) { return (
diff --git a/src/__tests__/interlinearizer.web-view.test.tsx b/src/__tests__/interlinearizer.web-view.test.tsx index 5eaf2704..94a13003 100644 --- a/src/__tests__/interlinearizer.web-view.test.tsx +++ b/src/__tests__/interlinearizer.web-view.test.tsx @@ -251,9 +251,7 @@ describe('InterlinearizerWebView', () => { // defaultScrRef is GEN 1:1, so verse 1 is active const { container } = render(); - const activeSegments = Array.from(container.querySelectorAll('button')).filter((el) => - el.className.includes('tw-bg-muted/50'), - ); + const activeSegments = container.querySelectorAll('button[aria-current="true"]'); expect(activeSegments).toHaveLength(1); }); @@ -340,11 +338,14 @@ describe('InterlinearizerWebView', () => { it('passes a book-stable ref to BookUSJ so verse changes do not re-fetch the book', () => { const mockBookUSJ = jest.fn().mockReturnValue([{}, jest.fn(), false]); jest.mocked(useProjectData).mockImplementation(() => ({ BookUSJ: mockBookUSJ })); - render(); - - expect(mockBookUSJ).toHaveBeenCalledWith( - { book: 'GEN', chapterNum: 1, verseNum: 1 }, - undefined, + const { rerender } = render(); + rerender( + , ); + + const refsPassed = mockBookUSJ.mock.calls.map((c) => c[0]); + refsPassed.forEach((ref) => expect(ref).toEqual({ book: 'GEN', chapterNum: 1, verseNum: 1 })); }); }); diff --git a/src/__tests__/parsers/papi/bookTokenizer.test.ts b/src/__tests__/parsers/papi/bookTokenizer.test.ts index a5905bcf..19e55adb 100644 --- a/src/__tests__/parsers/papi/bookTokenizer.test.ts +++ b/src/__tests__/parsers/papi/bookTokenizer.test.ts @@ -116,4 +116,15 @@ describe('tokenizeBook', () => { 'Invalid verse SID', ); }); + + it('classifies astral-plane letters (surrogate pairs) as word tokens', () => { + // Gothic letters U+10330–U+1034F are outside the BMP; each code point is two UTF-16 code + // units. Testing surfaceText[0] (a lone surrogate) against WORD_START_RE would fail — the + // fix is to test the full surfaceText string. + const text = '𐌰𐌱𐌲'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe(text); + }); }); diff --git a/src/__tests__/parsers/papi/usjBookExtractor.test.ts b/src/__tests__/parsers/papi/usjBookExtractor.test.ts index 4b879c10..5ee958d5 100644 --- a/src/__tests__/parsers/papi/usjBookExtractor.test.ts +++ b/src/__tests__/parsers/papi/usjBookExtractor.test.ts @@ -103,7 +103,7 @@ describe('extractBookFromUsj', () => { }; const { verses } = extractBookFromUsj(usj, WS); expect(verses).toHaveLength(1); - expect(verses[0].text).toBe('Blessed is the manwho walks not in the counsel of the wicked.'); + expect(verses[0].text).toBe('Blessed is the man who walks not in the counsel of the wicked.'); }); it('includes text inside inline char nodes', () => { @@ -168,6 +168,22 @@ describe('extractBookFromUsj', () => { expect(verses[1]).toEqual({ sid: 'GEN 1:2', text: 'Some text.' }); }); + it('captures text nested directly inside a verse node', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'GEN', content: [] }, + { + type: 'para', + marker: 'p', + content: [{ type: 'verse', sid: 'GEN 1:1', content: ['Inline verse content.'] }], + }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses).toHaveLength(1); + expect(verses[0]).toEqual({ sid: 'GEN 1:1', text: 'Inline verse content.' }); + }); + it('throws when a verse marker is missing its sid attribute', () => { const usj: UsjDocument = { content: [ @@ -184,4 +200,50 @@ describe('extractBookFromUsj', () => { const usj: UsjDocument = { content: [{ type: 'para', content: ['Some text.'] }] }; expect(() => extractBookFromUsj(usj, WS)).toThrow('no book marker'); }); + + it('flushes an open verse when a chapter boundary is crossed', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'GEN', content: [] }, + { + type: 'para', + marker: 'p', + content: [{ type: 'verse', sid: 'GEN 1:31' }, 'Last verse of chapter one.'], + }, + { type: 'chapter', number: '2', sid: 'GEN 2' }, + { + type: 'para', + marker: 'p', + content: [{ type: 'verse', sid: 'GEN 2:1' }, 'First verse of chapter two.'], + }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses).toHaveLength(2); + expect(verses[0]).toEqual({ sid: 'GEN 1:31', text: 'Last verse of chapter one.' }); + expect(verses[1]).toEqual({ sid: 'GEN 2:1', text: 'First verse of chapter two.' }); + }); + + it('traverses content nested directly inside a chapter node', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'GEN', content: [] }, + { + type: 'chapter', + number: '1', + sid: 'GEN 1', + content: [ + { + type: 'para', + marker: 'p', + content: [{ type: 'verse', sid: 'GEN 1:1' }, 'In the beginning.'], + }, + ], + }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses).toHaveLength(1); + expect(verses[0]).toEqual({ sid: 'GEN 1:1', text: 'In the beginning.' }); + }); }); diff --git a/src/interlinearizer.web-view.tsx b/src/interlinearizer.web-view.tsx index 3226f05e..d606dd32 100644 --- a/src/interlinearizer.web-view.tsx +++ b/src/interlinearizer.web-view.tsx @@ -25,6 +25,7 @@ function SegmentView({ return ( ; } @@ -40,7 +40,7 @@ export const BOOK_CHAPTER_CONTROL_STRING_KEYS: string[] = []; export function BookChapterControl({ scrRef, handleSubmit, -}: { +}: Readonly<{ scrRef: ScriptureRef; handleSubmit: (ref: ScriptureRef) => void; className?: string; @@ -48,7 +48,7 @@ export function BookChapterControl({ recentSearches?: ScriptureRef[]; onAddRecentSearch?: (scrRef: ScriptureRef) => void; id?: string; -}) { +}>) { return (
{scrRef.book} {scrRef.chapterNum}:{scrRef.verseNum} diff --git a/__mocks__/platform-bible-utils.ts b/__mocks__/platform-bible-utils.ts index 02e0e9f1..34c867fe 100644 --- a/__mocks__/platform-bible-utils.ts +++ b/__mocks__/platform-bible-utils.ts @@ -36,7 +36,7 @@ class UnsubscriberAsyncList { (unsubscriber as { dispose: UnsubscriberFn }).dispose.bind(unsubscriber) ); } else if (typeof unsubscriber === 'function') { - this.unsubscribers.add(unsubscriber as UnsubscriberFn); + this.unsubscribers.add(unsubscriber); } }); } @@ -63,6 +63,6 @@ interface PlatformError { } const isPlatformError = (value: unknown): value is PlatformError => - typeof value === 'object' && value !== null && 'platformErrorVersion' in (value as object); + typeof value === 'object' && value !== null && 'platformErrorVersion' in (value); export { UnsubscriberAsyncList, isPlatformError }; diff --git a/cspell.json b/cspell.json index edacb5b8..3244aae5 100644 --- a/cspell.json +++ b/cspell.json @@ -28,6 +28,7 @@ "guids", "hopkinson", "iframes", + "imte", "interlinearization", "interlinearizer", "localstorage", diff --git a/src/__tests__/interlinearizer.web-view.test.tsx b/src/__tests__/interlinearizer.web-view.test.tsx index 94a13003..0b6bc2ba 100644 --- a/src/__tests__/interlinearizer.web-view.test.tsx +++ b/src/__tests__/interlinearizer.web-view.test.tsx @@ -289,14 +289,27 @@ describe('InterlinearizerWebView', () => { expect(screen.getByText('In')).toBeInTheDocument(); }); - it('shows a no-verse message when tokenization throws', () => { + it('shows an error heading and message when tokenization throws an Error', () => { mockBookData({}); jest.mocked(tokenizeBook).mockImplementation(() => { throw new Error('parse failure'); }); render(); - expect(screen.getByText(/no verse data for gen 1\./i)).toBeInTheDocument(); + expect(screen.getByRole('heading', { name: /error processing book/i })).toBeInTheDocument(); + expect(screen.getByText('parse failure')).toBeInTheDocument(); + }); + + it('shows an error message when tokenization throws a non-Error value', () => { + mockBookData({}); + jest.mocked(tokenizeBook).mockImplementation(() => { + // eslint-disable-next-line no-throw-literal + throw 'unexpected string error'; + }); + render(); + + expect(screen.getByRole('heading', { name: /error processing book/i })).toBeInTheDocument(); + expect(screen.getByText('unexpected string error')).toBeInTheDocument(); }); it('renders non-word tokens as muted chips', () => { diff --git a/src/__tests__/main.test.ts b/src/__tests__/main.test.ts index 8cdf744d..b99af479 100644 --- a/src/__tests__/main.test.ts +++ b/src/__tests__/main.test.ts @@ -50,6 +50,41 @@ const { __mockLogger, } = papiBackendMock; +function isCallable(f: unknown): f is (...args: unknown[]) => unknown { + return typeof f === 'function'; +} + +function findRegisteredHandler(commandName: string): ((...args: unknown[]) => unknown) | undefined { + const call = jest.mocked(__mockRegisterCommand).mock.calls.find((c) => c[0] === commandName); + const rawHandler: unknown = call?.[1]; + return isCallable(rawHandler) ? rawHandler : undefined; +} + +async function getOpenForWebViewHandler(): Promise< + (webViewId?: string) => Promise +> { + const context = createTestActivationContext(); + await activate(context); + const rawHandler = findRegisteredHandler('interlinearizer.openForWebView'); + if (!rawHandler) throw new Error('Handler not found for interlinearizer.openForWebView'); + return async (webViewId?: string): Promise => { + const result: unknown = await rawHandler(webViewId); + return typeof result === 'string' ? result : undefined; + }; +} + +function getOpenWebViewCallback(): (event: { webView: SavedWebViewDefinition }) => void { + const cb: unknown = __mockOnDidOpenWebView.mock.calls[0]?.[0]; + if (!isCallable(cb)) throw new Error('onDidOpenWebView callback not found'); + return (event) => cb(event); +} + +function getCloseWebViewCallback(): (event: { webView: SavedWebViewDefinition }) => void { + const cb: unknown = __mockOnDidCloseWebView.mock.calls[0]?.[0]; + if (!isCallable(cb)) throw new Error('onDidCloseWebView callback not found'); + return (event) => cb(event); +} + describe('main', () => { const mainWebViewType = 'interlinearizer.mainWebView'; @@ -216,32 +251,7 @@ describe('main', () => { }); }); - function isCallable(f: unknown): f is (...args: unknown[]) => unknown { - return typeof f === 'function'; - } - - function findRegisteredHandler( - commandName: string, - ): ((...args: unknown[]) => unknown) | undefined { - const call = jest.mocked(__mockRegisterCommand).mock.calls.find((c) => c[0] === commandName); - const rawHandler: unknown = call?.[1]; - return isCallable(rawHandler) ? rawHandler : undefined; - } - describe('interlinearizer.openForWebView command', () => { - async function getOpenForWebViewHandler(): Promise< - (webViewId?: string) => Promise - > { - const context = createTestActivationContext(); - await activate(context); - const rawHandler = findRegisteredHandler('interlinearizer.openForWebView'); - if (!rawHandler) throw new Error('Handler not found for interlinearizer.openForWebView'); - return async (webViewId?: string): Promise => { - const result: unknown = await rawHandler(webViewId); - return typeof result === 'string' ? result : undefined; - }; - } - it('looks up the projectId from the given WebView and opens the Interlinearizer', async () => { __mockGetOpenWebViewDefinition.mockResolvedValue({ id: 'some-webview', @@ -340,18 +350,6 @@ describe('main', () => { expect(__mockOnDidCloseWebView).toHaveBeenCalledTimes(1); }); - function getOpenWebViewCallback(): (event: { webView: SavedWebViewDefinition }) => void { - const cb: unknown = __mockOnDidOpenWebView.mock.calls[0]?.[0]; - if (!isCallable(cb)) throw new Error('onDidOpenWebView callback not found'); - return (event) => cb(event); - } - - function getCloseWebViewCallback(): (event: { webView: SavedWebViewDefinition }) => void { - const cb: unknown = __mockOnDidCloseWebView.mock.calls[0]?.[0]; - if (!isCallable(cb)) throw new Error('onDidCloseWebView callback not found'); - return (event) => cb(event); - } - describe('onDidOpenWebView callback', () => { it('adds the webView to the project map so subsequent opens reuse the existing tab', async () => { __mockSelectProject.mockResolvedValue('my-project'); diff --git a/src/__tests__/parsers/papi/usjBookExtractor.test.ts b/src/__tests__/parsers/papi/usjBookExtractor.test.ts index 5ee958d5..48510c4c 100644 --- a/src/__tests__/parsers/papi/usjBookExtractor.test.ts +++ b/src/__tests__/parsers/papi/usjBookExtractor.test.ts @@ -246,4 +246,21 @@ describe('extractBookFromUsj', () => { expect(verses).toHaveLength(1); expect(verses[0]).toEqual({ sid: 'GEN 1:1', text: 'In the beginning.' }); }); + + it('skips content of heading para markers encountered inside a verse', () => { + const usj: UsjDocument = { + content: [ + { type: 'book', code: 'PSA', content: [] }, + { + type: 'para', + marker: 'p', + content: [{ type: 'verse', sid: 'PSA 119:176' }, 'I have gone astray'], + }, + { type: 'para', marker: 's1', content: ['A section heading'] }, + ], + }; + const { verses } = extractBookFromUsj(usj, WS); + expect(verses).toHaveLength(1); + expect(verses[0].text).toBe('I have gone astray'); + }); }); diff --git a/src/interlinearizer.web-view.tsx b/src/interlinearizer.web-view.tsx index dc1407bc..e91501b3 100644 --- a/src/interlinearizer.web-view.tsx +++ b/src/interlinearizer.web-view.tsx @@ -10,7 +10,7 @@ import { useMemo } from 'react'; import { BookChapterControl, BOOK_CHAPTER_CONTROL_STRING_KEYS } from 'platform-bible-react'; import { extractBookFromUsj } from 'parsers/papi/usjBookExtractor'; import { tokenizeBook } from 'parsers/papi/bookTokenizer'; -import type { Segment } from 'interlinearizer'; +import type { Book, Segment } from 'interlinearizer'; import { logger } from '@papi/frontend'; /** Renders the tokens of a single segment as inline chips. */ @@ -18,11 +18,11 @@ function SegmentView({ segment, isActive, onClick, -}: { +}: Readonly<{ segment: Segment; isActive?: boolean; onClick?: () => void; -}) { +}>) { return (
)} - {!bookError && isLoading &&

Loading…

} + {tokenizeError && ( +
+

Error processing book

+ 
+            {tokenizeError}
+
+
+ )} + + {!bookError && !tokenizeError && isLoading && ( +

Loading…

+ )} - {!bookError && !isLoading && chapterSegments.length === 0 && ( + {!bookError && !tokenizeError && !isLoading && chapterSegments.length === 0 && (

No verse data for {scrRef.book} {scrRef.chapterNum}.

)} - {!bookError && !isLoading && chapterSegments.length > 0 && ( + {!bookError && !tokenizeError && !isLoading && chapterSegments.length > 0 && (
{chapterSegments.map((seg) => ( 0 && + !state.currentVerse.text.endsWith(' ') + ) + state.currentVerse.text += ' '; + if (node.content) traverse(node.content, state); +} + +const NODE_HANDLERS: Partial void>> = { + book: handleBookNode, + chapter: handleChapterNode, + verse: handleVerseNode, + note: () => {}, // skip note/footnote content — not part of the baseline text + para: handleParaNode, +}; + /** * Recursively walks a USJ content array, accumulating verse text into `state`. * @@ -72,33 +150,11 @@ function traverse(nodes: MarkerContent[], state: TraversalState): void { nodes.forEach((node) => { if (typeof node === 'string') { if (state.currentVerse !== undefined) state.currentVerse.text += node; - } else if (node.type === 'book') { - if (node.code) state.bookCode = node.code; - if (node.content) traverse(node.content, state); - } else if (node.type === 'chapter') { - if (state.currentVerse !== undefined) { - state.verses.push(state.currentVerse); - state.currentVerse = undefined; - } - if (node.content) traverse(node.content, state); - } else if (node.type === 'verse') { - if (state.currentVerse !== undefined) state.verses.push(state.currentVerse); - if (!node.sid) throw new Error('Invalid USJ: verse marker missing required sid attribute'); - state.currentVerse = { sid: node.sid, text: '' }; - if (node.content) traverse(node.content, state); - } else if (node.type === 'note') { - // Skip note and footnote content — not part of the baseline text. - } else if (node.type === 'para') { - if ( - state.currentVerse !== undefined && - state.currentVerse.text.length > 0 && - !state.currentVerse.text.endsWith(' ') - ) - state.currentVerse.text += ' '; - if (node.content) traverse(node.content, state); - } else if (node.content) { - traverse(node.content, state); + return; } + const handler = NODE_HANDLERS[node.type]; + if (handler) handler(node, state); + else if (node.content) traverse(node.content, state); }); } @@ -123,19 +179,16 @@ function stableStringify(value: unknown): string { /** * FNV-1a 32-bit hash — sufficient for one-way internal content versioning. * - * Iterates over UTF-16 code units (matching `String.prototype.charCodeAt`) rather than Unicode code - * points. This is intentional: the hash only needs to be stable within a single JS runtime, not - * portable across languages. - * * @param s - String to hash. * @returns Lowercase hex string of the unsigned 32-bit FNV-1a digest. */ function fnv1a32(s: string): string { - let h = 2166136261; - for (let i = 0; i < s.length; i++) { + const h = [...s].reduce( + /* v8 ignore next -- codePointAt(0) on a spread char is always defined */ // eslint-disable-next-line no-bitwise - h = Math.imul(h ^ s.charCodeAt(i), 16777619); - } + (acc, char) => Math.imul(acc ^ (char.codePointAt(0) ?? 0), 16777619), + 2166136261, + ); // eslint-disable-next-line no-bitwise return (h >>> 0).toString(16); } diff --git a/src/parsers/pt9/interlinearXmlParser.ts b/src/parsers/pt9/interlinearXmlParser.ts index 63f0dc9d..6a77ad4d 100644 --- a/src/parsers/pt9/interlinearXmlParser.ts +++ b/src/parsers/pt9/interlinearXmlParser.ts @@ -142,7 +142,7 @@ interface ParsedInterlinearXml { * * @param clusterElement - Parsed Cluster from fast-xml-parser (may have Lexeme array or none). * @returns Array of LexemeData with LexemeId (from Id) and SenseId (from GlossId, or ''). - * @throws {Error} If any Lexeme element is missing the required Id attribute. + * @throws {SyntaxError} If any Lexeme element is missing the required Id attribute. */ function extractLexemesFromCluster(clusterElement: ParsedCluster): LexemeData[] { const elements = clusterElement.Lexeme ?? []; @@ -150,7 +150,7 @@ function extractLexemesFromCluster(clusterElement: ParsedCluster): LexemeData[] return elements.map((el) => { const lexemeId = el['@_Id']; if (!lexemeId) { - throw new Error('Invalid XML: Lexeme missing required Id attribute'); + throw new SyntaxError('Invalid XML: Lexeme missing required Id attribute'); } return { LexemeId: lexemeId, SenseId: el['@_GlossId'] ?? '' }; }); @@ -193,7 +193,8 @@ function extractPunctuationsFromVerse(verseDataElement: ParsedVerseData): Punctu * @param verseDataElement - Parsed VerseData from fast-xml-parser (may have Cluster array or none). * @returns Array of ClusterData: TextRange from Cluster's Range, Lexemes from Lexeme children, * LexemesId (slash-joined), Id (LexemesId/Index-Length or Index-Length when no lexemes). - * @throws {Error} If a Cluster is missing its Range element or Range is missing Index or Length. + * @throws {SyntaxError} If a Cluster is missing its Range element or Range is missing Index or + * Length. */ function extractClustersFromVerse(verseDataElement: ParsedVerseData): ClusterData[] { const clusterElements = verseDataElement.Cluster ?? []; @@ -201,13 +202,13 @@ function extractClustersFromVerse(verseDataElement: ParsedVerseData): ClusterDat return clusterElements.map((el) => { const rangeElement = el.Range; if (!rangeElement) { - throw new Error('Invalid XML: Cluster missing required Range element'); + throw new SyntaxError('Invalid XML: Cluster missing required Range element'); } const index = Number(rangeElement['@_Index']); const length = Number(rangeElement['@_Length']); if (!Number.isFinite(index) || !Number.isFinite(length)) { - throw new Error('Invalid XML: Range missing required Index or Length attributes'); + throw new SyntaxError('Invalid XML: Range missing required Index or Length attributes'); } const textRange: StringRange = { Index: index, Length: length }; @@ -272,26 +273,27 @@ export class InterlinearXmlParser { * entries. * @returns Parsed interlinear data: ScrTextName, GlossLanguage, BookId, and Verses (record of * verse key to {@link VerseData} with Hash, Clusters, Punctuations). - * @throws {Error} If the root element, required attributes (GlossLanguage, BookId), required - * structure (Verses, Cluster Range, Lexeme Id), or duplicate verse reference is present. + * @throws {SyntaxError} If the root element, required attributes (GlossLanguage, BookId), or + * required structure (Verses, Cluster Range, Lexeme Id) is missing. + * @throws {RangeError} If a verse reference appears more than once. */ parse(xml: string): InterlinearData { const parsed: ParsedInterlinearXml = this.parser.parse(xml); const root = parsed.InterlinearData; if (!root) { - throw new Error('Invalid XML: Missing InterlinearData root element'); + throw new SyntaxError('Invalid XML: Missing InterlinearData root element'); } const scrTextName = root['@_ScrTextName'] ?? ''; const glossLanguage = root['@_GlossLanguage'] ?? ''; const bookId = root['@_BookId'] ?? ''; if (!glossLanguage || !bookId) { - throw new Error('Invalid XML: Missing required attributes GlossLanguage or BookId'); + throw new SyntaxError('Invalid XML: Missing required attributes GlossLanguage or BookId'); } const versesElement = root.Verses; if (!versesElement) { - throw new Error('Invalid XML: Missing Verses element'); + throw new SyntaxError('Invalid XML: Missing Verses element'); } const items = versesElement.item ?? []; @@ -301,7 +303,7 @@ export class InterlinearXmlParser { if (!verseKey) return acc; if (verseKey in acc) { - throw new Error( + throw new RangeError( `Invalid XML: Duplicate verse reference "${verseKey}". At most one VerseData per reference is allowed.`, ); } From 2076fd8ac77cc45cbea2fbf115daba223ab9cd61 Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Fri, 24 Apr 2026 16:40:33 -0600 Subject: [PATCH 09/32] Fix correctness bugs and tighten tests from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - usjBookExtractor: trim trailing whitespace from verse text when flushing across para/chapter/EOF boundaries; guard NODE_HANDLERS lookup with Object.hasOwn to avoid prototype-chain hits; pad fnv1a32 output to 8 hex chars; fix undefined-in-array serialisation in stableStringify; fix misplaced v8 ignore comment (next → next 2) for the codePointAt branch - interlinearXmlParser: throw SyntaxError (not RangeError) for duplicate verse references - interlinearizer.web-view: replace

with inside SegmentView to produce valid button content; move logger.error out of useMemo into useEffect so it fires only on error value changes - tests: assert non-empty before every() checks to prevent vacuous truth; add cross-segment token ID uniqueness assertion; add stable-ref object identity check to BookUSJ subscriber test --- .../interlinearizer.web-view.test.tsx | 2 ++ .../parsers/papi/bookTokenizer.test.ts | 20 +++++++++++++----- .../parsers/pt9/interlinearXmlParser.test.ts | 4 ++-- src/interlinearizer.web-view.tsx | 18 +++++++++------- src/parsers/papi/usjBookExtractor.ts | 21 +++++++++++++------ src/parsers/pt9/interlinearXmlParser.ts | 4 ++-- 6 files changed, 47 insertions(+), 22 deletions(-) diff --git a/src/__tests__/interlinearizer.web-view.test.tsx b/src/__tests__/interlinearizer.web-view.test.tsx index 0b6bc2ba..6d5ce362 100644 --- a/src/__tests__/interlinearizer.web-view.test.tsx +++ b/src/__tests__/interlinearizer.web-view.test.tsx @@ -360,5 +360,7 @@ describe('InterlinearizerWebView', () => { const refsPassed = mockBookUSJ.mock.calls.map((c) => c[0]); refsPassed.forEach((ref) => expect(ref).toEqual({ book: 'GEN', chapterNum: 1, verseNum: 1 })); + expect(mockBookUSJ.mock.calls.length).toBeGreaterThanOrEqual(1); + refsPassed.slice(1).forEach((ref) => expect(ref).toBe(refsPassed[0])); }); }); diff --git a/src/__tests__/parsers/papi/bookTokenizer.test.ts b/src/__tests__/parsers/papi/bookTokenizer.test.ts index 6b9a2d26..695f2468 100644 --- a/src/__tests__/parsers/papi/bookTokenizer.test.ts +++ b/src/__tests__/parsers/papi/bookTokenizer.test.ts @@ -54,12 +54,16 @@ describe('tokenizeBook', () => { it('labels word tokens as word', () => { const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'Hello world' }])); - expect(segments[0].tokens.every((t) => t.type === 'word')).toBe(true); + const { tokens } = segments[0]; + expect(tokens.length).toBeGreaterThan(0); + expect(tokens.every((t) => t.type === 'word')).toBe(true); }); it('labels punctuation tokens as punctuation', () => { const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: '., ;!' }])); - expect(segments[0].tokens.every((t) => t.type === 'punctuation')).toBe(true); + const { tokens } = segments[0]; + expect(tokens.length).toBeGreaterThan(0); + expect(tokens.every((t) => t.type === 'punctuation')).toBe(true); }); it('produces mixed word and punctuation tokens in the correct order', () => { @@ -85,12 +89,16 @@ describe('tokenizeBook', () => { { sid: 'GEN 1:1', text: 'Word.' }, { sid: 'GEN 1:2', text: 'Word.' }, ]); - // IDs are `${sid}:${charStart}`, so every token's id must start with its segment's id (the SID). - tokenizeBook(raw).segments.forEach((s) => { + const book = tokenizeBook(raw); + // Each token id must start with its segment's id (the SID). + book.segments.forEach((s) => { s.tokens.forEach((t) => { expect(t.id.startsWith(s.id)).toBe(true); }); }); + // All token ids across all segments must be globally unique. + const ids = book.segments.flatMap((s) => s.tokens.map((t) => t.id)); + expect(new Set(ids).size).toBe(ids.length); }); it('assigns writingSystem to every token', () => { @@ -99,7 +107,9 @@ describe('tokenizeBook', () => { writingSystem: 'kmr', }; const { segments } = tokenizeBook(raw); - expect(segments[0].tokens.every((t) => t.writingSystem === 'kmr')).toBe(true); + const { tokens } = segments[0]; + expect(tokens.length).toBeGreaterThan(0); + expect(tokens.every((t) => t.writingSystem === 'kmr')).toBe(true); }); it('produces an empty token list for an empty verse', () => { diff --git a/src/__tests__/parsers/pt9/interlinearXmlParser.test.ts b/src/__tests__/parsers/pt9/interlinearXmlParser.test.ts index aaf1d48a..ed42f239 100644 --- a/src/__tests__/parsers/pt9/interlinearXmlParser.test.ts +++ b/src/__tests__/parsers/pt9/interlinearXmlParser.test.ts @@ -1,8 +1,8 @@ /** @file Unit tests for {@link InterlinearXmlParser}. */ /// -import * as fs from 'fs'; -import * as path from 'path'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; import { InterlinearXmlParser } from 'parsers/pt9/interlinearXmlParser'; diff --git a/src/interlinearizer.web-view.tsx b/src/interlinearizer.web-view.tsx index e91501b3..e7e1fbbf 100644 --- a/src/interlinearizer.web-view.tsx +++ b/src/interlinearizer.web-view.tsx @@ -6,7 +6,7 @@ import { useRecentScriptureRefs, } from '@papi/frontend/react'; import { isPlatformError } from 'platform-bible-utils'; -import { useMemo } from 'react'; +import { useEffect, useMemo } from 'react'; import { BookChapterControl, BOOK_CHAPTER_CONTROL_STRING_KEYS } from 'platform-bible-react'; import { extractBookFromUsj } from 'parsers/papi/usjBookExtractor'; import { tokenizeBook } from 'parsers/papi/bookTokenizer'; @@ -34,9 +34,9 @@ function SegmentView({ } onClick={onClick} > -

+ {segment.startRef.verse} -

+
{segment.tokens.map((token) => token.type === 'word' ? ( @@ -90,15 +90,19 @@ function ProjectBookFetcher({ const ws = isPlatformError(writingSystem) ? 'und' : writingSystem || 'und'; return [tokenizeBook(extractBookFromUsj(bookResult, ws)), undefined]; } catch (err) { + return [undefined, err instanceof Error ? err.message : String(err)]; + } + }, [bookResult, writingSystem]); + + useEffect(() => { + if (tokenizeError) logger.error('Failed to parse/tokenize USJ book', { - err, + err: tokenizeError, writingSystem, projectId, book: scrRef.book, }); - return [undefined, err instanceof Error ? err.message : String(err)]; - } - }, [bookResult, writingSystem, projectId, scrRef.book]); + }, [tokenizeError, writingSystem, projectId, scrRef.book]); const chapterSegments = useMemo( () => diff --git a/src/parsers/papi/usjBookExtractor.ts b/src/parsers/papi/usjBookExtractor.ts index f36a1b88..98e44728 100644 --- a/src/parsers/papi/usjBookExtractor.ts +++ b/src/parsers/papi/usjBookExtractor.ts @@ -107,6 +107,7 @@ function handleBookNode(node: UsjNode, state: TraversalState): void { function handleChapterNode(node: UsjNode, state: TraversalState): void { if (state.currentVerse !== undefined) { + state.currentVerse.text = state.currentVerse.text.trimEnd(); state.verses.push(state.currentVerse); state.currentVerse = undefined; } @@ -114,7 +115,10 @@ function handleChapterNode(node: UsjNode, state: TraversalState): void { } function handleVerseNode(node: UsjNode, state: TraversalState): void { - if (state.currentVerse !== undefined) state.verses.push(state.currentVerse); + if (state.currentVerse !== undefined) { + state.currentVerse.text = state.currentVerse.text.trimEnd(); + state.verses.push(state.currentVerse); + } if (!node.sid) throw new Error('Invalid USJ: verse marker missing required sid attribute'); state.currentVerse = { sid: node.sid, text: '' }; if (node.content) traverse(node.content, state); @@ -152,7 +156,7 @@ function traverse(nodes: MarkerContent[], state: TraversalState): void { if (state.currentVerse !== undefined) state.currentVerse.text += node; return; } - const handler = NODE_HANDLERS[node.type]; + const handler = Object.hasOwn(NODE_HANDLERS, node.type) ? NODE_HANDLERS[node.type] : undefined; if (handler) handler(node, state); else if (node.content) traverse(node.content, state); }); @@ -169,7 +173,9 @@ function traverse(nodes: MarkerContent[], state: TraversalState): void { */ function stableStringify(value: unknown): string { if (!(value instanceof Object)) return JSON.stringify(value); - if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`; + if (Array.isArray(value)) + /* v8 ignore next -- stableStringify(elem) is undefined only when elem is undefined, which USJ arrays never contain */ + return `[${value.map((elem) => stableStringify(elem) ?? 'null').join(',')}]`; const sorted = Object.entries(value) .sort(([a], [b]) => (a < b ? -1 : 1)) .map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`); @@ -184,13 +190,13 @@ function stableStringify(value: unknown): string { */ function fnv1a32(s: string): string { const h = [...s].reduce( - /* v8 ignore next -- codePointAt(0) on a spread char is always defined */ + /* v8 ignore next 2 -- codePointAt(0) on a spread char is always defined */ // eslint-disable-next-line no-bitwise (acc, char) => Math.imul(acc ^ (char.codePointAt(0) ?? 0), 16777619), 2166136261, ); // eslint-disable-next-line no-bitwise - return (h >>> 0).toString(16); + return (h >>> 0).toString(16).padStart(8, '0'); } /** @@ -210,7 +216,10 @@ export function extractBookFromUsj(usj: UsjDocument, writingSystem: string): Raw traverse(usj.content, state); - if (state.currentVerse !== undefined) state.verses.push(state.currentVerse); + if (state.currentVerse !== undefined) { + state.currentVerse.text = state.currentVerse.text.trimEnd(); + state.verses.push(state.currentVerse); + } if (!state.bookCode) throw new Error('Invalid USJ: no book marker with a code attribute found'); diff --git a/src/parsers/pt9/interlinearXmlParser.ts b/src/parsers/pt9/interlinearXmlParser.ts index 6a77ad4d..34c3fe11 100644 --- a/src/parsers/pt9/interlinearXmlParser.ts +++ b/src/parsers/pt9/interlinearXmlParser.ts @@ -275,7 +275,7 @@ export class InterlinearXmlParser { * verse key to {@link VerseData} with Hash, Clusters, Punctuations). * @throws {SyntaxError} If the root element, required attributes (GlossLanguage, BookId), or * required structure (Verses, Cluster Range, Lexeme Id) is missing. - * @throws {RangeError} If a verse reference appears more than once. + * @throws {SyntaxError} If a verse reference appears more than once. */ parse(xml: string): InterlinearData { const parsed: ParsedInterlinearXml = this.parser.parse(xml); @@ -303,7 +303,7 @@ export class InterlinearXmlParser { if (!verseKey) return acc; if (verseKey in acc) { - throw new RangeError( + throw new SyntaxError( `Invalid XML: Duplicate verse reference "${verseKey}". At most one VerseData per reference is allowed.`, ); } From 37e78bf27c9d80079f028be58f16cc5cc270e227 Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Fri, 24 Apr 2026 16:53:37 -0600 Subject: [PATCH 10/32] Fix stableStringify undefined handling and add test Handle undefined values explicitly at the entry point rather than using ?? in the array branch, and add a test that exercises the path via a node with an optional property set to undefined. --- src/__tests__/parsers/papi/usjBookExtractor.test.ts | 8 ++++++++ src/parsers/papi/usjBookExtractor.ts | 5 ++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/__tests__/parsers/papi/usjBookExtractor.test.ts b/src/__tests__/parsers/papi/usjBookExtractor.test.ts index 48510c4c..4523512b 100644 --- a/src/__tests__/parsers/papi/usjBookExtractor.test.ts +++ b/src/__tests__/parsers/papi/usjBookExtractor.test.ts @@ -263,4 +263,12 @@ describe('extractBookFromUsj', () => { expect(verses).toHaveLength(1); expect(verses[0].text).toBe('I have gone astray'); }); + + it('produces a stable contentHash when a node has an optional property explicitly set to undefined', () => { + const usj: UsjDocument = { + content: [{ type: 'book', code: 'GEN', marker: undefined, content: [] }], + }; + const hash = extractBookFromUsj(usj, WS).contentHash; + expect(hash).toBe(extractBookFromUsj(usj, WS).contentHash); + }); }); diff --git a/src/parsers/papi/usjBookExtractor.ts b/src/parsers/papi/usjBookExtractor.ts index 98e44728..57189c93 100644 --- a/src/parsers/papi/usjBookExtractor.ts +++ b/src/parsers/papi/usjBookExtractor.ts @@ -172,10 +172,9 @@ function traverse(nodes: MarkerContent[], state: TraversalState): void { * @returns A stable JSON string with object keys in UTF-16 code-unit order. */ function stableStringify(value: unknown): string { + if (value === undefined) return 'null'; if (!(value instanceof Object)) return JSON.stringify(value); - if (Array.isArray(value)) - /* v8 ignore next -- stableStringify(elem) is undefined only when elem is undefined, which USJ arrays never contain */ - return `[${value.map((elem) => stableStringify(elem) ?? 'null').join(',')}]`; + if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`; const sorted = Object.entries(value) .sort(([a], [b]) => (a < b ? -1 : 1)) .map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`); From 2dc4e72e0ee82e2180725e3fe85bc1226fb8436d Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Fri, 24 Apr 2026 17:14:51 -0600 Subject: [PATCH 11/32] Documentation/Error type audit --- src/main.ts | 16 ++++++--- src/parsers/papi/bookTokenizer.ts | 21 +++++++++--- src/parsers/papi/usjBookExtractor.ts | 50 +++++++++++++++++++++++++--- src/types/interlinearizer.d.ts | 15 ++++++++- 4 files changed, 88 insertions(+), 14 deletions(-) diff --git a/src/main.ts b/src/main.ts index 0b57c532..30187c46 100644 --- a/src/main.ts +++ b/src/main.ts @@ -17,6 +17,7 @@ const mainWebViewType = 'interlinearizer.mainWebView'; /** Options passed to `openWebView` when opening the Interlinearizer. */ export interface InterlinearizerOpenOptions extends OpenWebViewOptions { + /** Paratext project ID to load in the Interlinearizer WebView. */ projectId?: string; } @@ -28,15 +29,15 @@ const mainWebViewProvider: IWebViewProvider = { * * @param savedWebView - Platform-provided definition (webViewType, etc.). * @param openWebViewOptions - Options passed by the caller; may include a projectId to link. - * @returns WebView definition with title, content, and styles, or undefined. - * @throws {Error} When savedWebView.webViewType is not the Interlinearizer type. + * @returns WebView definition with title, content, and styles. + * @throws {TypeError} When savedWebView.webViewType is not the Interlinearizer type. */ async getWebView( savedWebView: SavedWebViewDefinition, openWebViewOptions?: InterlinearizerOpenOptions, ): Promise { if (savedWebView.webViewType !== mainWebViewType) { - throw new Error( + throw new TypeError( `${mainWebViewType} provider received request to provide a ${savedWebView.webViewType} WebView`, ); } @@ -60,7 +61,10 @@ const openWebViewsByProject = new Map(); /** * Opens the Interlinearizer WebView for the given project. If no projectId is provided, shows a * project picker dialog. Each project gets its own tab; reopening an already-open project brings - * that tab to front. Returns the WebView ID, or undefined if the user cancels. + * that tab to front. + * + * @param projectId - Project to open; if omitted a picker dialog is shown. + * @returns The WebView ID of the opened (or focused) tab, or `undefined` if the user cancels. */ async function openInterlinearizer(projectId?: string): Promise { const resolvedProjectId = @@ -82,6 +86,9 @@ async function openInterlinearizer(projectId?: string): Promise { if (!webViewId) return openInterlinearizer(); @@ -95,6 +102,7 @@ async function openInterlinearizerForWebView(webViewId?: string): Promise { logger.debug('Interlinearizer extension is activating!'); diff --git a/src/parsers/papi/bookTokenizer.ts b/src/parsers/papi/bookTokenizer.ts index 27f6d729..9811c092 100644 --- a/src/parsers/papi/bookTokenizer.ts +++ b/src/parsers/papi/bookTokenizer.ts @@ -5,19 +5,27 @@ import type { Book, ScriptureRef, Segment, Token, TokenType } from 'interlineari import type { RawBook } from './usjBookExtractor'; -// Matches word tokens (Unicode letters / digits / combining marks) and punctuation tokens -// (any single non-word, non-whitespace character). Whitespace is not tokenized. +/** + * Matches word tokens (`\p{L}\p{N}\p{M}` runs) and punctuation tokens (any single non-word, + * non-whitespace character). Whitespace is not tokenized. + */ const TOKEN_RE = /[\p{L}\p{N}\p{M}]+|[^\p{L}\p{N}\p{M}\s]/gu; +/** + * Tests whether a matched token string starts with a word character, to classify it as `word` vs + * `punctuation`. + */ const WORD_START_RE = /[\p{L}\p{N}\p{M}]/u; /** * Parses a USJ verse SID (e.g. `"GEN 1:1"`) into a {@link ScriptureRef}. * - * @throws {Error} If `sid` is not a valid scripture reference string. + * @param sid - Verse SID string from the USJ `verse` marker (e.g. `"GEN 1:1"`). + * @returns A `ScriptureRef` with `book`, `chapter`, and `verse` populated. + * @throws {SyntaxError} If `sid` is not a valid scripture reference string. */ function parseSid(sid: string): ScriptureRef { const { success, verseRef } = VerseRef.tryParse(sid); - if (!success) throw new Error(`Invalid verse SID: "${sid}"`); + if (!success) throw new SyntaxError(`Invalid verse SID: "${sid}"`); return { book: verseRef.book, chapter: verseRef.chapterNum, verse: verseRef.verseNum }; } @@ -31,6 +39,8 @@ function parseSid(sid: string): ScriptureRef { * @param text - The verse's `baselineText` string. * @param sid - The verse SID used as the token ID prefix (e.g. `"GEN 1:1"`). * @param writingSystem - BCP 47 tag assigned to every token's `writingSystem` field. + * @returns Ordered array of {@link Token}s; empty when `text` contains no word or punctuation + * characters. */ function tokenizeVerse(text: string, sid: string, writingSystem: string): Token[] { return Array.from(text.matchAll(TOKEN_RE)).map((match) => { @@ -53,7 +63,8 @@ function tokenizeVerse(text: string, sid: string, writingSystem: string): Token[ * token.surfaceText`. * * @param rawBook - Extracted book data from {@link extractBookFromUsj}. - * @throws {Error} If any `RawVerse.sid` cannot be parsed as a valid scripture reference. + * @returns A `Book` with one `Segment` per verse, each containing its ordered `Token`s. + * @throws {SyntaxError} If any `RawVerse.sid` cannot be parsed as a valid scripture reference. */ export function tokenizeBook(rawBook: RawBook): Book { const segments: Segment[] = rawBook.verses.map(({ sid, text }) => { diff --git a/src/parsers/papi/usjBookExtractor.ts b/src/parsers/papi/usjBookExtractor.ts index 57189c93..040abbc9 100644 --- a/src/parsers/papi/usjBookExtractor.ts +++ b/src/parsers/papi/usjBookExtractor.ts @@ -37,11 +37,20 @@ type MarkerContent = string | UsjNode; /** A USJ marker node. Only the fields used during extraction are declared. */ interface UsjNode { + /** Node type string (e.g. `"book"`, `"chapter"`, `"verse"`, `"para"`, `"note"`). */ type: string; + /** USFM marker (e.g. `"p"`, `"s1"`, `"q"`). Present on `para` and `note` nodes. */ marker?: string; + /** Chapter or verse number string. Present on `chapter` nodes. */ number?: string; + /** 3-letter book code. Present on `book` nodes. */ code?: string; + /** + * Verse or chapter SID. Present on `verse` nodes (e.g. `"GEN 1:1"`) and `chapter` nodes (e.g. + * `"GEN 1"`). + */ sid?: string; + /** Child content items (strings or nested nodes). */ content?: MarkerContent[]; } @@ -95,16 +104,32 @@ const HEADING_PARA_MARKERS = new Set([ /** Mutable state threaded through the recursive USJ traversal. */ interface TraversalState { + /** 3-letter book code captured from the `book` marker (e.g. `"GEN"`). */ bookCode: string; + /** The verse currently being accumulated; `undefined` when outside a verse scope. */ currentVerse: { sid: string; text: string } | undefined; + /** Completed verses in document order. */ verses: RawVerse[]; } +/** + * Captures the book code from a `book` node, then recurses into its content. + * + * @param node - The `book` USJ node; `node.code` is the 3-letter book code. + * @param state - Shared traversal state updated in place. + */ function handleBookNode(node: UsjNode, state: TraversalState): void { if (node.code) state.bookCode = node.code; if (node.content) traverse(node.content, state); } +/** + * Closes the current open verse (if any) when a `chapter` node is encountered, then recurses into + * the chapter's content to pick up verses inside it. + * + * @param node - The `chapter` USJ node. + * @param state - Shared traversal state updated in place. + */ function handleChapterNode(node: UsjNode, state: TraversalState): void { if (state.currentVerse !== undefined) { state.currentVerse.text = state.currentVerse.text.trimEnd(); @@ -114,16 +139,31 @@ function handleChapterNode(node: UsjNode, state: TraversalState): void { if (node.content) traverse(node.content, state); } +/** + * Closes the previous open verse (if any) and opens a new one for a `verse` node. + * + * @param node - The `verse` USJ node; must carry a `sid` attribute (e.g. `"GEN 1:1"`). + * @param state - Shared traversal state updated in place. + * @throws {SyntaxError} If the `verse` node is missing its required `sid` attribute. + */ function handleVerseNode(node: UsjNode, state: TraversalState): void { if (state.currentVerse !== undefined) { state.currentVerse.text = state.currentVerse.text.trimEnd(); state.verses.push(state.currentVerse); } - if (!node.sid) throw new Error('Invalid USJ: verse marker missing required sid attribute'); + if (!node.sid) throw new SyntaxError('Invalid USJ: verse marker missing required sid attribute'); state.currentVerse = { sid: node.sid, text: '' }; if (node.content) traverse(node.content, state); } +/** + * Recurses into a `para` node's content, appending a space between adjacent para nodes when needed. + * Heading-class paragraphs (see {@link HEADING_PARA_MARKERS}) are skipped entirely so their text is + * not included in the verse baseline. + * + * @param node - The `para` USJ node; `node.marker` determines whether to skip or recurse. + * @param state - Shared traversal state updated in place. + */ function handleParaNode(node: UsjNode, state: TraversalState): void { if (node.marker && HEADING_PARA_MARKERS.has(node.marker)) return; if ( @@ -135,6 +175,7 @@ function handleParaNode(node: UsjNode, state: TraversalState): void { if (node.content) traverse(node.content, state); } +/** Dispatch table mapping USJ node `type` strings to their traversal handlers. */ const NODE_HANDLERS: Partial void>> = { book: handleBookNode, chapter: handleChapterNode, @@ -148,7 +189,6 @@ const NODE_HANDLERS: Partial { @@ -207,7 +247,8 @@ function fnv1a32(s: string): string { * * @param usj - USJ document returned by `useProjectData('platformScripture.USJ_Book', ...)`. * @param writingSystem - BCP 47 tag for the baseline, from `platform.languageTag`. - * @throws {Error} If no `book` marker with a `code` attribute is found in the document. + * @returns A `RawBook` with `bookCode`, `writingSystem`, `contentHash`, and `verses` populated. + * @throws {SyntaxError} If no `book` marker with a `code` attribute is found in the document. */ export function extractBookFromUsj(usj: UsjDocument, writingSystem: string): RawBook { const contentHash = fnv1a32(stableStringify(usj.content)); @@ -220,7 +261,8 @@ export function extractBookFromUsj(usj: UsjDocument, writingSystem: string): Raw state.verses.push(state.currentVerse); } - if (!state.bookCode) throw new Error('Invalid USJ: no book marker with a code attribute found'); + if (!state.bookCode) + throw new SyntaxError('Invalid USJ: no book marker with a code attribute found'); return { bookCode: state.bookCode, diff --git a/src/types/interlinearizer.d.ts b/src/types/interlinearizer.d.ts index d4ecd542..009c1668 100644 --- a/src/types/interlinearizer.d.ts +++ b/src/types/interlinearizer.d.ts @@ -71,7 +71,7 @@ declare module 'interlinearizer' { export type TokenType = 'word' | 'punctuation'; /** - * How an analysis was produced. + * Confidence level of an analysis. * * - `high` — human-created or human-confirmed * - `medium` — tool-assisted, reasonably confident @@ -105,8 +105,11 @@ declare module 'interlinearizer' { * text. When `charIndex` is absent the reference is verse-level only. */ export interface ScriptureRef { + /** 3-letter SIL book code (e.g. `"GEN"`). */ book: string; + /** 1-based chapter number. */ chapter: number; + /** 1-based verse number. */ verse: number; /** Zero-based character offset within the verse's baseline text. */ charIndex?: number; @@ -232,6 +235,7 @@ declare module 'interlinearizer' { * `InterlinearText`. `Alignment` records become `AlignmentLink`s. */ export interface InterlinearAlignment { + /** Unique identifier for this alignment pair. */ id: string; /** @@ -280,6 +284,7 @@ declare module 'interlinearizer' { * `senseIds`. Analysis is typically in a single language. */ export interface InterlinearText { + /** Unique identifier for this interlinear text. */ id: string; /** Writing system of the baseline text. */ @@ -318,6 +323,7 @@ declare module 'interlinearizer' { * from token checksums at import time. */ export interface Book { + /** Unique identifier for this book; typically equal to `bookRef`. */ id: string; /** Book identifier (e.g. `"GEN"`, `"MAT"`). */ @@ -519,6 +525,10 @@ declare module 'interlinearizer' { * synthesized. */ export interface SegmentAnalysis { + /** + * Unique within the owning `TextAnalysis` — used as a stable reference for this analysis + * record. + */ id: string; /** @@ -749,6 +759,7 @@ declare module 'interlinearizer' { * share the same gloss / sense. */ export type Phrase = { + /** Unique within the owning `TextAnalysis` — used as a stable reference for this phrase record. */ id: string; /** Ordered `Token.id` values that compose this phrase. */ @@ -813,6 +824,7 @@ declare module 'interlinearizer' { * Eflomal-generated alignments leave `originNum` and `statusNum` unset (default 0, CREATED). */ export interface AlignmentLink { + /** Unique within the owning `InterlinearAlignment` — stable reference for this link. */ id: string; /** Source-side endpoints (one or more tokens / morphemes). */ @@ -821,6 +833,7 @@ declare module 'interlinearizer' { /** Target-side endpoints (one or more tokens / morphemes). */ targetEndpoints: AlignmentEndpoint[]; + /** Review status of this alignment link. */ status: AssignmentStatus; /** How the alignment was created (manual, automatic tool, etc.). */ From b12f00f5315d5ddc106955e0e666578cf0795408 Mon Sep 17 00:00:00 2001 From: "D. Ror." Date: Mon, 27 Apr 2026 10:34:26 -0600 Subject: [PATCH 12/32] Clean-up incomplete changes (#32) * Clean-up incomplete changes * Update overlooked path --- README.md | 2 +- jest.config.ts | 5 +---- src/parsers/pt9/pt9-xml.md | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5bd167a7..19095041 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ The general file structure for an extension is as follows: - `src/` contains the source code for the extension - `src/main.ts` is the main entry file for the extension (registers commands and wires interlinear XML) - `src/types/interlinearizer.d.ts` is this extension's types file that defines how other extensions can use this extension through the `papi`. It is copied into the build folder - - `src/parsers/interlinearXmlParser.ts` parses interlinear XML into structured data (uses fast-xml-parser). The PT9 XML schema and parsed output are documented in `src/parsers/pt9-xml.md` + - `src/parsers/pt9/` contains parser and schema for parsing PT9 interlinear XML into structured data - `*.web-view.tsx` files will be treated as React WebViews - `*.web-view.scss` files provide styles for WebViews - `*.web-view.html` files are a conventional way to provide HTML WebViews (no special functionality) diff --git a/jest.config.ts b/jest.config.ts index b12ee344..7bbd34bb 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -92,15 +92,12 @@ const config: Config = { '^@papi/frontend/react$': '/__mocks__/papi-frontend-react.ts', /** Mock so test-helpers get UnsubscriberAsyncList without loading ESM deps. */ '^platform-bible-utils$': '/__mocks__/platform-bible-utils.ts', - /** Mock platform-bible-react and lucide-react — both ship ESM that Jest cannot parse. */ + /** Mock ESM deps that Jest cannot parse. */ '^platform-bible-react$': '/__mocks__/platform-bible-react.tsx', - '^lucide-react$': '/__mocks__/lucide-react.tsx', /** Resolve webpack ?inline imports. */ '^(.+)\\.web-view\\?inline$': '/__mocks__/web-view-inline.ts', /** Resolve webpack ?inline imports: SCSS content. */ '^(.+)\\.(scss|sass|css)\\?inline$': '/__mocks__/styleInlineMock.ts', - /** Resolve webpack ?raw import for test XML in web-view. */ - '^(.+)/Interlinear_en_MAT\\.xml\\?raw$': '/__mocks__/interlinearXmlContent.ts', }, /** Exclude dist from module resolution to avoid Haste naming collision with root package.json. */ diff --git a/src/parsers/pt9/pt9-xml.md b/src/parsers/pt9/pt9-xml.md index 56d0baac..b570279e 100644 --- a/src/parsers/pt9/pt9-xml.md +++ b/src/parsers/pt9/pt9-xml.md @@ -1,6 +1,6 @@ # Paratext 9 XML schema -The extension reads PT9 interlinear data from XML files (e.g. `Interlinear__.xml` in project data). The parser in `src/parsers/interlinearXmlParser.ts` expects the following structure. Sample files live in `test-data/` (e.g. `Interlinear_en_MAT.xml`). +The extension reads PT9 interlinear data from XML files (e.g. `Interlinear__.xml` in project data). The parser in `src/parsers/pt9/interlinearXmlParser.ts` expects the following structure. Sample files live in `test-data/` (e.g. `Interlinear_en_MAT.xml`). ## Document structure From 5598f716b442b957b236595a7a36b6a5d97bb726 Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Mon, 27 Apr 2026 10:42:29 -0600 Subject: [PATCH 13/32] Fix CodeRabbit nitpicks --- src/__tests__/interlinearizer.web-view.test.tsx | 2 +- src/parsers/papi/usjBookExtractor.ts | 3 ++- src/parsers/pt9/interlinearXmlParser.ts | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/__tests__/interlinearizer.web-view.test.tsx b/src/__tests__/interlinearizer.web-view.test.tsx index 6d5ce362..5dd7a675 100644 --- a/src/__tests__/interlinearizer.web-view.test.tsx +++ b/src/__tests__/interlinearizer.web-view.test.tsx @@ -360,7 +360,7 @@ describe('InterlinearizerWebView', () => { const refsPassed = mockBookUSJ.mock.calls.map((c) => c[0]); refsPassed.forEach((ref) => expect(ref).toEqual({ book: 'GEN', chapterNum: 1, verseNum: 1 })); - expect(mockBookUSJ.mock.calls.length).toBeGreaterThanOrEqual(1); + expect(mockBookUSJ.mock.calls.length).toBeGreaterThanOrEqual(2); refsPassed.slice(1).forEach((ref) => expect(ref).toBe(refsPassed[0])); }); }); diff --git a/src/parsers/papi/usjBookExtractor.ts b/src/parsers/papi/usjBookExtractor.ts index 040abbc9..d0a41ece 100644 --- a/src/parsers/papi/usjBookExtractor.ts +++ b/src/parsers/papi/usjBookExtractor.ts @@ -216,7 +216,8 @@ function stableStringify(value: unknown): string { if (!(value instanceof Object)) return JSON.stringify(value); if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`; const sorted = Object.entries(value) - .sort(([a], [b]) => (a < b ? -1 : 1)) + // eslint-disable-next-line no-nested-ternary + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) .map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`); return `{${sorted.join(',')}}`; } diff --git a/src/parsers/pt9/interlinearXmlParser.ts b/src/parsers/pt9/interlinearXmlParser.ts index 34c3fe11..1c5c9fb6 100644 --- a/src/parsers/pt9/interlinearXmlParser.ts +++ b/src/parsers/pt9/interlinearXmlParser.ts @@ -302,7 +302,7 @@ export class InterlinearXmlParser { const verseKey = item.string; if (!verseKey) return acc; - if (verseKey in acc) { + if (Object.prototype.hasOwnProperty.call(acc, verseKey)) { throw new SyntaxError( `Invalid XML: Duplicate verse reference "${verseKey}". At most one VerseData per reference is allowed.`, ); From 1f4cde50a5289f272e0ccba27cda973d1c3fa6ca Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Mon, 27 Apr 2026 11:19:31 -0600 Subject: [PATCH 14/32] Fix nitpicks, revert previous nitpick --- .../interlinearizer.web-view.test.tsx | 10 ++- .../parsers/papi/bookTokenizer.test.ts | 5 +- .../parsers/pt9/interlinearXmlParser.test.ts | 68 ++++++++++++++++--- src/parsers/papi/usjBookExtractor.ts | 5 +- src/parsers/pt9/interlinearXmlParser.ts | 4 +- 5 files changed, 75 insertions(+), 17 deletions(-) diff --git a/src/__tests__/interlinearizer.web-view.test.tsx b/src/__tests__/interlinearizer.web-view.test.tsx index 5dd7a675..907b6a63 100644 --- a/src/__tests__/interlinearizer.web-view.test.tsx +++ b/src/__tests__/interlinearizer.web-view.test.tsx @@ -18,6 +18,12 @@ import { tokenizeBook } from 'parsers/papi/bookTokenizer'; jest.mock('parsers/papi/usjBookExtractor'); jest.mock('parsers/papi/bookTokenizer'); +/** + * Matches the PlatformError shape from platform-bible-utils (discriminated by + * platformErrorVersion). + */ +type PlatformError = { platformErrorVersion: number; message: string }; + /** * Load the WebView module; it assigns the component to globalThis.webViewComponent. This pattern is * required by the Platform.Bible WebView framework: the WebView entry is built with a ?inline query @@ -162,9 +168,7 @@ function mockBookData(value: unknown, isLoading = false): void { } /** Configures useProjectSetting to return the given writing system tag. */ -function mockWritingSystem( - tag: string | { platformErrorVersion: number; message: string } = 'en', -): void { +function mockWritingSystem(tag: string | PlatformError = 'en'): void { jest.mocked(useProjectSetting).mockReturnValue([tag, jest.fn(), jest.fn(), false]); } diff --git a/src/__tests__/parsers/papi/bookTokenizer.test.ts b/src/__tests__/parsers/papi/bookTokenizer.test.ts index 695f2468..97172750 100644 --- a/src/__tests__/parsers/papi/bookTokenizer.test.ts +++ b/src/__tests__/parsers/papi/bookTokenizer.test.ts @@ -143,7 +143,10 @@ describe('tokenizeBook', () => { it('throws on an invalid verse SID', () => { expect(() => tokenizeBook(makeRawBook([{ sid: 'not-a-ref', text: 'text' }]))).toThrow( - 'Invalid verse SID', + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining('Invalid verse SID'), + }), ); }); diff --git a/src/__tests__/parsers/pt9/interlinearXmlParser.test.ts b/src/__tests__/parsers/pt9/interlinearXmlParser.test.ts index ed42f239..e65ed07d 100644 --- a/src/__tests__/parsers/pt9/interlinearXmlParser.test.ts +++ b/src/__tests__/parsers/pt9/interlinearXmlParser.test.ts @@ -630,7 +630,12 @@ describe('InterlinearXmlParser', () => { `; - expect(() => parser.parse(xml)).toThrow('Invalid XML: Missing InterlinearData root element'); + expect(() => parser.parse(xml)).toThrow( + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining('Invalid XML: Missing InterlinearData root element'), + }), + ); }); it('throws when GlossLanguage is missing', () => { @@ -645,7 +650,12 @@ describe('InterlinearXmlParser', () => { `; expect(() => parser.parse(xml)).toThrow( - 'Invalid XML: Missing required attributes GlossLanguage or BookId', + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining( + 'Invalid XML: Missing required attributes GlossLanguage or BookId', + ), + }), ); }); @@ -661,7 +671,12 @@ describe('InterlinearXmlParser', () => { `; expect(() => parser.parse(xml)).toThrow( - 'Invalid XML: Missing required attributes GlossLanguage or BookId', + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining( + 'Invalid XML: Missing required attributes GlossLanguage or BookId', + ), + }), ); }); @@ -677,7 +692,12 @@ describe('InterlinearXmlParser', () => { `; expect(() => parser.parse(xml)).toThrow( - 'Invalid XML: Missing required attributes GlossLanguage or BookId', + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining( + 'Invalid XML: Missing required attributes GlossLanguage or BookId', + ), + }), ); }); @@ -686,7 +706,12 @@ describe('InterlinearXmlParser', () => { `; - expect(() => parser.parse(xml)).toThrow('Invalid XML: Missing Verses element'); + expect(() => parser.parse(xml)).toThrow( + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining('Invalid XML: Missing Verses element'), + }), + ); }); it('throws when Cluster is missing Range element', () => { @@ -705,7 +730,10 @@ describe('InterlinearXmlParser', () => { `; expect(() => parser.parse(xml)).toThrow( - 'Invalid XML: Cluster missing required Range element', + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining('Invalid XML: Cluster missing required Range element'), + }), ); }); @@ -726,7 +754,12 @@ describe('InterlinearXmlParser', () => { `; expect(() => parser.parse(xmlNoIndex)).toThrow( - 'Invalid XML: Range missing required Index or Length attributes', + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining( + 'Invalid XML: Range missing required Index or Length attributes', + ), + }), ); const xmlNoLength = ` @@ -745,7 +778,12 @@ describe('InterlinearXmlParser', () => { `; expect(() => parser.parse(xmlNoLength)).toThrow( - 'Invalid XML: Range missing required Index or Length attributes', + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining( + 'Invalid XML: Range missing required Index or Length attributes', + ), + }), ); }); @@ -765,7 +803,12 @@ describe('InterlinearXmlParser', () => { `; - expect(() => parser.parse(xml)).toThrow('Invalid XML: Lexeme missing required Id attribute'); + expect(() => parser.parse(xml)).toThrow( + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining('Invalid XML: Lexeme missing required Id attribute'), + }), + ); }); it('throws when the same verse reference appears in more than one item', () => { @@ -803,7 +846,12 @@ describe('InterlinearXmlParser', () => { `; expect(() => parser.parse(xml)).toThrow( - 'Invalid XML: Duplicate verse reference "MAT 1:1". At most one VerseData per reference is allowed.', + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining( + 'Invalid XML: Duplicate verse reference "MAT 1:1". At most one VerseData per reference is allowed.', + ), + }), ); }); }); diff --git a/src/parsers/papi/usjBookExtractor.ts b/src/parsers/papi/usjBookExtractor.ts index d0a41ece..64777b06 100644 --- a/src/parsers/papi/usjBookExtractor.ts +++ b/src/parsers/papi/usjBookExtractor.ts @@ -208,6 +208,8 @@ function traverse(nodes: MarkerContent[], state: TraversalState): void { * Produces the same output regardless of engine locale, making the result safe to feed into a hash * function. Arrays preserve their original order; only object keys are sorted. * + * Intended for plain JSON-shaped structures only; does not special-case Date, Map, Set, or RegExp. + * * @param value - Any JSON-serializable value. * @returns A stable JSON string with object keys in UTF-16 code-unit order. */ @@ -216,8 +218,7 @@ function stableStringify(value: unknown): string { if (!(value instanceof Object)) return JSON.stringify(value); if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`; const sorted = Object.entries(value) - // eslint-disable-next-line no-nested-ternary - .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .sort(([a], [b]) => (a < b ? -1 : 1)) .map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`); return `{${sorted.join(',')}}`; } diff --git a/src/parsers/pt9/interlinearXmlParser.ts b/src/parsers/pt9/interlinearXmlParser.ts index 1c5c9fb6..c2c4f30a 100644 --- a/src/parsers/pt9/interlinearXmlParser.ts +++ b/src/parsers/pt9/interlinearXmlParser.ts @@ -298,15 +298,17 @@ export class InterlinearXmlParser { const items = versesElement.item ?? []; + const seen = new Set(); const verses = items.reduce>((acc, item) => { const verseKey = item.string; if (!verseKey) return acc; - if (Object.prototype.hasOwnProperty.call(acc, verseKey)) { + if (seen.has(verseKey)) { throw new SyntaxError( `Invalid XML: Duplicate verse reference "${verseKey}". At most one VerseData per reference is allowed.`, ); } + seen.add(verseKey); const verseDataElement = item.VerseData; if (!verseDataElement) { From 2024a89be12a0be12d5a1d22e20f7cd5381a23c4 Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Mon, 27 Apr 2026 11:28:31 -0600 Subject: [PATCH 15/32] Improve docstring coverage --- src/interlinearizer.web-view.tsx | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/interlinearizer.web-view.tsx b/src/interlinearizer.web-view.tsx index e7e1fbbf..9bcf257e 100644 --- a/src/interlinearizer.web-view.tsx +++ b/src/interlinearizer.web-view.tsx @@ -13,7 +13,15 @@ import { tokenizeBook } from 'parsers/papi/bookTokenizer'; import type { Book, Segment } from 'interlinearizer'; import { logger } from '@papi/frontend'; -/** Renders the tokens of a single segment as inline chips. */ +/** + * Renders the tokens of a single segment as inline chips. + * + * @param props - Component props + * @param props.segment - The segment whose tokens to render + * @param props.isActive - Whether this segment is the currently selected verse + * @param props.onClick - Callback invoked when the segment button is clicked + * @returns A button containing the segment's verse label and token chips + */ function SegmentView({ segment, isActive, @@ -63,6 +71,13 @@ function SegmentView({ /** * Fetches the USJ book for the given project, tokenizes it, and renders all segments in the current * chapter. Shows loading / error states while data is in flight or unavailable. + * + * @param props - Component props + * @param props.projectId - PAPI project ID whose USJ book to fetch and tokenize + * @param props.scrRef - Current scripture reference shared via the scroll group + * @param props.setScrRef - Setter to update the scroll-group scripture reference when a segment is + * clicked + * @returns A column of {@link SegmentView} chips, or an appropriate loading / error message */ function ProjectBookFetcher({ projectId, @@ -174,6 +189,13 @@ function ProjectBookFetcher({ /** * Root WebView component for the Interlinearizer. Renders a sticky reference picker at the top and * delegates book fetching to {@link ProjectBookFetcher} in the scrollable content area below. + * + * @param props - WebView props injected by the PAPI host + * @param props.projectId - PAPI project ID passed from the host; undefined when the WebView is + * opened outside a project context + * @param props.useWebViewScrollGroupScrRef - Hook that exposes the shared scroll-group scripture + * reference and its setter + * @returns The full interlinearizer WebView layout */ globalThis.webViewComponent = function InterlinearizerWebView({ projectId, From 5e693448156002bb9310d04ca2f99e0b65c93278 Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Mon, 27 Apr 2026 12:03:33 -0600 Subject: [PATCH 16/32] Fix div-in-button issue, reintroduce export for backend mock to ensure no redeclaration of mocks, adjust `usjExtractor` test to better test handling of undefined markers, filter out undefined values in `stableStringify` --- __mocks__/papi-backend.ts | 3 +++ .../parsers/papi/usjBookExtractor.test.ts | 10 +++++++--- src/interlinearizer.web-view.tsx | 14 ++++++++------ src/parsers/papi/usjBookExtractor.ts | 1 + 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/__mocks__/papi-backend.ts b/__mocks__/papi-backend.ts index 09aaf6e9..4f9cab20 100644 --- a/__mocks__/papi-backend.ts +++ b/__mocks__/papi-backend.ts @@ -60,3 +60,6 @@ module.exports = { __mockOnDidCloseWebView: mockOnDidCloseWebView, __mockLogger: mockLogger, }; + +/** Marks this file as a module so top-level const/let are module-scoped; avoids TS "redeclare" when both papi-backend and papi-frontend mocks are in the project (they are used mutually exclusively by Jest). */ +export {}; diff --git a/src/__tests__/parsers/papi/usjBookExtractor.test.ts b/src/__tests__/parsers/papi/usjBookExtractor.test.ts index 4523512b..85057c84 100644 --- a/src/__tests__/parsers/papi/usjBookExtractor.test.ts +++ b/src/__tests__/parsers/papi/usjBookExtractor.test.ts @@ -265,10 +265,14 @@ describe('extractBookFromUsj', () => { }); it('produces a stable contentHash when a node has an optional property explicitly set to undefined', () => { - const usj: UsjDocument = { + const withUndefined: UsjDocument = { content: [{ type: 'book', code: 'GEN', marker: undefined, content: [] }], }; - const hash = extractBookFromUsj(usj, WS).contentHash; - expect(hash).toBe(extractBookFromUsj(usj, WS).contentHash); + const withoutUndefined: UsjDocument = { + content: [{ type: 'book', code: 'GEN', content: [] }], + }; + + const hash = extractBookFromUsj(withUndefined, WS).contentHash; + expect(hash).toBe(extractBookFromUsj(withoutUndefined, WS).contentHash); }); }); diff --git a/src/interlinearizer.web-view.tsx b/src/interlinearizer.web-view.tsx index 9bcf257e..39b760aa 100644 --- a/src/interlinearizer.web-view.tsx +++ b/src/interlinearizer.web-view.tsx @@ -1,13 +1,13 @@ import type { WebViewProps } from '@papi/core'; import { + useLocalizedStrings, useProjectData, useProjectSetting, - useLocalizedStrings, useRecentScriptureRefs, } from '@papi/frontend/react'; import { isPlatformError } from 'platform-bible-utils'; import { useEffect, useMemo } from 'react'; -import { BookChapterControl, BOOK_CHAPTER_CONTROL_STRING_KEYS } from 'platform-bible-react'; +import { BOOK_CHAPTER_CONTROL_STRING_KEYS, BookChapterControl } from 'platform-bible-react'; import { extractBookFromUsj } from 'parsers/papi/usjBookExtractor'; import { tokenizeBook } from 'parsers/papi/bookTokenizer'; import type { Book, Segment } from 'interlinearizer'; @@ -45,7 +45,7 @@ function SegmentView({ {segment.startRef.verse} -
+ {segment.tokens.map((token) => token.type === 'word' ? ( ), )} -
+ ); } @@ -110,13 +110,15 @@ function ProjectBookFetcher({ }, [bookResult, writingSystem]); useEffect(() => { - if (tokenizeError) + if (tokenizeError) { + const ws = isPlatformError(writingSystem) ? 'und' : writingSystem || 'und'; logger.error('Failed to parse/tokenize USJ book', { err: tokenizeError, - writingSystem, + writingSystem: ws, projectId, book: scrRef.book, }); + } }, [tokenizeError, writingSystem, projectId, scrRef.book]); const chapterSegments = useMemo( diff --git a/src/parsers/papi/usjBookExtractor.ts b/src/parsers/papi/usjBookExtractor.ts index 64777b06..961a2b1a 100644 --- a/src/parsers/papi/usjBookExtractor.ts +++ b/src/parsers/papi/usjBookExtractor.ts @@ -218,6 +218,7 @@ function stableStringify(value: unknown): string { if (!(value instanceof Object)) return JSON.stringify(value); if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`; const sorted = Object.entries(value) + .filter(([, v]) => v !== undefined) .sort(([a], [b]) => (a < b ? -1 : 1)) .map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`); return `{${sorted.join(',')}}`; From e5ed09c3ea17aebdc1e9cd7f04153221b8193da3 Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Mon, 27 Apr 2026 13:24:52 -0600 Subject: [PATCH 17/32] Improve parsing of index and length in pt9 XML Parser, reject duplicate verse ids during extraction in `usjBookExtractor.ts` --- src/parsers/papi/usjBookExtractor.ts | 12 +++++++++++- src/parsers/pt9/interlinearXmlParser.ts | 18 ++++++++++++------ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/parsers/papi/usjBookExtractor.ts b/src/parsers/papi/usjBookExtractor.ts index 961a2b1a..6b4b758d 100644 --- a/src/parsers/papi/usjBookExtractor.ts +++ b/src/parsers/papi/usjBookExtractor.ts @@ -106,6 +106,8 @@ const HEADING_PARA_MARKERS = new Set([ interface TraversalState { /** 3-letter book code captured from the `book` marker (e.g. `"GEN"`). */ bookCode: string; + /** Verse SIDs seen so far; used to reject duplicates. */ + seenVerseIds: Set; /** The verse currently being accumulated; `undefined` when outside a verse scope. */ currentVerse: { sid: string; text: string } | undefined; /** Completed verses in document order. */ @@ -152,6 +154,9 @@ function handleVerseNode(node: UsjNode, state: TraversalState): void { state.verses.push(state.currentVerse); } if (!node.sid) throw new SyntaxError('Invalid USJ: verse marker missing required sid attribute'); + if (state.seenVerseIds.has(node.sid)) + throw new SyntaxError(`Invalid USJ: duplicate verse SID "${node.sid}"`); + state.seenVerseIds.add(node.sid); state.currentVerse = { sid: node.sid, text: '' }; if (node.content) traverse(node.content, state); } @@ -255,7 +260,12 @@ function fnv1a32(s: string): string { */ export function extractBookFromUsj(usj: UsjDocument, writingSystem: string): RawBook { const contentHash = fnv1a32(stableStringify(usj.content)); - const state: TraversalState = { bookCode: '', currentVerse: undefined, verses: [] }; + const state: TraversalState = { + bookCode: '', + seenVerseIds: new Set(), + currentVerse: undefined, + verses: [], + }; traverse(usj.content, state); diff --git a/src/parsers/pt9/interlinearXmlParser.ts b/src/parsers/pt9/interlinearXmlParser.ts index c2c4f30a..78ac2b68 100644 --- a/src/parsers/pt9/interlinearXmlParser.ts +++ b/src/parsers/pt9/interlinearXmlParser.ts @@ -156,6 +156,12 @@ function extractLexemesFromCluster(clusterElement: ParsedCluster): LexemeData[] }); } +const parseStrictNumber = (raw: string): number | undefined => { + if (raw === undefined || raw.trim() === '') return undefined; + const n = Number(raw); + return Number.isFinite(n) ? n : undefined; +}; + /** * Maps a parsed VerseData's Punctuation array to {@link PunctuationData} array. * @@ -174,9 +180,9 @@ function extractPunctuationsFromVerse(verseDataElement: ParsedVerseData): Punctu return elements.flatMap((el) => { const rangeElement = el.Range; if (!rangeElement) return []; - const index = Number(rangeElement['@_Index']); - const length = Number(rangeElement['@_Length']); - if (!Number.isFinite(index) || !Number.isFinite(length)) return []; + const index = parseStrictNumber(rangeElement['@_Index']); + const length = parseStrictNumber(rangeElement['@_Length']); + if (index === undefined || length === undefined) return []; return [ { TextRange: { Index: index, Length: length }, @@ -205,9 +211,9 @@ function extractClustersFromVerse(verseDataElement: ParsedVerseData): ClusterDat throw new SyntaxError('Invalid XML: Cluster missing required Range element'); } - const index = Number(rangeElement['@_Index']); - const length = Number(rangeElement['@_Length']); - if (!Number.isFinite(index) || !Number.isFinite(length)) { + const index = parseStrictNumber(rangeElement['@_Index']); + const length = parseStrictNumber(rangeElement['@_Length']); + if (index === undefined || length === undefined) { throw new SyntaxError('Invalid XML: Range missing required Index or Length attributes'); } From 46ab515955794e9ade71cf29908787738ddea7c4 Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Mon, 27 Apr 2026 14:18:33 -0600 Subject: [PATCH 18/32] Tighten range/index validation to exclude non-whole numbers --- src/parsers/pt9/interlinearXmlParser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parsers/pt9/interlinearXmlParser.ts b/src/parsers/pt9/interlinearXmlParser.ts index 78ac2b68..9179f9a1 100644 --- a/src/parsers/pt9/interlinearXmlParser.ts +++ b/src/parsers/pt9/interlinearXmlParser.ts @@ -159,7 +159,7 @@ function extractLexemesFromCluster(clusterElement: ParsedCluster): LexemeData[] const parseStrictNumber = (raw: string): number | undefined => { if (raw === undefined || raw.trim() === '') return undefined; const n = Number(raw); - return Number.isFinite(n) ? n : undefined; + return Number.isInteger(n) && n >= 0 ? n : undefined; }; /** From 477ec4f9f5a28eb611e22dbf0bd5eb8241fa810d Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Tue, 28 Apr 2026 11:58:02 -0600 Subject: [PATCH 19/32] Add TabToolbar to WebView, audit mocks --- __mocks__/papi-backend.ts | 8 ---- __mocks__/papi-frontend-react.ts | 53 +++++++----------------- __mocks__/papi-frontend.ts | 47 ++------------------- __mocks__/platform-bible-react.tsx | 65 ++++++++++++++++++------------ src/interlinearizer.web-view.tsx | 47 ++++++++++++++------- 5 files changed, 89 insertions(+), 131 deletions(-) diff --git a/__mocks__/papi-backend.ts b/__mocks__/papi-backend.ts index 4f9cab20..c8d3a9f4 100644 --- a/__mocks__/papi-backend.ts +++ b/__mocks__/papi-backend.ts @@ -51,14 +51,6 @@ module.exports = { __esModule: true, default: defaultExport, logger: mockLogger, - __mockRegisterWebViewProvider: mockRegisterWebViewProvider, - __mockRegisterCommand: mockRegisterCommand, - __mockOpenWebView: mockOpenWebView, - __mockSelectProject: mockSelectProject, - __mockGetOpenWebViewDefinition: mockGetOpenWebViewDefinition, - __mockOnDidOpenWebView: mockOnDidOpenWebView, - __mockOnDidCloseWebView: mockOnDidCloseWebView, - __mockLogger: mockLogger, }; /** Marks this file as a module so top-level const/let are module-scoped; avoids TS "redeclare" when both papi-backend and papi-frontend mocks are in the project (they are used mutually exclusively by Jest). */ diff --git a/__mocks__/papi-frontend-react.ts b/__mocks__/papi-frontend-react.ts index cce42fc1..16ad2abc 100644 --- a/__mocks__/papi-frontend-react.ts +++ b/__mocks__/papi-frontend-react.ts @@ -1,28 +1,17 @@ /** - * @file Jest mock for @papi/frontend/react. Provides stub implementations of various PAPI React hooks so + * @file Jest mock for @papi/frontend/react. Provides stub implementations of PAPI React hooks so * WebView/frontend components can be unit-tested without the real Platform API. */ -/** - * useData('providerName') returns an object whose keys are data type names and values are hooks. - * Mock: any property returns a function that returns [undefined, setter, false]. - */ -const createUseDataLikeHook = () => - jest.fn(() => - new Proxy( - {}, - { - get: () => () => [undefined, jest.fn(), false], - }, - ), - ); +const useProjectData = jest.fn(() => + new Proxy( + {}, + { + get: () => () => [undefined, jest.fn(), false], + }, + ), +); -const useDataProvider = jest.fn().mockReturnValue(undefined); -const useData = createUseDataLikeHook(); -const useScrollGroupScrRef = jest.fn().mockReturnValue([undefined, jest.fn()]); -const useSetting = jest.fn().mockImplementation((_key: string, defaultState: unknown) => [defaultState, jest.fn()]); -const useProjectData = createUseDataLikeHook(); -const useProjectDataProvider = jest.fn().mockReturnValue(undefined); const useProjectSetting = jest .fn() .mockImplementation((_projectDataProviderSource: unknown, _key: string, defaultState: unknown) => [ @@ -31,14 +20,13 @@ const useProjectSetting = jest jest.fn(), false, ]); -const useDialogCallback = jest.fn().mockReturnValue(jest.fn()); -const useDataProviderMulti = jest.fn().mockReturnValue([]); + /** Returns [record, isLoading] tuple; maps each key to itself so tests get a predictable string. */ const useLocalizedStrings = jest.fn().mockImplementation((keys: string[]) => [ Array.isArray(keys) ? keys.reduce>((acc, k) => ({ ...acc, [k]: k }), {}) : {}, false, ]); -const useWebViewController = jest.fn().mockReturnValue(undefined); + /** Returns { recentScriptureRefs, addRecentScriptureRef } matching the real hook signature. */ const useRecentScriptureRefs = jest .fn() @@ -46,24 +34,11 @@ const useRecentScriptureRefs = jest module.exports = { __esModule: true, - useDataProvider, - useData, - useScrollGroupScrRef, - useSetting, useProjectData, - useProjectDataProvider, useProjectSetting, - useDialogCallback, - useDataProviderMulti, useLocalizedStrings, - useWebViewController, useRecentScriptureRefs, - __mockUseDataProvider: useDataProvider, - __mockUseData: useData, - __mockUseLocalizedStrings: useLocalizedStrings, - __mockUseSetting: useSetting, - __mockUseProjectData: useProjectData, - __mockUseProjectDataProvider: useProjectDataProvider, - __mockUseProjectSetting: useProjectSetting, - __mockUseWebViewController: useWebViewController, }; + +/** Marks this file as a module so top-level const/let are module-scoped. */ +export {}; diff --git a/__mocks__/papi-frontend.ts b/__mocks__/papi-frontend.ts index 7536d654..009859da 100644 --- a/__mocks__/papi-frontend.ts +++ b/__mocks__/papi-frontend.ts @@ -1,7 +1,6 @@ /** - * @file Jest mock for @papi/frontend. Provides papi, logger, network, projectDataProviders, and other - * renderer API stubs so WebView/frontend code can be unit-tested without loading the real - * Platform API. + * @file Jest mock for @papi/frontend. Provides a logger stub so WebView/frontend code can be + * unit-tested without loading the real Platform API. */ const mockLogger = { @@ -11,50 +10,10 @@ const mockLogger = { warn: jest.fn(), }; -const mockNetwork = { - request: jest.fn(), - subscribe: jest.fn().mockReturnValue({ dispose: jest.fn() }), -}; - -const mockProjectDataProviders = { - get: jest.fn().mockResolvedValue(undefined), - register: jest.fn().mockResolvedValue({ dispose: jest.fn() }), -}; - -const mockWebViews = { - getWebView: jest.fn(), - openWebView: jest.fn().mockResolvedValue(undefined), -}; - -/** Default papi object shape used in renderer/WebViews. Only commonly used services are stubbed. */ -const papi = { - logger: mockLogger, - network: mockNetwork, - projectDataProviders: mockProjectDataProviders, - webViews: mockWebViews, - react: {}, // Re-export of @papi/frontend/react; tests usually import that module directly. -}; - -const defaultExport = { - ...papi, - __mockLogger: mockLogger, - __mockNetwork: mockNetwork, - __mockProjectDataProviders: mockProjectDataProviders, - __mockWebViews: mockWebViews, -}; - module.exports = { __esModule: true, - default: defaultExport, logger: mockLogger, - network: mockNetwork, - projectDataProviders: mockProjectDataProviders, - webViews: mockWebViews, - __mockLogger: mockLogger, - __mockNetwork: mockNetwork, - __mockProjectDataProviders: mockProjectDataProviders, - __mockWebViews: mockWebViews, }; -/** Marks this file as a module so top-level const/let are module-scoped; avoids TS "redeclare" when both papi-backend and papi-frontend mocks are in the project (they are used mutually exclusively by Jest). */ +/** Marks this file as a module so top-level const/let are module-scoped. */ export {}; diff --git a/__mocks__/platform-bible-react.tsx b/__mocks__/platform-bible-react.tsx index c8322b8d..ef701bbf 100644 --- a/__mocks__/platform-bible-react.tsx +++ b/__mocks__/platform-bible-react.tsx @@ -1,33 +1,11 @@ /** * @file Jest mock for platform-bible-react. The real package ships ESM which Jest cannot parse * without extra transform configuration. This stub provides the subset used by extension - * components: `cn`, `Button`, `BookChapterControl`, and `BOOK_CHAPTER_CONTROL_STRING_KEYS`. + * components: `BookChapterControl`, `BOOK_CHAPTER_CONTROL_STRING_KEYS`, `TabToolbar`, and + * `ScrollGroupSelector`. */ -import type { ButtonHTMLAttributes, ReactNode } from 'react'; - -function flattenCn(arg: unknown): string[] { - if (typeof arg === 'string') return arg.length > 0 ? [arg] : []; - if (Array.isArray(arg)) return arg.flatMap(flattenCn); - if (arg !== null && typeof arg === 'object') - return Object.entries(arg as Record) - .filter(([, v]) => Boolean(v)) - .map(([k]) => k); - return []; -} - -export const cn = (...args: unknown[]): string => args.flatMap(flattenCn).join(' '); - -interface ButtonProps extends ButtonHTMLAttributes { - variant?: string; - size?: string; - children?: ReactNode; - asChild?: boolean; -} - -export function Button({ children, variant: _v, size: _s, asChild: _a, ...rest }: Readonly) { - return ; -} +import type { ReactNode, ReactElement } from 'react'; interface ScriptureRef { book: string; @@ -37,6 +15,41 @@ interface ScriptureRef { export const BOOK_CHAPTER_CONTROL_STRING_KEYS: string[] = []; +export function TabToolbar({ + startAreaChildren, + endAreaChildren, +}: Readonly<{ + className?: string; + startAreaChildren?: ReactNode; + endAreaChildren?: ReactNode; + onSelectProjectMenuItem?: () => void; + onSelectViewInfoMenuItem?: () => void; +}>): ReactElement { + return ( +
+
{startAreaChildren}
+
{endAreaChildren}
+
+ ); +} + +export function ScrollGroupSelector({ + scrollGroupId, + onChangeScrollGroupId, +}: Readonly<{ + availableScrollGroupIds?: (number | undefined)[]; + scrollGroupId?: number; + onChangeScrollGroupId?: (id: number | undefined) => void; +}>): ReactElement { + return ( + ); } diff --git a/src/main.ts b/src/main.ts index b0d7444b..0e69f74a 100644 --- a/src/main.ts +++ b/src/main.ts @@ -128,7 +128,7 @@ export async function activate(context: ExecutionActivationContext): Promise Date: Wed, 29 Apr 2026 13:46:02 -0600 Subject: [PATCH 24/32] Improve token regexes, minor adjustments, add docstring to `parseStrictNumber()` --- .../parsers/papi/bookTokenizer.test.ts | 112 ++++++++++++++++++ src/interlinearizer.web-view.tsx | 1 - src/parsers/papi/bookTokenizer.ts | 42 ++++++- src/parsers/papi/usjBookExtractor.ts | 2 +- src/parsers/pt9/interlinearXmlParser.ts | 9 ++ 5 files changed, 160 insertions(+), 6 deletions(-) diff --git a/src/__tests__/parsers/papi/bookTokenizer.test.ts b/src/__tests__/parsers/papi/bookTokenizer.test.ts index 74cf326b..7a6cded9 100644 --- a/src/__tests__/parsers/papi/bookTokenizer.test.ts +++ b/src/__tests__/parsers/papi/bookTokenizer.test.ts @@ -150,6 +150,118 @@ describe('tokenizeBook', () => { ); }); + describe('word-internal joiners', () => { + it("tokenizes don't (ASCII apostrophe) as a single word token", () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: "don't" }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe("don't"); + }); + + it('tokenizes don’t (U+2019 right single quote) as a single word token', () => { + const text = 'don’t'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe(text); + }); + + it("tokenizes l'homme as a single word token", () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: "l'homme" }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe("l'homme"); + }); + + it('tokenizes well-known as a single word token', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'well-known' }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe('well-known'); + }); + + it('tokenizes "it\'s well-known" as two word tokens', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: "it's well-known" }])); + const wordTokens = segments[0].tokens.filter((t) => t.type === 'word'); + expect(wordTokens).toHaveLength(2); + expect(wordTokens[0].surfaceText).toBe("it's"); + expect(wordTokens[1].surfaceText).toBe('well-known'); + }); + + it("tokenizes 'hello' as word then punctuation (leading apostrophe absorbed into word)", () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: "'hello'" }])); + const types = segments[0].tokens.map((t) => t.type); + expect(types).toEqual(['word', 'punctuation']); + expect(segments[0].tokens[0].surfaceText).toBe("'hello"); + expect(segments[0].tokens[1].surfaceText).toBe("'"); + }); + + it('tokenizes a standalone apostrophe as punctuation (no following word character)', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: "'" }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('punctuation'); + }); + + it("tokenizes word-initial U+0027 as part of the word (e.g. Hebrew aleph romanisation 'Elohim)", () => { + const text = "'Elohim"; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe(text); + }); + + it('tokenizes word-initial U+2019 as part of the word', () => { + const text = '’Elohim'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe(text); + }); + + it('tokenizes U+02BC (modifier letter apostrophe) as a word character regardless of position', () => { + // U+02BC is \p{L} so it is inherently a word character — no special handling needed. + const text = 'ʼelohim'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe(text); + }); + + it('tokenizes end- as word then punctuation (trailing joiner is not absorbed)', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'end-' }])); + expect(segments[0].tokens).toHaveLength(2); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe('end'); + expect(segments[0].tokens[1].type).toBe('punctuation'); + expect(segments[0].tokens[1].surfaceText).toBe('-'); + }); + + it('tokenizes -start as punctuation then word (leading joiner is not absorbed)', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: '-start' }])); + expect(segments[0].tokens).toHaveLength(2); + expect(segments[0].tokens[0].type).toBe('punctuation'); + expect(segments[0].tokens[0].surfaceText).toBe('-'); + expect(segments[0].tokens[1].type).toBe('word'); + expect(segments[0].tokens[1].surfaceText).toBe('start'); + }); + + // Double joiners between word chars are absorbed greedily: a--b → one token "a--b". + it('tokenizes a--b as a single word token (greedy double-joiner absorption)', () => { + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: 'a--b' }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe('a--b'); + }); + + it('upholds the charStart/charEnd invariant for joiner-containing tokens', () => { + const text = "it's well-known, don’t you think?"; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + segments[0].tokens.forEach((token) => + expect(text.slice(token.charStart, token.charEnd)).toBe(token.surfaceText), + ); + }); + }); + it('classifies astral-plane letters (surrogate pairs) as word tokens', () => { // Gothic letters U+10330–U+1034F are outside the BMP; each code point is two UTF-16 code // units. Testing surfaceText[0] (a lone surrogate) against WORD_START_RE would fail — the diff --git a/src/interlinearizer.web-view.tsx b/src/interlinearizer.web-view.tsx index d12f27e3..1860a500 100644 --- a/src/interlinearizer.web-view.tsx +++ b/src/interlinearizer.web-view.tsx @@ -230,7 +230,6 @@ globalThis.webViewComponent = function InterlinearizerWebView({ scrRef={scrRef} handleSubmit={(ref) => { setScrRef(ref); - onAddRecentRef(ref); }} localizedStrings={localizedStrings} recentSearches={recentRefs} diff --git a/src/parsers/papi/bookTokenizer.ts b/src/parsers/papi/bookTokenizer.ts index 0f7e1a0b..0abaf725 100644 --- a/src/parsers/papi/bookTokenizer.ts +++ b/src/parsers/papi/bookTokenizer.ts @@ -6,15 +6,49 @@ import type { Book, ScriptureRef, Segment, Token, TokenType } from 'interlineari import type { RawBook } from './usjBookExtractor'; /** - * Matches word tokens (`\p{L}\p{N}\p{M}` runs) and punctuation tokens (any single non-word, - * non-whitespace character). Whitespace is not tokenized. + * Unicode property classes that define a "word character" for tokenization purposes. + * + * Includes letters, numbers, combining marks, and join-control characters (U+200C ZWNJ / U+200D + * ZWJ) so that Arabic, Farsi, and Indic script ligatures are not split mid-token. */ -const TOKEN_RE = /[\p{L}\p{N}\p{M}]+|[^\p{L}\p{N}\p{M}\s]/gu; +const CHAR_SET = String.raw`\p{L}\p{N}\p{M}\p{Join_Control}`; + +/** + * Matches word tokens and punctuation tokens. Whitespace is not tokenized. + * + * A word token is a run of word characters (`\p{L}\p{N}\p{M}\p{Join_Control}`) optionally extended + * through word-internal joiners (apostrophes and hyphens). A joiner is absorbed into the + * surrounding word only when it is both preceded by a word character (structural: it follows the + * initial run or a prior joiner+word extension) and followed by another word character (lookahead). + * Trailing joiners are left as standalone punctuation tokens. + * + * U+0027 and U+2019 are additionally absorbed at word-initial position (before the first word + * character) to handle languages where these characters represent a phonemic glottal stop or + * similar feature (e.g. Hebrew aleph in romanization, various indigenous-language orthographies). + * Multiple leading apostrophes are absorbed greedily. Leading hyphens/dashes are NOT absorbed. + * + * Note: U+02BC (modifier letter apostrophe) and U+02BB (ʻokina) are `\p{L}` characters and are + * therefore always treated as word characters regardless of position. + * + * Multiple consecutive word-internal joiners between word characters are absorbed greedily (e.g. + * `a--b` → one token `a--b`). + * + * Word-internal joiners: U+002D hyphen-minus, U+0027 apostrophe, U+2019 right single quote, U+02BC + * modifier-letter apostrophe, U+2010-U+2015 Unicode hyphens/dashes. + * + * `\uXXXX` escapes are used for joiner characters to prevent auto-formatters from converting them + * to typographic quotes or other Unicode variants. + */ +const TOKEN_RE = new RegExp( + String.raw`(?:[\u0027\u2019]+(?=[${CHAR_SET}]))?[${CHAR_SET}]+(?:[\u002D\u0027\u2019\u02BC\u2010-\u2015]+(?=[${CHAR_SET}])[${CHAR_SET}]+)*|[^${CHAR_SET}\s]`, + 'gv', +); + /** * Tests whether a matched token string starts with a word character, to classify it as `word` vs * `punctuation`. */ -const WORD_START_RE = /[\p{L}\p{N}\p{M}]/u; +const WORD_START_RE = new RegExp(`[${CHAR_SET}]`, 'v'); /** * Parses a USJ verse SID (e.g. `"GEN 1:1"`) into a {@link ScriptureRef}. diff --git a/src/parsers/papi/usjBookExtractor.ts b/src/parsers/papi/usjBookExtractor.ts index 8ddae018..66c564ee 100644 --- a/src/parsers/papi/usjBookExtractor.ts +++ b/src/parsers/papi/usjBookExtractor.ts @@ -224,7 +224,7 @@ function stableStringify(value: unknown): string { if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`; const sorted = Object.entries(value) .filter(([, v]) => v !== undefined) - .sort(([a], [b]) => (a < b ? -1 : 1)) + .sort(([a], [b]) => +(a > b) - +(a < b)) .map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`); return `{${sorted.join(',')}}`; } diff --git a/src/parsers/pt9/interlinearXmlParser.ts b/src/parsers/pt9/interlinearXmlParser.ts index 9179f9a1..3391cc93 100644 --- a/src/parsers/pt9/interlinearXmlParser.ts +++ b/src/parsers/pt9/interlinearXmlParser.ts @@ -156,6 +156,15 @@ function extractLexemesFromCluster(clusterElement: ParsedCluster): LexemeData[] }); } +/** + * Parses a string to a non-negative integer, returning `undefined` for empty, non-integer, or + * negative values. Used to validate XML attribute strings before converting them to numeric + * ranges. + * + * @param raw - The string to parse (e.g. an XML attribute value). + * @returns The parsed non-negative integer, or `undefined` if the input is empty, non-integer, or + * negative. + */ const parseStrictNumber = (raw: string): number | undefined => { if (raw === undefined || raw.trim() === '') return undefined; const n = Number(raw); From da05e57b6b5c5aa9b696ca66dbbcdf57e959c933 Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Wed, 29 Apr 2026 14:24:47 -0600 Subject: [PATCH 25/32] Improve `parseStrictNumber()`, improve hook mock in `papi-frontend-react`, add clarifying comment to method result to ensure it isn't flagged as problematic in the future --- __mocks__/papi-frontend-react.ts | 2 +- src/main.ts | 4 ++++ src/parsers/pt9/interlinearXmlParser.ts | 5 +++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/__mocks__/papi-frontend-react.ts b/__mocks__/papi-frontend-react.ts index fb00c10c..52c6e6c0 100644 --- a/__mocks__/papi-frontend-react.ts +++ b/__mocks__/papi-frontend-react.ts @@ -56,7 +56,7 @@ const useLocalizedStrings = jest.fn().mockImplementation((keys: string[]) => [ */ const useRecentScriptureRefs = jest .fn() - .mockReturnValue({ recentScriptureRefs: [], addRecentScriptureRef: jest.fn() }); + .mockImplementation(() => ({ recentScriptureRefs: [], addRecentScriptureRef: jest.fn() })); module.exports = { __esModule: true, diff --git a/src/main.ts b/src/main.ts index 0e69f74a..5793bcfc 100644 --- a/src/main.ts +++ b/src/main.ts @@ -126,6 +126,10 @@ export async function activate(context: ExecutionActivationContext): Promise { if (raw === undefined || raw.trim() === '') return undefined; - const n = Number(raw); - return Number.isInteger(n) && n >= 0 ? n : undefined; + if (!/^\d+$/.test(raw.trim())) return undefined; + const n = Number.parseInt(raw, 10); + return n >= 0 ? n : undefined; }; /** From 5fc175c815340dfadd715916bf7ee9099b641fef Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Wed, 29 Apr 2026 14:51:02 -0600 Subject: [PATCH 26/32] Expand coverage collection to all source files - Previously only covered parsers, main.ts, and the WebView component. - Now includes any new source files without manual config updates. --- jest.config.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/jest.config.ts b/jest.config.ts index 7bbd34bb..ed3741d7 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -23,15 +23,10 @@ const config: Config = { */ collectCoverage: false, - /** - * Collect coverage from parsers, main entry (main.ts), and WebView UI - * (interlinearizer.web-view.tsx). Excludes test files, type declarations, and build artifacts. - */ + /** Collect coverage from all source files. Excludes type declarations and test files. */ collectCoverageFrom: [ - 'src/parsers/**/*.ts', - 'src/main.ts', - 'src/**/*.web-view.tsx', - '!src/parsers/**/*.d.ts', + 'src/**/*.{ts,tsx}', + '!src/**/*.d.ts', '!src/**/__tests__/**', '!src/**/*.test.{ts,tsx}', '!src/**/*.spec.{ts,tsx}', From c6d91cbc62eb69dd506c2fc9a2e205b2f2ac1a7e Mon Sep 17 00:00:00 2001 From: "D. Ror." Date: Wed, 29 Apr 2026 15:06:07 -0600 Subject: [PATCH 27/32] Clean up regex (#34) Co-authored-by: alex-rawlings-yyc --- src/parsers/papi/bookTokenizer.ts | 54 +++++++++++++++++++------------ 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/src/parsers/papi/bookTokenizer.ts b/src/parsers/papi/bookTokenizer.ts index 0abaf725..0fc479e9 100644 --- a/src/parsers/papi/bookTokenizer.ts +++ b/src/parsers/papi/bookTokenizer.ts @@ -10,45 +10,57 @@ import type { RawBook } from './usjBookExtractor'; * * Includes letters, numbers, combining marks, and join-control characters (U+200C ZWNJ / U+200D * ZWJ) so that Arabic, Farsi, and Indic script ligatures are not split mid-token. + * + * Note: U+02BC (modifier letter apostrophe) and U+02BB (ʻokina) are included in `\p{L}` and are + * always word characters, despite appearing like punctuation. */ const CHAR_SET = String.raw`\p{L}\p{N}\p{M}\p{Join_Control}`; +/** + * Includes U+0027 and U+2019 at word-initial position (before the first word character) to handle + * languages where these characters represent a phonemic glottal stop or similar feature (e.g. + * Hebrew aleph in romanization, various indigenous-language orthographies). + * + * Doesn't include hyphens/dashes. + */ +const LEAD_SET = String.raw`\u0027\u2019`; + +/** + * Word-internal joiners: + * + * - \u0027 (Apostrophe) + * - \u002D (Hyphen-minus) + * - \u2010-\u2015 (Unicode hyphens/dashes) + * - \u2019 (Right single quote) + * + * `\uXXXX` escapes are used for joiner characters to prevent auto-formatters from converting them + * to typographic quotes or other Unicode variants. + */ +const JOIN_SET = String.raw`\u0027\u002D\u2010-\u2015\u2019`; + /** * Matches word tokens and punctuation tokens. Whitespace is not tokenized. * - * A word token is a run of word characters (`\p{L}\p{N}\p{M}\p{Join_Control}`) optionally extended - * through word-internal joiners (apostrophes and hyphens). A joiner is absorbed into the - * surrounding word only when it is both preceded by a word character (structural: it follows the - * initial run or a prior joiner+word extension) and followed by another word character (lookahead). - * Trailing joiners are left as standalone punctuation tokens. + * A word token is a run of word characters, optionally extended through some leading characters and + * word-internal joiners (e.g., apostrophes and hyphens). A joiner is absorbed into the surrounding + * word only when it is both preceded and followed by word characters. Trailing joiners are left as + * standalone punctuation tokens. * - * U+0027 and U+2019 are additionally absorbed at word-initial position (before the first word - * character) to handle languages where these characters represent a phonemic glottal stop or - * similar feature (e.g. Hebrew aleph in romanization, various indigenous-language orthographies). * Multiple leading apostrophes are absorbed greedily. Leading hyphens/dashes are NOT absorbed. * - * Note: U+02BC (modifier letter apostrophe) and U+02BB (ʻokina) are `\p{L}` characters and are - * therefore always treated as word characters regardless of position. - * * Multiple consecutive word-internal joiners between word characters are absorbed greedily (e.g. * `a--b` → one token `a--b`). - * - * Word-internal joiners: U+002D hyphen-minus, U+0027 apostrophe, U+2019 right single quote, U+02BC - * modifier-letter apostrophe, U+2010-U+2015 Unicode hyphens/dashes. - * - * `\uXXXX` escapes are used for joiner characters to prevent auto-formatters from converting them - * to typographic quotes or other Unicode variants. */ const TOKEN_RE = new RegExp( - String.raw`(?:[\u0027\u2019]+(?=[${CHAR_SET}]))?[${CHAR_SET}]+(?:[\u002D\u0027\u2019\u02BC\u2010-\u2015]+(?=[${CHAR_SET}])[${CHAR_SET}]+)*|[^${CHAR_SET}\s]`, + String.raw`(?:[${LEAD_SET}]+(?=[${CHAR_SET}]))?[${CHAR_SET}]+(?:[${JOIN_SET}]+(?=[${CHAR_SET}])[${CHAR_SET}]+)*|[^${CHAR_SET}\s]`, 'gv', ); /** - * Tests whether a matched token string starts with a word character, to classify it as `word` vs + * Tests whether a matched token string contains a word character, to classify it as `word` vs * `punctuation`. */ -const WORD_START_RE = new RegExp(`[${CHAR_SET}]`, 'v'); +const WORD_CONTAIN_RE = new RegExp(`[${CHAR_SET}]`, 'v'); /** * Parses a USJ verse SID (e.g. `"GEN 1:1"`) into a {@link ScriptureRef}. @@ -81,7 +93,7 @@ function tokenizeVerse(text: string, sid: string, writingSystem: string): Token[ const surfaceText = match[0]; const charStart = match.index; const charEnd = charStart + surfaceText.length; - const type: TokenType = WORD_START_RE.test(surfaceText) ? 'word' : 'punctuation'; + const type: TokenType = WORD_CONTAIN_RE.test(surfaceText) ? 'word' : 'punctuation'; return { id: `${sid}:${charStart}`, surfaceText, writingSystem, type, charStart, charEnd }; }); } From f391011995ca6547f63cd0aceb28193df1794b2c Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Wed, 29 Apr 2026 15:12:27 -0600 Subject: [PATCH 28/32] Clarify `SyntaxError` when `Range` from `Cluster` is invalid, improve error handling when attempting to open Interlinearizer WebView when `projectId` can't be resolved from `webViewId` --- src/main.ts | 8 +++++++- src/parsers/pt9/interlinearXmlParser.ts | 4 +++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main.ts b/src/main.ts index 5793bcfc..2cbe7bc8 100644 --- a/src/main.ts +++ b/src/main.ts @@ -93,7 +93,13 @@ async function openInterlinearizer(projectId?: string): Promise { if (!webViewId) return openInterlinearizer(); const webViewDefinition = await papi.webViews.getOpenWebViewDefinition(webViewId); - return openInterlinearizer(webViewDefinition?.projectId); + if (!webViewDefinition?.projectId) { + logger.warn( + `Could not resolve projectId from webViewId "${webViewId}" while opening Interlinearizer`, + ); + return undefined; + } + return openInterlinearizer(webViewDefinition.projectId); } /** diff --git a/src/parsers/pt9/interlinearXmlParser.ts b/src/parsers/pt9/interlinearXmlParser.ts index 8e219e0b..765b6287 100644 --- a/src/parsers/pt9/interlinearXmlParser.ts +++ b/src/parsers/pt9/interlinearXmlParser.ts @@ -224,7 +224,9 @@ function extractClustersFromVerse(verseDataElement: ParsedVerseData): ClusterDat const index = parseStrictNumber(rangeElement['@_Index']); const length = parseStrictNumber(rangeElement['@_Length']); if (index === undefined || length === undefined) { - throw new SyntaxError('Invalid XML: Range missing required Index or Length attributes'); + throw new SyntaxError( + 'Invalid XML: Range missing or invalid Index/Length attributes (must be non-negative integers)', + ); } const textRange: StringRange = { Index: index, Length: length }; From 494d322ca1dd0aec639c5479ee7e3e53404095e6 Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Wed, 29 Apr 2026 15:38:38 -0600 Subject: [PATCH 29/32] Revert `openInterlinearizerForWebView`, as Platform.Bible has a project picker for when `undefined` is returned, add `user-event` package for better react testing, other minor adjustments --- package-lock.json | 15 +++++++++++++++ package.json | 1 + src/__tests__/interlinearizer.web-view.test.tsx | 11 ++++++----- src/__tests__/parsers/papi/bookTokenizer.test.ts | 10 ++++++++++ .../parsers/pt9/interlinearXmlParser.test.ts | 4 ++-- src/interlinearizer.web-view.tsx | 4 +--- src/main.ts | 8 +------- src/parsers/papi/bookTokenizer.ts | 5 ++++- 8 files changed, 40 insertions(+), 18 deletions(-) diff --git a/package-lock.json b/package-lock.json index ab4c249c..8db7b58d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "@tailwindcss/typography": "^0.5.16", "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", + "@testing-library/user-event": "^14.6.1", "@types/jest": "^30.0.0", "@types/node": "^22.19.13", "@types/react": "^18.3.18", @@ -2827,6 +2828,20 @@ } } }, + "node_modules/@testing-library/user-event": { + "version": "14.6.1", + "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.1.tgz", + "integrity": "sha512-vq7fv0rnt+QTXgPxr5Hjc210p6YKq2kmdziLgnsZGgLJ9e6VAShx1pACLuRjd/AS/sr7phAR58OIIpf0LlmQNw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12", + "npm": ">=6" + }, + "peerDependencies": { + "@testing-library/dom": ">=7.21.4" + } + }, "node_modules/@tsconfig/node10": { "version": "1.0.12", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", diff --git a/package.json b/package.json index dcbd2e8f..30612746 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "@tailwindcss/typography": "^0.5.16", "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", + "@testing-library/user-event": "^14.6.1", "@types/jest": "^30.0.0", "@types/node": "^22.19.13", "@types/react": "^18.3.18", diff --git a/src/__tests__/interlinearizer.web-view.test.tsx b/src/__tests__/interlinearizer.web-view.test.tsx index d460208b..eaca59c0 100644 --- a/src/__tests__/interlinearizer.web-view.test.tsx +++ b/src/__tests__/interlinearizer.web-view.test.tsx @@ -4,7 +4,8 @@ import type { WebViewProps } from '@papi/core'; import type { SerializedVerseRef } from '@sillsdev/scripture'; -import { render, screen, fireEvent } from '@testing-library/react'; +import { render, screen } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; import { useProjectData, useProjectSetting, @@ -326,7 +327,7 @@ describe('InterlinearizerWebView', () => { expect(screen.getByText('.')).toBeInTheDocument(); }); - it('calls setScrRef and addRecentScriptureRef when the verse picker submits', () => { + it('calls setScrRef and addRecentScriptureRef when the verse picker submits', async () => { mockBookData({}); const mockSetScrRef = jest.fn(); const mockAddRecentRef = jest.fn(); @@ -336,20 +337,20 @@ describe('InterlinearizerWebView', () => { }); render(); - fireEvent.click(screen.getByRole('button', { name: /submit reference/i })); + await userEvent.click(screen.getByRole('button', { name: /submit reference/i })); expect(mockSetScrRef).toHaveBeenCalledWith(defaultScrRef); expect(mockAddRecentRef).toHaveBeenCalledWith(defaultScrRef); }); - it('calls setScrRef with the segment ref when a verse box is clicked', () => { + it('calls setScrRef with the segment ref when a verse box is clicked', async () => { mockBookData({}); jest.mocked(tokenizeBook).mockReturnValue(GEN_1_MULTI_BOOK); const mockSetScrRef = jest.fn(); // Start at verse 1; click verse 2's token to select it render(); - fireEvent.click(screen.getByText('And')); + await userEvent.click(screen.getByText('And')); expect(mockSetScrRef).toHaveBeenCalledWith({ book: 'GEN', chapterNum: 1, verseNum: 2 }); }); diff --git a/src/__tests__/parsers/papi/bookTokenizer.test.ts b/src/__tests__/parsers/papi/bookTokenizer.test.ts index 7a6cded9..e6488678 100644 --- a/src/__tests__/parsers/papi/bookTokenizer.test.ts +++ b/src/__tests__/parsers/papi/bookTokenizer.test.ts @@ -141,6 +141,16 @@ describe('tokenizeBook', () => { expect(segments[0].tokens[0].surfaceText).toBe('ñ'); }); + it('throws when a verse SID book code does not match rawBook.bookCode', () => { + const raw: RawBook = { ...makeRawBook([{ sid: 'EXO 1:1', text: 'text' }]), bookCode: 'GEN' }; + expect(() => tokenizeBook(raw)).toThrow( + expect.objectContaining({ + name: 'SyntaxError', + message: expect.stringContaining('does not match book code'), + }), + ); + }); + it('throws on an invalid verse SID', () => { expect(() => tokenizeBook(makeRawBook([{ sid: 'not-a-ref', text: 'text' }]))).toThrow( expect.objectContaining({ diff --git a/src/__tests__/parsers/pt9/interlinearXmlParser.test.ts b/src/__tests__/parsers/pt9/interlinearXmlParser.test.ts index e65ed07d..c80befcf 100644 --- a/src/__tests__/parsers/pt9/interlinearXmlParser.test.ts +++ b/src/__tests__/parsers/pt9/interlinearXmlParser.test.ts @@ -757,7 +757,7 @@ describe('InterlinearXmlParser', () => { expect.objectContaining({ name: 'SyntaxError', message: expect.stringContaining( - 'Invalid XML: Range missing required Index or Length attributes', + 'Invalid XML: Range missing or invalid Index/Length attributes (must be non-negative integers)', ), }), ); @@ -781,7 +781,7 @@ describe('InterlinearXmlParser', () => { expect.objectContaining({ name: 'SyntaxError', message: expect.stringContaining( - 'Invalid XML: Range missing required Index or Length attributes', + 'Invalid XML: Range missing or invalid Index/Length attributes (must be non-negative integers)', ), }), ); diff --git a/src/interlinearizer.web-view.tsx b/src/interlinearizer.web-view.tsx index 1860a500..4c4a120e 100644 --- a/src/interlinearizer.web-view.tsx +++ b/src/interlinearizer.web-view.tsx @@ -228,9 +228,7 @@ globalThis.webViewComponent = function InterlinearizerWebView({ startAreaChildren={ { - setScrRef(ref); - }} + handleSubmit={setScrRef} localizedStrings={localizedStrings} recentSearches={recentRefs} onAddRecentSearch={onAddRecentRef} diff --git a/src/main.ts b/src/main.ts index 2cbe7bc8..5793bcfc 100644 --- a/src/main.ts +++ b/src/main.ts @@ -93,13 +93,7 @@ async function openInterlinearizer(projectId?: string): Promise { if (!webViewId) return openInterlinearizer(); const webViewDefinition = await papi.webViews.getOpenWebViewDefinition(webViewId); - if (!webViewDefinition?.projectId) { - logger.warn( - `Could not resolve projectId from webViewId "${webViewId}" while opening Interlinearizer`, - ); - return undefined; - } - return openInterlinearizer(webViewDefinition.projectId); + return openInterlinearizer(webViewDefinition?.projectId); } /** diff --git a/src/parsers/papi/bookTokenizer.ts b/src/parsers/papi/bookTokenizer.ts index 0fc479e9..bd6baf3c 100644 --- a/src/parsers/papi/bookTokenizer.ts +++ b/src/parsers/papi/bookTokenizer.ts @@ -89,7 +89,7 @@ function parseSid(sid: string): ScriptureRef { * characters. */ function tokenizeVerse(text: string, sid: string, writingSystem: string): Token[] { - return Array.from(text.matchAll(TOKEN_RE)).map((match) => { + return Array.from(text.matchAll(TOKEN_RE), (match) => { const surfaceText = match[0]; const charStart = match.index; const charEnd = charStart + surfaceText.length; @@ -115,6 +115,9 @@ function tokenizeVerse(text: string, sid: string, writingSystem: string): Token[ export function tokenizeBook(rawBook: RawBook): Book { const segments: Segment[] = rawBook.verses.map(({ sid, text }) => { const ref = parseSid(sid); + if (ref.book !== rawBook.bookCode) { + throw new SyntaxError(`Verse SID "${sid}" does not match book code "${rawBook.bookCode}"`); + } return { id: sid, startRef: { ...ref }, From 2aff42b26d4496a4849e14224c86e4a5b3460112 Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Wed, 29 Apr 2026 16:14:47 -0600 Subject: [PATCH 30/32] Allow word-final apostrophe glottal markers to be captured --- .../parsers/papi/bookTokenizer.test.ts | 24 +++++++++++++++---- src/parsers/papi/bookTokenizer.ts | 21 ++++++++-------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/__tests__/parsers/papi/bookTokenizer.test.ts b/src/__tests__/parsers/papi/bookTokenizer.test.ts index e6488678..0e993539 100644 --- a/src/__tests__/parsers/papi/bookTokenizer.test.ts +++ b/src/__tests__/parsers/papi/bookTokenizer.test.ts @@ -198,12 +198,11 @@ describe('tokenizeBook', () => { expect(wordTokens[1].surfaceText).toBe('well-known'); }); - it("tokenizes 'hello' as word then punctuation (leading apostrophe absorbed into word)", () => { + it("tokenizes 'hello' as a single word token (both leading and trailing apostrophes absorbed)", () => { const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text: "'hello'" }])); - const types = segments[0].tokens.map((t) => t.type); - expect(types).toEqual(['word', 'punctuation']); - expect(segments[0].tokens[0].surfaceText).toBe("'hello"); - expect(segments[0].tokens[1].surfaceText).toBe("'"); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe("'hello'"); }); it('tokenizes a standalone apostrophe as punctuation (no following word character)', () => { @@ -228,6 +227,21 @@ describe('tokenizeBook', () => { expect(segments[0].tokens[0].surfaceText).toBe(text); }); + it("tokenizes word-final U+0027 as part of the word (e.g. Hebrew aleph romanisation bara')", () => { + const text = "bara'"; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe(text); + }); + + it('tokenizes word-final U+2019 as part of the word (e.g. Hebrew aleph romanisation bara’)', () => { + const text = 'bara’'; + const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); + expect(segments[0].tokens).toHaveLength(1); + expect(segments[0].tokens[0].type).toBe('word'); + expect(segments[0].tokens[0].surfaceText).toBe(text); + }); it('tokenizes U+02BC (modifier letter apostrophe) as a word character regardless of position', () => { // U+02BC is \p{L} so it is inherently a word character — no special handling needed. const text = 'ʼelohim'; diff --git a/src/parsers/papi/bookTokenizer.ts b/src/parsers/papi/bookTokenizer.ts index bd6baf3c..abca6c51 100644 --- a/src/parsers/papi/bookTokenizer.ts +++ b/src/parsers/papi/bookTokenizer.ts @@ -17,13 +17,13 @@ import type { RawBook } from './usjBookExtractor'; const CHAR_SET = String.raw`\p{L}\p{N}\p{M}\p{Join_Control}`; /** - * Includes U+0027 and U+2019 at word-initial position (before the first word character) to handle - * languages where these characters represent a phonemic glottal stop or similar feature (e.g. - * Hebrew aleph in romanization, various indigenous-language orthographies). + * Includes U+0027 and U+2019 at word-initial and word-final positions to handle languages where + * these characters represent a phonemic glottal stop or similar feature (e.g. Hebrew aleph in + * romanization, various indigenous-language orthographies). * * Doesn't include hyphens/dashes. */ -const LEAD_SET = String.raw`\u0027\u2019`; +const GLOTTAL_SET = String.raw`\u0027\u2019`; /** * Word-internal joiners: @@ -41,18 +41,19 @@ const JOIN_SET = String.raw`\u0027\u002D\u2010-\u2015\u2019`; /** * Matches word tokens and punctuation tokens. Whitespace is not tokenized. * - * A word token is a run of word characters, optionally extended through some leading characters and - * word-internal joiners (e.g., apostrophes and hyphens). A joiner is absorbed into the surrounding - * word only when it is both preceded and followed by word characters. Trailing joiners are left as - * standalone punctuation tokens. + * A word token is a run of word characters, optionally extended through some leading/trailing + * glottal characters and word-internal joiners (e.g., apostrophes and hyphens). A joiner is + * absorbed into the surrounding word only when it is both preceded and followed by word characters. + * Trailing joiners that are not in GLOTTAL_SET are left as standalone punctuation tokens. * - * Multiple leading apostrophes are absorbed greedily. Leading hyphens/dashes are NOT absorbed. + * Multiple leading or trailing glottal characters are absorbed greedily. Leading hyphens/dashes are + * NOT absorbed. * * Multiple consecutive word-internal joiners between word characters are absorbed greedily (e.g. * `a--b` → one token `a--b`). */ const TOKEN_RE = new RegExp( - String.raw`(?:[${LEAD_SET}]+(?=[${CHAR_SET}]))?[${CHAR_SET}]+(?:[${JOIN_SET}]+(?=[${CHAR_SET}])[${CHAR_SET}]+)*|[^${CHAR_SET}\s]`, + String.raw`(?:[${GLOTTAL_SET}]+(?=[${CHAR_SET}]))?[${CHAR_SET}]+(?:[${JOIN_SET}]+(?=[${CHAR_SET}])[${CHAR_SET}]+)*[${GLOTTAL_SET}]*|[^${CHAR_SET}\s]`, 'gv', ); From 13acc79478a5b0cdf9d20472838bc0785e3aad3c Mon Sep 17 00:00:00 2001 From: alex-rawlings-yyc Date: Wed, 29 Apr 2026 16:20:44 -0600 Subject: [PATCH 31/32] Update comment --- src/__tests__/parsers/papi/bookTokenizer.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/__tests__/parsers/papi/bookTokenizer.test.ts b/src/__tests__/parsers/papi/bookTokenizer.test.ts index 0e993539..d36238fd 100644 --- a/src/__tests__/parsers/papi/bookTokenizer.test.ts +++ b/src/__tests__/parsers/papi/bookTokenizer.test.ts @@ -288,7 +288,7 @@ describe('tokenizeBook', () => { it('classifies astral-plane letters (surrogate pairs) as word tokens', () => { // Gothic letters U+10330–U+1034F are outside the BMP; each code point is two UTF-16 code - // units. Testing surfaceText[0] (a lone surrogate) against WORD_START_RE would fail — the + // units. Testing surfaceText[0] (a lone surrogate) against WORD_CONTAIN_RE would fail — the // fix is to test the full surfaceText string. const text = '𐌰𐌱𐌲'; const { segments } = tokenizeBook(makeRawBook([{ sid: 'GEN 1:1', text }])); From 503883c36264a161b8e1da3231ce835857284a74 Mon Sep 17 00:00:00 2001 From: "D. Ror" Date: Thu, 30 Apr 2026 06:21:53 -0400 Subject: [PATCH 32/32] Tweak tests --- src/__tests__/interlinearizer.web-view.test.tsx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/__tests__/interlinearizer.web-view.test.tsx b/src/__tests__/interlinearizer.web-view.test.tsx index eaca59c0..fd6f65f5 100644 --- a/src/__tests__/interlinearizer.web-view.test.tsx +++ b/src/__tests__/interlinearizer.web-view.test.tsx @@ -7,17 +7,17 @@ import type { SerializedVerseRef } from '@sillsdev/scripture'; import { render, screen } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import { + useLocalizedStrings, useProjectData, useProjectSetting, - useLocalizedStrings, useRecentScriptureRefs, } from '@papi/frontend/react'; import type { Book } from 'interlinearizer'; import { extractBookFromUsj } from 'parsers/papi/usjBookExtractor'; import { tokenizeBook } from 'parsers/papi/bookTokenizer'; -jest.mock('parsers/papi/usjBookExtractor'); jest.mock('parsers/papi/bookTokenizer'); +jest.mock('parsers/papi/usjBookExtractor'); /** * Matches the PlatformError shape from platform-bible-utils (discriminated by @@ -68,7 +68,7 @@ const GEN_1_1_BOOK: Book = { }; /** Pre-built Book with no segments — used by the no-verse-data test. */ -const EMPTY_BOOK: Book = { id: 'GEN', bookRef: 'GEN', textVersion: 'v1', segments: [] }; +const GEN_EMPTY_BOOK: Book = { id: 'GEN', bookRef: 'GEN', textVersion: 'v1', segments: [] }; /** Book with two segments in GEN 1 — used by chapter-display tests. */ const GEN_1_MULTI_BOOK: Book = { @@ -235,7 +235,7 @@ describe('InterlinearizerWebView', () => { it('shows a no-verse message when the tokenized book has no segments at all', () => { mockBookData({}); - jest.mocked(tokenizeBook).mockReturnValue(EMPTY_BOOK); + jest.mocked(tokenizeBook).mockReturnValue(GEN_EMPTY_BOOK); render(); expect(screen.getByText(/no verse data for gen 1\./i)).toBeInTheDocument(); @@ -355,13 +355,13 @@ describe('InterlinearizerWebView', () => { expect(mockSetScrRef).toHaveBeenCalledWith({ book: 'GEN', chapterNum: 1, verseNum: 2 }); }); - it('passes a book-stable ref to BookUSJ so verse changes do not re-fetch the book', () => { + it('passes a book-stable ref to BookUSJ so chapter and verse changes do not re-fetch the book', () => { const mockBookUSJ = jest.fn().mockReturnValue([{}, jest.fn(), false]); jest.mocked(useProjectData).mockImplementation(() => ({ BookUSJ: mockBookUSJ })); const { rerender } = render(); rerender( , );