Preserve DOCX manual line breaks

Author: dipeshbabu

sdk/typescript/src/knowledge-base.ts

@@ -193,6 +193,7 @@ function extractDocx(path: string, bytes: Uint8Array): string {
     return decodeXml(
       xml
         .replace(/<\/(?:\w+:)?p\s*>/gu, "\n")
+        .replace(/<(?:\w+:)?br\b[^>]*\/>/gu, "\n")
         .replace(/<(?:\w+:)?tab\b[^>]*\/>/gu, "\t")
         .replace(/<[^>]+>/gu, ""),
     );

sdk/typescript/tests-ts/knowledge-base.test.ts

@@ -41,10 +41,10 @@ async function extractedDocuments(path: string): Promise<string[]> {
   );
 }
 
-function docx(text: string): Uint8Array {
+function docx(text: string, secondLine?: string): Uint8Array {
   return zipSync({
     "word/document.xml": strToU8(
-      `<?xml version="1.0"?><w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body><w:p><w:r><w:t>${text}</w:t></w:r></w:p></w:body></w:document>`,
+      `<?xml version="1.0"?><w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body><w:p><w:r><w:t>${text}</w:t></w:r>${secondLine === undefined ? "" : `<w:br/><w:r><w:t>${secondLine}</w:t></w:r>`}</w:p></w:body></w:document>`,
     ),
   });
 }
@@ -111,14 +111,17 @@ describe("scan knowledge bases", () => {
       join(root, "architecture.pdf"),
       pdf("Payment service boundary"),
     );
-    await writeFile(join(root, "threat-model.docx"), docx("SSRF &amp; IDOR"));
+    await writeFile(
+      join(root, "threat-model.docx"),
+      docx("SSRF &amp; IDOR", "Review authentication"),
+    );
 
     const knowledgeBase = await prepareKnowledgeBase([root]);
     temporaryDirectories.push(knowledgeBase.path);
     const documents = await extractedDocuments(knowledgeBase.path);
 
     expect(documents).toContain("Payment service boundary");
-    expect(documents).toContain("SSRF & IDOR\n");
+    expect(documents).toContain("SSRF & IDOR\nReview authentication\n");
   });
 
   test("cleans up documents and rediscovers directory contents on later runs", async () => {