From b4f4a4403a59c4047d78a4e43621a3d547ae26d1 Mon Sep 17 00:00:00 2001 From: Zachary Travis Date: Fri, 5 Apr 2024 13:39:16 -0700 Subject: [PATCH] Fix issues with unicode ANSI skipping The current implementation of unicode ANSI replacement sequences is slightly wrong - according to https://www.biblioscape.com/rtf15_spec.htm * A scope delimiter (i.e. "{" or "}") should end the current skippable data * Control words or symbols should be considered a single skipped character (and in my testing with MS office, they're ignored) * Any binary data is also considered a single skipped character --- .../parser/standard/StandardRtfParser.java | 34 ++++++++++++------ .../standard/StandardRtfParserTest.java | 31 ++++++++++++++++ .../parser/standard/data/testSkipBinary.rtf | Bin 0 -> 38 bytes .../parser/standard/data/testSkipBinary.xml | 1 + .../parser/standard/data/testSkipCombo.rtf | Bin 0 -> 74 bytes .../parser/standard/data/testSkipCombo.xml | 1 + .../parser/standard/data/testSkipCommand.rtf | Bin 0 -> 29 bytes .../parser/standard/data/testSkipCommand.xml | 1 + .../parser/standard/data/testSkipGroupEnd.rtf | Bin 0 -> 30 bytes .../parser/standard/data/testSkipGroupEnd.xml | 1 + .../standard/data/testSkipGroupStart.rtf | Bin 0 -> 33 bytes .../standard/data/testSkipGroupStart.xml | 1 + 12 files changed, 60 insertions(+), 10 deletions(-) create mode 100644 src/test/resources/com/rtfparserkit/parser/standard/data/testSkipBinary.rtf create mode 100644 src/test/resources/com/rtfparserkit/parser/standard/data/testSkipBinary.xml create mode 100644 src/test/resources/com/rtfparserkit/parser/standard/data/testSkipCombo.rtf create mode 100644 src/test/resources/com/rtfparserkit/parser/standard/data/testSkipCombo.xml create mode 100644 src/test/resources/com/rtfparserkit/parser/standard/data/testSkipCommand.rtf create mode 100644 src/test/resources/com/rtfparserkit/parser/standard/data/testSkipCommand.xml create mode 100644 src/test/resources/com/rtfparserkit/parser/standard/data/testSkipGroupEnd.rtf create mode 100644 src/test/resources/com/rtfparserkit/parser/standard/data/testSkipGroupEnd.xml create mode 100644 src/test/resources/com/rtfparserkit/parser/standard/data/testSkipGroupStart.rtf create mode 100644 src/test/resources/com/rtfparserkit/parser/standard/data/testSkipGroupStart.xml diff --git a/src/main/java/com/rtfparserkit/parser/standard/StandardRtfParser.java b/src/main/java/com/rtfparserkit/parser/standard/StandardRtfParser.java index 4f320e6..b55eca2 100644 --- a/src/main/java/com/rtfparserkit/parser/standard/StandardRtfParser.java +++ b/src/main/java/com/rtfparserkit/parser/standard/StandardRtfParser.java @@ -59,6 +59,8 @@ public void processGroupStart() handleEvent(GROUP_START); stack.push(state); state = new ParserState(state); + // Clear any remaining skippable bytes on group start. + skipBytes = 0; } /** @@ -69,6 +71,8 @@ public void processGroupEnd() { handleEvent(GROUP_END); state = stack.pop(); + // Clear any remaining skippable bytes on group end. + skipBytes = 0; } /** @@ -77,21 +81,21 @@ public void processGroupEnd() @Override public void processCharacterBytes(byte[] data) { - try + if (skipBytes < data.length) { - if (data.length != 0) + try { - if (skipBytes < data.length) - { - handleEvent(new StringEvent(new String(data, skipBytes, data.length - skipBytes, currentEncoding()))); - } - skipBytes = 0; + handleEvent(new StringEvent(new String(data, skipBytes, data.length - skipBytes, currentEncoding()))); + } + catch (UnsupportedEncodingException ex) + { + throw new RuntimeException(ex); } + skipBytes = 0; } - - catch (UnsupportedEncodingException ex) + else { - throw new RuntimeException(ex); + skipBytes -= data.length; } } @@ -133,6 +137,11 @@ public void processDocumentEnd() @Override public void processBinaryBytes(byte[] data) { + if (skipBytes > 0) { + // Treat all binary data as a single skippable character. + skipBytes--; + return; + } handleEvent(new BinaryBytesEvent(data)); } @@ -151,6 +160,11 @@ public void processString(String string) @Override public void processCommand(Command command, int parameter, boolean hasParameter, boolean optional) { + if (skipBytes > 0) { + // A command should be treated as a single skippable character (and skipped). + skipBytes--; + return; + } if (command.getCommandType() == CommandType.Encoding) { processEncoding(command, hasParameter, parameter); diff --git a/src/test/java/com/rtfparserkit/parser/standard/StandardRtfParserTest.java b/src/test/java/com/rtfparserkit/parser/standard/StandardRtfParserTest.java index 014f3c9..39ce4f8 100644 --- a/src/test/java/com/rtfparserkit/parser/standard/StandardRtfParserTest.java +++ b/src/test/java/com/rtfparserkit/parser/standard/StandardRtfParserTest.java @@ -154,4 +154,35 @@ public void testNecCharacters() throws Exception { TestUtilities.assertRtfParserDumpMatches(this, new StandardRtfParser(), "testNecCharacters"); } + + @Test + public void testSkipGroupStart() throws Exception + { + TestUtilities.assertRtfParserDumpMatches(this, new StandardRtfParser(), "testSkipGroupStart"); + } + + @Test + public void testSkipGroupEnd() throws Exception + { + TestUtilities.assertRtfParserDumpMatches(this, new StandardRtfParser(), "testSkipGroupEnd"); + } + + @Test + public void testSkipBinary() throws Exception + { + TestUtilities.assertRtfParserDumpMatches(this, new StandardRtfParser(), "testSkipBinary"); + } + + @Test + public void testSkipCommand() throws Exception + { + TestUtilities.assertRtfParserDumpMatches(this, new StandardRtfParser(), "testSkipCommand"); + } + + @Test + public void testSkipCombo() throws Exception + { + TestUtilities.assertRtfParserDumpMatches(this, new StandardRtfParser(), "testSkipCombo"); + } + } diff --git a/src/test/resources/com/rtfparserkit/parser/standard/data/testSkipBinary.rtf b/src/test/resources/com/rtfparserkit/parser/standard/data/testSkipBinary.rtf new file mode 100644 index 0000000000000000000000000000000000000000..3f8e3266c1a19e77794a7b4cd314775874dc9c6d GIT binary patch literal 38 tcmb=9DJn@baxyz \ No newline at end of file diff --git a/src/test/resources/com/rtfparserkit/parser/standard/data/testSkipCombo.rtf b/src/test/resources/com/rtfparserkit/parser/standard/data/testSkipCombo.rtf new file mode 100644 index 0000000000000000000000000000000000000000..57c2c89c83b84be3be352bfcabd83446f414913e GIT binary patch literal 74 zcmb=9DJn@bZL50L( Rkiw+Is#;zyAY$O<0sx4d7OVgO literal 0 HcmV?d00001 diff --git a/src/test/resources/com/rtfparserkit/parser/standard/data/testSkipCombo.xml b/src/test/resources/com/rtfparserkit/parser/standard/data/testSkipCombo.xml new file mode 100644 index 0000000..8624a34 --- /dev/null +++ b/src/test/resources/com/rtfparserkit/parser/standard/data/testSkipCombo.xml @@ -0,0 +1 @@ +afooaar abaz \ No newline at end of file diff --git a/src/test/resources/com/rtfparserkit/parser/standard/data/testSkipCommand.rtf b/src/test/resources/com/rtfparserkit/parser/standard/data/testSkipCommand.rtf new file mode 100644 index 0000000000000000000000000000000000000000..51c106b8ac4904a7e85bfe824142c7add983a919 GIT binary patch literal 29 hcmb=9DJn@bk=H;s8ac \ No newline at end of file diff --git a/src/test/resources/com/rtfparserkit/parser/standard/data/testSkipGroupEnd.rtf b/src/test/resources/com/rtfparserkit/parser/standard/data/testSkipGroupEnd.rtf new file mode 100644 index 0000000000000000000000000000000000000000..6c12c551414a08698a9edf72fefb6071712df6bc GIT binary patch literal 30 lcmb=9DJn@bafoo \ No newline at end of file diff --git a/src/test/resources/com/rtfparserkit/parser/standard/data/testSkipGroupStart.rtf b/src/test/resources/com/rtfparserkit/parser/standard/data/testSkipGroupStart.rtf new file mode 100644 index 0000000000000000000000000000000000000000..33b6b5fb124d62abe8f88f909cc79165e3ffbee4 GIT binary patch literal 33 mcmb=9DJn@bKy=Lrq~ literal 0 HcmV?d00001 diff --git a/src/test/resources/com/rtfparserkit/parser/standard/data/testSkipGroupStart.xml b/src/test/resources/com/rtfparserkit/parser/standard/data/testSkipGroupStart.xml new file mode 100644 index 0000000..d4931be --- /dev/null +++ b/src/test/resources/com/rtfparserkit/parser/standard/data/testSkipGroupStart.xml @@ -0,0 +1 @@ +afoo \ No newline at end of file