Merge pull request #473 from LebedevRI/tree

`PrefixCodeLUTDecoder<???, PrefixCodeTreeDecoder<???>>` support

Author: LebedevRI

fuzz/all-fuzzers.txt

@@ -22,17 +22,23 @@ PanasonicV6DecompressorFuzzer
 PanasonicV7DecompressorFuzzer
 PentaxDecompressorFuzzer
 PhaseOneDecompressorFuzzer
+PrefixCodeDecoderFuzzer-LUTWithLookupVsLUTWithTree
 PrefixCodeDecoderFuzzer-LUTWithLookupVsLUTWithVector
 PrefixCodeDecoderFuzzer-LUTWithLookupVsLookup
 PrefixCodeDecoderFuzzer-LUTWithLookupVsTree
 PrefixCodeDecoderFuzzer-LUTWithLookupVsVector
+PrefixCodeDecoderFuzzer-LUTWithTreeVsLUTWithVector
+PrefixCodeDecoderFuzzer-LUTWithTreeVsLookup
+PrefixCodeDecoderFuzzer-LUTWithTreeVsTree
+PrefixCodeDecoderFuzzer-LUTWithTreeVsVector
 PrefixCodeDecoderFuzzer-LUTWithVectorVsLookup
 PrefixCodeDecoderFuzzer-LUTWithVectorVsTree
 PrefixCodeDecoderFuzzer-LUTWithVectorVsVector
 PrefixCodeDecoderFuzzer-LookupVsTree
 PrefixCodeDecoderFuzzer-LookupVsVector
 PrefixCodeDecoderFuzzer-TreeVsVector
 PrefixCodeLUTWithLookupDecoderFuzzer
+PrefixCodeLUTWithTreeDecoderFuzzer
 PrefixCodeLUTWithVectorDecoderFuzzer
 PrefixCodeLookupDecoderFuzzer
 PrefixCodeTreeDecoderFuzzer

fuzz/librawspeed/codes/PrefixCodeDecoder/CMakeLists.txt

@@ -67,9 +67,8 @@ function(add_ht_dual_fuzzer impl0 impl1)
   add_dependencies(PrefixCodeDecoderFuzzers ${fuzzer})
 endfunction()
 
-set(IMPL     "Tree")
 set(FRONTEND "LUT")
-set(BACKEND  "Lookup" "Vector")
+set(BACKEND  "Tree" "Vector" "Lookup")
 
 foreach(backend ${BACKEND})
   list(APPEND IMPL "${backend}")

src/librawspeed/adt/BitIterator.h

@@ -0,0 +1,65 @@
+/*
+    RawSpeed - RAW file decoder.
+
+    Copyright (C) 2023 Roman Lebedev
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+*/
+
+#pragma once
+
+#include "adt/Invariant.h" // for invariant
+#include "common/Common.h" // for bitwidth
+#include <iterator>        // for input_iterator_tag
+#include <utility>         // for enable_if_t, is_unsigned_v
+
+namespace rawspeed {
+
+template <typename T, typename = std::enable_if_t<std::is_unsigned_v<T>>>
+struct BitMSBIterator {
+  T bitsPat;
+  int bitIdx;
+
+  using iterator_category = std::input_iterator_tag;
+  using difference_type = std::ptrdiff_t;
+  using value_type = bool;
+  using pointer = const value_type*;   // Unusable, but must be here.
+  using reference = const value_type&; // Unusable, but must be here.
+
+  BitMSBIterator(T bitsPat_, int bitIdx_) : bitsPat(bitsPat_), bitIdx(bitIdx_) {
+    invariant(bitIdx < static_cast<int>(bitwidth<T>()) && bitIdx >= -1);
+  }
+
+  value_type operator*() const {
+    invariant(static_cast<unsigned>(bitIdx) < bitwidth<T>() &&
+              "Iterator overflow");
+    return (bitsPat >> bitIdx) & 0b1;
+  }
+  BitMSBIterator& operator++() {
+    --bitIdx; // We go from MSB to LSB.
+    invariant(bitIdx >= -1);
+    return *this;
+  }
+  friend inline bool operator==(const BitMSBIterator& a,
+                                const BitMSBIterator& b) {
+    invariant(a.bitsPat == b.bitsPat && "Comparing unrelated iterators.");
+    return a.bitIdx == b.bitIdx;
+  }
+  friend bool operator!=(const BitMSBIterator& a, const BitMSBIterator& b) {
+    return !(a == b);
+  }
+};
+
+} // namespace rawspeed

src/librawspeed/adt/CMakeLists.txt

@@ -2,6 +2,7 @@ FILE(GLOB SOURCES
   "AlignedAllocator.h"
   "Array1DRef.h"
   "Array2DRef.h"
+  "BitIterator.h"
   "CroppedArray1DRef.h"
   "CroppedArray2DRef.h"
   "DefaultInitAllocatorAdaptor.h"

src/librawspeed/codes/AbstractPrefixCode.h

@@ -20,6 +20,8 @@
 
 #pragma once
 
+#include "adt/BitIterator.h"              // for BitMSBIterator
+#include "adt/iterator_range.h"           // for iterator_range
 #include "common/Common.h"                // for bitwidth
 #include "decoders/RawDecoderException.h" // for ThrowException, ThrowRDE
 #include <cstdint>                        // for uint16_t
@@ -150,6 +152,11 @@ template <typename CodeTag> class AbstractPrefixCode {
       assert(code <= ((1U << code_len) - 1U));
     }
 
+    [[nodiscard]] iterator_range<BitMSBIterator<typename Traits::CodeTy>>
+    getBitsMSB() const {
+      return {{code, code_len - 1}, {code, -1}};
+    }
+
     static bool HaveCommonPrefix(const CodeSymbol& symbol,
                                  const CodeSymbol& partial) {
       assert(partial.code_len <= symbol.code_len);

src/librawspeed/codes/BinaryPrefixTree.h

@@ -87,20 +87,12 @@ void BinaryPrefixTree<CodeTag>::add(const CodeSymbol symbol, CodeTy value) {
   invariant(symbol.code_len > 0);
   invariant(symbol.code_len <= Traits::MaxCodeLenghtBits);
 
-  auto getSymbolsNthMSB = [&symbol](int msbBitIdx) {
-    invariant(msbBitIdx >= 0 && msbBitIdx < symbol.code_len);
-    unsigned MSBs = extractHighBits(symbol.code, 1 + msbBitIdx,
-                                    /*effectiveBitwidth=*/symbol.code_len);
-    return MSBs & 0b1;
-  };
-
   CodeSymbol partial;
   partial.code = 0;
   partial.code_len = 0;
 
   std::reference_wrapper<std::unique_ptr<Node>> newBud = root;
-  for (int depth = 0; depth < symbol.code_len; ++depth) {
-    unsigned bit = getSymbolsNthMSB(depth);
+  for (unsigned bit : symbol.getBitsMSB()) {
     ++partial.code_len;
     partial.code = (partial.code << 1) | bit;
     std::unique_ptr<Node>& bud = newBud;

src/librawspeed/codes/PrefixCodeTreeDecoder.h

@@ -30,14 +30,15 @@
 #include <initializer_list>                  // for initializer_list
 #include <iterator>                          // for advance, next
 #include <memory>                            // for unique_ptr, make_unique
+#include <optional>                          // for optional
 #include <tuple>                             // for tie
 #include <utility>                           // for pair
 #include <vector>                            // for vector, vector<>::co...
 
 namespace rawspeed {
 
 template <typename CodeTag>
-class PrefixCodeTreeDecoder final : public AbstractPrefixCodeDecoder<CodeTag> {
+class PrefixCodeTreeDecoder : public AbstractPrefixCodeDecoder<CodeTag> {
 public:
   using Tag = CodeTag;
   using Base = AbstractPrefixCodeDecoder<CodeTag>;
@@ -48,29 +49,23 @@ class PrefixCodeTreeDecoder final : public AbstractPrefixCodeDecoder<CodeTag> {
 private:
   BinaryPrefixTree<CodeTag> tree;
 
+protected:
   template <typename BIT_STREAM>
-  inline std::pair<typename Base::CodeSymbol,
-                   typename Traits::CodeValueTy /*codeValue*/>
-  readSymbol(BIT_STREAM& bs) const {
-    static_assert(
-        BitStreamTraits<typename BIT_STREAM::tag>::canUseWithPrefixCodeDecoder,
-        "This BitStream specialization is not marked as usable here");
+  inline std::pair<typename Base::CodeSymbol, int /*codeValue*/>
+  finishReadingPartialSymbol(BIT_STREAM& bs,
+                             typename Base::CodeSymbol initialPartial) const {
     typename Base::CodeSymbol partial;
+    partial.code = 0;
+    partial.code_len = 0;
 
     const auto* top = &(tree.root->getAsBranch());
 
-    // Read bits until either find the code or detect the incorrect code
-    for (partial.code = 0, partial.code_len = 1;; ++partial.code_len) {
-      invariant(partial.code_len <= Traits::MaxCodeLenghtBits);
-
-      // Read one more bit
-      const bool bit = bs.getBitsNoFill(1);
-
-      // codechecker_false_positive [core.uninitialized.Assign]
+    auto walkBinaryTree = [&partial, &top](bool bit)
+        -> std::optional<
+            std::pair<typename Base::CodeSymbol, int /*codeValue*/>> {
       partial.code <<= 1;
       partial.code |= bit;
-
-      // What is the last bit, which we have just read?
+      partial.code_len++;
 
       // NOTE: The order *IS* important! Left to right, zero to one!
       const auto& newNode = top->buds[bit];
@@ -84,17 +79,50 @@ class PrefixCodeTreeDecoder final : public AbstractPrefixCodeDecoder<CodeTag> {
       if (static_cast<typename decltype(tree)::Node::Type>(*newNode) ==
           decltype(tree)::Node::Type::Leaf) {
         // Ok, great, hit a Leaf. This is it.
-        return {partial, newNode->getAsLeaf().value};
+        return {{partial, newNode->getAsLeaf().value}};
       }
 
       // Else, this is a branch, continue looking.
       top = &(newNode->getAsBranch());
+      return std::nullopt;
+    };
+
+    // First, translate pre-existing code bits.
+    for (unsigned bit : initialPartial.getBitsMSB()) {
+      if (auto sym = walkBinaryTree(bit))
+        return *sym;
+    }
+
+    // Read bits until either find the code or detect the incorrect code
+    while (true) {
+      invariant(partial.code_len <= Traits::MaxCodeLenghtBits);
+
+      // Read one more bit
+      const bool bit = bs.getBitsNoFill(1);
+
+      if (auto sym = walkBinaryTree(bit))
+        return *sym;
     }
 
     // We have either returned the found symbol, or thrown on incorrect symbol.
     __builtin_unreachable();
   }
 
+  template <typename BIT_STREAM>
+  inline std::pair<typename Base::CodeSymbol, int /*codeValue*/>
+  readSymbol(BIT_STREAM& bs) const {
+    static_assert(
+        BitStreamTraits<typename BIT_STREAM::tag>::canUseWithPrefixCodeDecoder,
+        "This BitStream specialization is not marked as usable here");
+
+    // Start from completely unknown symbol.
+    typename Base::CodeSymbol partial;
+    partial.code_len = 0;
+    partial.code = 0;
+
+    return finishReadingPartialSymbol(bs, partial);
+  }
+
 public:
   void setup(bool fullDecode_, bool fixDNGBug16_) {
     AbstractPrefixCodeDecoder<CodeTag>::setup(fullDecode_, fixDNGBug16_);