Make init message serialization legacy-compatible

Older lightning clients expect every feature flags to appear only in
either the global or local feature set of the init message.

This commit makes DNL compatible with these clients while still
supporting modern clients.

Author: canndrew

src/DotNetLightning.Core/Serialize/Features.fs

@@ -89,7 +89,7 @@ module internal Feature =
         |> Map.add (Feature.BasicMultiPartPayment) ([Feature.PaymentSecret])
         |> Map.add (Feature.PaymentSecret) ([Feature.VariableLengthOnion])
 
-    let private isFeatureOn(features: BitArray) (bit: int) =
+    let isFeatureOn(features: BitArray) (bit: int) =
         (features.Length > bit) && features.Reverse().[bit]
 
     let hasFeature(features: BitArray) (f: Feature) (support: FeaturesSupport option) =
@@ -173,14 +173,12 @@ module internal Feature =
 /// Uses regular class instead of F# type for caching byte[] representation
 [<StructuredFormatDisplay("{PrettyPrint}")>]
 type FeatureBit private (bitArray) =
-    let mutable bytes = null
     member val BitArray: BitArray = bitArray with get, set
     member this.ByteArray
         with get() =
-            if isNull bytes then
-                bytes <- this.BitArray.ToByteArray()
-            bytes
-        and set(v: byte[]) = bytes <- v
+            bitArray.ToByteArray()
+        and set(bytes: byte[]) =
+            this.BitArray <- BitArray.FromBytes(bytes)
     new() =
         FeatureBit(
             let b: bool array = [||]
@@ -199,12 +197,8 @@ type FeatureBit private (bitArray) =
         }
     static member Zero = FeatureBit()
     static member TryCreate(bytes: byte[]) =
-        result {
-            let! fb = FeatureBit.TryCreate(BitArray.FromBytes(bytes))
-            fb.ByteArray <- bytes
-            return fb
-        }
-        
+        FeatureBit.TryCreate(BitArray.FromBytes(bytes))
+
     static member TryCreate(v: int64) =
         BitArray.FromInt64(v) |> FeatureBit.TryCreate
 
@@ -233,6 +227,29 @@ type FeatureBit private (bitArray) =
     override this.ToString() =
         this.BitArray.PrintBits()
 
+    member this.SetFeatureBit(index: int, on: bool): unit =
+        let length = this.BitArray.Length
+        if length <= index then
+            this.BitArray.Length <- index + 1
+
+            //this.BitArray.RightShift(index - length + 1)
+
+            // NOTE: Calling RightShift gives me:
+            // "The field, constructor or member 'RightShift' is not defined."
+            // So I just re-implement it here
+            for i in (length - 1) .. -1 .. 0 do
+                this.BitArray.[i + index - length + 1] <- this.BitArray.[i]
+
+            // NOTE: this probably wouldn't be necessary if we were using
+            // RightShift, but the dotnet docs don't actualy specify that
+            // RightShift sets the leading bits to zero.
+            for i in 0 .. (index - length) do
+                this.BitArray.[i] <- false
+        this.BitArray.[this.BitArray.Length - index - 1] <- on
+
+    member this.GetFeatureBit(index: int): bool =
+        Feature.isFeatureOn this.BitArray index
+
     member this.HasFeature(f, ?featureType) =
         Feature.hasFeature this.BitArray (f) (featureType)
 

src/DotNetLightning.Core/Serialize/Msgs/Msgs.fs

@@ -9,6 +9,7 @@ open NBitcoin
 open DotNetLightning.Utils
 open DotNetLightning.Utils.Primitives
 open DotNetLightning.Utils.OnionError
+open DotNetLightning.Core.Utils.Extensions
 
 open DotNetLightning.Serialize
 
@@ -375,7 +376,8 @@ type Init =
         interface ISetupMsg
         interface ILightningSerializable<Init> with
             member this.Deserialize(ls: LightningReaderStream) =
-                // For backwards compatibility reason, we must consider legacy `global features` section. (see bolt 1)
+                // For backwards compatibility reason, we must consider legacy
+                // `global features` section. (see bolt 1)
                 let globalFeatures = ls.ReadWithLen()
                 let localFeatures = ls.ReadWithLen()
                 let oredFeatures =
@@ -396,24 +398,30 @@ type Init =
                     oredFeatures
                 this.Features <- oredFeatures |> FeatureBit.CreateUnsafe
                 this.TLVStream <- ls.ReadTLVStream() |> Array.map(InitTLV.FromGenericTLV)
+
             member this.Serialize(ls) =
-                // Trim any leading zero bytes from the from of the local features
+                // For legacy compatiblity we treat the VariableLengthOnion feature specially.
+                // Older versions of the lightning protocol spec had a
+                // distinction between global and local features, of which
+                // VariableLengthOnion was the only global feature. Although
+                // this distinction has since been removed, older clients still
+                // expect VariableLengthOnion to appear only in the
+                // global_features field of an init message and other features
+                // to appear only in the local_features field. This is still
+                // compatible with newer clients since those clients simply OR
+                // the two feature fields.
                 let localFeatures =
-                    let features = this.Features.ToByteArray()
-                    let startIndex =
-                        match Array.tryFindIndex (fun (b: byte) -> b <> 0uy) features with
-                        | Some index -> index
-                        | None -> features.Length
-                    features.[startIndex..]
-                // For legacy compatibility, the last (lowest) 13 bits get
-                // duplicated in the globalFeatures array.
+                    let localFeatures = this.Features
+                    localFeatures.SetFeatureBit(Feature.VariableLengthOnion.MandatoryBitPosition, false)
+                    localFeatures.SetFeatureBit(Feature.VariableLengthOnion.OptionalBitPosition, false)
+                    localFeatures.ToByteArray()
                 let globalFeatures =
-                    if localFeatures.Length < 2 then
-                        localFeatures
-                    else
-                        let features = localFeatures.[(localFeatures.Length - 2)..]
-                        features.[0] <- features.[0] &&& 0b00111111uy
-                        features
+                    let globalFeatures = FeatureBit.Zero
+                    if this.Features.GetFeatureBit(Feature.VariableLengthOnion.MandatoryBitPosition) then
+                        globalFeatures.SetFeatureBit(Feature.VariableLengthOnion.MandatoryBitPosition, true)
+                    if this.Features.GetFeatureBit(Feature.VariableLengthOnion.OptionalBitPosition) then
+                        globalFeatures.SetFeatureBit(Feature.VariableLengthOnion.OptionalBitPosition, true)
+                    globalFeatures.ToByteArray()
                 ls.WriteWithLen(globalFeatures)
                 ls.WriteWithLen(localFeatures)
                 ls.WriteTLVStream(this.TLVStream |> Array.map(fun tlv -> tlv.ToGenericTLV()))

src/DotNetLightning.Core/Utils/Extensions.fs

@@ -101,11 +101,34 @@ type BitArrayExtensions() =
 type System.Collections.BitArray with
     member this.ToByteArray() =
         if this.Length = 0 then [||] else
-        let ret: byte[] = Array.zeroCreate (((this.Length - 1) / 8) + 1)
-        let boolArray: bool[] = this.Reverse()
-        let t = BitArray(boolArray)
-        t.CopyTo(ret, 0)
-        ret |> Array.rev
+
+        let leadingZeros =
+            match (Seq.tryFindIndex (fun b -> b) (Seq.cast this)) with
+            | Some i -> i
+            | None -> this.Length
+        let trueLength = this.Length - leadingZeros
+        let desiredLength = ((trueLength + 7) / 8) * 8
+        let difference = desiredLength - this.Length
+        let bitArray =
+            if difference < 0 then
+                // Drop zeroes from the front of the array until we have a multiple of 8 bits
+                let shortenedBitArray = BitArray(desiredLength)
+                for i in 0 .. (desiredLength - 1) do
+                    shortenedBitArray.[i] <- this.[i - difference]
+                shortenedBitArray
+            else if difference > 0 then
+                // Push zeroes to the front of the array until we have a multiple of 8 bits
+                let lengthenedBitArray = BitArray(desiredLength)
+                for i in 0 .. (this.Length - 1) do
+                    lengthenedBitArray.[i + difference] <- this.[i]
+                lengthenedBitArray
+            else
+                this
+
+        // Copy the bit array to a byte array, then flip the bytes.
+        let byteArray: byte[] = Array.zeroCreate(desiredLength / 8)
+        bitArray.CopyTo(byteArray, 0)
+        byteArray |> Array.map (fun b -> b.FlipBit())
 
     static member From5BitEncoding(b: byte[]) =
         let bitArray = System.Collections.BitArray(b.Length * 5)