diff --git a/src/FastExpressionCompiler.LightExpression/FlatExpression.cs b/src/FastExpressionCompiler.LightExpression/FlatExpression.cs index 52654ef8..a6b79907 100644 --- a/src/FastExpressionCompiler.LightExpression/FlatExpression.cs +++ b/src/FastExpressionCompiler.LightExpression/FlatExpression.cs @@ -49,17 +49,14 @@ public enum ExprNodeKind : byte [StructLayout(LayoutKind.Explicit, Size = 24)] public struct ExprNode { - private const int NodeTypeShift = 56; - private const int TagShift = 48; - private const int NextShift = 32; + private const int NodeTypeShift = 24; + private const int TagShift = 16; private const int CountShift = 16; - private const ulong IndexMask = 0xFFFF; - private const ulong KindMask = 0x0F; - private const ulong NextMask = IndexMask << NextShift; - private const ulong ChildCountMask = IndexMask << CountShift; - private const ulong ChildInfoMask = ChildCountMask | IndexMask; - private const ulong KeepWithoutNextMask = ~NextMask; - private const ulong KeepWithoutChildInfoMask = ~ChildInfoMask; + private const uint IndexMask = 0xFFFF; + private const uint KindMask = 0x0F; + private const uint NextMask = IndexMask; + private const uint KeepWithoutNextMask = ~NextMask; + private const uint KeepWithoutTagAndNextMask = ~(NextMask | (0xFFU << TagShift)); private const int FlagsShift = 4; /// Gets or sets the runtime type of the represented node. @@ -70,18 +67,21 @@ public struct ExprNode [FieldOffset(8)] public object Obj; [FieldOffset(16)] - private ulong _data; + private uint _data; + + [FieldOffset(20)] + private uint _nodeTypeAndKind; /// Gets the expression kind encoded for this node. - public ExpressionType NodeType => (ExpressionType)((_data >> NodeTypeShift) & 0xFF); + public ExpressionType NodeType => (ExpressionType)((_nodeTypeAndKind >> NodeTypeShift) & 0xFF); /// Gets the payload classification for this node. - public ExprNodeKind Kind => (ExprNodeKind)((_data >> TagShift) & KindMask); + public ExprNodeKind Kind => (ExprNodeKind)((_nodeTypeAndKind >> TagShift) & KindMask); - internal byte Flags => (byte)(((byte)(_data >> TagShift)) >> FlagsShift); + internal byte Flags => (byte)(((byte)(_nodeTypeAndKind >> TagShift)) >> FlagsShift); /// Gets the next sibling node index in the intrusive child chain. - public int NextIdx => (int)((_data >> NextShift) & IndexMask); + public int NextIdx => (int)(_nodeTypeAndKind & IndexMask); /// Gets the number of direct children linked from this node. public int ChildCount => (int)((_data >> CountShift) & IndexMask); @@ -89,28 +89,42 @@ public struct ExprNode /// Gets the first child index or an auxiliary payload index. public int ChildIdx => (int)(_data & IndexMask); + internal int Value32 => unchecked((int)_data); + internal ExprNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind kind, byte flags = 0, int childIdx = 0, int childCount = 0, int nextIdx = 0) { Type = type; Obj = obj; var tag = (byte)((flags << FlagsShift) | (byte)kind); - _data = ((ulong)(byte)nodeType << NodeTypeShift) - | ((ulong)tag << TagShift) - | ((ulong)(ushort)nextIdx << NextShift) - | ((ulong)(ushort)childCount << CountShift) + _data = ((uint)(ushort)childCount << CountShift) | (ushort)childIdx; + _nodeTypeAndKind = ((uint)(byte)nodeType << NodeTypeShift) + | ((uint)tag << TagShift) + | (ushort)nextIdx; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void SetNextIdx(int nextIdx) => - _data = (_data & KeepWithoutNextMask) | ((ulong)(ushort)nextIdx << NextShift); + internal void SetNextIdx(int nextIdx, bool pointsToParent = false) + { + var tag = (byte)(_nodeTypeAndKind >> TagShift); + var nextPointsToParentMask = (byte)(ExprTree.NextPointsToParentFlag << FlagsShift); + tag = pointsToParent + ? (byte)(tag | nextPointsToParentMask) + : (byte)(tag & ~nextPointsToParentMask); + _nodeTypeAndKind = (_nodeTypeAndKind & KeepWithoutTagAndNextMask) + | ((uint)tag << TagShift) + | (ushort)nextIdx; + } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void SetChildInfo(int childIdx, int childCount) => - _data = (_data & KeepWithoutChildInfoMask) - | ((ulong)(ushort)childCount << CountShift) + _data = ((uint)(ushort)childCount << CountShift) | (ushort)childIdx; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void SetValue32(int value) => + _data = unchecked((uint)value); + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool Is(ExprNodeKind kind) => Kind == kind; @@ -120,22 +134,73 @@ internal void SetChildInfo(int childIdx, int childCount) => [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool HasFlag(byte flag) => (Flags & flag) != 0; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool NextPointsToParent() => HasFlag(ExprTree.NextPointsToParentFlag); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool IsParameterDeclaration() => + NodeType == ExpressionType.Parameter && HasFlag(ExprTree.ParameterDeclarationFlag); + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool ShouldCloneWhenLinked() => Kind == ExprNodeKind.LabelTarget || NodeType == ExpressionType.Parameter || Kind == ExprNodeKind.ObjectReference || ChildCount == 0; } +/// Stores owner-side metadata for a lambda node. +public struct LambdaInfo +{ + /// Gets or sets the lambda owner node index. + public int LambdaIndex; + + /// Gets or sets the lambda body node index. + public int BodyIndex; + + /// Gets or sets the declaration node indexes owned by the lambda. + public ChildList ParameterDeclarations; + + internal LambdaInfo(int lambdaIndex, int bodyIndex, in ChildList parameterDeclarations) + { + LambdaIndex = lambdaIndex; + BodyIndex = bodyIndex; + ParameterDeclarations = parameterDeclarations; + } +} + +/// Stores owner-side metadata for a block node. +public struct BlockInfo +{ + /// Gets or sets the block owner node index. + public int BlockIndex; + + /// Gets or sets the child-list node index containing the block body expressions. + public int BodyExpressionsIndex; + + /// Gets or sets the declaration node indexes owned by the block. + public ChildList VariableDeclarations; + + internal BlockInfo(int blockIndex, int bodyExpressionsIndex, in ChildList variableDeclarations) + { + BlockIndex = blockIndex; + BodyExpressionsIndex = bodyExpressionsIndex; + VariableDeclarations = variableDeclarations; + } +} + /// Stores an expression tree as a flat node array plus out-of-line closure constants. public struct ExprTree { private static readonly object ClosureConstantMarker = new(); - private const byte ParameterByRefFlag = 1; + internal const byte ParameterByRefFlag = 1; + internal const byte ParameterDeclarationFlag = 2; + private const byte ConstantInlineValue32Flag = 1; private const byte BinaryLiftedToNullFlag = 1; private const byte LoopHasBreakFlag = 1; private const byte LoopHasContinueFlag = 2; private const byte CatchHasVariableFlag = 1; private const byte CatchHasFilterFlag = 2; private const byte TryFaultFlag = 1; + internal const byte NextPointsToParentFlag = 8; + private const ushort UnboundParameterScopeIndex = ushort.MaxValue; /// Gets or sets the root node index. public int RootIndex; @@ -146,11 +211,38 @@ public struct ExprTree /// Gets or sets closure constants that are referenced from constant nodes. public SmallList, NoArrayPool> ClosureConstants; + /// Gets or sets owner-side metadata for lambda nodes, populated during FE construction. + public SmallList, NoArrayPool> Lambdas; + + /// Gets or sets owner-side metadata for block nodes, populated during FE construction. + public SmallList, NoArrayPool> Blocks; + + private SmallMap16 _lambdaInfoIndexesByNodeIndex; + private SmallMap16 _blockInfoIndexesByNodeIndex; + + /// Returns the owner-side metadata for the specified lambda node, if present. + public bool TryGetLambdaInfo(int lambdaIndex, out LambdaInfo info) + { + ref var infoIndex = ref _lambdaInfoIndexesByNodeIndex.Map.TryGetValueRef(lambdaIndex, out var found); + info = found ? Lambdas.GetSurePresentRef(infoIndex) : default; + return found; + } + + /// Returns the owner-side metadata for the specified block node, if present. + public bool TryGetBlockInfo(int blockIndex, out BlockInfo info) + { + ref var infoIndex = ref _blockInfoIndexesByNodeIndex.Map.TryGetValueRef(blockIndex, out var found); + info = found ? Blocks.GetSurePresentRef(infoIndex) : default; + return found; + } + /// Adds a parameter node and returns its index. public int Parameter(Type type, string name = null) { - var id = Nodes.Count + 1; - return AddRawLeafExpressionNode(type, name, ExpressionType.Parameter, type.IsByRef ? ParameterByRefFlag : (byte)0, childIdx: id); + var parameterType = type.IsByRef ? type.GetElementType() ?? type : type; + return AddRawLeafExpressionNode(parameterType, name, ExpressionType.Parameter, + (byte)((type.IsByRef ? ParameterByRefFlag : 0) | ParameterDeclarationFlag), + childIdx: UnboundParameterScopeIndex); } /// Adds a typed parameter node and returns its index. @@ -169,18 +261,23 @@ public int Constant(object value) => /// Adds a constant node with an explicit constant type. public int Constant(object value, Type type) { - if (ShouldInlineConstant(value, type)) - return AddRawExpressionNode(type, value, ExpressionType.Constant); + if (TryGetInlineConstantValue32(value, type, out var value32)) + return AddRawInlineConstantNode(type, value32); + + if (ShouldStoreConstantInClosureConstants(value, type)) + { + var constantIndex = ClosureConstants.Add(value); + return AddRawExpressionNodeWithChildIndex(type, ClosureConstantMarker, ExpressionType.Constant, constantIndex); + } - var constantIndex = ClosureConstants.Add(value); - return AddRawExpressionNodeWithChildIndex(type, ClosureConstantMarker, ExpressionType.Constant, constantIndex); + return AddRawExpressionNode(type, value, ExpressionType.Constant); } /// Adds a null constant node. public int ConstantNull(Type type = null) => AddRawExpressionNode(type ?? typeof(object), null, ExpressionType.Constant); /// Adds an constant node. - public int ConstantInt(int value) => AddRawExpressionNode(typeof(int), value, ExpressionType.Constant); + public int ConstantInt(int value) => AddRawInlineConstantNode(typeof(int), value); /// Adds a typed constant node. public int ConstantOf(T value) => Constant(value, typeof(T)); @@ -306,23 +403,34 @@ public int Block(params int[] expressions) => /// Adds a block node with optional explicit result type and variables. public int Block(Type type, IEnumerable variables, params int[] expressions) { + // Block shape: + // - without variables: children = [body-expressions-child-list] + // - with variables: children = [declarations-child-list, body-expressions-child-list] + // The declarations child list contains parameter declaration nodes. + // Each declaration node is later bound to this block via ChildIdx = blockIndex and ChildCount = declaration position. + // Parameter references inside the body are separate usage nodes with ChildIdx = declaration index. if (expressions == null || expressions.Length == 0) throw new ArgumentException("Block should contain at least one expression.", nameof(expressions)); ChildList children = default; + ChildList variableDeclarations = default; if (variables != null) { - ChildList variableChildren = default; foreach (var variable in variables) - variableChildren.Add(variable); - if (variableChildren.Count != 0) - children.Add(AddChildListNode(in variableChildren)); + variableDeclarations.Add(RequireParameterDeclarationIndex(variable)); + if (variableDeclarations.Count != 0) + children.Add(AddChildListNode(in variableDeclarations)); } ChildList bodyChildren = default; for (var i = 0; i < expressions.Length; ++i) - bodyChildren.Add(expressions[i]); - children.Add(AddChildListNode(in bodyChildren)); - return AddFactoryExpressionNode(type ?? Nodes[expressions[expressions.Length - 1]].Type, null, ExpressionType.Block, in children); + bodyChildren.Add(CloneChild(expressions[i])); + var bodyExpressionsIndex = AddChildListNode(in bodyChildren); + children.Add(bodyExpressionsIndex); + var blockIndex = AddNode(type ?? Nodes.GetSurePresentRef(expressions[expressions.Length - 1]).Type, null, ExpressionType.Block, ExprNodeKind.Expression, 0, in children); + if (variableDeclarations.Count != 0) + BindParameterDeclarations(blockIndex, variableDeclarations); + AddBlockInfo(blockIndex, bodyExpressionsIndex, variableDeclarations); + return blockIndex; } /// Adds a typed lambda node. @@ -330,10 +438,35 @@ public int Lambda(int body, params int[] parameters) where TDelegate Lambda(typeof(TDelegate), body, parameters); /// Adds a lambda node. - public int Lambda(Type delegateType, int body, params int[] parameters) => - parameters == null || parameters.Length == 0 - ? AddFactoryExpressionNode(delegateType, null, ExpressionType.Lambda, 0, body) - : AddFactoryExpressionNode(delegateType, null, ExpressionType.Lambda, PrependToChildList(body, parameters)); + public int Lambda(Type delegateType, int body, params int[] parameters) + { + // Lambda shape: children = [body, declaration0, declaration1, ...] + // The lambda stores declaration nodes directly after the body. + // Each declaration node is later bound to this lambda via ChildIdx = lambdaIndex and ChildCount = parameter position. + // Parameter references inside the body are separate usage nodes with ChildIdx = declaration index. + if (parameters == null || parameters.Length == 0) + { + var lambdaIndexWithoutParameters = AddFactoryExpressionNode(delegateType, null, ExpressionType.Lambda, 0, body); + ChildList emptyDeclarations = default; + AddLambdaInfo(lambdaIndexWithoutParameters, body, emptyDeclarations); + return lambdaIndexWithoutParameters; + } + + ChildList children = default; + children.Add(CloneChild(body)); + ChildList declarations = default; + for (var i = 0; i < parameters.Length; ++i) + { + var declarationIndex = RequireParameterDeclarationIndex(parameters[i]); + declarations.Add(declarationIndex); + children.Add(declarationIndex); + } + + var lambdaIndex = AddNode(delegateType, null, ExpressionType.Lambda, ExprNodeKind.Expression, 0, in children); + BindParameterDeclarations(lambdaIndex, declarations); + AddLambdaInfo(lambdaIndex, body, declarations); + return lambdaIndex; + } /// Adds a member-assignment binding node. public int Bind(System.Reflection.MemberInfo member, int expression) => @@ -614,6 +747,10 @@ private int AddRawExpressionNode(Type type, object obj, ExpressionType nodeType, private int AddRawLeafExpressionNode(Type type, object obj, ExpressionType nodeType, byte flags = 0, int childIdx = 0, int childCount = 0) => AddLeafNode(type, obj, nodeType, ExprNodeKind.Expression, flags, childIdx, childCount); + private int AddRawInlineConstantNode(Type type, int value32) => + AddRawLeafExpressionNode(type, null, ExpressionType.Constant, ConstantInlineValue32Flag, + childIdx: (ushort)value32, childCount: (ushort)(value32 >> 16)); + private int AddRawExpressionNodeWithChildIndex(Type type, object obj, ExpressionType nodeType, int childIdx) => AddRawLeafExpressionNode(type, obj, nodeType, childIdx: childIdx); @@ -677,7 +814,7 @@ private static ChildList PrependToChildList(int first, int[] rest) /// Builds the flat representation while preserving parameter and label identity with stack-friendly maps. private struct Builder { - private SmallMap16> _parameterIds; + private SmallMap16> _parameterDeclarations; private SmallMap16> _labelIds; private ExprTree _tree; @@ -698,34 +835,46 @@ private int AddExpression(SysExpr expression) case ExpressionType.Parameter: { var parameter = (SysParameterExpression)expression; - return _tree.AddRawLeafExpressionNode(expression.Type, parameter.Name, expression.NodeType, - parameter.IsByRef ? ParameterByRefFlag : (byte)0, childIdx: GetId(ref _parameterIds, parameter)); + return AddParameterUsage(parameter); } case ExpressionType.Lambda: { var lambda = (System.Linq.Expressions.LambdaExpression)expression; - ChildList children = default; - children.Add(AddExpression(lambda.Body)); + ChildList declarations = default; for (var i = 0; i < lambda.Parameters.Count; ++i) - children.Add(AddExpression(lambda.Parameters[i])); - return _tree.AddRawExpressionNode(expression.Type, null, expression.NodeType, children); + declarations.Add(AddParameterDeclaration(lambda.Parameters[i])); + var body = AddExpression(lambda.Body); + ChildList children = default; + children.Add(body); + for (var i = 0; i < declarations.Count; ++i) + children.Add(declarations[i]); + var lambdaIndex = _tree.AddRawExpressionNode(expression.Type, null, expression.NodeType, children); + if (declarations.Count != 0) + _tree.BindParameterDeclarations(lambdaIndex, declarations); + _tree.AddLambdaInfo(lambdaIndex, body, declarations); + return lambdaIndex; } case ExpressionType.Block: { var block = (System.Linq.Expressions.BlockExpression)expression; ChildList children = default; + ChildList variables = default; if (block.Variables.Count != 0) { - ChildList variables = default; for (var i = 0; i < block.Variables.Count; ++i) - variables.Add(AddExpression(block.Variables[i])); + variables.Add(AddParameterDeclaration(block.Variables[i])); children.Add(_tree.AddChildListNode(in variables)); } ChildList expressions = default; for (var i = 0; i < block.Expressions.Count; ++i) expressions.Add(AddExpression(block.Expressions[i])); - children.Add(_tree.AddChildListNode(in expressions)); - return _tree.AddRawExpressionNode(expression.Type, null, expression.NodeType, in children); + var bodyExpressionsIndex = _tree.AddChildListNode(in expressions); + children.Add(bodyExpressionsIndex); + var blockIndex = _tree.AddRawExpressionNode(expression.Type, null, expression.NodeType, in children); + if (variables.Count != 0) + _tree.BindParameterDeclarations(blockIndex, variables); + _tree.AddBlockInfo(blockIndex, bodyExpressionsIndex, variables); + return blockIndex; } case ExpressionType.MemberAccess: { @@ -936,11 +1085,16 @@ private int AddExpression(SysExpr expression) private int AddConstant(System.Linq.Expressions.ConstantExpression constant) { - if (ShouldInlineConstant(constant.Value, constant.Type)) - return _tree.AddRawExpressionNode(constant.Type, constant.Value, constant.NodeType); + if (TryGetInlineConstantValue32(constant.Value, constant.Type, out var value32)) + return _tree.AddRawInlineConstantNode(constant.Type, value32); - var constantIndex = _tree.ClosureConstants.Add(constant.Value); - return _tree.AddRawExpressionNodeWithChildIndex(constant.Type, ClosureConstantMarker, constant.NodeType, constantIndex); + if (ShouldStoreConstantInClosureConstants(constant.Value, constant.Type)) + { + var constantIndex = _tree.ClosureConstants.Add(constant.Value); + return _tree.AddRawExpressionNodeWithChildIndex(constant.Type, ClosureConstantMarker, constant.NodeType, constantIndex); + } + + return _tree.AddRawExpressionNode(constant.Type, constant.Value, constant.NodeType); } private int AddSwitchCase(SysSwitchCase switchCase) @@ -1005,6 +1159,33 @@ private int AddElementInit(SysElementInit init) return _tree.AddRawAuxNode(init.AddMethod.DeclaringType, init.AddMethod, ExprNodeKind.ElementInit, children); } + private int AddParameterDeclaration(SysParameterExpression parameter) + { + ref var declarationIndex = ref _parameterDeclarations.Map.AddOrGetValueRef(parameter, out var found); + if (found) + return Throw.DuplicateParameterDeclaration(parameter.Name); + + var parameterType = parameter.IsByRef ? parameter.Type.GetElementType() ?? parameter.Type : parameter.Type; + declarationIndex = _tree.AddRawLeafExpressionNode(parameterType, parameter.Name, ExpressionType.Parameter, + (byte)((parameter.IsByRef ? ParameterByRefFlag : 0) | ParameterDeclarationFlag), + childIdx: UnboundParameterScopeIndex); + return declarationIndex; + } + + private int AddParameterUsage(SysParameterExpression parameter) + { + ref var declarationIndex = ref _parameterDeclarations.Map.AddOrGetValueRef(parameter, out var found); + if (!found) + { + var parameterType = parameter.IsByRef ? parameter.Type.GetElementType() ?? parameter.Type : parameter.Type; + declarationIndex = _tree.AddRawLeafExpressionNode(parameterType, parameter.Name, ExpressionType.Parameter, + (byte)((parameter.IsByRef ? ParameterByRefFlag : 0) | ParameterDeclarationFlag), + childIdx: UnboundParameterScopeIndex); + } + + return _tree.AddParameterUsageNode(in _tree.Nodes.GetSurePresentRef(declarationIndex), declarationIndex); + } + private static int GetId(ref SmallMap16> ids, object item) { ref var id = ref ids.Map.AddOrGetValueRef(item, out var found); @@ -1042,6 +1223,7 @@ private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind var nodeIndex = Nodes.Count; ref var newNode = ref Nodes.AddDefaultAndGetRef(); newNode = new ExprNode(type, obj, nodeType, kind, flags, child0, 1); + Nodes.GetSurePresentRef(child0).SetNextIdx(nodeIndex, pointsToParent: true); return nodeIndex; } @@ -1050,7 +1232,8 @@ private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind var nodeIndex = Nodes.Count; ref var newNode = ref Nodes.AddDefaultAndGetRef(); newNode = new ExprNode(type, obj, nodeType, kind, flags, c0, 2); - Nodes[c0].SetNextIdx(c1); + Nodes.GetSurePresentRef(c0).SetNextIdx(c1); + Nodes.GetSurePresentRef(c1).SetNextIdx(nodeIndex, pointsToParent: true); return nodeIndex; } @@ -1059,8 +1242,9 @@ private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind var nodeIndex = Nodes.Count; ref var newNode = ref Nodes.AddDefaultAndGetRef(); newNode = new ExprNode(type, obj, nodeType, kind, flags, c0, 3); - Nodes[c0].SetNextIdx(c1); - Nodes[c1].SetNextIdx(c2); + Nodes.GetSurePresentRef(c0).SetNextIdx(c1); + Nodes.GetSurePresentRef(c1).SetNextIdx(c2); + Nodes.GetSurePresentRef(c2).SetNextIdx(nodeIndex, pointsToParent: true); return nodeIndex; } @@ -1069,9 +1253,10 @@ private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind var nodeIndex = Nodes.Count; ref var newNode = ref Nodes.AddDefaultAndGetRef(); newNode = new ExprNode(type, obj, nodeType, kind, flags, c0, 4); - Nodes[c0].SetNextIdx(c1); - Nodes[c1].SetNextIdx(c2); - Nodes[c2].SetNextIdx(c3); + Nodes.GetSurePresentRef(c0).SetNextIdx(c1); + Nodes.GetSurePresentRef(c1).SetNextIdx(c2); + Nodes.GetSurePresentRef(c2).SetNextIdx(c3); + Nodes.GetSurePresentRef(c3).SetNextIdx(nodeIndex, pointsToParent: true); return nodeIndex; } @@ -1080,10 +1265,11 @@ private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind var nodeIndex = Nodes.Count; ref var newNode = ref Nodes.AddDefaultAndGetRef(); newNode = new ExprNode(type, obj, nodeType, kind, flags, c0, 5); - Nodes[c0].SetNextIdx(c1); - Nodes[c1].SetNextIdx(c2); - Nodes[c2].SetNextIdx(c3); - Nodes[c3].SetNextIdx(c4); + Nodes.GetSurePresentRef(c0).SetNextIdx(c1); + Nodes.GetSurePresentRef(c1).SetNextIdx(c2); + Nodes.GetSurePresentRef(c2).SetNextIdx(c3); + Nodes.GetSurePresentRef(c3).SetNextIdx(c4); + Nodes.GetSurePresentRef(c4).SetNextIdx(nodeIndex, pointsToParent: true); return nodeIndex; } @@ -1092,11 +1278,12 @@ private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind var nodeIndex = Nodes.Count; ref var newNode = ref Nodes.AddDefaultAndGetRef(); newNode = new ExprNode(type, obj, nodeType, kind, flags, c0, 6); - Nodes[c0].SetNextIdx(c1); - Nodes[c1].SetNextIdx(c2); - Nodes[c2].SetNextIdx(c3); - Nodes[c3].SetNextIdx(c4); - Nodes[c4].SetNextIdx(c5); + Nodes.GetSurePresentRef(c0).SetNextIdx(c1); + Nodes.GetSurePresentRef(c1).SetNextIdx(c2); + Nodes.GetSurePresentRef(c2).SetNextIdx(c3); + Nodes.GetSurePresentRef(c3).SetNextIdx(c4); + Nodes.GetSurePresentRef(c4).SetNextIdx(c5); + Nodes.GetSurePresentRef(c5).SetNextIdx(nodeIndex, pointsToParent: true); return nodeIndex; } @@ -1105,12 +1292,13 @@ private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind var nodeIndex = Nodes.Count; ref var newNode = ref Nodes.AddDefaultAndGetRef(); newNode = new ExprNode(type, obj, nodeType, kind, flags, c0, 7); - Nodes[c0].SetNextIdx(c1); - Nodes[c1].SetNextIdx(c2); - Nodes[c2].SetNextIdx(c3); - Nodes[c3].SetNextIdx(c4); - Nodes[c4].SetNextIdx(c5); - Nodes[c5].SetNextIdx(c6); + Nodes.GetSurePresentRef(c0).SetNextIdx(c1); + Nodes.GetSurePresentRef(c1).SetNextIdx(c2); + Nodes.GetSurePresentRef(c2).SetNextIdx(c3); + Nodes.GetSurePresentRef(c3).SetNextIdx(c4); + Nodes.GetSurePresentRef(c4).SetNextIdx(c5); + Nodes.GetSurePresentRef(c5).SetNextIdx(c6); + Nodes.GetSurePresentRef(c6).SetNextIdx(nodeIndex, pointsToParent: true); return nodeIndex; } @@ -1123,7 +1311,8 @@ private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind ref var newNode = ref Nodes.AddDefaultAndGetRef(); newNode = new ExprNode(type, obj, nodeType, kind, flags, children[0], children.Length); for (var i = 1; i < children.Length; ++i) - Nodes[children[i - 1]].SetNextIdx(children[i]); + Nodes.GetSurePresentRef(children[i - 1]).SetNextIdx(children[i]); + Nodes.GetSurePresentRef(children[children.Length - 1]).SetNextIdx(nodeIndex, pointsToParent: true); return nodeIndex; } @@ -1136,13 +1325,174 @@ private int AddNode(Type type, object obj, ExpressionType nodeType, ExprNodeKind ref var newNode = ref Nodes.AddDefaultAndGetRef(); newNode = new ExprNode(type, obj, nodeType, kind, flags, children[0], children.Count); for (var i = 1; i < children.Count; ++i) - Nodes[children[i - 1]].SetNextIdx(children[i]); + Nodes.GetSurePresentRef(children[i - 1]).SetNextIdx(children[i]); + Nodes.GetSurePresentRef(children[children.Count - 1]).SetNextIdx(nodeIndex, pointsToParent: true); return nodeIndex; } + private int RequireParameterDeclarationIndex(int index) + { + ref var node = ref Nodes.GetSurePresentRef(index); + if (node.IsParameterDeclaration()) + return index; + return Throw.ParameterDeclarationExpected(index); + } + + private void BindParameterDeclarations(int ownerIndex, in ChildList declarations) + { + for (var i = 0; i < declarations.Count; ++i) + BindParameterDeclaration(ownerIndex, declarations[i], i); + } + + private void BindParameterDeclaration(int ownerIndex, int declarationIndex, int position) + { + // Parameter node contract: + // - declaration node: ParameterDeclarationFlag set, ChildIdx = owning lambda/block index, ChildCount = position in that owner + // - usage node: ParameterDeclarationFlag clear, ChildIdx = declaration node index, ChildCount = 0 + ref var declaration = ref Nodes.GetSurePresentRef(declarationIndex); + if (!declaration.IsParameterDeclaration()) + Throw.ParameterDeclarationExpected(declarationIndex); + if (declaration.ChildIdx != UnboundParameterScopeIndex) + Throw.DuplicateParameterDeclaration(declaration.Obj as string); + + declaration.SetChildInfo(ownerIndex, position); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static bool ShouldInlineConstant(object value, Type type) => - value == null || value is string || value is Type || type.IsEnum || Type.GetTypeCode(type) != TypeCode.Object; + private void AddLambdaInfo(int lambdaIndex, int bodyIndex, in ChildList declarations) + { + ref var infoIndex = ref _lambdaInfoIndexesByNodeIndex.Map.AddOrGetValueRef(lambdaIndex, out var found); + var info = new LambdaInfo(lambdaIndex, bodyIndex, declarations); + if (found) + Lambdas.GetSurePresentRef(infoIndex) = info; + else + { + infoIndex = Lambdas.Count; + Lambdas.Add(info); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void AddBlockInfo(int blockIndex, int bodyExpressionsIndex, in ChildList declarations) + { + ref var infoIndex = ref _blockInfoIndexesByNodeIndex.Map.AddOrGetValueRef(blockIndex, out var found); + var info = new BlockInfo(blockIndex, bodyExpressionsIndex, declarations); + if (found) + Blocks.GetSurePresentRef(infoIndex) = info; + else + { + infoIndex = Blocks.Count; + Blocks.Add(info); + } + } + + private int AddParameterUsageNode(in ExprNode node, int declarationIndex) => + AddRawLeafExpressionNode(node.Type, node.Obj, ExpressionType.Parameter, + (byte)(node.Flags & ~(ParameterDeclarationFlag | NextPointsToParentFlag)), childIdx: declarationIndex); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryGetInlineConstantValue32(object value, Type type, out int value32) + { + if (value == null) + { + value32 = default; + return false; + } + + var valueType = type.IsEnum ? Enum.GetUnderlyingType(type) : type; + switch (Type.GetTypeCode(valueType)) + { + case TypeCode.Boolean: + value32 = (bool)value ? 1 : 0; + return true; + case TypeCode.Char: + value32 = (char)value; + return true; + case TypeCode.SByte: + value32 = (sbyte)value; + return true; + case TypeCode.Byte: + value32 = (byte)value; + return true; + case TypeCode.Int16: + value32 = (short)value; + return true; + case TypeCode.UInt16: + value32 = (ushort)value; + return true; + case TypeCode.Int32: + value32 = (int)value; + return true; + case TypeCode.UInt32: + value32 = unchecked((int)(uint)value); + return true; + case TypeCode.Single: + value32 = ConvertSingleToInt32Bits((float)value); + return true; + default: + value32 = default; + return false; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool ShouldStoreConstantInClosureConstants(object value, Type type) => + value != null && value is not string && value is not Type && !type.IsEnum && Type.GetTypeCode(type) == TypeCode.Object; + + private static object ReadInlineConstantValue32(Type type, int value32) + { + if (type.IsEnum) + { + var underlyingType = Enum.GetUnderlyingType(type); + return Enum.ToObject(type, ReadInlineConstantValue32(underlyingType, value32)); + } + + return Type.GetTypeCode(type) switch + { + TypeCode.Boolean => value32 != 0, + TypeCode.Char => (char)value32, + TypeCode.SByte => (sbyte)value32, + TypeCode.Byte => (byte)value32, + TypeCode.Int16 => (short)value32, + TypeCode.UInt16 => (ushort)value32, + TypeCode.Int32 => value32, + TypeCode.UInt32 => unchecked((uint)value32), + TypeCode.Single => ConvertInt32BitsToSingle(value32), + _ => Throw.UnsupportedInlineConstantType(type) + }; + } + + [StructLayout(LayoutKind.Explicit)] + private struct SingleInt32Union + { + [FieldOffset(0)] + public float Single; + + [FieldOffset(0)] + public int Int32; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ConvertSingleToInt32Bits(float value) => + new SingleInt32Union { Single = value }.Int32; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static float ConvertInt32BitsToSingle(int value) => + new SingleInt32Union { Int32 = value }.Single; + + private ChildList GetChildren(int index) + { + ref var node = ref Nodes.GetSurePresentRef(index); + var count = node.ChildCount; + ChildList children = default; + var childIndex = node.ChildIdx; + for (var i = 0; i < count; ++i) + { + children.Add(childIndex); + childIndex = Nodes.GetSurePresentRef(childIndex).NextIdx; + } + return children; + } private static Type GetMemberType(System.Reflection.MemberInfo member) => member switch { @@ -1183,9 +1533,12 @@ private static Type GetArrayElementType(Type arrayType, int depth) private int CloneChild(int index) { - ref var node = ref Nodes[index]; + ref var node = ref Nodes.GetSurePresentRef(index); + if (node.NodeType == ExpressionType.Parameter) + return AddParameterUsageNode(in node, node.IsParameterDeclaration() ? index : node.ChildIdx); + return node.ShouldCloneWhenLinked() - ? AddLeafNode(node.Type, node.Obj, node.NodeType, node.Kind, node.Flags, node.ChildIdx, node.ChildCount) + ? AddLeafNode(node.Type, node.Obj, node.NodeType, node.Kind, (byte)(node.Flags & ~NextPointsToParentFlag), node.ChildIdx, node.ChildCount) : index; } @@ -1212,13 +1565,13 @@ private ChildList CloneChildren(in ChildList children) private struct Reader { private readonly ExprTree _tree; - private SmallMap16 _parametersById; + private SmallMap16 _parametersByDeclarationIndex; private SmallMap16 _labelsById; public Reader(ExprTree tree) { _tree = tree; - _parametersById = default; + _parametersByDeclarationIndex = default; _labelsById = default; } @@ -1232,27 +1585,24 @@ public SysExpr ReadExpression(int index) switch (node.NodeType) { case ExpressionType.Constant: - return SysExpr.Constant(ReferenceEquals(node.Obj, ClosureConstantMarker) - ? _tree.ClosureConstants[node.ChildIdx] - : node.Obj, node.Type); + return SysExpr.Constant( + node.HasFlag(ConstantInlineValue32Flag) + ? ReadInlineConstantValue32(node.Type, node.Value32) + : ReferenceEquals(node.Obj, ClosureConstantMarker) + ? _tree.ClosureConstants[node.ChildIdx] + : node.Obj, + node.Type); case ExpressionType.Default: return SysExpr.Default(node.Type); case ExpressionType.Parameter: - { - ref var parameter = ref _parametersById.Map.AddOrGetValueRef(node.ChildIdx, out var found); - if (found) - return parameter; - - var parameterType = node.HasFlag(ParameterByRefFlag) && !node.Type.IsByRef ? node.Type.MakeByRefType() : node.Type; - return parameter = SysExpr.Parameter(parameterType, (string)node.Obj); - } + return ReadParameter(index, in node); case ExpressionType.Lambda: { var children = GetChildren(index); - var body = ReadExpression(children[0]); var parameters = new SysParameterExpression[children.Count - 1]; for (var i = 1; i < children.Count; ++i) parameters[i - 1] = (SysParameterExpression)ReadExpression(children[i]); + var body = ReadExpression(children[0]); return SysExpr.Lambda(node.Type, body, parameters); } case ExpressionType.Block: @@ -1451,6 +1801,25 @@ public SysExpr ReadExpression(int index) } } + private SysParameterExpression ReadParameter(int index, in ExprNode node) + { + var declarationIndex = node.IsParameterDeclaration() ? index : node.ChildIdx; + if (!node.IsParameterDeclaration()) + { + ref var existing = ref _parametersByDeclarationIndex.Map.TryGetValueRef(declarationIndex, out var foundExisting); + return foundExisting && existing != null + ? existing + : ReadParameter(declarationIndex, in _tree.Nodes[declarationIndex]); + } + + ref var parameter = ref _parametersByDeclarationIndex.Map.AddOrGetValueRef(declarationIndex, out var found); + if (found && parameter != null) + return parameter; + + var parameterType = node.HasFlag(ParameterByRefFlag) && !node.Type.IsByRef ? node.Type.MakeByRefType() : node.Type; + return parameter = SysExpr.Parameter(parameterType, (string)node.Obj); + } + [RequiresUnreferencedCode(FastExpressionCompiler.LightExpression.Trimming.Message)] private SysSwitchCase ReadSwitchCase(int index) { @@ -1571,6 +1940,45 @@ private SysExpr[] ReadExpressions(in ChildList childIndexes) } +internal static class Throw +{ + public static void DuplicateParameterDeclaration(string name) => + throw new ArgumentException($"The parameter or variable `{name ?? "?"}` is declared more than once."); + + public static void ParameterDeclarationExpected(int index) => + throw new ArgumentException($"Expected a parameter declaration node at index {index}."); + + public static void UndeclaredParameterUsage(string name) => + throw new InvalidOperationException($"The parameter or variable `{name ?? "?"}` is used before it is declared."); + + public static void UnsupportedInlineConstantType(Type type) => + throw new NotSupportedException($"Inline 32-bit constant storage does not support `{type}`."); + + public static T DuplicateParameterDeclaration(string name) + { + DuplicateParameterDeclaration(name); + return default; + } + + public static T ParameterDeclarationExpected(int index) + { + ParameterDeclarationExpected(index); + return default; + } + + public static T UndeclaredParameterUsage(string name) + { + UndeclaredParameterUsage(name); + return default; + } + + public static T UnsupportedInlineConstantType(Type type) + { + UnsupportedInlineConstantType(type); + return default; + } +} + /// Provides conversions from System and LightExpression trees to . public static class FlatExpressionExtensions { diff --git a/test/FastExpressionCompiler.LightExpression.UnitTests/LightExpressionTests.cs b/test/FastExpressionCompiler.LightExpression.UnitTests/LightExpressionTests.cs index b91c23c0..270e2d5e 100644 --- a/test/FastExpressionCompiler.LightExpression.UnitTests/LightExpressionTests.cs +++ b/test/FastExpressionCompiler.LightExpression.UnitTests/LightExpressionTests.cs @@ -30,11 +30,13 @@ public int Run() Multiple_methods_in_block_should_be_aligned_when_output_to_csharp(); Can_roundtrip_light_expression_through_flat_expression(); Flat_expression_preserves_parameter_and_label_identity_and_collects_closure_constants(); + Flat_expression_uses_parameter_declarations_parent_links_and_inline_constants(); + Flat_expression_collects_lambda_and_block_owner_metadata_when_flattening_system_expression(); Can_convert_dynamic_runtime_variables_and_debug_info_to_light_expression_and_flat_expression(); Can_build_flat_expression_directly_with_light_expression_like_api(); Can_build_flat_expression_control_flow_directly(); Can_property_test_generated_flat_expression_roundtrip_structurally(); - return 17; + return 19; } @@ -420,6 +422,87 @@ public void Flat_expression_preserves_parameter_and_label_identity_and_collects_ Asserts.AreSame(@goto.Target, label.Target); } + public void Flat_expression_uses_parameter_declarations_parent_links_and_inline_constants() + { + var fe = default(ExprTree); + var parameter = fe.Parameter(typeof(int), "p"); + var variable = fe.Variable(typeof(int), "v"); + var assign = fe.Assign(variable, parameter); + var smallConstant = fe.ConstantOf(12.5f); + var largeConstant = fe.ConstantOf(42L); + var block = fe.Block(typeof(long), new[] { variable }, assign, smallConstant, largeConstant); + fe.RootIndex = fe.Lambda>(block, parameter); + + Asserts.AreEqual(1, fe.Lambdas.Count); + Asserts.IsTrue(fe.TryGetLambdaInfo(fe.RootIndex, out var lambdaInfo)); + Asserts.AreEqual(fe.RootIndex, lambdaInfo.LambdaIndex); + Asserts.AreEqual(block, lambdaInfo.BodyIndex); + Asserts.AreEqual(1, lambdaInfo.ParameterDeclarations.Count); + Asserts.AreEqual(parameter, lambdaInfo.ParameterDeclarations[0]); + + Asserts.AreEqual(1, fe.Blocks.Count); + Asserts.IsTrue(fe.TryGetBlockInfo(block, out var blockInfo)); + Asserts.AreEqual(block, blockInfo.BlockIndex); + Asserts.AreEqual(1, blockInfo.VariableDeclarations.Count); + Asserts.AreEqual(variable, blockInfo.VariableDeclarations[0]); + Asserts.AreEqual(3, fe.Nodes[blockInfo.BodyExpressionsIndex].ChildCount); + + var parameterDeclaration = fe.Nodes[parameter]; + Asserts.AreEqual(fe.RootIndex, parameterDeclaration.ChildIdx); + Asserts.AreEqual(0, parameterDeclaration.ChildCount); + + var variableDeclaration = fe.Nodes[variable]; + Asserts.AreEqual(block, variableDeclaration.ChildIdx); + Asserts.AreEqual(0, variableDeclaration.ChildCount); + + var assignNode = fe.Nodes[assign]; + var leftUsageIndex = assignNode.ChildIdx; + var rightUsageIndex = fe.Nodes[leftUsageIndex].NextIdx; + Asserts.AreEqual(variable, fe.Nodes[leftUsageIndex].ChildIdx); + Asserts.AreEqual(parameter, fe.Nodes[rightUsageIndex].ChildIdx); + Asserts.AreEqual(assign, fe.Nodes[rightUsageIndex].NextIdx); + + Asserts.AreEqual(0, fe.ClosureConstants.Count); + Asserts.AreEqual(null, fe.Nodes[smallConstant].Obj); + Asserts.AreEqual(42L, fe.Nodes[largeConstant].Obj); + + var roundtrip = (System.Linq.Expressions.Expression>)fe.ToExpression(); + var compiled = roundtrip.Compile(); + Asserts.AreEqual(42L, compiled(7)); + } + + public void Flat_expression_collects_lambda_and_block_owner_metadata_when_flattening_system_expression() + { + var outer = SysExpr.Parameter(typeof(int), "outer"); + var local = SysExpr.Parameter(typeof(int), "local"); + var nested = SysExpr.Lambda>( + SysExpr.Block( + typeof(int), + new[] { local }, + SysExpr.Assign(local, outer), + local)); + var lambda = SysExpr.Lambda>>(nested, outer); + + var fe = lambda.ToFlatExpression(); + + Asserts.AreEqual(2, fe.Lambdas.Count); + Asserts.AreEqual(1, fe.Blocks.Count); + + Asserts.IsTrue(fe.TryGetLambdaInfo(fe.RootIndex, out var outerLambda)); + Asserts.AreEqual(fe.RootIndex, outerLambda.LambdaIndex); + Asserts.AreEqual(1, outerLambda.ParameterDeclarations.Count); + + Asserts.IsTrue(fe.TryGetLambdaInfo(outerLambda.BodyIndex, out var nestedLambda)); + Asserts.AreEqual(ExpressionType.Lambda, fe.Nodes[nestedLambda.LambdaIndex].NodeType); + Asserts.AreEqual(0, nestedLambda.ParameterDeclarations.Count); + Asserts.AreEqual(ExpressionType.Block, fe.Nodes[nestedLambda.BodyIndex].NodeType); + + Asserts.IsTrue(fe.TryGetBlockInfo(nestedLambda.BodyIndex, out var blockInfo)); + Asserts.AreEqual(nestedLambda.BodyIndex, blockInfo.BlockIndex); + Asserts.AreEqual(1, blockInfo.VariableDeclarations.Count); + Asserts.AreEqual(2, fe.Nodes[blockInfo.BodyExpressionsIndex].ChildCount); + } + public void Can_convert_dynamic_runtime_variables_and_debug_info_to_light_expression_and_flat_expression() { var runtimeParameter = SysExpr.Parameter(typeof(int), "runtime");