Decompiler: add SSA dead-code elimination

Author: SamboyCoding

Cpp2IL.Core.Tests/Analysis/DeadCodeEliminationTests.cs

@@ -0,0 +1,74 @@
+using System.Collections.Generic;
+using System.Linq;
+using Cpp2IL.Core.Analysis;
+using Cpp2IL.Core.Graphs;
+using Cpp2IL.Core.ISIL;
+
+namespace Cpp2IL.Core.Tests.Analysis;
+
+public class DeadCodeEliminationTests
+{
+    private static List<Instruction> Live(ISILControlFlowGraph graph)
+        => graph.Blocks.SelectMany(b => b.Instructions).Where(i => i.OpCode != OpCode.Nop).ToList();
+
+    [Test]
+    public void RemovesDeadDefinitionButKeepsLiveOnes()
+    {
+        var x = new LocalVariable("x", new Register(null, "x"));
+        var dead = new LocalVariable("dead", new Register(null, "dead"));
+
+        var graph = new ISILControlFlowGraph(new List<Instruction>
+        {
+            new(0, OpCode.Move, x, 5),
+            new(1, OpCode.Subtract, dead, x, 1), // dead's result is never read
+            new(2, OpCode.Return, x),
+        });
+
+        DeadCodeEliminator.Run(graph);
+
+        var live = Live(graph);
+        Assert.That(live.Any(i => i.OpCode == OpCode.Subtract), Is.False, "dead computation should be removed");
+        Assert.That(live.Any(i => i.OpCode == OpCode.Move), Is.True, "live definition should remain");
+        Assert.That(live.Any(i => i.OpCode == OpCode.Return), Is.True, "terminator should remain");
+    }
+
+    [Test]
+    public void RemovesDeadChainToFixpoint()
+    {
+        // x = 5; temp = x - 1; flag = temp < 0; return x
+        // 'flag' is unused -> dead; that makes 'temp' unused -> dead too (cascade).
+        var x = new LocalVariable("x", new Register(null, "x"));
+        var temp = new LocalVariable("temp", new Register(null, "temp"));
+        var flag = new LocalVariable("flag", new Register(null, "flag"));
+
+        var graph = new ISILControlFlowGraph(new List<Instruction>
+        {
+            new(0, OpCode.Move, x, 5),
+            new(1, OpCode.Subtract, temp, x, 1),
+            new(2, OpCode.CheckLess, flag, temp, 0),
+            new(3, OpCode.Return, x),
+        });
+
+        DeadCodeEliminator.Run(graph);
+
+        var live = Live(graph);
+        Assert.That(live.Any(i => i.OpCode == OpCode.CheckLess), Is.False, "dead flag should be removed");
+        Assert.That(live.Any(i => i.OpCode == OpCode.Subtract), Is.False, "now-dead temp should be removed (cascade)");
+        Assert.That(live.Count, Is.EqualTo(2), "only the live Move and Return should remain");
+    }
+
+    [Test]
+    public void KeepsCallsEvenWhenResultUnused()
+    {
+        // A call with no observed result must be kept (side effects).
+        var graph = new ISILControlFlowGraph(new List<Instruction>
+        {
+            new(0, OpCode.CallVoid, 0xDEADBEEFUL),
+            new(1, OpCode.Return),
+        });
+
+        DeadCodeEliminator.Run(graph);
+
+        Assert.That(Live(graph).Any(i => i.OpCode == OpCode.CallVoid), Is.True, "calls must never be removed");
+    }
+}

Cpp2IL.Core/Analysis/DeadCodeEliminator.cs

@@ -0,0 +1,113 @@
+using System.Collections.Generic;
+using Cpp2IL.Core.Graphs;
+using Cpp2IL.Core.ISIL;
+using Cpp2IL.Core.Model.Contexts;
+
+namespace Cpp2IL.Core.Analysis;
+
+/// <summary>
+/// Removes pure instructions whose result is never used. This eliminates, among other things, the
+/// dead flag/temporary computations the x86 lifter emits eagerly for every comparison - a single
+/// <c>cmp</c>/<c>test</c> produces all of CF/OF/SF/ZF/PF plus scratch temporaries, but the branch
+/// that follows only consumes one of them.
+///
+/// Must run while the graph is still in SSA form (every local is assigned exactly once), so that a
+/// global use count of zero is sufficient to prove a definition dead. Instructions are turned into
+/// nops rather than spliced out; the structural cleanup happens later, out of SSA, where it is safe
+/// for phi nodes.
+/// </summary>
+public static class DeadCodeEliminator
+{
+    public static void Run(MethodAnalysisContext method) => Run(method.ControlFlowGraph!);
+
+    public static void Run(ISILControlFlowGraph cfg)
+    {
+        // Removing a dead definition can make its operands dead in turn, so iterate to a fixpoint.
+        // This is monotonic (each pass only nops instructions) and therefore always terminates.
+        var changed = true;
+        while (changed)
+        {
+            changed = false;
+
+            var useCounts = CountUses(cfg);
+
+            foreach (var block in cfg.Blocks)
+            {
+                foreach (var instruction in block.Instructions)
+                {
+                    if (!IsRemovable(instruction.OpCode))
+                        continue;
+
+                    // Only definitions of a register local are candidates. Stores have a memory or
+                    // field destination (Destination is not a local) and are never dead.
+                    if (instruction.Destination is not LocalVariable destination)
+                        continue;
+
+                    if (useCounts.TryGetValue(destination, out var count) && count > 0)
+                        continue;
+
+                    instruction.OpCode = OpCode.Nop;
+                    instruction.Operands = [];
+                    changed = true;
+                }
+            }
+        }
+    }
+
+    private static Dictionary<LocalVariable, int> CountUses(ISILControlFlowGraph cfg)
+    {
+        var counts = new Dictionary<LocalVariable, int>();
+
+        foreach (var block in cfg.Blocks)
+            foreach (var instruction in block.Instructions)
+                foreach (var used in UsedLocals(instruction))
+                    counts[used] = counts.TryGetValue(used, out var c) ? c + 1 : 1;
+
+        return counts;
+    }
+
+    /// <summary>
+    /// Every local read by the instruction. The single write position - a plain local destination -
+    /// is excluded. Memory and field operands always contribute their address/object locals as
+    /// reads, even when they are the destination of a store.
+    /// </summary>
+    private static IEnumerable<LocalVariable> UsedLocals(Instruction instruction)
+    {
+        var destination = instruction.Destination as LocalVariable;
+
+        foreach (var operand in instruction.Operands)
+        {
+            switch (operand)
+            {
+                case LocalVariable local when !ReferenceEquals(local, destination):
+                    yield return local;
+                    break;
+                case MemoryOperand memory:
+                    if (memory.Base is LocalVariable baseLocal)
+                        yield return baseLocal;
+                    if (memory.Index is LocalVariable indexLocal)
+                        yield return indexLocal;
+                    break;
+                case FieldReference field when field.Local is { } fieldLocal:
+                    yield return fieldLocal;
+                    break;
+            }
+        }
+    }
+
+    /// <summary>
+    /// Opcodes with no side effects, so removing a never-read result is safe. Calls, stores,
+    /// returns and branches are intentionally excluded.
+    /// </summary>
+    private static bool IsRemovable(OpCode opCode) =>
+        opCode switch
+        {
+            OpCode.Move or OpCode.Phi
+                or OpCode.Add or OpCode.Subtract or OpCode.Multiply or OpCode.Divide
+                or OpCode.ShiftLeft or OpCode.ShiftRight
+                or OpCode.And or OpCode.Or or OpCode.Xor
+                or OpCode.Not or OpCode.Negate
+                or OpCode.CheckEqual or OpCode.CheckGreater or OpCode.CheckLess => true,
+            _ => false
+        };
+}

Cpp2IL.Core/Model/Contexts/MethodAnalysisContext.cs

@@ -356,6 +356,11 @@ public void Analyze()
         // Create locals
         SsaForm.Build(this);
         LocalVariables.CreateAll(this);
+
+        // Eliminate dead computations (e.g. the unused flags emitted for every comparison) while
+        // still in SSA form, where a zero use count proves a definition dead.
+        DeadCodeEliminator.Run(this);
+
         SsaForm.Remove(this);
 
         MetadataResolver.ResolveAll(this);