use chained implicits, in order to let go of .start (#1007)

* WIP

* WIP: works for Expression.file

* refactor, add typeDecl

* more steps

* WIP - probably want to revert

* Revert "WIP - probably want to revert"

This reverts commit e7e46e4.

* fix test

* literal

* identifier

* more implicits, refactor

* more

* more

* more

* call is also an expression - make implicits unique

* more

* remove many .start usages and trim imports

* also chain cfgNode implicits: needed to make some implicits lower priority

* remove superfluous steps which used to use .start and now led to recursive loop

* remove more recursive loops

* remove (almost) all .start invocations and (now) superfluous extension methods

* format

Author: mpollmeier

dataflowengineoss/src/main/scala/io/shiftleft/dataflowengineoss/dotgenerator/DdgGenerator.scala

@@ -4,9 +4,9 @@ import io.shiftleft.codepropertygraph.generated.{EdgeKeys, EdgeTypes, nodes}
 import io.shiftleft.dataflowengineoss.semanticsloader.Semantics
 import io.shiftleft.semanticcpg.dotgenerator.DotSerializer.{Edge, Graph}
 import overflowdb.Node
-import overflowdb.traversal._
 import io.shiftleft.semanticcpg.language._
 import io.shiftleft.dataflowengineoss.language._
+import overflowdb.traversal.jIteratortoTraversal
 
 import scala.collection.mutable
 
@@ -18,7 +18,7 @@ class DdgGenerator {
   def generate(methodNode: nodes.Method)(implicit semantics: Semantics): Graph = {
     val entryNode = methodNode
     val paramNodes = methodNode.parameter.l
-    val allOtherNodes = methodNode.start.cfgNode.l
+    val allOtherNodes = methodNode.cfgNode.l
     val exitNode = methodNode.methodReturn
     val allNodes: List[nodes.StoredNode] = List(entryNode, exitNode) ++ paramNodes ++ allOtherNodes
     val visibleNodes = allNodes.filter(shouldBeDisplayed)
@@ -39,16 +39,15 @@ class DdgGenerator {
         e.copy(src = surroundingCall(e.src), dst = surroundingCall(e.dst))
       }
       .filter(e => e.src != e.dst)
-      .dedup
-      .l
+      .distinct
 
     edgeCache.clear
     Graph(ddgNodes, ddgEdges)
   }
 
   private def surroundingCall(node: nodes.StoredNode): nodes.StoredNode = {
     node match {
-      case arg: nodes.Expression => arg.start.inCall.headOption.getOrElse(node)
+      case arg: nodes.Expression => arg.inCall.headOption.getOrElse(node)
       case _                     => node
     }
   }

dataflowengineoss/src/main/scala/io/shiftleft/dataflowengineoss/language/Path.scala

@@ -1,7 +1,6 @@
 package io.shiftleft.dataflowengineoss.language
 
 import io.shiftleft.codepropertygraph.generated.nodes
-import overflowdb.traversal._
 import overflowdb.traversal.help.Table
 import io.shiftleft.semanticcpg.language._
 
@@ -16,7 +15,7 @@ object Path {
         val method = trackingPoint.method
         val methodName = method.name
         val lineNumber = trackingPoint.cfgNode.lineNumber.getOrElse("N/A").toString
-        val fileName = method.start.file.name.headOption.getOrElse("N/A")
+        val fileName = method.file.name.headOption.getOrElse("N/A")
 
         val trackedSymbol = trackingPoint match {
           case _: nodes.MethodParameterIn =>

dataflowengineoss/src/main/scala/io/shiftleft/dataflowengineoss/language/nodemethods/ExpressionMethods.scala

@@ -2,8 +2,8 @@ package io.shiftleft.dataflowengineoss.language.nodemethods
 
 import io.shiftleft.codepropertygraph.generated.nodes
 import io.shiftleft.dataflowengineoss.semanticsloader.{FlowSemantic, Semantics}
-import io.shiftleft.semanticcpg.language.{NoResolve, toExpression}
-import overflowdb.traversal.NodeOps
+import io.shiftleft.semanticcpg.language._
+import overflowdb.traversal.Traversal
 
 class ExpressionMethods[NodeType <: nodes.Expression](val node: NodeType) extends AnyVal {
 
@@ -28,15 +28,15 @@ class ExpressionMethods[NodeType <: nodes.Expression](val node: NodeType) extend
   /**
     * Retrieve flow semantic for the call this argument is a part of.
     * */
-  def semanticsForCallByArg(implicit semantics: Semantics): List[FlowSemantic] = {
+  def semanticsForCallByArg(implicit semantics: Semantics): Traversal[FlowSemantic] = {
     argToMethods(node).flatMap { method =>
       semantics.forMethod(method.fullName)
     }
   }
 
-  private def argToMethods(arg: nodes.Expression): List[nodes.Method] = {
-    arg.start.inCall.l.flatMap { call =>
-      NoResolve.getCalledMethods(call).toList
+  private def argToMethods(arg: nodes.Expression): Traversal[nodes.Method] = {
+    arg.inCall.flatMap { call =>
+      NoResolve.getCalledMethods(call)
     }
   }
 

dataflowengineoss/src/main/scala/io/shiftleft/dataflowengineoss/language/package.scala

@@ -16,8 +16,9 @@ package object language {
   implicit def expressionMethods[NodeType <: nodes.Expression](node: NodeType): ExpressionMethods[NodeType] =
     new ExpressionMethods(node)
 
-  implicit def toTrackingPoint[NodeType <: nodes.TrackingPointBase](traversal: Traversal[NodeType]): TrackingPoint =
-    new TrackingPoint(traversal.cast[nodes.TrackingPoint])
+  implicit def toTrackingPoint[A, NodeType <: nodes.TrackingPointBase](a: A)(
+      implicit f: A => Traversal[NodeType]): TrackingPoint =
+    new TrackingPoint(f(a).cast[nodes.TrackingPoint])
 
   implicit def trackingPointToAstNodeMethods(node: nodes.TrackingPoint) =
     new AstNodeMethods(trackingPointToAstNode(node))
@@ -28,10 +29,11 @@ package object language {
     case _                              => ??? //TODO markus/fabs?
   }
 
-  implicit def trackingPointToAstBase(trav: Traversal[nodes.TrackingPoint]): AstNode[nodes.AstNode] =
-    new AstNode(trav.map(trackingPointToAstNode))
+  implicit def trackingPointToAstBase[A](a: A)(
+      implicit f: A => Traversal[nodes.TrackingPoint]): AstNode[nodes.AstNode] =
+    new AstNode(f(a).map(trackingPointToAstNode))
 
-  implicit def toDdgNodeDot(trav: Traversal[nodes.Method]): DdgNodeDot =
-    new DdgNodeDot(trav)
+  implicit def toDdgNodeDot[A](a: A)(implicit f: A => Traversal[nodes.Method]): DdgNodeDot =
+    new DdgNodeDot(f(a))
 
 }

dataflowengineoss/src/main/scala/io/shiftleft/dataflowengineoss/layers/dataflows/DumpCpg14.scala

@@ -4,7 +4,6 @@ import better.files.File
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.semanticcpg.language._
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions}
-import overflowdb.traversal._
 import io.shiftleft.dataflowengineoss.language._
 import io.shiftleft.dataflowengineoss.semanticsloader.Semantics
 
@@ -27,7 +26,7 @@ class DumpCpg14(options: Cpg14DumpOptions)(implicit semantics: Semantics) extend
     val cpg = context.cpg
     cpg.method.zipWithIndex.foreach {
       case (method, i) =>
-        val str = method.start.dotCpg14.head
+        val str = method.dotCpg14.head
         (File(options.outDir) / s"${i}-cpg.dot").write(str)
     }
   }

dataflowengineoss/src/main/scala/io/shiftleft/dataflowengineoss/layers/dataflows/DumpDdg.scala

@@ -4,7 +4,6 @@ import better.files.File
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.semanticcpg.language._
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions}
-import overflowdb.traversal._
 import io.shiftleft.dataflowengineoss.language._
 import io.shiftleft.dataflowengineoss.semanticsloader.Semantics
 
@@ -27,7 +26,7 @@ class DumpDdg(options: DdgDumpOptions)(implicit semantics: Semantics) extends La
     val cpg = context.cpg
     cpg.method.zipWithIndex.foreach {
       case (method, i) =>
-        val str = method.start.dotDdg.head
+        val str = method.dotDdg.head
         (File(options.outDir) / s"${i}-ddg.dot").write(str)
     }
   }

dataflowengineoss/src/main/scala/io/shiftleft/dataflowengineoss/layers/dataflows/DumpPdg.scala

@@ -4,7 +4,6 @@ import better.files.File
 import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.semanticcpg.language._
 import io.shiftleft.semanticcpg.layers.{LayerCreator, LayerCreatorContext, LayerCreatorOptions}
-import overflowdb.traversal._
 import io.shiftleft.dataflowengineoss.language._
 import io.shiftleft.dataflowengineoss.semanticsloader.Semantics
 
@@ -27,7 +26,7 @@ class DumpPdg(options: PdgDumpOptions)(implicit semantics: Semantics) extends La
     val cpg = context.cpg
     cpg.method.zipWithIndex.foreach {
       case (method, i) =>
-        val str = method.start.dotPdg.head
+        val str = method.dotPdg.head
         (File(options.outDir) / s"${i}-pdg.dot").write(str)
     }
   }

dataflowengineoss/src/main/scala/io/shiftleft/dataflowengineoss/passes/reachingdef/ReachingDefPass.scala

@@ -4,7 +4,6 @@ import io.shiftleft.codepropertygraph.Cpg
 import io.shiftleft.codepropertygraph.generated.{nodes, _}
 import io.shiftleft.passes.{DiffGraph, ParallelCpgPass}
 import io.shiftleft.semanticcpg.language._
-import overflowdb.traversal._
 
 /**
   * A pass that calculates reaching definitions ("data dependencies").
@@ -79,7 +78,7 @@ class ReachingDefPass(cpg: Cpg) extends ParallelCpgPass[nodes.Method](cpg) {
           }
 
         case methodReturn: nodes.MethodReturn =>
-          methodReturn.start.cfgPrev.isReturn.foreach { ret =>
+          methodReturn.cfgPrev.isReturn.foreach { ret =>
             addEdge(ret, methodReturn, "<RET>")
           }
 

dataflowengineoss/src/main/scala/io/shiftleft/dataflowengineoss/passes/reachingdef/ReachingDefProblem.scala

@@ -2,12 +2,12 @@ package io.shiftleft.dataflowengineoss.passes.reachingdef
 
 import io.shiftleft.codepropertygraph.generated.nodes
 import io.shiftleft.semanticcpg.accesspath.{AccessPath, MatchResult, TrackedBase}
-import io.shiftleft.semanticcpg.language.nodemethods.TrackingPointMethodsBase.ImplicitsAPI
 import io.shiftleft.semanticcpg.language._
+import io.shiftleft.semanticcpg.language.nodemethods.TrackingPointMethodsBase.ImplicitsAPI
 import org.slf4j.{Logger, LoggerFactory}
+import overflowdb.traversal._
 
 import scala.jdk.CollectionConverters._
-import overflowdb.traversal._
 
 object Definition {
 
@@ -49,8 +49,8 @@ class ReachingDefFlowGraph(method: nodes.Method) extends FlowGraph {
   private def initSucc(ns: List[nodes.StoredNode]): Map[nodes.StoredNode, List[nodes.StoredNode]] = {
     ns.map {
       case n @ (ret: nodes.Return)              => n -> List(ret.method.methodReturn)
-      case n @ (cfgNode: nodes.CfgNode)         => n -> cfgNode.start.cfgNext.l
-      case n @ (param: nodes.MethodParameterIn) => n -> param.start.method.cfgFirst.l
+      case n @ (cfgNode: nodes.CfgNode)         => n -> cfgNode.cfgNext.l
+      case n @ (param: nodes.MethodParameterIn) => n -> param.method.cfgFirst.l
       case n =>
         logger.warn(s"Node type ${n.getClass.getSimpleName} should not be part of the CFG");
         n -> List()
@@ -63,9 +63,9 @@ class ReachingDefFlowGraph(method: nodes.Method) extends FlowGraph {
   private def initPred(ns: List[nodes.StoredNode],
                        method: nodes.Method): Map[nodes.StoredNode, List[nodes.StoredNode]] = {
     ns.map {
-      case n @ (_: nodes.CfgNode) if method.start.cfgFirst.headOption.contains(n) =>
+      case n @ (_: nodes.CfgNode) if method.cfgFirst.headOption.contains(n) =>
         n -> method.parameter.l.sortBy(_.order).lastOption.toList
-      case n @ (cfgNode: nodes.CfgNode)     => n -> cfgNode.start.cfgPrev.l
+      case n @ (cfgNode: nodes.CfgNode)     => n -> cfgNode.cfgPrev.l
       case n @ (_: nodes.MethodParameterIn) => n -> List(method)
       case n =>
         logger.warn(s"Node type ${n.getClass.getSimpleName} should not be part of the CFG");
@@ -97,17 +97,17 @@ class ReachingDefTransferFunction(method: nodes.Method) extends TransferFunction
   def initGen(method: nodes.Method): Map[nodes.StoredNode, Set[Definition]] = {
 
     def defsMadeByCall(call: nodes.Call): Set[Definition] = {
-      (Set(call) ++ call.start.argument.toSet
+      (Set(call) ++ call.start.argument
         .filterNot(_.isInstanceOf[nodes.Literal])
         .filterNot(_.isInstanceOf[nodes.FieldIdentifier]))
         .map(x => Definition.fromNode(x.asInstanceOf[nodes.StoredNode]))
     }
 
-    val defsForParams = method.start.parameter.l.map { param =>
+    val defsForParams = method.parameter.l.map { param =>
       param -> Set(Definition.fromNode(param.asInstanceOf[nodes.StoredNode]))
     }
 
-    val defsForCalls = method.start.call.l.map { call =>
+    val defsForCalls = method.call.l.map { call =>
       call -> defsMadeByCall(call)
     }
 
@@ -121,7 +121,7 @@ class ReachingDefTransferFunction(method: nodes.Method) extends TransferFunction
   def initKill(method: nodes.Method,
                gen: Map[nodes.StoredNode, Set[Definition]]): Map[nodes.StoredNode, Set[Definition]] = {
 
-    val baseToCalls: Map[TrackedBase, List[(nodes.Call, AccessPath)]] = method.start.call.l
+    val baseToCalls: Map[TrackedBase, List[(nodes.Call, AccessPath)]] = method.call.l
       .map { call =>
         val (base, path) = call.trackedBaseAndAccessPath
         (base, (call, path))
@@ -153,7 +153,7 @@ class ReachingDefTransferFunction(method: nodes.Method) extends TransferFunction
 
     // We are also adding nodes here that may not even be definitions, but that's
     // fine since `kill` is only subtracted
-    method.start.call.map { call =>
+    method.call.map { call =>
       val killedDefs = gen(call)
         .map { d =>
           allOtherInstancesOf(d.node)

dataflowengineoss/src/main/scala/io/shiftleft/dataflowengineoss/queryengine/Engine.scala

@@ -168,8 +168,8 @@ object Engine {
   private def elemForArgument(e: Edge, curNode: nodes.Expression)(
       implicit semantics: Semantics): Option[PathElement] = {
     val parentNode = e.outNode().asInstanceOf[nodes.Expression]
-    val parentNodeCall = parentNode.start.inCall.l
-    val sameCallSite = parentNode.start.inCall.l == curNode.start.inCall.l
+    val parentNodeCall = parentNode.inCall.l
+    val sameCallSite = parentNode.inCall.l == curNode.start.inCall.l
 
     if (sameCallSite && parentNode.isUsed && curNode.isDefined ||
         !sameCallSite && curNode.isUsed) {
@@ -189,7 +189,7 @@ object Engine {
   }
 
   def argToMethods(arg: nodes.Expression): List[nodes.Method] = {
-    arg.start.inCall.l.flatMap { call =>
+    arg.inCall.l.flatMap { call =>
       methodsForCall(call)
     }
   }