upgrade codegen, fix most deprecation warnings (#1097)

mpollmeier · web-flow · commit c71dc53fba72 · 2021-01-24T22:51:17.000+13:00
* more * back to latest released codegen for now * more fixes, use latest released codegen * more fixes * more * fmt * fix remaining deprecation warnings * reenable fatal-warnings * Revert "reenable fatal-warnings" This reverts commit 3eddaec. * help/helpVerbose parens * reenable fatal-warnings, but somehow can't do that for console Might be a bug in scalac, but maybe I'm blind. Not super important right now, I left a comment for later * latest odb * properly disable Xfatal for console
diff --git a/build.sbt b/build.sbt
@@ -1,7 +1,7 @@
 name := "codepropertygraph"
 
 // parsed by project/Versions.scala, updated by updateDependencies.sh
-val overflowdbVersion = "1.29"
+val overflowdbVersion = "1.30"
 
 inThisBuild(
   List(
@@ -71,7 +71,7 @@ lazy val fuzzyc2cpgtests = Projects.fuzzyc2cpgtests
 ThisBuild/scalacOptions ++= Seq(
   "-deprecation",
   "-feature",
-  // "-Xfatal-warnings", // TODO MP reenable
+  "-Xfatal-warnings",
   "-language:implicitConversions",
   "-Ycache-macro-class-loader:last-modified",
   "-Ybackend-parallelism", "4")
diff --git a/console/build.sbt b/console/build.sbt
@@ -16,7 +16,6 @@ scalacOptions ++= Seq(
   "-language:implicitConversions",     // Allow definition of implicit functions called views
   "-unchecked",                        // Enable additional warnings where generated code depends on assumptions.
   "-Xcheckinit",                       // Wrap field accessors to throw an exception on uninitialized access.
-  // "-Xfatal-warnings",   // TODO MP reenable               // Fail the compilation if there are any warnings.
   "-Xlint:adapted-args",               // Warn if an argument list is modified to match the receiver.
   "-Xlint:constant",                   // Evaluation of a constant arithmetic expression results in an error.
   "-Xlint:delayedinit-select",         // Selecting member of DelayedInit.
@@ -41,6 +40,9 @@ scalacOptions ++= Seq(
   "-Ywarn-unused:privates"             // Warn if a private member is unused.
 )
 
+// would love to reenable, but somehow StorageBackend.scala triggers a strange `[warn] method with a single empty parameter list overrides method without any parameter list` that doesn't make sense to me...
+scalacOptions -= "-Xfatal-warnings"
+
 val ScoptVersion = "3.7.1"
 val BetterFilesVersion = "3.8.0"
 val CaskVersion = "0.7.8"
diff --git a/console/src/main/scala/io/shiftleft/console/PPrinter.scala b/console/src/main/scala/io/shiftleft/console/PPrinter.scala
@@ -5,7 +5,7 @@ import pprint.{PPrinter, Renderer, Result, Tree, Truncated}
 
 object pprinter {
 
-  val AnsiEncodedRegexp = """\u001b\[[\d;]+m""".r
+  val AnsiEncodedRegexp = "\u001b\\[[\\d;]+m".r
   def isAnsiEncoded(s: String): Boolean =
     AnsiEncodedRegexp.findFirstIn(s).isDefined
 
@@ -15,10 +15,10 @@ object pprinter {
     * */
   def fixForFansi(ansiEncoded: String): String =
     ansiEncoded
-      .replaceAll("""\u001b\[m""", """\u001b[39m""") //encoding ends with [39m for fansi instead of [m
-      .replaceAll("""\u001b\[0(\d)m""", """\u001b[$1m""") // `[01m` is encoded as `[1m` in fansi for all single digit numbers
-      .replaceAll("""\u001b\[0?(\d+);0?(\d+)m""", """\u001b[$1m\u001b[$2m""") // `[01;34m` is encoded as `[1m[34m` in fansi
-      .replaceAll("""\u001b\[[00]+;0?(\d+);0?(\d+);0?(\d+)m""", """\u001b[$1;$2;$3m""") // `[00;38;05;70m` is encoded as `[38;5;70m` in fansi - 8bit color encoding
+      .replaceAll("\u001b\\[m", "\u001b[39m") //encoding ends with [39m for fansi instead of [m
+      .replaceAll("\u001b\\[0(\\d)m", "\u001b[$1m") // `[01m` is encoded as `[1m` in fansi for all single digit numbers
+      .replaceAll("\u001b\\[0?(\\d+);0?(\\d+)m", "\u001b[$1m\u001b[$2m") // `[01;34m` is encoded as `[1m[34m` in fansi
+      .replaceAll("\u001b\\[[00]+;0?(\\d+);0?(\\d+);0?(\\d+)m", "\u001b[$1;$2;$3m") // `[00;38;05;70m` is encoded as `[38;5;70m` in fansi - 8bit color encoding
 
   def create(original: PPrinter): PPrinter =
     new PPrinter(defaultHeight = 99999, additionalHandlers = myAdditionalHandlers(original)) {
diff --git a/console/src/test/scala/io/shiftleft/console/scripting/AmmoniteExecutorTest.scala b/console/src/test/scala/io/shiftleft/console/scripting/AmmoniteExecutorTest.scala
@@ -34,7 +34,9 @@ class AmmoniteExecutorTest extends AnyWordSpec with Matchers {
     "pass arguments to a script" in withExecutor { executor =>
       val script = getScriptPath("scripts/general/arguments-concatenate.sc")
 
-      executor.runScript(script, Map("one" -> "hello", "two" -> "world"), Cpg.emptyCpg).unsafeRunSync() shouldBe "hello world"
+      executor
+        .runScript(script, Map("one" -> "hello", "two" -> "world"), Cpg.emptyCpg)
+        .unsafeRunSync() shouldBe "hello world"
     }
 
     "execute multiple scripts" in withExecutor { executor =>
diff --git a/fuzzyc2cpg/src/test/scala/io/shiftleft/fuzzyc2cpg/passes/CfgCreationPassTests.scala b/fuzzyc2cpg/src/test/scala/io/shiftleft/fuzzyc2cpg/passes/CfgCreationPassTests.scala
@@ -483,9 +483,7 @@ class CfgFixture(file1Code: String) {
   }
 
   def succOf(code: String): Set[String] = {
-    codeToNode(code)
-      ._cfgOut()
-      .asScala
+    codeToNode(code)._cfgOut.asScala
       .map(_.asInstanceOf[nodes.CfgNode])
       .toSet
       .map[String](_.code)
diff --git a/project/meta-build.sbt b/project/meta-build.sbt
@@ -1,6 +1,6 @@
 libraryDependencies ++= Seq(
   "com.github.pathikrit" %% "better-files" % "3.8.0",
-  "io.shiftleft" %% "overflowdb-codegen" % "1.39",
+  "io.shiftleft" %% "overflowdb-codegen" % "1.44",
 )
 
 resolvers += Resolver.bintrayRepo("shiftleft", "maven")
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CdgGenerator.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/CdgGenerator.scala
@@ -10,8 +10,7 @@ class CdgGenerator extends CfgGenerator {
   override val edgeType: String = EdgeTypes.CDG
 
   override def expand(v: nodes.StoredNode): Iterator[Edge] = {
-    v._cdgOut()
-      .asScala
+    v._cdgOut.asScala
       .filter(_.isInstanceOf[nodes.StoredNode])
       .map(node => Edge(v, node, edgeType = edgeType))
   }
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/DotSerializer.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/dotgenerator/DotSerializer.scala
@@ -83,7 +83,7 @@ object DotSerializer {
   }
 
   private def escape(str: String): String = {
-    str.replaceAllLiterally("\"", "\\\"")
+    str.replace("\"", "\\\"")
   }
 
   private def graphEnd(sb: StringBuilder): String = {
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/LocationCreator.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/LocationCreator.scala
@@ -111,7 +111,7 @@ object LocationCreator {
           methodRef.code,
           methodRef.label,
           methodRef.lineNumber,
-          methodRef._methodViaContainsIn.next
+          methodRef._methodViaContainsIn.next()
         )
       case source: nodes.Source =>
         apply(source.node)
@@ -138,7 +138,7 @@ object LocationCreator {
       val namespaceOption = for {
         tpe <- typeOption
         namespaceBlock <- tpe._namespaceBlockViaAstIn
-        namespace <- namespaceBlock._namespaceViaRefOut.nextOption
+        namespace <- namespaceBlock._namespaceViaRefOut.nextOption()
       } yield namespace.name
       val namespaceName = namespaceOption.getOrElse("")
 
@@ -162,7 +162,7 @@ object LocationCreator {
 
   @tailrec
   private def findVertex(node: nodes.StoredNode, instanceCheck: nodes.StoredNode => Boolean): Option[nodes.StoredNode] =
-    node._astIn.nextOption match {
+    node._astIn.nextOption() match {
       case Some(head) if instanceCheck(head) => Some(head)
       case Some(head)                        => findVertex(head, instanceCheck)
       case None                              => None
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewNodeSteps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/NewNodeSteps.scala
@@ -11,7 +11,7 @@ trait HasStoreMethod {
 class NewNodeSteps[A <: NewNode](val traversal: Traversal[A]) extends HasStoreMethod {
 
   override def store()(implicit diffBuilder: DiffGraph.Builder): Unit =
-    traversal.sideEffect(storeRecursively).iterate
+    traversal.sideEffect(storeRecursively).iterate()
 
   private def storeRecursively(newNode: NewNode)(implicit diffBuilder: DiffGraph.Builder): Unit = {
     diffBuilder.addNode(newNode)
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Steps.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/Steps.scala
@@ -49,18 +49,18 @@ class Steps[A](val traversal: Traversal[A]) extends AnyVal {
     Execute the traversal and convert it into a Java list (as opposed
     to the Scala list obtained via `toList`)
     */
-  def jl(): JList[A] = b.asJava
+  def jl(): JList[A] = b().asJava
 
   /**
     * Print help/documentation based on the current elementType `A`.
     * Relies on all step extensions being annotated with @TraversalExt / @Doc
     * Note that this works independently of tab completion and implicit conversions in scope - it will simply list
     * all documented steps in the classpath
     * */
-  def help()(implicit elementType: ClassTag[A]): String =
+  def help(implicit elementType: ClassTag[A]): String =
     Steps.help.forElementSpecificSteps(elementType.runtimeClass, verbose = false)
 
-  def helpVerbose()(implicit elementType: ClassTag[A]): String =
+  def helpVerbose(implicit elementType: ClassTag[A]): String =
     Steps.help.forElementSpecificSteps(elementType.runtimeClass, verbose = true)
 
   /**
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CfgNodeMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/CfgNodeMethods.scala
@@ -25,7 +25,7 @@ class CfgNodeMethods(val node: nodes.CfgNode) extends AnyVal {
     * */
   def controlledBy: Traversal[nodes.CfgNode] = {
     expandExhaustively { v =>
-      v._cdgIn().asScala
+      v._cdgIn.asScala
     }
   }
 
@@ -35,7 +35,7 @@ class CfgNodeMethods(val node: nodes.CfgNode) extends AnyVal {
     * */
   def controls: Traversal[nodes.CfgNode] = {
     expandExhaustively { v =>
-      v._cdgOut().asScala
+      v._cdgOut.asScala
     }
   }
 
@@ -45,7 +45,7 @@ class CfgNodeMethods(val node: nodes.CfgNode) extends AnyVal {
     * */
   def dominatedBy: Traversal[nodes.CfgNode] = {
     expandExhaustively { v =>
-      v._dominateIn().asScala
+      v._dominateIn.asScala
     }
   }
 
@@ -55,7 +55,7 @@ class CfgNodeMethods(val node: nodes.CfgNode) extends AnyVal {
     * */
   def dominates: Traversal[nodes.CfgNode] = {
     expandExhaustively { v =>
-      v._dominateOut().asScala
+      v._dominateOut.asScala
     }
   }
 
@@ -65,7 +65,7 @@ class CfgNodeMethods(val node: nodes.CfgNode) extends AnyVal {
     * */
   def postDominatedBy: Traversal[nodes.CfgNode] = {
     expandExhaustively { v =>
-      v._postDominateIn().asScala
+      v._postDominateIn.asScala
     }
   }
 
@@ -75,7 +75,7 @@ class CfgNodeMethods(val node: nodes.CfgNode) extends AnyVal {
     * */
   def postDominates: Traversal[nodes.CfgNode] = {
     expandExhaustively { v =>
-      v._postDominateOut().asScala
+      v._postDominateOut.asScala
     }
   }
 
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/StoredNodeMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/StoredNodeMethods.scala
@@ -6,6 +6,6 @@ import scala.jdk.CollectionConverters._
 
 class StoredNodeMethods(val node: nodes.StoredNode) extends AnyVal {
   def tag: Traversal[nodes.Tag] = {
-    node._taggedByOut().asScala.map(_.asInstanceOf[nodes.Tag]).to(Traversal)
+    node._taggedByOut.asScala.map(_.asInstanceOf[nodes.Tag]).to(Traversal)
   }
 }
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/TrackingPointMethods.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/nodemethods/TrackingPointMethods.scala
@@ -114,7 +114,7 @@ object TrackingPointMethodsBase {
       case node: nodes.MethodParameterOut => (TrackedNamedVariable(node.name), Nil)
       case node: nodes.ImplicitCall       => (TrackedReturnValue(node), Nil)
       case node: nodes.PostExecutionCall =>
-        toTrackedBaseAndAccessPathInternal(node._refOut().next.asInstanceOf[nodes.TrackingPoint])
+        toTrackedBaseAndAccessPathInternal(node._refOut.next().asInstanceOf[nodes.TrackingPoint])
       case node: nodes.Identifier    => (TrackedNamedVariable(node.name), Nil)
       case node: nodes.Literal       => (TrackedLiteral(node), Nil)
       case node: nodes.MethodRef     => (TrackedMethodOrTypeRef(node), Nil)
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/passes/codepencegraph/CpgPostDomTreeAdapter.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/passes/codepencegraph/CpgPostDomTreeAdapter.scala
@@ -8,6 +8,6 @@ import overflowdb.traversal._
 class CpgPostDomTreeAdapter extends DomTreeAdapter[Node] {
 
   override def immediateDominator(cfgNode: Node): Option[Node] = {
-    cfgNode.in(EdgeTypes.POST_DOMINATE).nextOption
+    cfgNode.in(EdgeTypes.POST_DOMINATE).nextOption()
   }
 }
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/passes/compat/argumentcompat/ArgumentCompat.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/passes/compat/argumentcompat/ArgumentCompat.scala
@@ -26,7 +26,7 @@ class ArgumentCompat(cpg: Cpg) extends CpgPass(cpg) {
 
   private def addArgumentEdges(callOrReturn: nodes.AstNode, diffGraph: DiffGraph.Builder): Unit = {
     callOrReturn._astOut.asScala.foreach { argument =>
-      if (!argument._argumentIn().hasNext) {
+      if (!argument._argumentIn.hasNext) {
         diffGraph.addEdgeInOriginal(callOrReturn, argument, EdgeTypes.ARGUMENT)
       }
     }
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/passes/containsedges/ContainsEdgePass.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/passes/containsedges/ContainsEdgePass.scala
@@ -27,7 +27,7 @@ class ContainsEdgePass(cpg: Cpg) extends ParallelCpgPass[nodes.AstNode](cpg) {
           dstGraph.addEdgeInOriginal(source, destination, EdgeTypes.CONTAINS)
         }
       }
-      .iterate
+      .iterate()
 
     Iterator(dstGraph.build())
   }
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/passes/linking/calllinker/CallLinker.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/passes/linking/calllinker/CallLinker.scala
@@ -55,11 +55,7 @@ class CallLinker(cpg: Cpg) extends CpgPass(cpg) {
           val receiver = receiverIt.next
           receiver match {
             case _ =>
-              val receiverTypeDecl = receiver
-                ._evalTypeOut()
-                .onlyChecked
-                ._refOut
-                .onlyChecked
+              val receiverTypeDecl = receiver._evalTypeOut.onlyChecked._refOut.onlyChecked
 
               val resolvedMethodOption = receiverTypeDecl._bindsOut.asScala.collectFirst {
                 case binding: nodes.Binding if binding.name == call.name && binding.signature == call.signature =>
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/passes/linking/linker/Linker.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/passes/linking/linker/Linker.scala
@@ -227,7 +227,7 @@ object Linker {
           }
         }
       } else {
-        srcNode.out(edgeType).property(NodeKeys.FULL_NAME).nextOption match {
+        srcNode.out(edgeType).property(NodeKeys.FULL_NAME).nextOption() match {
           case Some(dstFullName) => srcNode.setProperty(dstFullNameKey, dstFullName)
           case None              => logger.warn(s"Missing outgoing edge of type ${edgeType} from node ${srcNode}")
         }
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/passes/linking/memberaccesslinker/MemberAccessLinker.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/passes/linking/memberaccesslinker/MemberAccessLinker.scala
@@ -52,7 +52,7 @@ class MemberAccessLinker(cpg: Cpg) extends CpgPass(cpg) {
         }
         .asInstanceOf[List[nodes.StoredNode]]
       if (args.size != 2) throw new RuntimeException("member/field access-style operators need two arguments")
-      val typ = args(0)._evalTypeOut.nextOption match {
+      val typ = args(0)._evalTypeOut.nextOption() match {
         case Some(t: nodes.Type) => t
         case _                   => return
       }
diff --git a/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/passes/receiveredges/ReceiverEdgePass.scala b/semanticcpg/src/main/scala/io/shiftleft/semanticcpg/passes/receiveredges/ReceiverEdgePass.scala
@@ -27,17 +27,19 @@ class ReceiverEdgePass(cpg: Cpg) extends CpgPass(cpg) {
     var loggedDeprecationWarning = false
     val dstGraph = DiffGraph.newBuilder
 
-    cpg.call.sideEffect { call =>
-      call._astOut.asScala.find { case node: HasArgumentIndex => node.argumentIndex == 0 }.foreach { instance =>
-        if (!instance._receiverIn.hasNext) {
-          dstGraph.addEdgeInOriginal(call, instance, EdgeTypes.RECEIVER)
-          if (!loggedDeprecationWarning) {
-            logger.warn("Using deprecated CPG format without RECEIVER edge between CALL and instance nodes.")
-            loggedDeprecationWarning = true
+    cpg.call
+      .sideEffect { call =>
+        call._astOut.asScala.find { case node: HasArgumentIndex => node.argumentIndex == 0 }.foreach { instance =>
+          if (!instance._receiverIn.hasNext) {
+            dstGraph.addEdgeInOriginal(call, instance, EdgeTypes.RECEIVER)
+            if (!loggedDeprecationWarning) {
+              logger.warn("Using deprecated CPG format without RECEIVER edge between CALL and instance nodes.")
+              loggedDeprecationWarning = true
+            }
           }
         }
       }
-    }.iterate
+      .iterate()
 
     Iterator(dstGraph.build())
   }

Original file line number	Diff line number	Diff line change
`@@ -34,7 +34,9 @@ class AmmoniteExecutorTest extends AnyWordSpec with Matchers {`
`34`	`34`	`"pass arguments to a script" in withExecutor { executor =>`
`35`	`35`	`val script = getScriptPath("scripts/general/arguments-concatenate.sc")`
`36`	`36`
`37`		`- executor.runScript(script, Map("one" -> "hello", "two" -> "world"), Cpg.emptyCpg).unsafeRunSync() shouldBe "hello world"`
	`37`	`+ executor`
	`38`	`+ .runScript(script, Map("one" -> "hello", "two" -> "world"), Cpg.emptyCpg)`
	`39`	`+ .unsafeRunSync() shouldBe "hello world"`
`38`	`40`	`}`
`39`	`41`
`40`	`42`	`"execute multiple scripts" in withExecutor { executor =>`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`libraryDependencies ++= Seq(`
`2`	`2`	`"com.github.pathikrit" %% "better-files" % "3.8.0",`
`3`		`- "io.shiftleft" %% "overflowdb-codegen" % "1.39",`
	`3`	`+ "io.shiftleft" %% "overflowdb-codegen" % "1.44",`
`4`	`4`	`)`
`5`	`5`
`6`	`6`	`resolvers += Resolver.bintrayRepo("shiftleft", "maven")`
Original file line number	Diff line number	Diff line change
`@@ -10,8 +10,7 @@ class CdgGenerator extends CfgGenerator {`
`10`	`10`	`override val edgeType: String = EdgeTypes.CDG`
`11`	`11`
`12`	`12`	`override def expand(v: nodes.StoredNode): Iterator[Edge] = {`
`13`		`- v._cdgOut()`
`14`		`- .asScala`
	`13`	`+ v._cdgOut.asScala`
`15`	`14`	`.filter(_.isInstanceOf[nodes.StoredNode])`
`16`	`15`	`.map(node => Edge(v, node, edgeType = edgeType))`
`17`	`16`	`}`
Original file line number	Diff line number	Diff line change
`@@ -83,7 +83,7 @@ object DotSerializer {`
`83`	`83`	`}`
`84`	`84`
`85`	`85`	`private def escape(str: String): String = {`
`86`		`- str.replaceAllLiterally("\"", "\\\"")`
	`86`	`+ str.replace("\"", "\\\"")`
`87`	`87`	`}`
`88`	`88`
`89`	`89`	`private def graphEnd(sb: StringBuilder): String = {`