Skip to content

Commit 4c5afda

Browse files
mpollmeiermpollmeier
authored
flatgraph port (#344)
* port to flatgraph * upgrade to latest
1 parent 7404576 commit 4c5afda

16 files changed

Lines changed: 77 additions & 81 deletions

‎build.sbt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
val cpgVersion = "1.6.11"
2-
val joernVersion = "2.0.335"
1+
val cpgVersion = "1.7.3"
2+
val joernVersion = "4.0.8"
33

44
val gitCommitString = SettingKey[String]("gitSha")
55

@@ -16,7 +16,7 @@ Global / excludeLintKeys += Fast / configuration
1616
Global / excludeLintKeys += gitCommitString
1717

1818
lazy val commonSettings = Seq(
19-
scalaVersion := "3.4.1",
19+
scalaVersion := "3.4.2",
2020
organization := "io.shiftleft",
2121
scalacOptions ++= Seq("-Xtarget:8"),
2222
resolvers ++= Seq(

‎project/build.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
sbt.version=1.9.9
1+
sbt.version=1.10.1

‎src/main/scala/io/shiftleft/js2cpg/astcreation/AstCreator.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ import com.oracle.js.parser.ir.{
4444
WithNode
4545
}
4646
import com.oracle.js.parser.ir.LiteralNode.ArrayLiteralNode
47+
import flatgraph.DiffGraphBuilder
4748
import io.shiftleft.codepropertygraph.generated.nodes.{
4849
NewBlock,
4950
NewCall,
@@ -64,7 +65,6 @@ import io.shiftleft.js2cpg.datastructures.scope._
6465
import io.shiftleft.js2cpg.passes.{Defines, EcmaBuiltins, PassHelpers}
6566
import io.shiftleft.js2cpg.passes.PassHelpers.ParamNodeInitKind
6667
import io.shiftleft.js2cpg.parser.{GeneralizingAstVisitor, JsSource}
67-
import overflowdb.BatchedUpdate.DiffGraphBuilder
6868
import org.slf4j.LoggerFactory
6969

7070
import scala.collection.mutable
@@ -429,7 +429,7 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes:
429429
astNodeBuilder.createTypeNode(methodName, methodFullName)
430430

431431
val astParentType = parentNodeId.label
432-
val astParentFullName = parentNodeId.properties("FULL_NAME").toString
432+
val astParentFullName = parentNodeId.propertiesMap.get("FULL_NAME").toString
433433

434434
val functionTypeDeclId =
435435
astNodeBuilder.createTypeDeclNode(methodName, methodFullName, astParentType, astParentFullName, Some(Defines.Any))
@@ -457,7 +457,7 @@ class AstCreator(diffGraph: DiffGraphBuilder, source: JsSource, usedIdentNodes:
457457
// need to be resolved first, we for now dont handle the class
458458
// hierarchy.
459459
val astParentType = methodAstParentStack.head.label
460-
val astParentFullName = methodAstParentStack.head.properties("FULL_NAME").toString
460+
val astParentFullName = methodAstParentStack.head.propertiesMap.get("FULL_NAME").toString
461461

462462
val typeDeclId =
463463
astNodeBuilder.createTypeDeclNode(typeName, typeFullName, astParentType, astParentFullName, inheritsFrom = None)

‎src/main/scala/io/shiftleft/js2cpg/astcreation/AstEdgeBuilder.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
package io.shiftleft.js2cpg.astcreation
22

3+
import flatgraph.DiffGraphBuilder
34
import io.shiftleft.codepropertygraph.generated.EdgeTypes
45
import io.shiftleft.codepropertygraph.generated.nodes._
56
import io.shiftleft.js2cpg.datastructures.OrderTracker
6-
import overflowdb.BatchedUpdate.DiffGraphBuilder
77
import org.slf4j.LoggerFactory
88

99
class AstEdgeBuilder(private val diffGraph: DiffGraphBuilder) {

‎src/main/scala/io/shiftleft/js2cpg/astcreation/AstNodeBuilder.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
package io.shiftleft.js2cpg.astcreation
22

33
import com.oracle.js.parser.ir._
4+
import flatgraph.DiffGraphBuilder
45
import io.shiftleft.codepropertygraph.generated.nodes._
56
import io.shiftleft.codepropertygraph.generated.{DispatchTypes, EvaluationStrategies, Operators}
67
import io.shiftleft.js2cpg.datastructures.{LineAndColumn, OrderTracker}
78
import io.shiftleft.js2cpg.datastructures.scope.{MethodScope, Scope}
89
import io.shiftleft.js2cpg.passes.Defines
910
import io.shiftleft.js2cpg.parser.JsSource
1011
import io.shiftleft.js2cpg.parser.JsSource.shortenCode
11-
import overflowdb.BatchedUpdate.DiffGraphBuilder
1212

1313
class AstNodeBuilder(
1414
private val diffGraph: DiffGraphBuilder,

‎src/main/scala/io/shiftleft/js2cpg/astcreation/NewCompositeNode.scala‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
package io.shiftleft.js2cpg.astcreation
22

3-
import io.shiftleft.codepropertygraph.generated.nodes.NewNode
3+
import flatgraph.BatchedUpdateInterface
4+
import io.shiftleft.codepropertygraph.generated.nodes.{NewNode, StoredNode}
45

6+
import java.util
57
import scala.collection.mutable.ListBuffer
68

7-
class NewCompositeNode(underlying: ListBuffer[NewNode] = ListBuffer.empty[NewNode]) extends NewNode {
9+
class NewCompositeNode(underlying: ListBuffer[NewNode] = ListBuffer.empty[NewNode]) extends NewNode(nodeKind = -1) {
810
override def label: String = "COMPOSITE"
911

10-
override def properties: Map[String, Any] = ??? // we do not need this
11-
1212
override def canEqual(that: Any): Boolean =
1313
that.isInstanceOf[NewCompositeNode]
1414

@@ -29,8 +29,9 @@ class NewCompositeNode(underlying: ListBuffer[NewNode] = ListBuffer.empty[NewNod
2929
underlying.foreach(func)
3030
}
3131

32-
def isValidInNeighbor(edgeLabel: String, node: NewNode): Boolean = ??? // we do not need this
33-
34-
def isValidOutNeighbor(edgeLabel: String, node: NewNode): Boolean = ??? // we do not need this
35-
32+
override def isValidInNeighbor(edgeLabel: String, node: NewNode): Boolean = ??? // we do not need this
33+
override def isValidOutNeighbor(edgeLabel: String, node: NewNode): Boolean = ??? // we do not need this
34+
override def propertiesMap: util.Map[String, Any] = ??? // we do not need this
35+
override def countAndVisitProperties(interface: BatchedUpdateInterface): Unit = ???
36+
override type StoredNodeType = StoredNode
3637
}

‎src/main/scala/io/shiftleft/js2cpg/passes/AstCreationPass.scala‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@ import java.nio.file.Path
44
import better.files.File
55
import com.oracle.js.parser.Source
66
import com.oracle.js.parser.ir.FunctionNode
7-
import io.shiftleft.codepropertygraph.Cpg
7+
import io.shiftleft.codepropertygraph.generated.Cpg
88
import io.shiftleft.js2cpg.core.Report
99
import io.shiftleft.js2cpg.astcreation.AstCreator
1010
import io.shiftleft.js2cpg.io.{FileUtils, JsFileChecks}
1111
import io.shiftleft.js2cpg.parser.{JavaScriptParser, JsSource}
12-
import io.shiftleft.passes.ConcurrentWriterCpgPass
12+
import io.shiftleft.passes.ForkJoinParallelCpgPass
1313
import org.slf4j.LoggerFactory
1414
import io.shiftleft.js2cpg.utils.SourceWrapper._
1515
import io.shiftleft.js2cpg.utils.TimeUtils
@@ -20,7 +20,7 @@ import scala.util.{Failure, Success, Try}
2020
* in parallel.
2121
*/
2222
class AstCreationPass(srcDir: File, filenames: List[(Path, Path)], cpg: Cpg, report: Report)
23-
extends ConcurrentWriterCpgPass[(Path, Path)](cpg) {
23+
extends ForkJoinParallelCpgPass[(Path, Path)](cpg) {
2424

2525
private val logger = LoggerFactory.getLogger(getClass)
2626

@@ -41,7 +41,7 @@ class AstCreationPass(srcDir: File, filenames: List[(Path, Path)], cpg: Cpg, rep
4141

4242
parseResult.map { case (parseResult, usedIdentNodes) =>
4343
val (result, duration) = {
44-
TimeUtils.time(generateCpg(parseResult, new DiffGraphBuilder, usedIdentNodes))
44+
TimeUtils.time(generateCpg(parseResult, Cpg.newDiffGraphBuilder, usedIdentNodes))
4545
}
4646
val path = parseResult.jsSource.originalFilePath
4747
result match {

‎src/main/scala/io/shiftleft/js2cpg/passes/ConfigPass.scala‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,19 @@
11
package io.shiftleft.js2cpg.passes
22

3-
import io.shiftleft.codepropertygraph.Cpg
3+
import io.shiftleft.codepropertygraph.generated.Cpg
44
import io.shiftleft.codepropertygraph.generated.nodes.NewConfigFile
55
import io.shiftleft.js2cpg.core.Report
66
import io.shiftleft.js2cpg.io.FileDefaults
77
import io.shiftleft.js2cpg.io.FileUtils
88
import io.shiftleft.js2cpg.utils.TimeUtils
9-
import io.shiftleft.passes.ConcurrentWriterCpgPass
9+
import io.shiftleft.passes.ForkJoinParallelCpgPass
1010
import io.shiftleft.utils.IOUtils
1111
import org.slf4j.{Logger, LoggerFactory}
1212

1313
import java.nio.file.Path
1414

1515
class ConfigPass(filenames: List[(Path, Path)], cpg: Cpg, report: Report)
16-
extends ConcurrentWriterCpgPass[(Path, Path)](cpg) {
16+
extends ForkJoinParallelCpgPass[(Path, Path)](cpg) {
1717

1818
private val logger: Logger = LoggerFactory.getLogger(getClass)
1919

@@ -37,7 +37,7 @@ class ConfigPass(filenames: List[(Path, Path)], cpg: Cpg, report: Report)
3737
val fileName = relativeFile.toString
3838
val content = fileContent(filePath)
3939
val (result, time) = TimeUtils.time {
40-
val localDiff = new DiffGraphBuilder
40+
val localDiff = Cpg.newDiffGraphBuilder
4141
logger.debug(s"Adding file '$relativeFile' as config file.")
4242
val configNode = NewConfigFile().name(fileName).content(content.mkString("\n"))
4343
report.addReportInfo(fileName, fileStatistics.linesOfCode, parsed = true, cpgGen = true, isConfig = true)

‎src/test/scala/io/shiftleft/js2cpg/io/ExcludeTest.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ class ExcludeTest extends AnyWordSpec with Matchers with TableDrivenPropertyChec
4040
defaultArgs.map(_.toArg).toArray
4141
)
4242

43-
val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString))
43+
val cpg = Cpg.withStorage(cpgPath.pathAsString, deserializeOnClose = false)
4444

4545
fileNames(cpg) should contain theSameElementsAs expectedFiles.map(_.replace("/", java.io.File.separator))
4646
cpg.close()

‎src/test/scala/io/shiftleft/js2cpg/io/PrivateModulesTest.scala‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers {
2424
val cpgPath = tmpDir / "cpg.bin.zip"
2525
Js2CpgMain.main(Array(tmpProjectPath.pathAsString, "--output", cpgPath.pathAsString, "--no-babel"))
2626

27-
val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString))
27+
val cpg = Cpg.withStorage(cpgPath.pathAsString, deserializeOnClose = false)
2828

2929
fileNames(cpg) should contain allElementsOf Set(
3030
s"@privateA${java.io.File.separator}a.js",
@@ -52,7 +52,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers {
5252
)
5353
)
5454

55-
val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString))
55+
val cpg = Cpg.withStorage(cpgPath.pathAsString, deserializeOnClose = false)
5656

5757
fileNames(cpg) should contain allElementsOf Set(
5858
s"@privateA${java.io.File.separator}a.js",
@@ -82,7 +82,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers {
8282
)
8383
)
8484

85-
val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString))
85+
val cpg = Cpg.withStorage(cpgPath.pathAsString, deserializeOnClose = false)
8686

8787
fileNames(cpg) should contain only s"@privateB${java.io.File.separator}b.js"
8888
cpg.close()
@@ -98,7 +98,7 @@ class PrivateModulesTest extends AnyWordSpec with Matchers {
9898
val cpgPath = tmpDir / "cpg.bin.zip"
9999
Js2CpgMain.main(Array(tmpProjectPath.pathAsString, "--output", cpgPath.pathAsString, "--no-babel"))
100100

101-
val cpg = Cpg.withConfig(overflowdb.Config.withoutOverflow.withStorageLocation(cpgPath.pathAsString))
101+
val cpg = Cpg.withStorage(cpgPath.pathAsString, deserializeOnClose = false)
102102

103103
fileNames(cpg) should contain only "index.js"
104104
cpg.close()

0 commit comments

Comments
 (0)