Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ MATCH (firstGitFile:Git&File&!Repository)-[gitChange:CHANGED_TOGETHER_WITH]-(sec
WHERE elementId(firstGitFile) < elementId(secondGitFile)
MATCH (firstGitFile)-[:RESOLVES_TO]->(firstCodeFile:File&!Git&!Repository)
MATCH (secondGitFile)-[:RESOLVES_TO]->(secondCodeFile:File&!Git&!Repository)
WHERE firstGitFile <> secondGitFile
CALL (firstCodeFile, secondCodeFile, gitChange) {
MERGE (firstCodeFile)-[pairwiseChange:CHANGED_TOGETHER_WITH]-(secondCodeFile)
SET pairwiseChange = properties(gitChange)
} IN TRANSACTIONS
} IN TRANSACTIONS OF 750 ROWS
RETURN count(*) AS pairCount
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ WHERE updateCommitCount > 2
WITH *
,fileCombination[0] AS firstFile
,fileCombination[1] AS secondFile
WHERE firstFile <> secondFile
WITH *
// Get the lowest number of git update commits of both files (file pair)
,CASE WHEN firstFile.updateCommitCount < secondFile.updateCommitCount
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ WHERE gitFileName ENDS WITH codeFileName
CALL { WITH git_file, code_file
MERGE (git_file)-[:RESOLVES_TO]->(code_file)
ON CREATE SET git_file.resolved = true
} IN TRANSACTIONS
} IN TRANSACTIONS OF 1000 ROWS
RETURN count(DISTINCT codeFileName) AS numberOfCodeFiles
,collect(DISTINCT codeFileName + ' <-> ' + gitFileName + '\n')[0..4] AS examples
// RETURN codeFileName, gitFileName
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ WHERE codeFileName ENDS WITH gitFileName
CALL { WITH git_file, code_file
MERGE (git_file)-[:RESOLVES_TO]->(code_file)
ON CREATE SET git_file.resolved = true
} IN TRANSACTIONS
} IN TRANSACTIONS OF 1000 ROWS
RETURN count(DISTINCT codeFileName) AS numberOfCodeFiles
,collect(DISTINCT codeFileName + ' <-> ' + gitFileName + '\n')[0..4] AS examples
// RETURN codeFileName, gitFileName
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// List of all NPM packages and their dependencies with build levels for GraphViz Visualization

MATCH (sourceForStatistics:NPM:Package)-[dependencyForStatistics:DEPENDS_ON]->(targetForStatistics:NPM:Package)
WHERE sourceForStatistics.maxDistanceFromSource IS NOT NULL
AND targetForStatistics.maxDistanceFromSource IS NOT NULL
WITH min(dependencyForStatistics.weightByDependencyType) AS minWeight
,max(dependencyForStatistics.weightByDependencyType) AS maxWeight
,max(targetForStatistics.maxDistanceFromSource) AS maxLevel
MATCH (source:NPM:Package)-[dependency:DEPENDS_ON]->(target:NPM:Package)
WHERE source.maxDistanceFromSource IS NOT NULL
AND target.maxDistanceFromSource IS NOT NULL
WITH *, CASE
WHEN maxWeight = minWeight THEN 0.0
ELSE toFloat(dependency.weightByDependencyType - minWeight) / toFloat(maxWeight - minWeight)
END AS normalizedWeight
WITH *, round((normalizedWeight * 5) + 1, 2) AS penWidth
WITH *, "\\n(level " + coalesce(source.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS sourceLevelInfo
WITH *, "\\n(level " + coalesce(target.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS targetLevelInfo
WITH *, source.name + sourceLevelInfo AS fullSourceName
WITH *, target.name + targetLevelInfo AS fullTargetName
WITH *, "\" -> \"" + fullTargetName
+ "\" [label = " + dependency.weightByDependencyType + ";"
+ " penwidth = " + penWidth + ";"
+ " ];" AS graphVizDotNotationEdge
WITH *, "\"" + fullSourceName + coalesce(graphVizDotNotationEdge, "\" [];") AS graphVizDotNotationLine
ORDER BY dependency.weightByDependencyType DESC, target.maxDistanceFromSource DESC
RETURN graphVizDotNotationLine
//Debugging
//,source.name AS sourceName
//,target.name AS targetName
//,penWidth
//,normalizedWeight
//,dependency.weightByDependencyType AS weight
//,minWeight
//,maxWeight
LIMIT 440
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@
WITH min(dependencyForStatistics.weight) AS minWeight
,max(dependencyForStatistics.weight) AS maxWeight
,max(targetForStatistics.maxDistanceFromSource) AS maxLevel
MATCH (source:TS:Module)-[dependency:DEPENDS_ON]->(target:TS:Module)
MATCH (source:TS:Module)-[dependency:DEPENDS_ON]->(target:TS:Module)
WHERE source.maxDistanceFromSource IS NOT NULL
AND target.maxDistanceFromSource IS NOT NULL
WITH *, toFloat(dependency.cardinality - minWeight) / toFloat(maxWeight - minWeight) AS normalizedWeight
WITH *, round((normalizedWeight * 5) + 1, 2) AS penWidth
WITH *, source.rootProjectName + "\\n" + source.name + "\\n(level " + coalesce(source.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS fullSourceName
WITH *, target.rootProjectName + "\\n" + target.name + "\\n(level " + coalesce(target.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS fullTargetName
WITH *, "\\n(level " + coalesce(source.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS sourceLevelInfo
WITH *, "\\n(level " + coalesce(target.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS targetLevelInfo
WITH *, source.rootProjectName + "\\n" + source.name + sourceLevelInfo AS fullSourceName
WITH *, target.rootProjectName + "\\n" + target.name + targetLevelInfo AS fullTargetName
WITH *, "\" -> \"" + fullTargetName
+ "\" [label = " + dependency.cardinality + ";"
+ " penwidth = " + penWidth + ";"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Path Finding - Longest path - Stream - List all dependencies for nodes contributing to longest paths and highlight those paths in the Visualization with GraphViz.
// Path Finding - Longest path - Stream - List all dependencies for nodes contributing to longest paths and highlight those paths in the Visualization with GraphViz. Recommended prerequisite: Topological_Sort_Write.cypher

// Gather global statistics about dependency weights and levels for normalization and node details
MATCH (sourceNodeForStatistics)-[dependencyForStatistics:DEPENDS_ON]->(targetNodeForStatistics)
Expand Down Expand Up @@ -55,8 +55,12 @@
WITH *, dependency[$dependencies_projection_weight_property] AS weight
WITH *, toFloat(weight - minWeight) * weightNormalizationFactor AS normalizedWeight
WITH *, round((normalizedWeight * 5) + 1, 2) AS penWidth
WITH *, source.name + "\\n(level " + source.maxDistanceFromSource + "/" + maxLevel + ")" AS startNodeTitle
WITH *, target.name + "\\n(level " + target.maxDistanceFromSource + "/" + maxLevel + ")" AS endNodeTitle
WITH *, coalesce("\\n(level " + source.maxDistanceFromSource + "/" + maxLevel + ")", "") AS startNodeLevelInfo
WITH *, coalesce("\\n" + source.rootProjectName, "") AS startNodeProjectInfo
WITH *, coalesce("\\n(level " + target.maxDistanceFromSource + "/" + maxLevel + ")", "") AS endNodeLevelInfo
WITH *, coalesce("\\n" + target.rootProjectName, "") AS endNodeProjectInfo
WITH *, source.name + startNodeProjectInfo + startNodeLevelInfo AS startNodeTitle
WITH *, target.name + endNodeProjectInfo + endNodeLevelInfo AS endNodeTitle
// The longest path will be highlighted in red.
WITH *, CASE WHEN isPartOfLongestPath THEN "; color=\"red\""
// Dependencies contributing to the longest path will be highlighted in dark orange.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Path Finding - Longest path - Stream - Find the top 100 dependencies contributing to the longest paths for Visualization with GraphViz
// Path Finding - Longest path - Stream - Find the top 100 dependencies contributing to the longest paths for Visualization with GraphViz. Recommended prerequisite: Topological_Sort_Write.cypher

MATCH (sourceNodeForStatistics)-[dependencyForStatistics:DEPENDS_ON]->(targetNodeForStatistics)
WHERE $dependencies_projection_node IN LABELS(sourceNodeForStatistics)
Expand All @@ -19,8 +19,12 @@
WITH *, dependency[$dependencies_projection_weight_property] AS weight
WITH *, toFloat(weight - minWeight) * weightNormalizationFactor AS normalizedWeight
WITH *, round((normalizedWeight * 5) + 1, 2) AS penWidth
WITH *, startNode.name + "\\n(level " + startNode.maxDistanceFromSource + "/" + maxLevel + ")" AS startNodeTitle
WITH *, endNode.name + "\\n(level " + endNode.maxDistanceFromSource + "/" + maxLevel + ")" AS endNodeTitle
WITH *, coalesce("\\n(level " + startNode.maxDistanceFromSource + "/" + maxLevel + ")", "") AS startNodeLevelInfo
WITH *, coalesce("\\n" + startNode.rootProjectName, "") AS startNodeProjectInfo
WITH *, coalesce("\\n(level " + endNode.maxDistanceFromSource + "/" + maxLevel + ")", "") AS endNodeLevelInfo
WITH *, coalesce("\\n" + endNode.rootProjectName, "") AS endNodeProjectInfo
WITH *, startNode.name + startNodeProjectInfo + startNodeLevelInfo AS startNodeTitle
WITH *, endNode.name + endNodeProjectInfo + endNodeLevelInfo AS endNodeTitle
WITH *, "[label=" + weight + "; penwidth=" + penWidth + "; ];" AS graphVizEdgeAttributes
WITH *, "\"" + startNodeTitle + "\" -> \"" + endNodeTitle + "\" " + graphVizEdgeAttributes AS graphVizDotNotationLine
RETURN graphVizDotNotationLine
Expand Down
8 changes: 8 additions & 0 deletions cypher/Path_Finding/Set_Parameters_NPM.cypher
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
// Example on how to set the parameters for path finding in this case for NPM Packages

:params {
"dependencies_projection_language":"NPM",
"dependencies_projection": "npm-package-path-finding",
"dependencies_projection_node": "Package",
"dependencies_projection_weight_property": "weightByDependencyType",
}
8 changes: 8 additions & 0 deletions cypher/Path_Finding/Set_Parameters_Typescript_Module.cypher
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
// Example on how to set the parameters for path finding in this case for TypeScript modules

:params {
"dependencies_projection_language":"Typescript",
"dependencies_projection": "typescript-module-path-finding",
"dependencies_projection_node": "Module",
"dependencies_projection_weight_property": "lowCouplingElement25PercentWeight",
}
10 changes: 10 additions & 0 deletions cypher/Topological_Sort/Topological_Sort_Exists.cypher
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
// Return the first node with a "maxDistanceFromSource" if it exists

MATCH (codeUnit)
WHERE codeUnit.maxDistanceFromSource IS NOT NULL
AND codeUnit.topologicalSortIndex IS NOT NULL
AND $dependencies_projection_node IN LABELS(codeUnit)
RETURN codeUnit.name AS shortCodeUnitName
,elementId(codeUnit) AS nodeElementId
,codeUnit.maxDistanceFromSource AS maxDistanceFromSource
LIMIT 1
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,5 @@
ON MATCH SET resolvedDependsOn = dependsOn // Overwrites existing properties
,resolvedDependsOn.cardinality = existingDependency.cardinality + dependsOn.cardinality // Add cardinalities
,resolvedDependsOn.updated = true
} IN TRANSACTIONS
} IN TRANSACTIONS OF 1000 ROWS
RETURN count(*) as resolvedDependencies
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
WITH externalDeclaration, internalDeclaration
CALL { WITH externalDeclaration, internalDeclaration
MERGE (externalDeclaration)-[:IS_IMPLEMENTED_IN]->(internalDeclaration)
} IN TRANSACTIONS
} IN TRANSACTIONS OF 1000 ROWS
RETURN count( DISTINCT externalDeclaration.globalFqn + ' -> ' + internalDeclaration.globalFqn) AS linkedDeclarationCount
,collect(DISTINCT externalDeclaration.globalFqn + ' -> ' + internalDeclaration.globalFqn)[0..4] AS linkedDeclarationExamples
//Debugging
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ WHERE equalGlobalFqn
OR equalNameAndNpmPackage
CALL { WITH module, externalModule
MERGE (externalModule)-[:IS_IMPLEMENTED_IN]->(module)
} IN TRANSACTIONS
} IN TRANSACTIONS OF 1000 ROWS
RETURN CASE WHEN equalGlobalFqn THEN 'equalGlobalFqn'
WHEN equalModule THEN 'equalModule'
WHEN equalNameWithoutNamespace THEN 'equalNameWithoutNamespace'
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// Enrich NPM:Package nodes with incoming and outgoing dependency counts
// Requires Link_npm_packages_with_depends_on_relationships.cypher

MATCH (package:NPM:Package)
WITH package,
// Private packages often contain test and example code. They will be left out of the main dependency graph analysis.
CASE WHEN package.private = true OR package.private = "true" THEN 1 ELSE 0 END AS testMarkerInteger,
COUNT {(package)<-[:DEPENDS_ON]-(:NPM:Package)} AS incomingDependencies,
COUNT {(package)-[:DEPENDS_ON]->(:NPM:Package)} AS outgoingDependencies
SET package.incomingDependencies = incomingDependencies,
package.outgoingDependencies = outgoingDependencies,
package.testMarkerInteger = testMarkerInteger
Comment thread
JohT marked this conversation as resolved.
RETURN count(DISTINCT package) AS numberOfPackagesEnriched
,sum(incomingDependencies) AS totalIncomingDependencies
,sum(outgoingDependencies) AS totalOutgoingDependencies
,avg(incomingDependencies) AS averageIncomingDependencies
,avg(outgoingDependencies) AS averageOutgoingDependencies
// Debugging
// RETURN package.name, package.incomingDependencies, package.outgoingDependencies
// LIMIT 20
Comment thread
JohT marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ UNWIND external_modules AS external_module
MERGE (external_module)-[:PROVIDED_BY_NPM_DEPENDENCY]->(npm_dependency)
SET external_module.npmPackage = npm_dependency.name
,external_module.npmPackageVersion = npm_dependency.dependency
} IN TRANSACTIONS
} IN TRANSACTIONS OF 1000 ROWS
RETURN totalNumberOfExternalModules
,count(DISTINCT external_module.globalFqn) AS numberOfLinkedExternalModules
,count(DISTINCT project.name) AS numberOfProjects
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
// Link npm dependencies to the npm package that describe them if it exists
// Link npm dependencies to the npm package that describes them, if it exists

MATCH (npm_dependency:NPM:Dependency)
MATCH (npm_package:NPM:Package)
WHERE npm_package.name = npm_dependency.name
AND npm_package <> npm_dependency
AND NOT npm_package.name CONTAINS '{{'
AND NOT npm_package.name CONTAINS '}}'
CALL { WITH npm_package, npm_dependency
MERGE (npm_dependency)-[:IS_DESCRIBED_IN_NPM_PACKAGE]->(npm_package)
} IN TRANSACTIONS
} IN TRANSACTIONS OF 1000 ROWS
RETURN count(*) AS numberOfWrittenRelationships
,count(DISTINCT npm_dependency) AS numberOfDistinctNpmDependencies
,count(DISTINCT npm_package) AS numberOfDistinctNpmPackages
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
// Link npm packages with DEPENDS_ON relationships. Requires Link_npm_dependencies_to_npm_packages.
// This creates direct package-to-package dependencies by following the chain:
// (source:NPM:Package)-[dependency_relationship]->(NPM:Dependency)-[:IS_DESCRIBED_IN_NPM_PACKAGE]->(target:NPM:Package)

MATCH (source:NPM:Package)-[dependency_relationship]->(npm_dependency:NPM:Dependency)
MATCH (npm_dependency)-[:IS_DESCRIBED_IN_NPM_PACKAGE]->(target:NPM:Package)
WHERE source <> target
AND source.name IS NOT NULL
AND target.name IS NOT NULL
WITH source
,target
,
// Weight peer dependencies as 3, regular dependencies as 2, and dev dependencies as 1.
// The highest weight is used when multiple dependency types exist for the same package pair.
max(CASE
WHEN dependency_relationship:DECLARES_PEER_DEPENDENCY THEN 3
WHEN dependency_relationship:DECLARES_DEPENDENCY THEN 2
WHEN dependency_relationship:DECLARES_DEV_DEPENDENCY THEN 1
ELSE 1
END) AS weightByDependencyType
CALL { WITH source, target, weightByDependencyType
MERGE (source)-[dependsOnRelationship:DEPENDS_ON]->(target)
SET dependsOnRelationship.weightByDependencyType = weightByDependencyType
} IN TRANSACTIONS OF 1000 ROWS
Comment thread
JohT marked this conversation as resolved.
RETURN count(*) AS numberOfWrittenRelationships
,count(DISTINCT source) AS numberOfDistinctSourcePackages
,count(DISTINCT target) AS numberOfDistinctTargetPackages
// Debugging
// RETURN source.name, target.name, weightByDependencyType
// LIMIT 10
4 changes: 3 additions & 1 deletion scripts/prepareAnalysis.sh
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,10 @@ execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Add_name_to_property_on_scan_nodes.cyph
# Preparation - Cleanup Graph for Typescript by removing duplicate relationships
execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Remove_duplicate_CONTAINS_relations_between_files.cypher"

# Preparation - Enrich Graph for Typescript by adding relationships between corresponding TS:Project and NPM:Package nodes
# Preparation - Enrich Graph for Typescript NPM data (link dependencies to packages, add package-to-package DEPENDS_ON relationships, enrich dependency counts, and link TS:Project nodes to NPM:Package nodes)
execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Link_npm_dependencies_to_npm_packages.cypher"
execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Link_npm_packages_with_depends_on_relationships.cypher"
execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Enrich_npm_packages_with_dependency_counts.cypher"
Comment thread
JohT marked this conversation as resolved.
execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Link_projects_to_npm_packages.cypher"
dataVerificationResult=$( execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Verify_projects_linked_to_npm_packages.cypher" "${@}")
if is_csv_column_greater_zero "${dataVerificationResult}" "unresolvedProjectsCount"; then
Expand Down
15 changes: 15 additions & 0 deletions scripts/reports/CentralityCsv.sh
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,21 @@ if createUndirectedDependencyProjection "${MODULE_LANGUAGE}" "${MODULE_PROJECTIO
runUndirectedCentralityAlgorithms "${MODULE_PROJECTION_UNDIRECTED}" "${MODULE_NODE}"
fi

# -- NPM Package Centrality ---------------------------------------

NPM_LANGUAGE="dependencies_projection_language=NPM"
NPM_PROJECTION="dependencies_projection=npm-package-centrality"
NPM_PROJECTION_UNDIRECTED="dependencies_projection=npm-package-centrality-undirected"
NPM_NODE="dependencies_projection_node=Package"
NPM_WEIGHT="dependencies_projection_weight_property=weightByDependencyType"

if createDirectedDependencyProjection "${NPM_LANGUAGE}" "${NPM_PROJECTION}" "${NPM_NODE}" "${NPM_WEIGHT}"; then
runCentralityAlgorithms "${NPM_PROJECTION}" "${NPM_NODE}" "${NPM_WEIGHT}"
fi
if createUndirectedDependencyProjection "${NPM_LANGUAGE}" "${NPM_PROJECTION_UNDIRECTED}" "${NPM_NODE}" "${NPM_WEIGHT}"; then
runUndirectedCentralityAlgorithms "${NPM_PROJECTION_UNDIRECTED}" "${NPM_NODE}"
fi

# ---------------------------------------------------------------

# Clean-up after report generation. Empty reports will be deleted.
Expand Down
5 changes: 5 additions & 0 deletions scripts/reports/InternalDependenciesVisualization.sh
Original file line number Diff line number Diff line change
Expand Up @@ -52,5 +52,10 @@ reportName="${FULL_REPORT_DIRECTORY}/TypeScriptModuleBuildLevels"
execute_cypher "${INTERNAL_DEPENDENCIES_CYPHER_DIR}/Typescript_Module_build_levels_for_graphviz.cypher" > "${reportName}.csv"
source "${VISUALIZATION_SCRIPTS_DIR}/visualizeQueryResults.sh" "${reportName}.csv"

# NPM Packages: Dependencies Visualization
reportName="${FULL_REPORT_DIRECTORY}/NpmPackageBuildLevels"
execute_cypher "${INTERNAL_DEPENDENCIES_CYPHER_DIR}/NPM_Package_build_levels_for_graphviz.cypher" > "${reportName}.csv"
source "${VISUALIZATION_SCRIPTS_DIR}/visualizeQueryResults.sh" "${reportName}.csv"

# Clean-up after report generation. Empty reports will be deleted.
source "${SCRIPTS_DIR}/cleanupAfterReportGeneration.sh" "${FULL_REPORT_DIRECTORY}"
10 changes: 10 additions & 0 deletions scripts/reports/PathFindingCsv.sh
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,16 @@ if createDirectedDependencyProjection "${MODULE_LANGUAGE}" "${MODULE_PROJECTION}
runPathFindingAlgorithms "${MODULE_PROJECTION}" "${MODULE_NODE}" "${MODULE_WEIGHT}"
fi

# -- NPM Package Path Finding -------------------------------

NPM_LANGUAGE="dependencies_projection_language=NPM"
NPM_PROJECTION="dependencies_projection=npm-package-path-finding"
NPM_NODE="dependencies_projection_node=Package"
NPM_WEIGHT="dependencies_projection_weight_property=weightByDependencyType"

if createDirectedDependencyProjection "${NPM_LANGUAGE}" "${NPM_PROJECTION}" "${NPM_NODE}" "${NPM_WEIGHT}"; then
runPathFindingAlgorithms "${NPM_PROJECTION}" "${NPM_NODE}" "${NPM_WEIGHT}"
fi
# ---------------------------------------------------------------

# Clean-up after report generation. Empty reports will be deleted.
Expand Down
Loading
Loading