diff --git a/cypher/GitLog/Add_CHANGED_TOGETHER_WITH_relationships_to_code_files.cypher b/cypher/GitLog/Add_CHANGED_TOGETHER_WITH_relationships_to_code_files.cypher index d9abf9bc6..fe7f505b9 100644 --- a/cypher/GitLog/Add_CHANGED_TOGETHER_WITH_relationships_to_code_files.cypher +++ b/cypher/GitLog/Add_CHANGED_TOGETHER_WITH_relationships_to_code_files.cypher @@ -4,8 +4,9 @@ MATCH (firstGitFile:Git&File&!Repository)-[gitChange:CHANGED_TOGETHER_WITH]-(sec WHERE elementId(firstGitFile) < elementId(secondGitFile) MATCH (firstGitFile)-[:RESOLVES_TO]->(firstCodeFile:File&!Git&!Repository) MATCH (secondGitFile)-[:RESOLVES_TO]->(secondCodeFile:File&!Git&!Repository) +WHERE firstGitFile <> secondGitFile CALL (firstCodeFile, secondCodeFile, gitChange) { MERGE (firstCodeFile)-[pairwiseChange:CHANGED_TOGETHER_WITH]-(secondCodeFile) SET pairwiseChange = properties(gitChange) - } IN TRANSACTIONS + } IN TRANSACTIONS OF 750 ROWS RETURN count(*) AS pairCount \ No newline at end of file diff --git a/cypher/GitLog/Add_CHANGED_TOGETHER_WITH_relationships_to_git_files.cypher b/cypher/GitLog/Add_CHANGED_TOGETHER_WITH_relationships_to_git_files.cypher index 79cad2774..8a8eecbf2 100644 --- a/cypher/GitLog/Add_CHANGED_TOGETHER_WITH_relationships_to_git_files.cypher +++ b/cypher/GitLog/Add_CHANGED_TOGETHER_WITH_relationships_to_git_files.cypher @@ -39,6 +39,7 @@ WHERE updateCommitCount > 2 WITH * ,fileCombination[0] AS firstFile ,fileCombination[1] AS secondFile + WHERE firstFile <> secondFile WITH * // Get the lowest number of git update commits of both files (file pair) ,CASE WHEN firstFile.updateCommitCount < secondFile.updateCommitCount diff --git a/cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Java.cypher b/cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Java.cypher index 804e8649d..a918497d2 100644 --- a/cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Java.cypher +++ b/cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Java.cypher @@ -20,7 +20,7 @@ WHERE gitFileName ENDS WITH codeFileName CALL { WITH git_file, code_file MERGE (git_file)-[:RESOLVES_TO]->(code_file) ON CREATE SET git_file.resolved = true - } IN TRANSACTIONS + } IN TRANSACTIONS OF 1000 ROWS RETURN count(DISTINCT codeFileName) AS numberOfCodeFiles ,collect(DISTINCT codeFileName + ' <-> ' + gitFileName + '\n')[0..4] AS examples // RETURN codeFileName, gitFileName diff --git a/cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Typescript.cypher b/cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Typescript.cypher index 82059ad04..2f5ba2aec 100644 --- a/cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Typescript.cypher +++ b/cypher/GitLog/Add_RESOLVES_TO_relationships_to_git_files_for_Typescript.cypher @@ -22,7 +22,7 @@ WHERE codeFileName ENDS WITH gitFileName CALL { WITH git_file, code_file MERGE (git_file)-[:RESOLVES_TO]->(code_file) ON CREATE SET git_file.resolved = true - } IN TRANSACTIONS + } IN TRANSACTIONS OF 1000 ROWS RETURN count(DISTINCT codeFileName) AS numberOfCodeFiles ,collect(DISTINCT codeFileName + ' <-> ' + gitFileName + '\n')[0..4] AS examples // RETURN codeFileName, gitFileName diff --git a/cypher/Internal_Dependencies/NPM_Package_build_levels_for_graphviz.cypher b/cypher/Internal_Dependencies/NPM_Package_build_levels_for_graphviz.cypher new file mode 100644 index 000000000..887c2fff1 --- /dev/null +++ b/cypher/Internal_Dependencies/NPM_Package_build_levels_for_graphviz.cypher @@ -0,0 +1,36 @@ +// List of all NPM packages and their dependencies with build levels for GraphViz Visualization + + MATCH (sourceForStatistics:NPM:Package)-[dependencyForStatistics:DEPENDS_ON]->(targetForStatistics:NPM:Package) + WHERE sourceForStatistics.maxDistanceFromSource IS NOT NULL + AND targetForStatistics.maxDistanceFromSource IS NOT NULL + WITH min(dependencyForStatistics.weightByDependencyType) AS minWeight + ,max(dependencyForStatistics.weightByDependencyType) AS maxWeight + ,max(targetForStatistics.maxDistanceFromSource) AS maxLevel + MATCH (source:NPM:Package)-[dependency:DEPENDS_ON]->(target:NPM:Package) + WHERE source.maxDistanceFromSource IS NOT NULL + AND target.maxDistanceFromSource IS NOT NULL + WITH *, CASE + WHEN maxWeight = minWeight THEN 0.0 + ELSE toFloat(dependency.weightByDependencyType - minWeight) / toFloat(maxWeight - minWeight) + END AS normalizedWeight + WITH *, round((normalizedWeight * 5) + 1, 2) AS penWidth + WITH *, "\\n(level " + coalesce(source.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS sourceLevelInfo + WITH *, "\\n(level " + coalesce(target.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS targetLevelInfo + WITH *, source.name + sourceLevelInfo AS fullSourceName + WITH *, target.name + targetLevelInfo AS fullTargetName + WITH *, "\" -> \"" + fullTargetName + + "\" [label = " + dependency.weightByDependencyType + ";" + + " penwidth = " + penWidth + ";" + + " ];" AS graphVizDotNotationEdge + WITH *, "\"" + fullSourceName + coalesce(graphVizDotNotationEdge, "\" [];") AS graphVizDotNotationLine + ORDER BY dependency.weightByDependencyType DESC, target.maxDistanceFromSource DESC +RETURN graphVizDotNotationLine + //Debugging + //,source.name AS sourceName + //,target.name AS targetName + //,penWidth + //,normalizedWeight + //,dependency.weightByDependencyType AS weight + //,minWeight + //,maxWeight +LIMIT 440 \ No newline at end of file diff --git a/cypher/Internal_Dependencies/Typescript_Module_build_levels_for_graphviz.cypher b/cypher/Internal_Dependencies/Typescript_Module_build_levels_for_graphviz.cypher index 667b7e883..262067d47 100644 --- a/cypher/Internal_Dependencies/Typescript_Module_build_levels_for_graphviz.cypher +++ b/cypher/Internal_Dependencies/Typescript_Module_build_levels_for_graphviz.cypher @@ -6,13 +6,15 @@ WITH min(dependencyForStatistics.weight) AS minWeight ,max(dependencyForStatistics.weight) AS maxWeight ,max(targetForStatistics.maxDistanceFromSource) AS maxLevel -MATCH (source:TS:Module)-[dependency:DEPENDS_ON]->(target:TS:Module) + MATCH (source:TS:Module)-[dependency:DEPENDS_ON]->(target:TS:Module) WHERE source.maxDistanceFromSource IS NOT NULL AND target.maxDistanceFromSource IS NOT NULL WITH *, toFloat(dependency.cardinality - minWeight) / toFloat(maxWeight - minWeight) AS normalizedWeight WITH *, round((normalizedWeight * 5) + 1, 2) AS penWidth - WITH *, source.rootProjectName + "\\n" + source.name + "\\n(level " + coalesce(source.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS fullSourceName - WITH *, target.rootProjectName + "\\n" + target.name + "\\n(level " + coalesce(target.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS fullTargetName + WITH *, "\\n(level " + coalesce(source.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS sourceLevelInfo + WITH *, "\\n(level " + coalesce(target.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS targetLevelInfo + WITH *, source.rootProjectName + "\\n" + source.name + sourceLevelInfo AS fullSourceName + WITH *, target.rootProjectName + "\\n" + target.name + targetLevelInfo AS fullTargetName WITH *, "\" -> \"" + fullTargetName + "\" [label = " + dependency.cardinality + ";" + " penwidth = " + penWidth + ";" diff --git a/cypher/Path_Finding/Path_Finding_6_Longest_paths_contributors_for_graphviz.cypher b/cypher/Path_Finding/Path_Finding_6_Longest_paths_contributors_for_graphviz.cypher index 81765164a..4a226c3eb 100644 --- a/cypher/Path_Finding/Path_Finding_6_Longest_paths_contributors_for_graphviz.cypher +++ b/cypher/Path_Finding/Path_Finding_6_Longest_paths_contributors_for_graphviz.cypher @@ -1,4 +1,4 @@ -// Path Finding - Longest path - Stream - List all dependencies for nodes contributing to longest paths and highlight those paths in the Visualization with GraphViz. +// Path Finding - Longest path - Stream - List all dependencies for nodes contributing to longest paths and highlight those paths in the Visualization with GraphViz. Recommended prerequisite: Topological_Sort_Write.cypher // Gather global statistics about dependency weights and levels for normalization and node details MATCH (sourceNodeForStatistics)-[dependencyForStatistics:DEPENDS_ON]->(targetNodeForStatistics) @@ -55,8 +55,12 @@ WITH *, dependency[$dependencies_projection_weight_property] AS weight WITH *, toFloat(weight - minWeight) * weightNormalizationFactor AS normalizedWeight WITH *, round((normalizedWeight * 5) + 1, 2) AS penWidth - WITH *, source.name + "\\n(level " + source.maxDistanceFromSource + "/" + maxLevel + ")" AS startNodeTitle - WITH *, target.name + "\\n(level " + target.maxDistanceFromSource + "/" + maxLevel + ")" AS endNodeTitle + WITH *, coalesce("\\n(level " + source.maxDistanceFromSource + "/" + maxLevel + ")", "") AS startNodeLevelInfo + WITH *, coalesce("\\n" + source.rootProjectName, "") AS startNodeProjectInfo + WITH *, coalesce("\\n(level " + target.maxDistanceFromSource + "/" + maxLevel + ")", "") AS endNodeLevelInfo + WITH *, coalesce("\\n" + target.rootProjectName, "") AS endNodeProjectInfo + WITH *, source.name + startNodeProjectInfo + startNodeLevelInfo AS startNodeTitle + WITH *, target.name + endNodeProjectInfo + endNodeLevelInfo AS endNodeTitle // The longest path will be highlighted in red. WITH *, CASE WHEN isPartOfLongestPath THEN "; color=\"red\"" // Dependencies contributing to the longest path will be highlighted in dark orange. diff --git a/cypher/Path_Finding/Path_Finding_6_Longest_paths_for_graphviz.cypher b/cypher/Path_Finding/Path_Finding_6_Longest_paths_for_graphviz.cypher index 4f1152733..b9ae47ad5 100644 --- a/cypher/Path_Finding/Path_Finding_6_Longest_paths_for_graphviz.cypher +++ b/cypher/Path_Finding/Path_Finding_6_Longest_paths_for_graphviz.cypher @@ -1,4 +1,4 @@ -// Path Finding - Longest path - Stream - Find the top 100 dependencies contributing to the longest paths for Visualization with GraphViz +// Path Finding - Longest path - Stream - Find the top 100 dependencies contributing to the longest paths for Visualization with GraphViz. Recommended prerequisite: Topological_Sort_Write.cypher MATCH (sourceNodeForStatistics)-[dependencyForStatistics:DEPENDS_ON]->(targetNodeForStatistics) WHERE $dependencies_projection_node IN LABELS(sourceNodeForStatistics) @@ -19,8 +19,12 @@ WITH *, dependency[$dependencies_projection_weight_property] AS weight WITH *, toFloat(weight - minWeight) * weightNormalizationFactor AS normalizedWeight WITH *, round((normalizedWeight * 5) + 1, 2) AS penWidth - WITH *, startNode.name + "\\n(level " + startNode.maxDistanceFromSource + "/" + maxLevel + ")" AS startNodeTitle - WITH *, endNode.name + "\\n(level " + endNode.maxDistanceFromSource + "/" + maxLevel + ")" AS endNodeTitle + WITH *, coalesce("\\n(level " + startNode.maxDistanceFromSource + "/" + maxLevel + ")", "") AS startNodeLevelInfo + WITH *, coalesce("\\n" + startNode.rootProjectName, "") AS startNodeProjectInfo + WITH *, coalesce("\\n(level " + endNode.maxDistanceFromSource + "/" + maxLevel + ")", "") AS endNodeLevelInfo + WITH *, coalesce("\\n" + endNode.rootProjectName, "") AS endNodeProjectInfo + WITH *, startNode.name + startNodeProjectInfo + startNodeLevelInfo AS startNodeTitle + WITH *, endNode.name + endNodeProjectInfo + endNodeLevelInfo AS endNodeTitle WITH *, "[label=" + weight + "; penwidth=" + penWidth + "; ];" AS graphVizEdgeAttributes WITH *, "\"" + startNodeTitle + "\" -> \"" + endNodeTitle + "\" " + graphVizEdgeAttributes AS graphVizDotNotationLine RETURN graphVizDotNotationLine diff --git a/cypher/Path_Finding/Set_Parameters_NPM.cypher b/cypher/Path_Finding/Set_Parameters_NPM.cypher new file mode 100644 index 000000000..8ca29d24a --- /dev/null +++ b/cypher/Path_Finding/Set_Parameters_NPM.cypher @@ -0,0 +1,8 @@ +// Example on how to set the parameters for path finding in this case for NPM Packages + +:params { + "dependencies_projection_language":"NPM", + "dependencies_projection": "npm-package-path-finding", + "dependencies_projection_node": "Package", + "dependencies_projection_weight_property": "weightByDependencyType", +} \ No newline at end of file diff --git a/cypher/Path_Finding/Set_Parameters_Typescript_Module.cypher b/cypher/Path_Finding/Set_Parameters_Typescript_Module.cypher new file mode 100644 index 000000000..0d2ac9fc3 --- /dev/null +++ b/cypher/Path_Finding/Set_Parameters_Typescript_Module.cypher @@ -0,0 +1,8 @@ +// Example on how to set the parameters for path finding in this case for TypeScript modules + +:params { + "dependencies_projection_language":"Typescript", + "dependencies_projection": "typescript-module-path-finding", + "dependencies_projection_node": "Module", + "dependencies_projection_weight_property": "lowCouplingElement25PercentWeight", +} \ No newline at end of file diff --git a/cypher/Topological_Sort/Topological_Sort_Exists.cypher b/cypher/Topological_Sort/Topological_Sort_Exists.cypher new file mode 100644 index 000000000..0532dd54f --- /dev/null +++ b/cypher/Topological_Sort/Topological_Sort_Exists.cypher @@ -0,0 +1,10 @@ +// Return the first node with a "maxDistanceFromSource" if it exists + +MATCH (codeUnit) + WHERE codeUnit.maxDistanceFromSource IS NOT NULL + AND codeUnit.topologicalSortIndex IS NOT NULL + AND $dependencies_projection_node IN LABELS(codeUnit) +RETURN codeUnit.name AS shortCodeUnitName + ,elementId(codeUnit) AS nodeElementId + ,codeUnit.maxDistanceFromSource AS maxDistanceFromSource +LIMIT 1 \ No newline at end of file diff --git a/cypher/Typescript_Enrichment/Add_DEPENDS_ON_relationship_to_resolved_modules.cypher b/cypher/Typescript_Enrichment/Add_DEPENDS_ON_relationship_to_resolved_modules.cypher index de9170b17..de104f0de 100644 --- a/cypher/Typescript_Enrichment/Add_DEPENDS_ON_relationship_to_resolved_modules.cypher +++ b/cypher/Typescript_Enrichment/Add_DEPENDS_ON_relationship_to_resolved_modules.cypher @@ -12,5 +12,5 @@ ON MATCH SET resolvedDependsOn = dependsOn // Overwrites existing properties ,resolvedDependsOn.cardinality = existingDependency.cardinality + dependsOn.cardinality // Add cardinalities ,resolvedDependsOn.updated = true - } IN TRANSACTIONS + } IN TRANSACTIONS OF 1000 ROWS RETURN count(*) as resolvedDependencies \ No newline at end of file diff --git a/cypher/Typescript_Enrichment/Add_IS_IMPLEMENTED_IN_relationship_for_matching_declarations.cypher b/cypher/Typescript_Enrichment/Add_IS_IMPLEMENTED_IN_relationship_for_matching_declarations.cypher index d650d4591..68f88dd13 100644 --- a/cypher/Typescript_Enrichment/Add_IS_IMPLEMENTED_IN_relationship_for_matching_declarations.cypher +++ b/cypher/Typescript_Enrichment/Add_IS_IMPLEMENTED_IN_relationship_for_matching_declarations.cypher @@ -7,7 +7,7 @@ WITH externalDeclaration, internalDeclaration CALL { WITH externalDeclaration, internalDeclaration MERGE (externalDeclaration)-[:IS_IMPLEMENTED_IN]->(internalDeclaration) - } IN TRANSACTIONS + } IN TRANSACTIONS OF 1000 ROWS RETURN count( DISTINCT externalDeclaration.globalFqn + ' -> ' + internalDeclaration.globalFqn) AS linkedDeclarationCount ,collect(DISTINCT externalDeclaration.globalFqn + ' -> ' + internalDeclaration.globalFqn)[0..4] AS linkedDeclarationExamples //Debugging diff --git a/cypher/Typescript_Enrichment/Add_IS_IMPLEMENTED_IN_relationship_for_matching_modules.cypher b/cypher/Typescript_Enrichment/Add_IS_IMPLEMENTED_IN_relationship_for_matching_modules.cypher index a8dc829e7..cf3cf395f 100644 --- a/cypher/Typescript_Enrichment/Add_IS_IMPLEMENTED_IN_relationship_for_matching_modules.cypher +++ b/cypher/Typescript_Enrichment/Add_IS_IMPLEMENTED_IN_relationship_for_matching_modules.cypher @@ -48,7 +48,7 @@ WHERE equalGlobalFqn OR equalNameAndNpmPackage CALL { WITH module, externalModule MERGE (externalModule)-[:IS_IMPLEMENTED_IN]->(module) - } IN TRANSACTIONS + } IN TRANSACTIONS OF 1000 ROWS RETURN CASE WHEN equalGlobalFqn THEN 'equalGlobalFqn' WHEN equalModule THEN 'equalModule' WHEN equalNameWithoutNamespace THEN 'equalNameWithoutNamespace' diff --git a/cypher/Typescript_Enrichment/Enrich_npm_packages_with_dependency_counts.cypher b/cypher/Typescript_Enrichment/Enrich_npm_packages_with_dependency_counts.cypher new file mode 100644 index 000000000..f640093b9 --- /dev/null +++ b/cypher/Typescript_Enrichment/Enrich_npm_packages_with_dependency_counts.cypher @@ -0,0 +1,20 @@ +// Enrich NPM:Package nodes with incoming and outgoing dependency counts +// Requires Link_npm_packages_with_depends_on_relationships.cypher + +MATCH (package:NPM:Package) +WITH package, + // Private packages often contain test and example code. They will be left out of the main dependency graph analysis. + CASE WHEN package.private = true OR package.private = "true" THEN 1 ELSE 0 END AS testMarkerInteger, + COUNT {(package)<-[:DEPENDS_ON]-(:NPM:Package)} AS incomingDependencies, + COUNT {(package)-[:DEPENDS_ON]->(:NPM:Package)} AS outgoingDependencies +SET package.incomingDependencies = incomingDependencies, + package.outgoingDependencies = outgoingDependencies, + package.testMarkerInteger = testMarkerInteger +RETURN count(DISTINCT package) AS numberOfPackagesEnriched + ,sum(incomingDependencies) AS totalIncomingDependencies + ,sum(outgoingDependencies) AS totalOutgoingDependencies + ,avg(incomingDependencies) AS averageIncomingDependencies + ,avg(outgoingDependencies) AS averageOutgoingDependencies +// Debugging +// RETURN package.name, package.incomingDependencies, package.outgoingDependencies +// LIMIT 20 diff --git a/cypher/Typescript_Enrichment/Link_external_modules_to_corresponding_npm_dependency.cypher b/cypher/Typescript_Enrichment/Link_external_modules_to_corresponding_npm_dependency.cypher index 1fa5a29c3..eec7630c8 100644 --- a/cypher/Typescript_Enrichment/Link_external_modules_to_corresponding_npm_dependency.cypher +++ b/cypher/Typescript_Enrichment/Link_external_modules_to_corresponding_npm_dependency.cypher @@ -11,7 +11,7 @@ UNWIND external_modules AS external_module MERGE (external_module)-[:PROVIDED_BY_NPM_DEPENDENCY]->(npm_dependency) SET external_module.npmPackage = npm_dependency.name ,external_module.npmPackageVersion = npm_dependency.dependency - } IN TRANSACTIONS + } IN TRANSACTIONS OF 1000 ROWS RETURN totalNumberOfExternalModules ,count(DISTINCT external_module.globalFqn) AS numberOfLinkedExternalModules ,count(DISTINCT project.name) AS numberOfProjects diff --git a/cypher/Typescript_Enrichment/Link_npm_dependencies_to_npm_packages.cypher b/cypher/Typescript_Enrichment/Link_npm_dependencies_to_npm_packages.cypher index 080b0fb93..abd1f694a 100644 --- a/cypher/Typescript_Enrichment/Link_npm_dependencies_to_npm_packages.cypher +++ b/cypher/Typescript_Enrichment/Link_npm_dependencies_to_npm_packages.cypher @@ -1,12 +1,14 @@ -// Link npm dependencies to the npm package that describe them if it exists +// Link npm dependencies to the npm package that describes them, if it exists MATCH (npm_dependency:NPM:Dependency) MATCH (npm_package:NPM:Package) WHERE npm_package.name = npm_dependency.name AND npm_package <> npm_dependency + AND NOT npm_package.name CONTAINS '{{' + AND NOT npm_package.name CONTAINS '}}' CALL { WITH npm_package, npm_dependency MERGE (npm_dependency)-[:IS_DESCRIBED_IN_NPM_PACKAGE]->(npm_package) - } IN TRANSACTIONS + } IN TRANSACTIONS OF 1000 ROWS RETURN count(*) AS numberOfWrittenRelationships ,count(DISTINCT npm_dependency) AS numberOfDistinctNpmDependencies ,count(DISTINCT npm_package) AS numberOfDistinctNpmPackages diff --git a/cypher/Typescript_Enrichment/Link_npm_packages_with_depends_on_relationships.cypher b/cypher/Typescript_Enrichment/Link_npm_packages_with_depends_on_relationships.cypher new file mode 100644 index 000000000..4fdf1dbc2 --- /dev/null +++ b/cypher/Typescript_Enrichment/Link_npm_packages_with_depends_on_relationships.cypher @@ -0,0 +1,30 @@ +// Link npm packages with DEPENDS_ON relationships. Requires Link_npm_dependencies_to_npm_packages. +// This creates direct package-to-package dependencies by following the chain: +// (source:NPM:Package)-[dependency_relationship]->(NPM:Dependency)-[:IS_DESCRIBED_IN_NPM_PACKAGE]->(target:NPM:Package) + +MATCH (source:NPM:Package)-[dependency_relationship]->(npm_dependency:NPM:Dependency) +MATCH (npm_dependency)-[:IS_DESCRIBED_IN_NPM_PACKAGE]->(target:NPM:Package) +WHERE source <> target + AND source.name IS NOT NULL + AND target.name IS NOT NULL +WITH source + ,target + , + // Weight peer dependencies as 3, regular dependencies as 2, and dev dependencies as 1. + // The highest weight is used when multiple dependency types exist for the same package pair. + max(CASE + WHEN dependency_relationship:DECLARES_PEER_DEPENDENCY THEN 3 + WHEN dependency_relationship:DECLARES_DEPENDENCY THEN 2 + WHEN dependency_relationship:DECLARES_DEV_DEPENDENCY THEN 1 + ELSE 1 + END) AS weightByDependencyType + CALL { WITH source, target, weightByDependencyType + MERGE (source)-[dependsOnRelationship:DEPENDS_ON]->(target) + SET dependsOnRelationship.weightByDependencyType = weightByDependencyType + } IN TRANSACTIONS OF 1000 ROWS +RETURN count(*) AS numberOfWrittenRelationships + ,count(DISTINCT source) AS numberOfDistinctSourcePackages + ,count(DISTINCT target) AS numberOfDistinctTargetPackages +// Debugging +// RETURN source.name, target.name, weightByDependencyType +// LIMIT 10 diff --git a/scripts/prepareAnalysis.sh b/scripts/prepareAnalysis.sh index 4c972cef2..e623b26c7 100644 --- a/scripts/prepareAnalysis.sh +++ b/scripts/prepareAnalysis.sh @@ -75,8 +75,10 @@ execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Add_name_to_property_on_scan_nodes.cyph # Preparation - Cleanup Graph for Typescript by removing duplicate relationships execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Remove_duplicate_CONTAINS_relations_between_files.cypher" -# Preparation - Enrich Graph for Typescript by adding relationships between corresponding TS:Project and NPM:Package nodes +# Preparation - Enrich Graph for Typescript NPM data (link dependencies to packages, add package-to-package DEPENDS_ON relationships, enrich dependency counts, and link TS:Project nodes to NPM:Package nodes) execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Link_npm_dependencies_to_npm_packages.cypher" +execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Link_npm_packages_with_depends_on_relationships.cypher" +execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Enrich_npm_packages_with_dependency_counts.cypher" execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Link_projects_to_npm_packages.cypher" dataVerificationResult=$( execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Verify_projects_linked_to_npm_packages.cypher" "${@}") if is_csv_column_greater_zero "${dataVerificationResult}" "unresolvedProjectsCount"; then diff --git a/scripts/reports/CentralityCsv.sh b/scripts/reports/CentralityCsv.sh index cb9315fbd..6113ce049 100755 --- a/scripts/reports/CentralityCsv.sh +++ b/scripts/reports/CentralityCsv.sh @@ -439,6 +439,21 @@ if createUndirectedDependencyProjection "${MODULE_LANGUAGE}" "${MODULE_PROJECTIO runUndirectedCentralityAlgorithms "${MODULE_PROJECTION_UNDIRECTED}" "${MODULE_NODE}" fi +# -- NPM Package Centrality --------------------------------------- + +NPM_LANGUAGE="dependencies_projection_language=NPM" +NPM_PROJECTION="dependencies_projection=npm-package-centrality" +NPM_PROJECTION_UNDIRECTED="dependencies_projection=npm-package-centrality-undirected" +NPM_NODE="dependencies_projection_node=Package" +NPM_WEIGHT="dependencies_projection_weight_property=weightByDependencyType" + +if createDirectedDependencyProjection "${NPM_LANGUAGE}" "${NPM_PROJECTION}" "${NPM_NODE}" "${NPM_WEIGHT}"; then + runCentralityAlgorithms "${NPM_PROJECTION}" "${NPM_NODE}" "${NPM_WEIGHT}" +fi +if createUndirectedDependencyProjection "${NPM_LANGUAGE}" "${NPM_PROJECTION_UNDIRECTED}" "${NPM_NODE}" "${NPM_WEIGHT}"; then + runUndirectedCentralityAlgorithms "${NPM_PROJECTION_UNDIRECTED}" "${NPM_NODE}" +fi + # --------------------------------------------------------------- # Clean-up after report generation. Empty reports will be deleted. diff --git a/scripts/reports/InternalDependenciesVisualization.sh b/scripts/reports/InternalDependenciesVisualization.sh index a6a358d49..f899e24db 100755 --- a/scripts/reports/InternalDependenciesVisualization.sh +++ b/scripts/reports/InternalDependenciesVisualization.sh @@ -52,5 +52,10 @@ reportName="${FULL_REPORT_DIRECTORY}/TypeScriptModuleBuildLevels" execute_cypher "${INTERNAL_DEPENDENCIES_CYPHER_DIR}/Typescript_Module_build_levels_for_graphviz.cypher" > "${reportName}.csv" source "${VISUALIZATION_SCRIPTS_DIR}/visualizeQueryResults.sh" "${reportName}.csv" +# NPM Packages: Dependencies Visualization +reportName="${FULL_REPORT_DIRECTORY}/NpmPackageBuildLevels" +execute_cypher "${INTERNAL_DEPENDENCIES_CYPHER_DIR}/NPM_Package_build_levels_for_graphviz.cypher" > "${reportName}.csv" +source "${VISUALIZATION_SCRIPTS_DIR}/visualizeQueryResults.sh" "${reportName}.csv" + # Clean-up after report generation. Empty reports will be deleted. source "${SCRIPTS_DIR}/cleanupAfterReportGeneration.sh" "${FULL_REPORT_DIRECTORY}" \ No newline at end of file diff --git a/scripts/reports/PathFindingCsv.sh b/scripts/reports/PathFindingCsv.sh index ebe8aa493..3359641b6 100755 --- a/scripts/reports/PathFindingCsv.sh +++ b/scripts/reports/PathFindingCsv.sh @@ -140,6 +140,16 @@ if createDirectedDependencyProjection "${MODULE_LANGUAGE}" "${MODULE_PROJECTION} runPathFindingAlgorithms "${MODULE_PROJECTION}" "${MODULE_NODE}" "${MODULE_WEIGHT}" fi +# -- NPM Package Path Finding ------------------------------- + +NPM_LANGUAGE="dependencies_projection_language=NPM" +NPM_PROJECTION="dependencies_projection=npm-package-path-finding" +NPM_NODE="dependencies_projection_node=Package" +NPM_WEIGHT="dependencies_projection_weight_property=weightByDependencyType" + +if createDirectedDependencyProjection "${NPM_LANGUAGE}" "${NPM_PROJECTION}" "${NPM_NODE}" "${NPM_WEIGHT}"; then + runPathFindingAlgorithms "${NPM_PROJECTION}" "${NPM_NODE}" "${NPM_WEIGHT}" +fi # --------------------------------------------------------------- # Clean-up after report generation. Empty reports will be deleted. diff --git a/scripts/reports/PathFindingVisualization.sh b/scripts/reports/PathFindingVisualization.sh index 6fe72e609..dd0ff8afb 100755 --- a/scripts/reports/PathFindingVisualization.sh +++ b/scripts/reports/PathFindingVisualization.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # Executes selected "Path_Finding" Cypher queries for GraphViz visualization. -# Visualizes Java Artifact and TypeScript Module dependencies with their longest paths. +# Visualizes Java Artifact, TypeScript Module and NPM Package dependencies with their longest paths. +# # It requires an already running Neo4j graph database with already scanned and analyzed artifacts. # The reports (csv, dot and svg files) will be written into the sub directory reports/path-finding-visualization. @@ -12,29 +13,30 @@ set -o errexit -o pipefail # Overrideable Constants (defaults also defined in sub scripts) REPORTS_DIRECTORY=${REPORTS_DIRECTORY:-"reports"} - +SCRIPT_NAME="PathFindingVisualization" ## Get this "scripts/reports" directory if not already set # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes. # This way non-standard tools like readlink aren't needed. REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} -echo "PathFindingVisualization: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}" +echo "${SCRIPT_NAME}: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}" # Get the "scripts" directory by taking the path of this script and going one directory up. SCRIPTS_DIR=${SCRIPTS_DIR:-"${REPORTS_SCRIPT_DIR}/.."} # Repository directory containing the shell scripts -echo "PathFindingVisualization SCRIPTS_DIR=${SCRIPTS_DIR}" +echo "${SCRIPT_NAME}: SCRIPTS_DIR=${SCRIPTS_DIR}" # Get the "scripts/visualization" directory. VISUALIZATION_SCRIPTS_DIR=${VISUALIZATION_SCRIPTS_DIR:-"${SCRIPTS_DIR}/visualization"} # Repository directory containing the shell scripts for visualization -echo "PathFindingVisualization VISUALIZATION_SCRIPTS_DIR=${VISUALIZATION_SCRIPTS_DIR}" +echo "${SCRIPT_NAME}: VISUALIZATION_SCRIPTS_DIR=${VISUALIZATION_SCRIPTS_DIR}" # Get the "cypher" directory by taking the path of this script and going two directory up and then to "cypher". CYPHER_DIR=${CYPHER_DIR:-"${REPORTS_SCRIPT_DIR}/../../cypher"} -echo "PathFindingVisualization CYPHER_DIR=${CYPHER_DIR}" +echo "${SCRIPT_NAME}: CYPHER_DIR=${CYPHER_DIR}" PATH_FINDINGS_CYPHER_DIR="${CYPHER_DIR}/Path_Finding" +TOPOLOGICAL_SORT_CYPHER_DIR="${CYPHER_DIR}/Topological_Sort" -# Define functions to execute cypher queries from within a given file +# Define functions to execute cypher queries from within a given file like execute_cypher and execute_cypher_queries_until_results source "${SCRIPTS_DIR}/executeQueryFunctions.sh" # Define functions to create and delete Graph Projections like "createDirectedDependencyProjection" @@ -51,13 +53,17 @@ ARTIFACT_NODE="dependencies_projection_node=Artifact" ARTIFACT_WEIGHT="dependencies_projection_weight_property=weight" if createDirectedDependencyProjection "${ARTIFACT_PROJECTION}" "${ARTIFACT_NODE}" "${ARTIFACT_WEIGHT}"; then + # Determines topological sort max distance from source if not already done for level info in visualization. + execute_cypher_queries_until_results "${TOPOLOGICAL_SORT_CYPHER_DIR}/Topological_Sort_Exists.cypher" \ + "${TOPOLOGICAL_SORT_CYPHER_DIR}/Topological_Sort_Write.cypher" "${ARTIFACT_PROJECTION}" "${ARTIFACT_NODE}" + reportName="JavaArtifactLongestPathsIsolated" - echo "PathFindingVisualization: Creating visualization ${reportName}..." + echo "${SCRIPT_NAME}: Creating visualization ${reportName}..." execute_cypher "${PATH_FINDINGS_CYPHER_DIR}/Path_Finding_6_Longest_paths_for_graphviz.cypher" "${ARTIFACT_PROJECTION}" "${ARTIFACT_NODE}" "${ARTIFACT_WEIGHT}" > "${FULL_REPORT_DIRECTORY}/${reportName}.csv" source "${VISUALIZATION_SCRIPTS_DIR}/visualizeQueryResults.sh" "${FULL_REPORT_DIRECTORY}/${reportName}.csv" reportName="JavaArtifactLongestPaths" - echo "PathFindingVisualization: Creating visualization ${reportName}..." + echo "${SCRIPT_NAME}: Creating visualization ${reportName}..." execute_cypher "${PATH_FINDINGS_CYPHER_DIR}/Path_Finding_6_Longest_paths_contributors_for_graphviz.cypher" "${ARTIFACT_PROJECTION}" "${ARTIFACT_NODE}" "${ARTIFACT_WEIGHT}" > "${FULL_REPORT_DIRECTORY}/${reportName}.csv" source "${VISUALIZATION_SCRIPTS_DIR}/visualizeQueryResults.sh" "${FULL_REPORT_DIRECTORY}/${reportName}.csv" fi @@ -69,16 +75,42 @@ MODULE_NODE="dependencies_projection_node=Module" MODULE_WEIGHT="dependencies_projection_weight_property=lowCouplingElement25PercentWeight" if createDirectedDependencyProjection "${MODULE_LANGUAGE}" "${MODULE_PROJECTION}" "${MODULE_NODE}" "${MODULE_WEIGHT}"; then + # Determines topological sort max distance from source if not already done for level info in visualization. + execute_cypher_queries_until_results "${TOPOLOGICAL_SORT_CYPHER_DIR}/Topological_Sort_Exists.cypher" \ + "${TOPOLOGICAL_SORT_CYPHER_DIR}/Topological_Sort_Write.cypher" "${MODULE_PROJECTION}" "${MODULE_NODE}" + reportName="TypeScriptModuleLongestPathsIsolated" - echo "PathFindingVisualization: Creating visualization ${reportName}..." + echo "${SCRIPT_NAME}: Creating visualization ${reportName}..." execute_cypher "${PATH_FINDINGS_CYPHER_DIR}/Path_Finding_6_Longest_paths_for_graphviz.cypher" "${MODULE_PROJECTION}" "${MODULE_NODE}" "${MODULE_WEIGHT}" > "${FULL_REPORT_DIRECTORY}/${reportName}.csv" source "${VISUALIZATION_SCRIPTS_DIR}/visualizeQueryResults.sh" "${FULL_REPORT_DIRECTORY}/${reportName}.csv" reportName="TypeScriptModuleLongestPaths" - echo "PathFindingVisualization: Creating visualization ${reportName}..." + echo "${SCRIPT_NAME}: Creating visualization ${reportName}..." execute_cypher "${PATH_FINDINGS_CYPHER_DIR}/Path_Finding_6_Longest_paths_contributors_for_graphviz.cypher" "${MODULE_PROJECTION}" "${MODULE_NODE}" "${MODULE_WEIGHT}" > "${FULL_REPORT_DIRECTORY}/${reportName}.csv" source "${VISUALIZATION_SCRIPTS_DIR}/visualizeQueryResults.sh" "${FULL_REPORT_DIRECTORY}/${reportName}.csv" fi +# NPM Packages: Longest Paths Visualization +NPM_LANGUAGE="dependencies_projection_language=NPM" +NPM_PROJECTION="dependencies_projection=npm-package-path-finding" +NPM_NODE="dependencies_projection_node=Package" +NPM_WEIGHT="dependencies_projection_weight_property=weightByDependencyType" + +if createDirectedDependencyProjection "${NPM_LANGUAGE}" "${NPM_PROJECTION}" "${NPM_NODE}" "${NPM_WEIGHT}"; then + # Determines topological sort max distance from source if not already done for level info in visualization. + execute_cypher_queries_until_results "${TOPOLOGICAL_SORT_CYPHER_DIR}/Topological_Sort_Exists.cypher" \ + "${TOPOLOGICAL_SORT_CYPHER_DIR}/Topological_Sort_Write.cypher" "${NPM_PROJECTION}" "${NPM_NODE}" + + reportName="NpmPackageLongestPathsIsolated" + echo "${SCRIPT_NAME}: Creating visualization ${reportName}..." + execute_cypher "${PATH_FINDINGS_CYPHER_DIR}/Path_Finding_6_Longest_paths_for_graphviz.cypher" "${NPM_PROJECTION}" "${NPM_NODE}" "${NPM_WEIGHT}" > "${FULL_REPORT_DIRECTORY}/${reportName}.csv" + source "${VISUALIZATION_SCRIPTS_DIR}/visualizeQueryResults.sh" "${FULL_REPORT_DIRECTORY}/${reportName}.csv" + + reportName="NpmPackageLongestPaths" + echo "${SCRIPT_NAME}: Creating visualization ${reportName}..." + execute_cypher "${PATH_FINDINGS_CYPHER_DIR}/Path_Finding_6_Longest_paths_contributors_for_graphviz.cypher" "${NPM_PROJECTION}" "${NPM_NODE}" "${NPM_WEIGHT}" > "${FULL_REPORT_DIRECTORY}/${reportName}.csv" + source "${VISUALIZATION_SCRIPTS_DIR}/visualizeQueryResults.sh" "${FULL_REPORT_DIRECTORY}/${reportName}.csv" +fi + # Clean-up after report generation. Empty reports will be deleted. source "${SCRIPTS_DIR}/cleanupAfterReportGeneration.sh" "${FULL_REPORT_DIRECTORY}" \ No newline at end of file diff --git a/scripts/reports/TopologicalSortCsv.sh b/scripts/reports/TopologicalSortCsv.sh index 7e0ed8a5b..be67cc528 100755 --- a/scripts/reports/TopologicalSortCsv.sh +++ b/scripts/reports/TopologicalSortCsv.sh @@ -103,7 +103,16 @@ if createDirectedDependencyProjection "${MODULE_LANGUAGE}" "${MODULE_PROJECTION} time topologicalSort "${MODULE_PROJECTION}" "${MODULE_NODE}" "${MODULE_WEIGHT}" fi -# --------------------------------------------------------------- +# -- NPM Package Topology --------------------------------------- + +NPM_LANGUAGE="dependencies_projection_language=NPM" +NPM_PROJECTION="dependencies_projection=npm-package-topology" +NPM_NODE="dependencies_projection_node=Package" +NPM_WEIGHT="dependencies_projection_weight_property=weightByDependencyType" + +if createDirectedDependencyProjection "${NPM_LANGUAGE}" "${NPM_PROJECTION}" "${NPM_NODE}" "${NPM_WEIGHT}"; then + time topologicalSort "${NPM_PROJECTION}" "${NPM_NODE}" "${NPM_WEIGHT}" +fi # Clean-up after report generation. Empty reports will be deleted. source "${SCRIPTS_DIR}/cleanupAfterReportGeneration.sh" "${FULL_REPORT_DIRECTORY}"