Merge pull request #547 from JohT/feature/separate-npm-dev-dependencies

Distinguish between dev and non dev NPM package dependencies

Author: JohT

COMMANDS.md

@@ -76,10 +76,12 @@ The [analyze.sh](./scripts/analysis/analyze.sh) command comes with these command
 
 - `--profile Neo4jv5` uses the older long term support (march 2025) version v5.26.x of Neo4j and suitable compatible versions of plugins and JQAssistant. Without specifying a profile, the newest versions will be used. Other profiles can be found in the directory [scripts/profiles](./scripts/profiles/).
 
-- `--profile Neo4j-latest-continue-on-scan-errors` is based on the default profile (`Neo4j-latest`) but uses the jQAssistant configuration template [template-neo4j-remote-jqassistant-continue-on-error.yaml](./scripts/configuration/template-neo4j-remote-jqassistant-continue-on-error.yaml) to continue on scan error instead of failing fast. This is temporarily useful when there is a known error that needs to be ignored. It is still recommended to use the default profile and fail fast if there is something wrong. Other profiles can be found in the directory [scripts/profiles](./scripts/profiles/).
+- `--profile Neo4j-latest-continue-on-scan-errors` is based on the default profile (`Neo4j-latest`) but uses the jQAssistant configuration template [template-neo4jv5-jqassistant-continue-on-error.yaml](./scripts/configuration/template-neo4jv5-jqassistant-continue-on-error.yaml) to continue on scan error instead of failing fast. This is temporarily useful when there is a known error that needs to be ignored. It is still recommended to use the default profile and fail fast if there is something wrong. Other profiles can be found in the directory [scripts/profiles](./scripts/profiles/).
 
 - `--profile Neo4j-latest-low-memory` is based on the default profile (`Neo4j-latest`) but uses only half of the memory (RAM) as configured in [template-neo4j-low-memory.conf](./scripts/configuration/template-neo4j-low-memory.conf). This is useful for the analysis of smaller codebases with less resources. Other profiles can be found in the directory [scripts/profiles](./scripts/profiles/).
 
+- `--profile Neo4j-latest-low-memory-continue-on-scan-errors` is based on the default profile (`Neo4j-latest`) but uses only half of the memory (RAM) as configured in [template-neo4j-low-memory.conf](./scripts/configuration/template-neo4j-low-memory.conf) and the jQAssistant configuration template [template-neo4jv5-jqassistant-continue-on-error.yaml](./scripts/configuration/template-neo4jv5-jqassistant-continue-on-error.yaml) to continue on scan error instead of failing fast. This is temporarily useful when there is a known error that needs to be ignored. It is still recommended to use the default profile and fail fast if there is something wrong. Other profiles can be found in the directory [scripts/profiles](./scripts/profiles/).
+
 - `--profile Neo4j-latest-high-memory` is based on the default profile (`Neo4j-latest`) but uses more memory (RAM) as configured in [template-neo4j-high-memory.conf](./scripts/configuration/template-neo4j-high-memory.conf). This is useful for the analysis of larger codebases with more resources. Other profiles can be found in the directory [scripts/profiles](./scripts/profiles/).
 
 - `--explore` activates the "explore" mode where no reports are generated. Furthermore, Neo4j won't be stopped at the end of the script and will therefore continue running.  This makes it easy to just set everything up but then use the running Neo4j server to explore the data manually.

cypher/Internal_Dependencies/NPM_Package_build_levels_for_graphviz.cypher

@@ -16,8 +16,10 @@
   WITH *, round((normalizedWeight * 5) + 1, 2) AS penWidth
   WITH *, "\\n(level " + coalesce(source.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS sourceLevelInfo
   WITH *, "\\n(level " + coalesce(target.maxDistanceFromSource + "/" + maxLevel, "?") + ")" AS targetLevelInfo
-  WITH *, source.name + sourceLevelInfo AS fullSourceName
-  WITH *, target.name + targetLevelInfo AS fullTargetName
+  WITH *, CASE WHEN source:NpmDevPackage THEN "\\n[dev]" ELSE "" END AS sourceDevDependencyInfo
+  WITH *, CASE WHEN target:NpmDevPackage THEN "\\n[dev]" ELSE "" END AS targetDevDependencyInfo
+  WITH *, source.name + sourceLevelInfo + sourceDevDependencyInfo AS fullSourceName
+  WITH *, target.name + targetLevelInfo + targetDevDependencyInfo AS fullTargetName
   WITH *, "\" -> \"" + fullTargetName 
                    + "\" [label = "  + dependency.weightByDependencyType + ";"
                    + " penwidth = "  + penWidth + ";"

…r/Path_Finding/Set_Parameters_NPM.cypher …g/Set_Parameters_NonDevNpmPackage.cyphercypher/Path_Finding/Set_Parameters_NPM.cypher renamed to cypher/Path_Finding/Set_Parameters_NonDevNpmPackage.cypher cypher/Path_Finding/Set_Parameters_NPM.cypher renamed to cypher/Path_Finding/Set_Parameters_NonDevNpmPackage.cypher

@@ -2,7 +2,7 @@
 
 :params {
     "dependencies_projection_language":"NPM",
-    "dependencies_projection": "npm-package-path-finding",     
-    "dependencies_projection_node": "Package",
+    "dependencies_projection": "npm-non-dev-package-path-finding",     
+    "dependencies_projection_node": "NpmNonDevPackage",
     "dependencies_projection_weight_property": "weightByDependencyType",
 }

cypher/Typescript_Enrichment/Label_npm_packages_by_dep_type.cypher

@@ -0,0 +1,26 @@
+// Label NPM packages as NpmDevPackage or NpmNonDevPackage based on incoming DEPENDS_ON. Requires Link_npm_packages_with_depends_on_relationships.cypher
+// For each package node, calculate the maximum `weightByDependencyType` across incoming
+// `DEPENDS_ON` relationships. If the max is 1 -> label `NpmDevPackage`; if > 1 -> label
+// `NpmNonDevPackage`. If the property is missing or < 1 then no label is added.
+// weightByDependencyType= 1 is a dev dependency, 2 is a regular dependency, and 3 is a peer dependency.
+
+MATCH (packageNode:NPM:Package)
+OPTIONAL MATCH (packageNode)<-[dependsOnRel:DEPENDS_ON]-()
+WITH packageNode, max(dependsOnRel.weightByDependencyType) AS maxWeight
+
+CALL {
+  WITH packageNode, maxWeight
+  FOREACH (_ IN CASE WHEN maxWeight = 1 THEN [1] ELSE [] END |
+    SET packageNode:NpmDevPackage
+  )
+  
+  FOREACH (_ IN CASE WHEN maxWeight > 1 THEN [1] ELSE [] END |
+    SET packageNode:NpmNonDevPackage
+  )
+} IN TRANSACTIONS OF 1000 ROWS
+
+RETURN count(packageNode) AS evaluatedPackages,
+       sum(CASE WHEN maxWeight = 1 THEN 1 ELSE 0 END) AS numberOfNpmDevPackages,
+       sum(CASE WHEN maxWeight > 1 THEN 1 ELSE 0 END) AS numberOfNpmNonDevPackages,
+       sum(CASE WHEN maxWeight IS NULL THEN 1 ELSE 0 END) AS numberOfPackagesWithMissingWeight,
+       sum(CASE WHEN maxWeight < 1 AND maxWeight IS NOT NULL THEN 1 ELSE 0 END) AS numberOfPackagesWithWeightBelowOne

cypher/Typescript_Enrichment/Remove_npm_dependency_type_labels.cypher

@@ -0,0 +1,5 @@
+// Remove NPM package type labels (idempotent).
+   
+ MATCH (packageNode:NPM:Package)
+REMOVE packageNode:NpmDevPackage
+REMOVE packageNode:NpmNonDevPackage

scripts/findPathsToScan.sh

@@ -54,7 +54,18 @@ else
 fi
 
 if [ -d "./${SOURCE_DIRECTORY}" ] ; then
+    # Scan package.json files for npm (nodes package manager) in the source directory
+    # Since the following issue had been resolved, the scan can be done here again:
+    # https://github.com/jqassistant-plugin/jqassistant-npm-plugin/issues/5
+    npmPackageJsonFiles="$(findPackageJsonFiles "./${SOURCE_DIRECTORY}")"
+    if [ -n "${npmPackageJsonFiles}" ]; then
+       directoriesAndFilesToScan="$(appendNonEmpty "${directoriesAndFilesToScan}")${npmPackageJsonFiles}"
+    fi
+    
     # Scan Typescript analysis json data files in the source directory
+    # Since Typescript scan might produce duplicate fileName properties on File nodes,
+    # it is done after the npm package.json scan, that breaks the scan on duplicate fileName properties.
+    # This order fixes "XOException: Expected exactly one result, but got CompositeRowObject".
     typescriptAnalysisFiles="$(find -L "./${SOURCE_DIRECTORY}" \
                                     -type d -name "node_modules" -prune -o \
                                     -type d -name "dist" -prune -o \
@@ -69,14 +80,6 @@ if [ -d "./${SOURCE_DIRECTORY}" ] ; then
         directoriesAndFilesToScan="$(appendNonEmpty "${directoriesAndFilesToScan}")${typescriptAnalysisFiles}"
     fi
 
-    # Scan package.json files for npm (nodes package manager) in the source directory
-    # Since the following issue had been resolved, the scan be done here again:
-    # https://github.com/jqassistant-plugin/jqassistant-npm-plugin/issues/5
-    npmPackageJsonFiles="$(findPackageJsonFiles "./${SOURCE_DIRECTORY}")"
-    if [ -n "${npmPackageJsonFiles}" ]; then
-       directoriesAndFilesToScan="$(appendNonEmpty "${directoriesAndFilesToScan}")${npmPackageJsonFiles}"
-    fi
-
     # Scan git repositories in the artifacts directory
     if [ "${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}" = "" ] || [ "${IMPORT_GIT_LOG_DATA_IF_SOURCE_IS_PRESENT}" = "plugin" ] ; then
         gitDirectories="$(find -L "./${SOURCE_DIRECTORY}" \

scripts/prepareAnalysis.sh

@@ -79,6 +79,8 @@ execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Remove_duplicate_CONTAINS_relations_bet
 execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Link_npm_dependencies_to_npm_packages.cypher"
 execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Link_npm_packages_with_depends_on_relationships.cypher"
 execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Enrich_npm_packages_with_dependency_counts.cypher"
+execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Remove_npm_dependency_type_labels.cypher"
+execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Label_npm_packages_by_dep_type.cypher"
 execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Link_projects_to_npm_packages.cypher"
 dataVerificationResult=$( execute_cypher "${TYPESCRIPT_CYPHER_DIR}/Verify_projects_linked_to_npm_packages.cypher" "${@}")
 if is_csv_column_greater_zero "${dataVerificationResult}" "unresolvedProjectsCount"; then

scripts/profiles/Neo4j-latest-low-memory-continue-on-scan-errors.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# Sets all settings variables for a low memory, continue-on-error analysis with the latest version of Neo4j.
+# The chosen settings are tested to be compatible and working.
+
+NEO4J_VERSION=${NEO4J_VERSION:-"2026.01.4"} # Neo4j Graph Database Version. Current versions: >= 2025.03.0. Version 4.4.42 and 5.26.5 are the previous LTS (long term support) versions as of April 2025.
+NEO4J_HTTP_TRANSACTION_ENDPOINT=${NEO4J_HTTP_TRANSACTION_ENDPOINT:-"db/neo4j/tx/commit"}
+NEO4J_CONFIG_TEMPLATE=${NEO4J_CONFIG_TEMPLATE:-"template-neo4j-low-memory.conf"}
+
+# Awesome Procedures (APOC) Plugin for Neo4j
+NEO4J_APOC_PLUGIN_VERSION=${NEO4J_APOC_PLUGIN_VERSION:-"2026.01.4"}
+NEO4J_APOC_PLUGIN_EDITION=${NEO4J_APOC_PLUGIN_EDITION:-"core"}
+NEO4J_APOC_PLUGIN_GITHUB=${NEO4J_APOC_PLUGIN_GITHUB:-"neo4j/apoc"}
+
+NEO4J_GDS_PLUGIN_VERSION=${NEO4J_GDS_PLUGIN_VERSION:-"2.26.0"}
+NEO4J_OPEN_GDS_PLUGIN_VERSION=${NEO4J_OPEN_GDS_PLUGIN_VERSION:-"2.26.0"}
+NEO4J_GDS_PLUGIN_EDITION=${NEO4J_GDS_PLUGIN_EDITION:-"open"}
+
+JQASSISTANT_CLI_VERSION=${JQASSISTANT_CLI_VERSION:-"2.9.0"}
+JQASSISTANT_CLI_ARTIFACT=${JQASSISTANT_CLI_ARTIFACT:-"jqassistant-commandline-neo4jv5"}
+JQASSISTANT_CONFIG_TEMPLATE=${JQASSISTANT_CONFIG_TEMPLATE:-"template-neo4jv5-jqassistant-continue-on-error.yaml"}

scripts/reports/CentralityCsv.sh

@@ -439,12 +439,12 @@ if createUndirectedDependencyProjection "${MODULE_LANGUAGE}" "${MODULE_PROJECTIO
     runUndirectedCentralityAlgorithms "${MODULE_PROJECTION_UNDIRECTED}" "${MODULE_NODE}"
 fi
 
-# -- NPM Package Centrality ---------------------------------------
+# -- Non Dev NPM Package Centrality ---------------------------------------
 
 NPM_LANGUAGE="dependencies_projection_language=NPM"
-NPM_PROJECTION="dependencies_projection=npm-package-centrality" 
-NPM_PROJECTION_UNDIRECTED="dependencies_projection=npm-package-centrality-undirected" 
-NPM_NODE="dependencies_projection_node=Package" 
+NPM_PROJECTION="dependencies_projection=npm-non-dev-package-centrality" 
+NPM_PROJECTION_UNDIRECTED="dependencies_projection=npm-non-dev-package-centrality-undirected" 
+NPM_NODE="dependencies_projection_node=NpmNonDevPackage" 
 NPM_WEIGHT="dependencies_projection_weight_property=weightByDependencyType" 
 
 if createDirectedDependencyProjection "${NPM_LANGUAGE}" "${NPM_PROJECTION}" "${NPM_NODE}" "${NPM_WEIGHT}"; then

scripts/reports/PathFindingCsv.sh

@@ -140,16 +140,28 @@ if createDirectedDependencyProjection "${MODULE_LANGUAGE}" "${MODULE_PROJECTION}
     runPathFindingAlgorithms "${MODULE_PROJECTION}" "${MODULE_NODE}" "${MODULE_WEIGHT}"
 fi
 
-# -- NPM Package Path Finding -------------------------------
+# -- Non Dev NPM Package Path Finding -------------------------------
 
 NPM_LANGUAGE="dependencies_projection_language=NPM"
-NPM_PROJECTION="dependencies_projection=npm-package-path-finding"
-NPM_NODE="dependencies_projection_node=Package"
+NPM_PROJECTION="dependencies_projection=npm-non-dev-package-path-finding"
+NPM_NODE="dependencies_projection_node=NpmNonDevPackage"
 NPM_WEIGHT="dependencies_projection_weight_property=weightByDependencyType"
 
 if createDirectedDependencyProjection "${NPM_LANGUAGE}" "${NPM_PROJECTION}" "${NPM_NODE}" "${NPM_WEIGHT}"; then
     runPathFindingAlgorithms "${NPM_PROJECTION}" "${NPM_NODE}" "${NPM_WEIGHT}"
 fi
+
+# -- Dev NPM Package Path Finding -------------------------------
+
+NPM_DEV_LANGUAGE="dependencies_projection_language=NPM"
+NPM_DEV_PROJECTION="dependencies_projection=npm-dev-package-path-finding"
+NPM_DEV_NODE="dependencies_projection_node=NpmDevPackage"
+NPM_DEV_WEIGHT="dependencies_projection_weight_property=weightByDependencyType" 
+
+if createDirectedDependencyProjection "${NPM_DEV_LANGUAGE}" "${NPM_DEV_PROJECTION}" "${NPM_DEV_NODE}" "${NPM_DEV_WEIGHT}"; then
+    runPathFindingAlgorithms "${NPM_DEV_PROJECTION}" "${NPM_DEV_NODE}" "${NPM_DEV_WEIGHT}"
+fi
+
 # ---------------------------------------------------------------
 
 # Clean-up after report generation. Empty reports will be deleted.