Skip to content

Commit 4099946

Browse files
JohTJohT
committed
Add strongly connected components community detection
1 parent 94648f7 commit 4099946

7 files changed

Lines changed: 202 additions & 16 deletions

‎cypher/Community_Detection/Community_Detection_3a_StronglyConnectedComponents_Estimate.cypher‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
// Community Detection Strongly Connected Components Estimate
2+
3+
CALL gds.scc.write.estimate(
4+
$dependencies_projection + '-cleaned', {
5+
writeProperty: $dependencies_projection_write_property
6+
,consecutiveIds: true
7+
})
8+
YIELD requiredMemory
9+
,nodeCount
10+
,relationshipCount
11+
,bytesMin
12+
,bytesMax
13+
,heapPercentageMin
14+
,heapPercentageMax
15+
,treeView
16+
,mapView
17+
RETURN requiredMemory
18+
,nodeCount
19+
,relationshipCount
20+
,bytesMin
21+
,bytesMax
22+
,heapPercentageMin
23+
,heapPercentageMax
24+
,treeView
25+
//,mapView //doesn't work on Windows with git bash jq version jq-1.7-dirty

‎cypher/Community_Detection/Community_Detection_3b_StronglyConnectedComponents_Statistics.cypher‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// Community Detection Strongly Connected Components Statistics
2+
3+
CALL gds.scc.stats(
4+
$dependencies_projection + '-cleaned', {
5+
consecutiveIds: true
6+
})
7+
YIELD componentCount
8+
,preProcessingMillis
9+
,computeMillis
10+
,postProcessingMillis
11+
,componentDistribution
12+
RETURN componentCount
13+
,preProcessingMillis
14+
,computeMillis
15+
,postProcessingMillis
16+
,componentDistribution.min
17+
,componentDistribution.mean
18+
,componentDistribution.max
19+
,componentDistribution.p50
20+
,componentDistribution.p75
21+
,componentDistribution.p90
22+
,componentDistribution.p95
23+
,componentDistribution.p99
24+
,componentDistribution.p999

‎cypher/Community_Detection/Community_Detection_3c_StronglyConnectedComponents_Mutate.cypher‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Community Detection Strongly Connected Components Mutate
2+
3+
CALL gds.scc.mutate(
4+
$dependencies_projection + '-cleaned', {
5+
mutateProperty: $dependencies_projection_write_property
6+
,consecutiveIds: true
7+
})
8+
YIELD componentCount
9+
,nodePropertiesWritten
10+
,preProcessingMillis
11+
,computeMillis
12+
,mutateMillis
13+
,postProcessingMillis
14+
,componentDistribution
15+
RETURN componentCount
16+
,nodePropertiesWritten
17+
,preProcessingMillis
18+
,computeMillis
19+
,mutateMillis
20+
,postProcessingMillis
21+
,componentDistribution.min
22+
,componentDistribution.mean
23+
,componentDistribution.max
24+
,componentDistribution.p50
25+
,componentDistribution.p75
26+
,componentDistribution.p90
27+
,componentDistribution.p95
28+
,componentDistribution.p99
29+
,componentDistribution.p999

‎cypher/Community_Detection/Community_Detection_3d_StrongyConnectedComponents_Stream.cypher‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
// Community Detection Strongly Connected Components Stream
2+
3+
CALL gds.scc.stream(
4+
$dependencies_projection + '-cleaned', {
5+
consecutiveIds: true
6+
})
7+
YIELD nodeId, componentId
8+
WITH componentId
9+
,gds.util.asNode(nodeId) AS member
10+
WITH componentId
11+
,member
12+
,coalesce(member.fqn, member.fileName, member.name) AS memberName
13+
WITH componentId
14+
,count(DISTINCT member) AS memberCount
15+
,collect(DISTINCT memberName) AS memberNames
16+
RETURN componentId
17+
,memberCount
18+
,memberNames
19+
ORDER BY memberCount DESC, componentId ASC

‎cypher/Community_Detection/Community_Detection_3e_StronglyConnectedComponents_Write.cypher‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Community Detection Strongly Connected Components write node property communityStronglyConnectedComponentId
2+
3+
CALL gds.scc.write(
4+
$dependencies_projection + '-cleaned', {
5+
consecutiveIds: true
6+
,writeProperty: 'communityStronglyConnectedComponentId'
7+
})
8+
YIELD componentCount
9+
,preProcessingMillis
10+
,computeMillis
11+
,writeMillis
12+
,postProcessingMillis
13+
,nodePropertiesWritten
14+
,componentDistribution
15+
RETURN componentCount
16+
,preProcessingMillis
17+
,computeMillis
18+
,writeMillis
19+
,postProcessingMillis
20+
,nodePropertiesWritten
21+
,componentDistribution.min
22+
,componentDistribution.mean
23+
,componentDistribution.max
24+
,componentDistribution.p50
25+
,componentDistribution.p75
26+
,componentDistribution.p90
27+
,componentDistribution.p95
28+
,componentDistribution.p99
29+
,componentDistribution.p999

‎cypher/Community_Detection/Community_Detection_Summary.cypher‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,20 @@
22

33
MATCH (codeUnit)
44
WHERE (codeUnit.incomingDependencies > 0 OR codeUnit.outgoingDependencies > 0)
5+
AND (codeUnit.testMarkerInteger IS NULL OR codeUnit.testMarkerInteger = 0)
56
AND $dependencies_projection_node IN LABELS(codeUnit)
67
RETURN coalesce(codeUnit.fqn, codeUnit.fileName, codeUnit.signature, codeUnit.name) AS name
78
,codeUnit.name AS shortName
8-
,codeUnit.communityLouvainId AS louvainId
9-
,codeUnit.communityLouvainIntermediateIds AS louvainIntermediateIds
10-
,codeUnit.communityLeidenId AS leidenId
11-
,codeUnit.communityLeidenIntermediateIds AS leidenIntermediateIds
12-
,codeUnit.communityLeidenIdModularity AS leidenModularity
13-
,codeUnit.communityWeaklyConnectedComponentId AS weaklyConnectedComponentId
14-
,codeUnit.communityLabelPropagationId AS labelPropagationId
15-
,codeUnit.communityKCoreDecompositionValue AS kCoreDecompositionValue
16-
,codeUnit.communityMaximumKCutId AS maximumKCutId
17-
,codeUnit.incomingDependencies AS incomingDependencies
18-
,codeUnit.outgoingDependencies AS outgoingDependencies
9+
,codeUnit.communityLouvainId AS louvainId
10+
,codeUnit.communityLouvainIntermediateIds AS louvainIntermediateIds
11+
,codeUnit.communityLeidenId AS leidenId
12+
,codeUnit.communityLeidenIntermediateIds AS leidenIntermediateIds
13+
,codeUnit.communityLeidenIdModularity AS leidenModularity
14+
,codeUnit.communityStronglyConnectedComponentId AS stronglyConnectedComponentId
15+
,codeUnit.communityWeaklyConnectedComponentId AS weaklyConnectedComponentId
16+
,codeUnit.communityLabelPropagationId AS labelPropagationId
17+
,codeUnit.communityKCoreDecompositionValue AS kCoreDecompositionValue
18+
,codeUnit.communityMaximumKCutId AS maximumKCutId
19+
,codeUnit.communityFastRpHdbscanLabel AS communityFastRpHdbscanLabel
20+
,codeUnit.incomingDependencies AS incomingDependencies
21+
,codeUnit.outgoingDependencies AS outgoingDependencies

‎scripts/reports/CommunityCsv.sh‎

Lines changed: 62 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -132,15 +132,48 @@ detectCommunitiesWithLeiden() {
132132
calculateCommunityMetrics "${@}" "${writePropertyName}"
133133
}
134134

135+
# Community Detection using the Strongly Connected Components Algorithm
136+
#
137+
# Required Parameters:
138+
# - dependencies_projection=...
139+
# Name prefix for the in-memory projection name for dependencies. Example: "package"
140+
# - dependencies_projection_node=...
141+
# Label of the nodes that will be used for the projection. Example: "Package"
142+
detectCommunitiesWithStronglyConnectedComponents() {
143+
local COMMUNITY_DETECTION_CYPHER_DIR="${CYPHER_DIR}/Community_Detection"
144+
local PROJECTION_CYPHER_DIR="${CYPHER_DIR}/Dependencies_Projection"
145+
146+
local writePropertyName="dependencies_projection_write_property=communityStronglyConnectedComponentId"
147+
local writeLabelName="dependencies_projection_write_label=StronglyConnectedComponent"
148+
149+
# Statistics
150+
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_3a_StronglyConnectedComponents_Estimate.cypher" "${@}" "${writePropertyName}"
151+
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_3b_StronglyConnectedComponents_Statistics.cypher" "${@}"
152+
153+
# Run the algorithm and write the result into the in-memory projection ("mutate")
154+
execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_3c_StronglyConnectedComponents_Mutate.cypher" "${@}" "${writePropertyName}"
155+
156+
# Stream to CSV
157+
local nodeLabel
158+
nodeLabel=$( extractQueryParameter "dependencies_projection_node" "${@}")
159+
execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_8_Stream_Mutated_Grouped.cypher" "${@}" "${writePropertyName}" > "${FULL_REPORT_DIRECTORY}/${nodeLabel}_Communities_Strongly_Connected_Components.csv"
160+
#execute_cypher "${COMMUNITY_DETECTION_CYPHER_DIR}/Community_Detection_3d_StronglyConnectedComponents_Stream.cypher" "${@}" > "${FULL_REPORT_DIRECTORY}/${nodeLabel}_Communities_Strongly_Connected_Components.csv"
161+
162+
# Update Graph (node properties and labels) using the already mutated property projection
163+
execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_9_Write_Mutated.cypher" "${@}" "${writePropertyName}"
164+
execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_10_Delete_Label.cypher" "${@}" "${writePropertyName}" "${writeLabelName}"
165+
execute_cypher "${PROJECTION_CYPHER_DIR}/Dependencies_11_Add_Label.cypher" "${@}" "${writePropertyName}" "${writeLabelName}"
166+
167+
calculateCommunityMetrics "${@}" "${writePropertyName}"
168+
}
169+
135170
# Community Detection using the Weakly Connected Components Algorithm
136171
#
137172
# Required Parameters:
138173
# - dependencies_projection=...
139174
# Name prefix for the in-memory projection name for dependencies. Example: "package"
140175
# - dependencies_projection_node=...
141176
# Label of the nodes that will be used for the projection. Example: "Package"
142-
# - dependencies_projection_weight_property=...
143-
# Name of the node property that contains the dependency weight. Example: "weight"
144177
detectCommunitiesWithWeaklyConnectedComponents() {
145178
local COMMUNITY_DETECTION_CYPHER_DIR="${CYPHER_DIR}/Community_Detection"
146179
local PROJECTION_CYPHER_DIR="${CYPHER_DIR}/Dependencies_Projection"
@@ -470,7 +503,10 @@ detectCommunities() {
470503
time calculateLocalClusteringCoefficient "${@}"
471504

472505
compareCommunityDetectionResults "${@}"
473-
listAllResults "${@}"
506+
}
507+
508+
detectDirectedCommunities() {
509+
time detectCommunitiesWithStronglyConnectedComponents "${@}"
474510
}
475511

476512
# -- Java Artifact Community Detection ---------------------------
@@ -482,8 +518,14 @@ ARTIFACT_GAMMA="dependencies_leiden_gamma=1.11" # default = 1.00
482518
ARTIFACT_KCUT="dependencies_maxkcut=5" # default = 2
483519

484520
if createUndirectedDependencyProjection "${ARTIFACT_PROJECTION}" "${ARTIFACT_NODE}" "${ARTIFACT_WEIGHT}"; then
485-
detectCommunities "${ARTIFACT_PROJECTION}" "${ARTIFACT_NODE}" "${ARTIFACT_WEIGHT}" "${ARTIFACT_GAMMA}" "${ARTIFACT_KCUT}" # "${ARTIFACT_NODE_EMBEDDINGS}"
521+
detectCommunities "${ARTIFACT_PROJECTION}" "${ARTIFACT_NODE}" "${ARTIFACT_WEIGHT}" "${ARTIFACT_GAMMA}" "${ARTIFACT_KCUT}"
486522
writeLeidenModularity "${ARTIFACT_PROJECTION}" "${ARTIFACT_NODE}" "${ARTIFACT_WEIGHT}"
523+
524+
if createDirectedDependencyProjection "${ARTIFACT_PROJECTION}" "${ARTIFACT_NODE}" "${ARTIFACT_WEIGHT}"; then
525+
detectDirectedCommunities "${ARTIFACT_PROJECTION}" "${ARTIFACT_NODE}" "${ARTIFACT_WEIGHT}"
526+
fi
527+
528+
listAllResults "${ARTIFACT_PROJECTION}" "${ARTIFACT_NODE}" "${ARTIFACT_WEIGHT}" "${ARTIFACT_GAMMA}" "${ARTIFACT_KCUT}"
487529
fi
488530

489531
# -- Java Package Community Detection -------------------------------
@@ -500,6 +542,11 @@ if createUndirectedDependencyProjection "${PACKAGE_PROJECTION}" "${PACKAGE_NODE}
500542

501543
detectCommunitiesWithHDBSCAN "${PACKAGE_PROJECTION}" "${PACKAGE_NODE}" "${PACKAGE_WEIGHT}"
502544

545+
if createDirectedDependencyProjection "${PACKAGE_PROJECTION}" "${PACKAGE_NODE}" "${PACKAGE_WEIGHT}"; then
546+
detectDirectedCommunities "${PACKAGE_PROJECTION}" "${PACKAGE_NODE}" "${PACKAGE_WEIGHT}"
547+
fi
548+
listAllResults "${PACKAGE_PROJECTION}" "${PACKAGE_NODE}" "${PACKAGE_WEIGHT}" "${PACKAGE_GAMMA}" "${PACKAGE_KCUT}"
549+
503550
# Package Community Detection - Special CSV Queries after update
504551
execute_cypher "${CYPHER_DIR}/Community_Detection/Which_package_community_spans_several_artifacts_and_how_are_the_packages_distributed.cypher" > "${FULL_REPORT_DIRECTORY}/Package_Communities_Leiden_That_Span_Multiple_Artifacts.csv"
505552
fi
@@ -513,12 +560,18 @@ TYPE_GAMMA="dependencies_leiden_gamma=5.00" # default = 1.00
513560
TYPE_KCUT="dependencies_maxkcut=100" # default = 2
514561

515562
if createUndirectedJavaTypeDependencyProjection "${TYPE_PROJECTION}"; then
516-
detectCommunities "${TYPE_PROJECTION}" "${TYPE_NODE}" "${TYPE_WEIGHT}" "${TYPE_GAMMA}" "${TYPE_KCUT}" "${TYPE_NODE_EMBEDDINGS}"
563+
detectCommunities "${TYPE_PROJECTION}" "${TYPE_NODE}" "${TYPE_WEIGHT}" "${TYPE_GAMMA}" "${TYPE_KCUT}"
517564
detectCommunitiesWithHDBSCAN "${TYPE_PROJECTION}" "${TYPE_NODE}" "${TYPE_WEIGHT}"
565+
518566
# Type Community Detection - Special CSV Queries after update
519567
execute_cypher "${CYPHER_DIR}/Community_Detection/Which_type_community_spans_several_artifacts_and_how_are_the_types_distributed.cypher" > "${FULL_REPORT_DIRECTORY}/Type_Communities_Leiden_That_Span_Multiple_Artifacts.csv"
520568
execute_cypher "${CYPHER_DIR}/Community_Detection/Type_communities_with_few_members_in_foreign_packages.cypher" > "${FULL_REPORT_DIRECTORY}/Type_communities_with_few_members_in_foreign_packages.csv"
521569
execute_cypher "${CYPHER_DIR}/Community_Detection/Type_communities_that_span_the_most_packages_with_type_statistics.cypher" > "${FULL_REPORT_DIRECTORY}/Type_communities_that_span_the_most_packages_with_type_statistics.csv"
570+
571+
if createDirectedJavaTypeDependencyProjection "${TYPE_PROJECTION}"; then
572+
detectDirectedCommunities "${TYPE_PROJECTION}" "${TYPE_NODE}" "${TYPE_WEIGHT}"
573+
fi
574+
listAllResults "${TYPE_PROJECTION}" "${TYPE_NODE}" "${TYPE_WEIGHT}" "${TYPE_GAMMA}" "${TYPE_KCUT}"
522575
fi
523576

524577
# -- Typescript Module Community Detection -----------------------
@@ -532,6 +585,10 @@ MODULE_KCUT="dependencies_maxkcut=20" # default = 2
532585

533586
if createUndirectedDependencyProjection "${MODULE_LANGUAGE}" "${MODULE_PROJECTION}" "${MODULE_NODE}" "${MODULE_WEIGHT}"; then
534587
detectCommunities "${MODULE_PROJECTION}" "${MODULE_NODE}" "${MODULE_WEIGHT}" "${MODULE_GAMMA}" "${MODULE_KCUT}"
588+
if createDirectedDependencyProjection "${MODULE_PROJECTION}" "${MODULE_NODE}" "${MODULE_WEIGHT}"; then
589+
detectDirectedCommunities "${MODULE_PROJECTION}" "${MODULE_NODE}" "${MODULE_WEIGHT}"
590+
fi
591+
listAllResults "${MODULE_PROJECTION}" "${MODULE_NODE}" "${MODULE_WEIGHT}" "${MODULE_GAMMA}" "${MODULE_KCUT}"
535592
fi
536593

537594
# ---------------------------------------------------------------

0 commit comments

Comments
 (0)