Skip to content

Commit febc0b5

Browse files
ChrisPennerChrisPenner
authored
Merge pull request #57 from unisoncomputing/cp/sync-speedups
Speed up incremental syncs
2 parents 012b5cb + ec56e0e commit febc0b5

2 files changed

Lines changed: 130 additions & 3 deletions

File tree

‎sql/2025-04-15_non-history-causal-dependencies.sql‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
-- Takes a causal_id and returns a table of ALL hashes which are dependencies of that causal, EXCEPT for the
2+
-- ancestors of the causal.
3+
CREATE OR REPLACE FUNCTION dependencies_of_causals_without_ancestors(the_causal_ids INTEGER[]) RETURNS TABLE (hash TEXT) AS $$
4+
WITH RECURSIVE all_causals(causal_id, causal_hash, causal_namespace_hash_id) AS (
5+
-- Base causal
6+
SELECT DISTINCT causal.id, causal.hash, causal.namespace_hash_id
7+
FROM UNNEST(the_causal_ids) AS causal_id
8+
JOIN causals causal ON causal.id = causal_id
9+
UNION
10+
-- This nested CTE is required because RECURSIVE CTEs can't refer
11+
-- to the recursive table more than once.
12+
-- I don't fully understand why or how this works, but it does
13+
( WITH rec AS (
14+
SELECT tc.causal_id, tc.causal_namespace_hash_id
15+
FROM all_causals tc
16+
)
17+
SELECT child_causal.id, child_causal.hash, child_causal.namespace_hash_id
18+
FROM rec tc
19+
JOIN namespace_children nc ON tc.causal_namespace_hash_id = nc.parent_namespace_hash_id
20+
JOIN causals child_causal ON nc.child_causal_id = child_causal.id
21+
)
22+
), all_namespaces(namespace_hash_id, namespace_hash) AS (
23+
SELECT DISTINCT tc.causal_namespace_hash_id AS namespace_hash_id, bh.base32 as namespace_hash
24+
FROM all_causals tc
25+
JOIN branch_hashes bh ON tc.causal_namespace_hash_id = bh.id
26+
), all_patches(patch_id, patch_hash) AS (
27+
SELECT DISTINCT patch.id, patch.hash
28+
FROM all_namespaces an
29+
JOIN namespace_patches np ON an.namespace_hash_id = np.namespace_hash_id
30+
JOIN patches patch ON np.patch_id = patch.id
31+
),
32+
-- term components to start transitively joining dependencies to
33+
base_term_components(component_hash_id) AS (
34+
SELECT DISTINCT term.component_hash_id
35+
FROM all_namespaces an
36+
JOIN namespace_terms nt ON an.namespace_hash_id = nt.namespace_hash_id
37+
JOIN terms term ON nt.term_id = term.id
38+
UNION
39+
SELECT DISTINCT term.component_hash_id
40+
FROM all_patches ap
41+
JOIN patch_term_mappings ptm ON ap.patch_id = ptm.patch_id
42+
JOIN terms term ON ptm.to_term_id = term.id
43+
UNION
44+
-- term metadata
45+
SELECT DISTINCT term.component_hash_id
46+
FROM all_namespaces an
47+
JOIN namespace_terms nt ON an.namespace_hash_id = nt.namespace_hash_id
48+
JOIN namespace_term_metadata meta ON nt.id = meta.named_term
49+
JOIN terms term ON meta.metadata_term_id = term.id
50+
UNION
51+
-- type metadata
52+
SELECT DISTINCT term.component_hash_id
53+
FROM all_namespaces an
54+
JOIN namespace_types nt ON an.namespace_hash_id = nt.namespace_hash_id
55+
JOIN namespace_type_metadata meta ON nt.id = meta.named_type
56+
JOIN terms term ON meta.metadata_term_id = term.id
57+
),
58+
-- type components to start transitively joining dependencies to
59+
base_type_components(component_hash_id) AS (
60+
SELECT DISTINCT typ.component_hash_id
61+
FROM all_namespaces an
62+
JOIN namespace_types nt ON an.namespace_hash_id = nt.namespace_hash_id
63+
JOIN types typ ON nt.type_id = typ.id
64+
UNION
65+
SELECT DISTINCT typ.component_hash_id
66+
FROM all_namespaces an
67+
JOIN namespace_terms nt ON an.namespace_hash_id = nt.namespace_hash_id
68+
JOIN constructors con ON nt.constructor_id = con.id
69+
JOIN types typ ON con.type_id = typ.id
70+
UNION
71+
SELECT DISTINCT typ.component_hash_id
72+
FROM all_patches ap
73+
JOIN patch_type_mappings ptm ON ap.patch_id = ptm.patch_id
74+
JOIN types typ ON ptm.to_type_id = typ.id
75+
UNION
76+
SELECT DISTINCT typ.component_hash_id
77+
FROM all_patches ap
78+
JOIN patch_constructor_mappings pcm ON ap.patch_id = pcm.patch_id
79+
JOIN constructors con ON pcm.to_constructor_id = con.id
80+
JOIN types typ ON con.type_id = typ.id
81+
),
82+
-- All the dependencies we join in transitively from the known term & type components we depend on.
83+
all_components(component_hash_id) AS (
84+
SELECT DISTINCT btc.component_hash_id
85+
FROM base_term_components btc
86+
UNION
87+
SELECT DISTINCT btc.component_hash_id
88+
FROM base_type_components btc
89+
UNION
90+
( WITH rec AS (
91+
SELECT DISTINCT ac.component_hash_id
92+
FROM all_components ac
93+
)
94+
-- recursively union in term dependencies
95+
SELECT DISTINCT ref.component_hash_id
96+
FROM rec atc
97+
-- This joins in ALL the terms from the component, not just the one that caused the dependency on the
98+
-- component
99+
JOIN terms term ON atc.component_hash_id = term.component_hash_id
100+
JOIN term_local_component_references ref ON term.id = ref.term_id
101+
UNION
102+
-- recursively union in type dependencies
103+
SELECT DISTINCT ref.component_hash_id
104+
FROM rec atc
105+
-- This joins in ALL the types from the component, not just the one that caused the dependency on the
106+
-- component
107+
JOIN types typ ON atc.component_hash_id = typ.component_hash_id
108+
JOIN type_local_component_references ref ON typ.id = ref.type_id
109+
)
110+
)
111+
(SELECT ch.base32 AS hash
112+
FROM all_components ac
113+
JOIN component_hashes ch ON ac.component_hash_id = ch.id
114+
)
115+
UNION ALL
116+
(SELECT ap.patch_hash AS hash
117+
FROM all_patches ap
118+
)
119+
UNION ALL
120+
(SELECT an.namespace_hash AS hash
121+
FROM all_namespaces an
122+
)
123+
UNION ALL
124+
(SELECT ac.causal_hash AS hash
125+
FROM all_causals ac
126+
)
127+
$$ LANGUAGE SQL;

‎src/Share/Web/UCM/SyncV2/Queries.hs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,12 @@ allSerializedDependenciesOfCausalCursor cid exceptCausalHashes = do
3333
JOIN causals c ON tch.hash = c.hash
3434
), dependency_hashes(hash) AS (
3535
SELECT DISTINCT deps.hash
36-
FROM dependencies_of_causals((SELECT ARRAY_AGG(kci.causal_id) FROM known_causal_ids kci)) AS deps
36+
FROM dependencies_of_causals_without_ancestors((SELECT ARRAY_AGG(kci.causal_id) FROM known_causal_ids kci)) AS deps
3737
), do_causals AS (
3838
INSERT INTO except_causals(causal_id)
3939
SELECT DISTINCT causal.id
40-
FROM the_causal_hashes tch
41-
JOIN causals causal ON tch.hash = causal.hash
40+
FROM dependency_hashes dh
41+
JOIN causals causal ON dh.hash = causal.hash
4242
ON CONFLICT DO NOTHING
4343
), do_namespaces AS (
4444
INSERT INTO except_namespaces(branch_hash_ids)

0 commit comments

Comments
 (0)