Skip to content

Commit 52389ff

Browse files
maparentmaparent
authored
ENG-1433 The database views are too slow (#783)
* ENG-1433: Faster view queries * subquery to avoid duplicates * factor out common code * regenerate types * Also apply to embeddings * sqruff * can_view_content should be stable
1 parent e754eb7 commit 52389ff

6 files changed

Lines changed: 209 additions & 117 deletions

File tree

‎packages/database/src/dbTypes.ts‎

Lines changed: 15 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -891,44 +891,6 @@ export type Database = {
891891
source_local_id: string | null
892892
space_id: number | null
893893
}
894-
Insert: {
895-
arity?: number | null
896-
author_id?: number | null
897-
created?: string | null
898-
description?: string | null
899-
epistemic_status?:
900-
| Database["public"]["Enums"]["EpistemicStatus"]
901-
| null
902-
id?: number | null
903-
is_schema?: boolean | null
904-
last_modified?: string | null
905-
literal_content?: Json | null
906-
name?: string | null
907-
reference_content?: Json | null
908-
refs?: number[] | null
909-
schema_id?: number | null
910-
source_local_id?: string | null
911-
space_id?: number | null
912-
}
913-
Update: {
914-
arity?: number | null
915-
author_id?: number | null
916-
created?: string | null
917-
description?: string | null
918-
epistemic_status?:
919-
| Database["public"]["Enums"]["EpistemicStatus"]
920-
| null
921-
id?: number | null
922-
is_schema?: boolean | null
923-
last_modified?: string | null
924-
literal_content?: Json | null
925-
name?: string | null
926-
reference_content?: Json | null
927-
refs?: number[] | null
928-
schema_id?: number | null
929-
source_local_id?: string | null
930-
space_id?: number | null
931-
}
932894
Relationships: [
933895
{
934896
foreignKeyName: "Concept_author_id_fkey"
@@ -990,36 +952,6 @@ export type Database = {
990952
text: string | null
991953
variant: Database["public"]["Enums"]["ContentVariant"] | null
992954
}
993-
Insert: {
994-
author_id?: number | null
995-
created?: string | null
996-
creator_id?: number | null
997-
document_id?: number | null
998-
id?: number | null
999-
last_modified?: string | null
1000-
metadata?: Json | null
1001-
part_of_id?: number | null
1002-
scale?: Database["public"]["Enums"]["Scale"] | null
1003-
source_local_id?: string | null
1004-
space_id?: number | null
1005-
text?: string | null
1006-
variant?: Database["public"]["Enums"]["ContentVariant"] | null
1007-
}
1008-
Update: {
1009-
author_id?: number | null
1010-
created?: string | null
1011-
creator_id?: number | null
1012-
document_id?: number | null
1013-
id?: number | null
1014-
last_modified?: string | null
1015-
metadata?: Json | null
1016-
part_of_id?: number | null
1017-
scale?: Database["public"]["Enums"]["Scale"] | null
1018-
source_local_id?: string | null
1019-
space_id?: number | null
1020-
text?: string | null
1021-
variant?: Database["public"]["Enums"]["ContentVariant"] | null
1022-
}
1023955
Relationships: [
1024956
{
1025957
foreignKeyName: "Content_author_id_fkey"
@@ -1210,28 +1142,6 @@ export type Database = {
12101142
space_id: number | null
12111143
url: string | null
12121144
}
1213-
Insert: {
1214-
author_id?: number | null
1215-
contents?: unknown
1216-
created?: string | null
1217-
id?: number | null
1218-
last_modified?: string | null
1219-
metadata?: Json | null
1220-
source_local_id?: string | null
1221-
space_id?: number | null
1222-
url?: string | null
1223-
}
1224-
Update: {
1225-
author_id?: number | null
1226-
contents?: unknown
1227-
created?: string | null
1228-
id?: number | null
1229-
last_modified?: string | null
1230-
metadata?: Json | null
1231-
source_local_id?: string | null
1232-
space_id?: number | null
1233-
url?: string | null
1234-
}
12351145
Relationships: [
12361146
{
12371147
foreignKeyName: "Document_author_id_fkey"
@@ -1272,22 +1182,6 @@ export type Database = {
12721182
source_local_id: string | null
12731183
space_id: number | null
12741184
}
1275-
Insert: {
1276-
created?: string | null
1277-
filehash?: string | null
1278-
filepath?: string | null
1279-
last_modified?: string | null
1280-
source_local_id?: string | null
1281-
space_id?: number | null
1282-
}
1283-
Update: {
1284-
created?: string | null
1285-
filehash?: string | null
1286-
filepath?: string | null
1287-
last_modified?: string | null
1288-
source_local_id?: string | null
1289-
space_id?: number | null
1290-
}
12911185
Relationships: []
12921186
}
12931187
my_groups: {
@@ -1441,6 +1335,7 @@ export type Database = {
14411335
}
14421336
}
14431337
can_access_account: { Args: { account_uid: string }; Returns: boolean }
1338+
can_view_content: { Args: { content_id: number }; Returns: boolean }
14441339
can_view_specific_resource: {
14451340
Args: { source_local_id_: string; space_id_: number }
14461341
Returns: boolean
@@ -1660,6 +1555,16 @@ export type Database = {
16601555
text_content: string
16611556
}[]
16621557
}
1558+
my_accessible_resources: {
1559+
Args: never
1560+
Returns: Database["public"]["CompositeTypes"]["accessible_resource"][]
1561+
SetofOptions: {
1562+
from: "*"
1563+
to: "accessible_resource"
1564+
isOneToOne: false
1565+
isSetofReturn: true
1566+
}
1567+
}
16631568
my_permissions_in_space: {
16641569
Args: { space_id_: number }
16651570
Returns: Database["public"]["Enums"]["SpaceAccessPermissions"]
@@ -1796,6 +1701,10 @@ export type Database = {
17961701
task_status: "active" | "timeout" | "complete" | "failed"
17971702
}
17981703
CompositeTypes: {
1704+
accessible_resource: {
1705+
space_id: number | null
1706+
source_local_id: string | null
1707+
}
17991708
account_local_input: {
18001709
name: string | null
18011710
account_local_id: string | null
@@ -2058,4 +1967,3 @@ export const Constants = {
20581967
},
20591968
},
20601969
} as const
2061-

‎packages/database/supabase/migrations/20260214200836_optimize_view_queries.sql‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
CREATE TYPE public.accessible_resource AS (
2+
space_id bigint,
3+
source_local_id character varying
4+
);
5+
6+
CREATE OR REPLACE FUNCTION public.my_accessible_resources() RETURNS SETOF public.accessible_resource
7+
STRICT STABLE
8+
SET search_path = ''
9+
LANGUAGE sql
10+
AS $$
11+
SELECT DISTINCT space_id, source_local_id FROM public."ResourceAccess"
12+
JOIN public.my_user_accounts() ON (account_uid = my_user_accounts);
13+
$$;
14+
15+
CREATE OR REPLACE VIEW public.my_documents AS
16+
SELECT
17+
id,
18+
space_id,
19+
source_local_id,
20+
url,
21+
"created",
22+
metadata,
23+
last_modified,
24+
author_id,
25+
contents
26+
FROM
27+
public."Document"
28+
LEFT OUTER JOIN public.my_accessible_resources() AS ra USING (space_id, source_local_id)
29+
WHERE (
30+
space_id = any(public.my_space_ids('reader'))
31+
OR (space_id = any(public.my_space_ids('partial')) AND ra.space_id IS NOT NULL)
32+
);
33+
34+
CREATE OR REPLACE VIEW public.my_contents AS
35+
SELECT
36+
id,
37+
document_id,
38+
source_local_id,
39+
variant,
40+
author_id,
41+
creator_id,
42+
created,
43+
text,
44+
metadata,
45+
scale,
46+
space_id,
47+
last_modified,
48+
part_of_id
49+
FROM public."Content"
50+
LEFT OUTER JOIN public.my_accessible_resources() AS ra USING (space_id, source_local_id)
51+
WHERE (
52+
space_id = any(public.my_space_ids('reader'))
53+
OR (space_id = any(public.my_space_ids('partial')) AND ra.space_id IS NOT NULL)
54+
);
55+
56+
57+
CREATE OR REPLACE VIEW public.my_concepts AS
58+
SELECT
59+
id,
60+
epistemic_status,
61+
name,
62+
description,
63+
author_id,
64+
created,
65+
last_modified,
66+
space_id,
67+
arity,
68+
schema_id,
69+
literal_content,
70+
reference_content,
71+
refs,
72+
is_schema,
73+
source_local_id
74+
FROM public."Concept"
75+
LEFT OUTER JOIN public.my_accessible_resources() AS ra USING (space_id, source_local_id)
76+
WHERE (
77+
space_id = any(public.my_space_ids('reader'))
78+
OR (space_id = any(public.my_space_ids('partial')) AND ra.space_id IS NOT NULL)
79+
);
80+
81+
CREATE OR REPLACE VIEW public.my_file_references AS
82+
SELECT
83+
source_local_id,
84+
space_id,
85+
filepath,
86+
filehash,
87+
created,
88+
last_modified
89+
FROM public."FileReference"
90+
LEFT OUTER JOIN public.my_accessible_resources() AS ra USING (space_id, source_local_id)
91+
WHERE (
92+
space_id = any(public.my_space_ids('reader'))
93+
OR (space_id = any(public.my_space_ids('partial')) AND ra.space_id IS NOT NULL)
94+
);
95+
96+
CREATE OR REPLACE VIEW public.my_contents_with_embedding_openai_text_embedding_3_small_1536 AS
97+
SELECT
98+
ct.id,
99+
ct.document_id,
100+
ct.source_local_id,
101+
ct.variant,
102+
ct.author_id,
103+
ct.creator_id,
104+
ct.created,
105+
ct.text,
106+
ct.metadata,
107+
ct.scale,
108+
ct.space_id,
109+
ct.last_modified,
110+
ct.part_of_id,
111+
emb.model,
112+
emb.vector
113+
FROM public."Content" AS ct
114+
JOIN public."ContentEmbedding_openai_text_embedding_3_small_1536" AS emb ON (ct.id = emb.target_id)
115+
LEFT OUTER JOIN public.my_accessible_resources() AS ra USING (space_id, source_local_id)
116+
WHERE (
117+
ct.space_id = any(public.my_space_ids('reader'))
118+
OR (ct.space_id = any(public.my_space_ids('partial')) AND ra.space_id IS NOT NULL)
119+
)
120+
AND NOT emb.obsolete;
121+
122+
CREATE OR REPLACE FUNCTION public.can_view_content(content_id BIGINT) RETURNS BOOLEAN
123+
STABLE
124+
SET search_path = ''
125+
LANGUAGE sql
126+
AS $$
127+
SELECT public.can_view_specific_resource(space_id, source_local_id) FROM public."Content" WHERE id=content_id;
128+
$$;
129+
130+
DROP POLICY IF EXISTS embedding_openai_te3s_1536_policy ON public."ContentEmbedding_openai_text_embedding_3_small_1536";
131+
DROP POLICY IF EXISTS embedding_openai_te3s_1536_select_policy ON public."ContentEmbedding_openai_text_embedding_3_small_1536";
132+
CREATE POLICY embedding_openai_te3s_1536_select_policy ON public."ContentEmbedding_openai_text_embedding_3_small_1536"
133+
FOR SELECT USING (public.content_in_space(target_id) OR public.can_view_content(target_id));
134+
DROP POLICY IF EXISTS embedding_openai_te3s_1536_delete_policy ON public."ContentEmbedding_openai_text_embedding_3_small_1536";
135+
CREATE POLICY embedding_openai_te3s_1536_delete_policy ON public."ContentEmbedding_openai_text_embedding_3_small_1536"
136+
FOR DELETE USING (public.content_in_space(target_id));
137+
DROP POLICY IF EXISTS embedding_openai_te3s_1536_insert_policy ON public."ContentEmbedding_openai_text_embedding_3_small_1536";
138+
CREATE POLICY embedding_openai_te3s_1536_insert_policy ON public."ContentEmbedding_openai_text_embedding_3_small_1536"
139+
FOR INSERT WITH CHECK (public.content_in_space(target_id));
140+
DROP POLICY IF EXISTS embedding_openai_te3s_1536_update_policy ON public."ContentEmbedding_openai_text_embedding_3_small_1536";
141+
CREATE POLICY embedding_openai_te3s_1536_update_policy ON public."ContentEmbedding_openai_text_embedding_3_small_1536"
142+
FOR UPDATE USING (public.content_in_space(target_id));

‎packages/database/supabase/schemas/assets.sql‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,10 @@ SELECT
3131
created,
3232
last_modified
3333
FROM public."FileReference"
34+
LEFT OUTER JOIN public.my_accessible_resources() AS ra USING (space_id, source_local_id)
3435
WHERE (
3536
space_id = any(public.my_space_ids('reader'))
36-
OR public.can_view_specific_resource(space_id, source_local_id)
37+
OR (space_id = any(public.my_space_ids('partial')) AND ra.space_id IS NOT NULL)
3738
);
3839

3940
GRANT ALL ON TABLE public."FileReference" TO authenticated;
@@ -121,7 +122,7 @@ CREATE TRIGGER on_insert_file_reference_trigger AFTER INSERT ON public."FileRefe
121122
INSERT INTO storage.buckets
122123
(id, name, public)
123124
VALUES
124-
('assets', 'assets', false)
125+
('assets', 'assets', FALSE)
125126
ON CONFLICT (id) DO NOTHING;
126127

127128
DROP POLICY IF EXISTS "storage_insert_assets_authenticated" ON storage.objects;
@@ -140,7 +141,7 @@ DROP POLICY IF EXISTS "storage_delete_assets_noref" ON storage.objects;
140141
CREATE POLICY "storage_delete_assets_noref"
141142
ON storage.objects FOR DELETE TO authenticated USING (
142143
bucket_id = 'assets' AND NOT EXISTS (
143-
SELECT true FROM public."FileReference"
144+
SELECT TRUE FROM public."FileReference"
144145
WHERE filehash = name LIMIT 1
145146
)
146147
);

‎packages/database/supabase/schemas/concept.sql‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,9 +127,10 @@ SELECT
127127
is_schema,
128128
source_local_id
129129
FROM public."Concept"
130+
LEFT OUTER JOIN public.my_accessible_resources() AS ra USING (space_id, source_local_id)
130131
WHERE (
131132
space_id = any(public.my_space_ids('reader'))
132-
OR public.can_view_specific_resource(space_id, source_local_id)
133+
OR (space_id = any(public.my_space_ids('partial')) AND ra.space_id IS NOT null)
133134
);
134135

135136
-- following https://docs.postgrest.org/en/v13/references/api/resource_embedding.html#recursive-relationships

0 commit comments

Comments
 (0)