Skip to content

Commit 7349b12

Browse files
authored
Merge pull request #88 from OpenSPP/feat/registry-search-optimization
feat: optimize registry search for large datasets with split queries and trigram indexes
2 parents 32a5c78 + 0efa590 commit 7349b12

13 files changed

Lines changed: 378 additions & 95 deletions

spp_registry/__manifest__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
{
44
"name": "OpenSPP Registry",
55
"category": "OpenSPP/Core",
6-
"version": "19.0.2.0.0",
6+
"version": "19.0.2.1.0",
77
"sequence": 1,
88
"author": "OpenSPP.org",
99
"website": "https://github.com/OpenSPP/OpenSPP2",

spp_registry/models/phone_number.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ class SPPPhoneNumber(models.Model):
2525
index=True,
2626
domain=[("is_registrant", "=", True)],
2727
)
28-
phone_no = fields.Char("Phone Number", required=True)
28+
phone_no = fields.Char("Phone Number", required=True, index=True)
2929
phone_sanitized = fields.Char(compute="_compute_phone_sanitized", store=True)
3030
date_collected = fields.Date(
3131
default=fields.Date.today,
@@ -34,6 +34,14 @@ class SPPPhoneNumber(models.Model):
3434
disabled_by = fields.Many2one("res.users")
3535
country_id = fields.Many2one("res.country", "Country")
3636

37+
def init(self):
38+
"""Create trigram index for phone number ILIKE search."""
39+
self.env.cr.execute("""
40+
CREATE EXTENSION IF NOT EXISTS pg_trgm;
41+
CREATE INDEX IF NOT EXISTS spp_phone_number_phone_no_trgm_idx
42+
ON spp_phone_number USING gin (phone_no gin_trgm_ops);
43+
""")
44+
3745
@api.onchange("date_collected")
3846
def _check_date_collected(self):
3947
for record in self:

spp_registry/models/reg_id.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ class SPPRegistrantID(models.Model):
2323
)
2424
available_id_type_ids = fields.Many2many("spp.vocabulary.code", compute="_compute_available_id_type_ids")
2525
id_type_id = fields.Many2one("spp.vocabulary.code", "ID Type", required=True)
26-
value = fields.Char(size=100)
26+
value = fields.Char(size=100, index=True)
2727

2828
expiry_date = fields.Date()
2929
id_type_as_str = fields.Char(related="id_type_id.display")

spp_registry/models/registrant.py

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ class SPPRegistrant(models.Model):
3333
disabled_by = fields.Many2one("res.users")
3434

3535
reg_ids = fields.One2many("spp.registry.id", "partner_id", "Registrant IDs")
36-
is_registrant = fields.Boolean("Registrant")
37-
is_group = fields.Boolean("Group")
36+
is_registrant = fields.Boolean("Registrant", index=True)
37+
is_group = fields.Boolean("Group", index=True)
3838

3939
name = fields.Char(index=True)
4040

@@ -44,7 +44,7 @@ class SPPRegistrant(models.Model):
4444
phone_number_ids = fields.One2many("spp.phone.number", "partner_id", "Phone Numbers")
4545

4646
company_id = fields.Many2one("res.company", required=True, default=lambda self: self.env.company)
47-
registration_date = fields.Date(default=lambda self: fields.Date.today())
47+
registration_date = fields.Date(default=lambda self: fields.Date.today(), index=True)
4848
tags_ids = fields.Many2many(
4949
"spp.vocabulary.code",
5050
relation="res_partner_registrant_tag_rel",
@@ -79,6 +79,20 @@ class SPPRegistrant(models.Model):
7979
compute="_compute_relationships_count",
8080
)
8181

82+
def init(self):
83+
"""Create trigram indexes for ILIKE search performance.
84+
85+
Standard B-tree indexes cannot help with ILIKE '%term%' (leading wildcard).
86+
Trigram GIN indexes allow PostgreSQL to use indexes for substring matching.
87+
"""
88+
self.env.cr.execute("""
89+
CREATE EXTENSION IF NOT EXISTS pg_trgm;
90+
CREATE INDEX IF NOT EXISTS res_partner_name_trgm_idx
91+
ON res_partner USING gin (name gin_trgm_ops);
92+
CREATE INDEX IF NOT EXISTS res_partner_email_trgm_idx
93+
ON res_partner USING gin (email gin_trgm_ops);
94+
""")
95+
8296
@api.onchange("phone_number_ids")
8397
def phone_number_ids_change(self):
8498
phone = ""

spp_registry_search/__manifest__.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
{
33
"name": "OpenSPP Registry Search Portal",
44
"category": "OpenSPP/Registry",
5-
"version": "19.0.2.0.0",
5+
"version": "19.0.2.1.0",
66
"sequence": 1,
77
"author": "OpenSPP.org",
88
"website": "https://github.com/OpenSPP/OpenSPP2",
@@ -16,6 +16,8 @@
1616
"security/groups.xml",
1717
"security/ir.model.access.csv",
1818
"security/rules.xml",
19+
"data/config_parameters.xml",
20+
"views/res_config_settings_views.xml",
1921
"views/registry_search_actions.xml",
2022
"views/menu_views.xml",
2123
],
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
<?xml version="1.0" encoding="UTF-8" ?>
2+
<odoo noupdate="1">
3+
<record id="search_mode_default" model="ir.config_parameter">
4+
<field name="key">spp_registry_search.search_mode</field>
5+
<field name="value">unified</field>
6+
</record>
7+
<record id="target_field_default" model="ir.config_parameter">
8+
<field name="key">spp_registry_search.target_field</field>
9+
<field name="value">name</field>
10+
</record>
11+
<record id="result_limit_default" model="ir.config_parameter">
12+
<field name="key">spp_registry_search.result_limit</field>
13+
<field name="value">50</field>
14+
</record>
15+
<record id="min_chars_default" model="ir.config_parameter">
16+
<field name="key">spp_registry_search.min_chars</field>
17+
<field name="value">3</field>
18+
</record>
19+
</odoo>
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
# Part of OpenSPP. See LICENSE file for full copyright and licensing details.
22
from . import registry_view_history
3+
from . import res_config_settings
34
from . import res_partner

spp_registry_search/models/registry_view_history.py

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -101,15 +101,14 @@ def record_view(self, partner_id):
101101

102102
def _cleanup_old_records(self, max_records=50):
103103
"""Remove oldest records beyond max_records per user."""
104-
user_records = self.search(
104+
to_delete = self.search(
105105
[
106106
("user_id", "=", self.env.uid),
107107
],
108108
order="view_date desc",
109+
offset=max_records,
109110
)
110-
111-
if len(user_records) > max_records:
112-
to_delete = user_records[max_records:]
111+
if to_delete:
113112
to_delete.unlink()
114113

115114
@api.model
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Part of OpenSPP. See LICENSE file for full copyright and licensing details.
2+
3+
from odoo import fields, models
4+
5+
6+
class ResConfigSettings(models.TransientModel):
7+
_inherit = "res.config.settings"
8+
9+
registry_search_mode = fields.Selection(
10+
selection=[
11+
("unified", "Unified Search (search across all fields)"),
12+
("targeted", "Targeted Search (search a specific field)"),
13+
],
14+
string="Registry Search Mode",
15+
default="unified",
16+
config_parameter="spp_registry_search.search_mode",
17+
help="'Unified Search' searches name, ID number, phone, and email simultaneously. "
18+
"Convenient but slower on very large datasets. "
19+
"'Targeted Search' requires users to select which field to search — faster on large datasets.",
20+
)
21+
22+
registry_search_target_field = fields.Selection(
23+
selection=[
24+
("name", "Name"),
25+
("id_number", "ID Number"),
26+
("phone", "Phone Number"),
27+
("email", "Email"),
28+
],
29+
string="Default Search Field",
30+
default="name",
31+
config_parameter="spp_registry_search.target_field",
32+
help="When search mode is 'Targeted Search', this is the default field "
33+
"users will search against. Users can change this in the search portal.",
34+
)
35+
36+
registry_search_result_limit = fields.Integer(
37+
string="Maximum Search Results",
38+
default=50,
39+
config_parameter="spp_registry_search.result_limit",
40+
help="Maximum number of results returned per search. Lower values improve performance. Range: 10-200.",
41+
)
42+
43+
registry_search_min_chars = fields.Integer(
44+
string="Minimum Search Characters",
45+
default=3,
46+
config_parameter="spp_registry_search.min_chars",
47+
help="Minimum number of characters required before a search is triggered. "
48+
"Higher values reduce unnecessary queries. Range: 1-10.",
49+
)

spp_registry_search/models/res_partner.py

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,3 +59,135 @@ def action_unarchive(self):
5959
if registrants and not self._check_archive_permission():
6060
raise AccessError(_("You do not have the necessary permissions to unarchive registrants."))
6161
return super().action_unarchive()
62+
63+
@api.model
64+
def get_search_config(self):
65+
"""Return registry search configuration for the JS portal."""
66+
get_param = self.env["ir.config_parameter"].sudo().get_param # nosemgrep: odoo-sudo-without-context
67+
result_limit = int(get_param("spp_registry_search.result_limit", "50"))
68+
min_chars = int(get_param("spp_registry_search.min_chars", "3"))
69+
return {
70+
"search_mode": get_param("spp_registry_search.search_mode", "unified"),
71+
"target_field": get_param("spp_registry_search.target_field", "name"),
72+
"result_limit": max(10, min(200, result_limit)),
73+
"min_chars": max(1, min(10, min_chars)),
74+
}
75+
76+
@api.model
77+
def search_registrants(self, search_term, search_type="all", search_field=None, advanced_filters=None, limit=50):
78+
"""Optimized registrant search that respects configured search mode.
79+
80+
Instead of a single ORM domain with OR across JOINs, runs separate
81+
targeted queries per field and merges results. Each individual query
82+
can use indexes efficiently.
83+
84+
Args:
85+
search_term: The search string
86+
search_type: 'all', 'individuals', or 'groups'
87+
search_field: If targeted mode, which field to search ('name', 'id_number', 'phone', 'email')
88+
advanced_filters: Dict with optional date range filters
89+
limit: Maximum results to return
90+
91+
Returns:
92+
list: List of dicts with partner data
93+
"""
94+
if not search_term:
95+
return []
96+
97+
# Read config with fallback defaults
98+
get_param = self.env["ir.config_parameter"].sudo().get_param # nosemgrep: odoo-sudo-without-context
99+
config_limit = int(get_param("spp_registry_search.result_limit", "50"))
100+
limit = max(10, min(200, limit or config_limit))
101+
102+
search_mode = get_param("spp_registry_search.search_mode", "unified")
103+
104+
# If targeted mode and a field is specified, only search that field
105+
if search_mode == "targeted" and search_field:
106+
partner_ids = self._search_by_field(search_term, search_field, limit)
107+
else:
108+
# Unified mode: run separate queries per field and merge
109+
partner_ids = self._search_unified(search_term, limit)
110+
111+
if not partner_ids:
112+
return []
113+
114+
# Build final domain with type and advanced filters
115+
domain = [("id", "in", list(partner_ids)), ("is_registrant", "=", True)]
116+
117+
if search_type == "individuals":
118+
domain.append(("is_group", "=", False))
119+
elif search_type == "groups":
120+
domain.append(("is_group", "=", True))
121+
122+
if advanced_filters:
123+
if advanced_filters.get("registrationDateFrom"):
124+
domain.append(("registration_date", ">=", advanced_filters["registrationDateFrom"]))
125+
if advanced_filters.get("registrationDateTo"):
126+
domain.append(("registration_date", "<=", advanced_filters["registrationDateTo"]))
127+
128+
fields_to_read = ["name", "is_group", "phone", "email", "registration_date", "disabled"]
129+
return self.search_read(domain, fields_to_read, limit=limit, order="id desc")
130+
131+
def _search_unified(self, search_term, limit):
132+
"""Run separate indexed queries per field and merge results."""
133+
partner_ids = set()
134+
135+
# Query 1: Name match (uses trigram index)
136+
name_matches = self.search(
137+
[("is_registrant", "=", True), ("name", "ilike", search_term)],
138+
limit=limit,
139+
)
140+
partner_ids.update(name_matches.ids)
141+
142+
# Query 2: ID number exact match (uses B-tree index)
143+
reg_ids = self.env["spp.registry.id"].search(
144+
[("value", "=", search_term)],
145+
limit=limit,
146+
)
147+
partner_ids.update(reg_ids.mapped("partner_id").ids)
148+
149+
# Query 3: Phone number match (uses trigram index)
150+
phones = self.env["spp.phone.number"].search(
151+
[("phone_no", "ilike", search_term)],
152+
limit=limit,
153+
)
154+
partner_ids.update(phones.mapped("partner_id").ids)
155+
156+
# Query 4: Email match (uses trigram index)
157+
email_matches = self.search(
158+
[("is_registrant", "=", True), ("email", "ilike", search_term)],
159+
limit=limit,
160+
)
161+
partner_ids.update(email_matches.ids)
162+
163+
return partner_ids
164+
165+
def _search_by_field(self, search_term, search_field, limit):
166+
"""Search a single field (targeted mode)."""
167+
if search_field == "name":
168+
return set(
169+
self.search(
170+
[("is_registrant", "=", True), ("name", "ilike", search_term)],
171+
limit=limit,
172+
).ids
173+
)
174+
elif search_field == "id_number":
175+
reg_ids = self.env["spp.registry.id"].search(
176+
[("value", "=", search_term)],
177+
limit=limit,
178+
)
179+
return set(reg_ids.mapped("partner_id").ids)
180+
elif search_field == "phone":
181+
phones = self.env["spp.phone.number"].search(
182+
[("phone_no", "ilike", search_term)],
183+
limit=limit,
184+
)
185+
return set(phones.mapped("partner_id").ids)
186+
elif search_field == "email":
187+
return set(
188+
self.search(
189+
[("is_registrant", "=", True), ("email", "ilike", search_term)],
190+
limit=limit,
191+
).ids
192+
)
193+
return set()

0 commit comments

Comments
 (0)