Skip to content

Commit aca13d9

Browse files
authored
Merge pull request #45 from OpenSPP/feat/aggregation-metrics
feat: add metrics services and aggregation engine
2 parents a6a6a13 + a11688a commit aca13d9

50 files changed

Lines changed: 8961 additions & 0 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

spp_aggregation/__init__.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Part of OpenSPP. See LICENSE file for full copyright and licensing details.
2+
3+
from . import models

spp_aggregation/__manifest__.py

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Part of OpenSPP. See LICENSE file for full copyright and licensing details.
2+
{
3+
"name": "OpenSPP Aggregation Engine",
4+
"summary": "Unified aggregation service for statistics, simulations, and GIS queries",
5+
"category": "OpenSPP",
6+
"version": "19.0.2.0.0",
7+
"sequence": 1,
8+
"author": "OpenSPP.org",
9+
"website": "https://github.com/OpenSPP/OpenSPP2",
10+
"license": "LGPL-3",
11+
"development_status": "Alpha",
12+
"maintainers": ["jeremi"],
13+
"depends": [
14+
"base",
15+
"spp_cel_domain",
16+
"spp_area",
17+
"spp_registry",
18+
"spp_security",
19+
"spp_metrics_services",
20+
],
21+
"data": [
22+
# Security
23+
"security/aggregation_security.xml",
24+
"security/ir.model.access.csv",
25+
# Data
26+
"data/cron_cache_cleanup.xml",
27+
# Views
28+
"views/aggregation_scope_views.xml",
29+
"views/aggregation_access_views.xml",
30+
"views/menu.xml",
31+
],
32+
"assets": {},
33+
"demo": [],
34+
"images": [],
35+
"application": False,
36+
"installable": True,
37+
"auto_install": False,
38+
}
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
<?xml version="1.0" encoding="utf-8" ?>
2+
<odoo noupdate="1">
3+
<!-- Cron job to clean up expired cache entries -->
4+
<record id="ir_cron_cache_cleanup" model="ir.cron">
5+
<field name="name">Aggregation: Cache Cleanup</field>
6+
<field name="model_id" ref="model_spp_aggregation_cache_entry" />
7+
<field name="state">code</field>
8+
<field name="code">model.cron_cleanup_expired()</field>
9+
<field name="interval_number">1</field>
10+
<field name="interval_type">hours</field>
11+
<field name="active">True</field>
12+
</record>
13+
</odoo>

spp_aggregation/models/__init__.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Part of OpenSPP. See LICENSE file for full copyright and licensing details.
2+
3+
from . import aggregation_scope
4+
from . import aggregation_access
5+
from . import service_scope_resolver
6+
from . import service_cache
7+
from . import statistic_registry
8+
from . import service_aggregation
Lines changed: 318 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,318 @@
1+
# Part of OpenSPP. See LICENSE file for full copyright and licensing details.
2+
import logging
3+
4+
from odoo import _, api, fields, models
5+
from odoo.exceptions import ValidationError
6+
7+
_logger = logging.getLogger(__name__)
8+
9+
10+
class AggregationAccessRule(models.Model):
11+
"""
12+
Access control rules for aggregation queries.
13+
14+
Determines what level of data access a user/group has:
15+
- aggregate: Can only see counts and statistics (no individual records)
16+
- individual: Can see individual record IDs in results
17+
18+
Also controls k-anonymity thresholds and scope restrictions.
19+
"""
20+
21+
_name = "spp.aggregation.access.rule"
22+
_description = "Aggregation Access Rule"
23+
_order = "sequence, name"
24+
25+
name = fields.Char(
26+
required=True,
27+
help="Human-readable name for this access rule.",
28+
)
29+
description = fields.Text(
30+
help="Optional description of what this rule grants.",
31+
)
32+
sequence = fields.Integer(
33+
default=10,
34+
help="Lower sequence = higher priority when multiple rules match.",
35+
)
36+
active = fields.Boolean(
37+
default=True,
38+
index=True,
39+
)
40+
41+
# -------------------------------------------------------------------------
42+
# Who this rule applies to (one of user/group)
43+
# -------------------------------------------------------------------------
44+
user_id = fields.Many2one(
45+
comodel_name="res.users",
46+
string="User",
47+
ondelete="cascade",
48+
help="Specific user this rule applies to.",
49+
)
50+
group_id = fields.Many2one(
51+
comodel_name="res.groups",
52+
string="Security Group",
53+
ondelete="cascade",
54+
help="Security group this rule applies to.",
55+
)
56+
57+
# -------------------------------------------------------------------------
58+
# Access level
59+
# -------------------------------------------------------------------------
60+
access_level = fields.Selection(
61+
selection=[
62+
("aggregate", "Aggregates Only"),
63+
("individual", "Individual Records"),
64+
],
65+
required=True,
66+
default="aggregate",
67+
help=(
68+
"Aggregates Only: User can see counts and statistics but NOT individual record IDs. "
69+
"Individual Records: User can see individual record IDs in results."
70+
),
71+
)
72+
73+
# -------------------------------------------------------------------------
74+
# Privacy settings
75+
# -------------------------------------------------------------------------
76+
minimum_k_anonymity = fields.Integer(
77+
default=5,
78+
help="Minimum count for a cell before it's suppressed (k-anonymity). Higher = more private.",
79+
)
80+
81+
# -------------------------------------------------------------------------
82+
# Scope restrictions
83+
# -------------------------------------------------------------------------
84+
allowed_scope_types = fields.Selection(
85+
selection=[
86+
("all", "All Scope Types"),
87+
("area_only", "Area-based Only"),
88+
("predefined", "Predefined Scopes Only"),
89+
],
90+
default="all",
91+
help=(
92+
"Restrict which scope types this user can query. "
93+
"Predefined means they can only use saved scope IDs, not inline definitions."
94+
),
95+
)
96+
allow_inline_scopes = fields.Boolean(
97+
default=False,
98+
help=(
99+
"If False, user can only query predefined scope IDs, not create inline scope definitions. "
100+
"This prevents ad-hoc queries that could be used to narrow down individuals."
101+
),
102+
)
103+
allowed_scope_ids = fields.Many2many(
104+
comodel_name="spp.aggregation.scope",
105+
relation="spp_aggregation_access_rule_scope_rel",
106+
column1="rule_id",
107+
column2="scope_id",
108+
string="Allowed Scopes",
109+
help="If set, user can only query these specific scopes (for predefined mode).",
110+
)
111+
112+
# -------------------------------------------------------------------------
113+
# Area restrictions
114+
# -------------------------------------------------------------------------
115+
allowed_area_ids = fields.Many2many(
116+
comodel_name="spp.area",
117+
relation="spp_aggregation_access_rule_area_rel",
118+
column1="rule_id",
119+
column2="area_id",
120+
string="Allowed Areas",
121+
help="If set, user can only query data from these specific areas (and optionally their children).",
122+
)
123+
include_child_areas = fields.Boolean(
124+
default=True,
125+
help="If True, allowed_area_ids includes child areas. If False, only the exact areas are allowed.",
126+
)
127+
128+
# -------------------------------------------------------------------------
129+
# Dimension restrictions
130+
# -------------------------------------------------------------------------
131+
max_group_by_dimensions = fields.Integer(
132+
default=3,
133+
help="Maximum number of dimensions allowed in group_by. More dimensions = more granular = less private.",
134+
)
135+
allowed_dimension_ids = fields.Many2many(
136+
comodel_name="spp.demographic.dimension",
137+
relation="spp_aggregation_access_rule_dimension_rel",
138+
column1="rule_id",
139+
column2="dimension_id",
140+
string="Allowed Dimensions",
141+
help="If set, user can only group by these dimensions.",
142+
)
143+
144+
# -------------------------------------------------------------------------
145+
# Validation
146+
# -------------------------------------------------------------------------
147+
@api.constrains("user_id", "group_id")
148+
def _check_user_or_group(self):
149+
"""Ensure exactly one of user_id or group_id is set."""
150+
for rule in self:
151+
if rule.user_id and rule.group_id:
152+
raise ValidationError(_("A rule cannot apply to both a specific user and a group."))
153+
if not rule.user_id and not rule.group_id:
154+
raise ValidationError(_("A rule must apply to either a user or a group."))
155+
156+
@api.constrains("minimum_k_anonymity")
157+
def _check_k_anonymity(self):
158+
"""Ensure k-anonymity threshold is reasonable."""
159+
for rule in self:
160+
if rule.minimum_k_anonymity < 1:
161+
raise ValidationError(_("Minimum k-anonymity must be at least 1."))
162+
if rule.minimum_k_anonymity > 100:
163+
raise ValidationError(_("Minimum k-anonymity should not exceed 100."))
164+
165+
@api.constrains("max_group_by_dimensions")
166+
def _check_max_dimensions(self):
167+
"""Ensure max dimensions is reasonable."""
168+
for rule in self:
169+
if rule.max_group_by_dimensions < 0:
170+
raise ValidationError(_("Maximum group_by dimensions cannot be negative."))
171+
if rule.max_group_by_dimensions > 10:
172+
raise ValidationError(_("Maximum group_by dimensions should not exceed 10."))
173+
174+
# -------------------------------------------------------------------------
175+
# Public API
176+
# -------------------------------------------------------------------------
177+
@api.model
178+
def get_effective_rule_for_user(self, user=None):
179+
"""
180+
Get the most permissive applicable access rule for a user.
181+
182+
Rules are evaluated in sequence order. User-specific rules take precedence
183+
over group-based rules.
184+
185+
:param user: res.users record (defaults to current user)
186+
:returns: Access rule record or None if no rule matches
187+
:rtype: spp.aggregation.access.rule or None
188+
"""
189+
user = user or self.env.user
190+
191+
# First check for user-specific rule
192+
user_rule = self.search(
193+
[("user_id", "=", user.id), ("active", "=", True)],
194+
limit=1,
195+
order="sequence",
196+
)
197+
if user_rule:
198+
return user_rule
199+
200+
# Then check for group-based rules
201+
group_rule = self.search(
202+
[("group_id", "in", user.group_ids.ids), ("active", "=", True)],
203+
limit=1,
204+
order="sequence",
205+
)
206+
return group_rule
207+
208+
def check_scope_allowed(self, scope):
209+
"""
210+
Check if a scope is allowed under this rule.
211+
212+
:param scope: spp.aggregation.scope record or dict for inline scope
213+
:returns: True if allowed
214+
:raises: ValidationError if not allowed
215+
"""
216+
self.ensure_one()
217+
218+
# Check inline scope restriction
219+
if isinstance(scope, dict) and not self.allow_inline_scopes:
220+
raise ValidationError(_("Inline scope definitions are not allowed for your access level."))
221+
222+
# Get scope type
223+
scope_type = scope.get("scope_type") if isinstance(scope, dict) else scope.scope_type
224+
225+
# Check scope type restriction
226+
if self.allowed_scope_types == "predefined":
227+
if isinstance(scope, dict):
228+
raise ValidationError(_("Only predefined scopes are allowed for your access level."))
229+
if self.allowed_scope_ids and scope.id not in self.allowed_scope_ids.ids:
230+
raise ValidationError(_("This scope is not in your allowed scope list."))
231+
232+
if self.allowed_scope_types == "area_only":
233+
if scope_type not in ("area", "area_tag"):
234+
raise ValidationError(_("Only area-based scopes are allowed for your access level."))
235+
236+
# Check area restrictions for explicit scopes
237+
if scope_type == "explicit" and self.allowed_area_ids:
238+
partner_ids = (
239+
scope.get("explicit_partner_ids") if isinstance(scope, dict) else scope.explicit_partner_ids.ids
240+
)
241+
self._check_explicit_scope_area_compliance(partner_ids)
242+
243+
return True
244+
245+
def check_dimensions_allowed(self, dimension_names):
246+
"""
247+
Check if the requested dimensions are allowed.
248+
249+
:param dimension_names: List of dimension names
250+
:returns: True if allowed
251+
:raises: ValidationError if not allowed
252+
"""
253+
self.ensure_one()
254+
255+
if len(dimension_names) > self.max_group_by_dimensions:
256+
raise ValidationError(
257+
_("Too many dimensions: maximum %d allowed, %d requested.")
258+
% (self.max_group_by_dimensions, len(dimension_names))
259+
)
260+
261+
if self.allowed_dimension_ids:
262+
allowed_names = set(self.allowed_dimension_ids.mapped("name"))
263+
requested = set(dimension_names)
264+
disallowed = requested - allowed_names
265+
if disallowed:
266+
raise ValidationError(_("Dimensions not allowed: %s") % ", ".join(disallowed))
267+
268+
return True
269+
270+
def _check_explicit_scope_area_compliance(self, partner_ids):
271+
"""
272+
Check if explicit partner IDs are within allowed areas.
273+
274+
:param partner_ids: List of partner IDs
275+
:returns: True if allowed
276+
:raises: ValidationError if any partner is outside allowed areas
277+
"""
278+
self.ensure_one()
279+
280+
if not self.allowed_area_ids:
281+
# No area restrictions
282+
return True
283+
284+
if not partner_ids:
285+
# Empty list is always allowed
286+
return True
287+
288+
# Build set of allowed area IDs
289+
allowed_area_ids = set(self.allowed_area_ids.ids)
290+
291+
# If include_child_areas is True, expand to include all child areas
292+
if self.include_child_areas:
293+
# Collect all parent_path values first, then do a single search using
294+
# OR-chained domain conditions to avoid N+1 queries inside a loop.
295+
parent_paths = [area.parent_path for area in self.allowed_area_ids if area.parent_path]
296+
if parent_paths:
297+
domain = ["|"] * (len(parent_paths) - 1)
298+
for path in parent_paths:
299+
domain.append(("parent_path", "like", f"{path}%"))
300+
child_areas = self.env["spp.area"].sudo().search(domain) # nosemgrep: odoo-sudo-without-context
301+
allowed_area_ids.update(child_areas.ids)
302+
303+
# Get area_ids for the partners
304+
partners = self.env["res.partner"].sudo().browse(partner_ids) # nosemgrep: odoo-sudo-without-context, odoo-sudo-on-sensitive-models # noqa: E501 # fmt: skip
305+
partner_area_ids = set(partners.mapped("area_id").ids)
306+
307+
# Check if all partner areas are in allowed areas
308+
disallowed_area_ids = partner_area_ids - allowed_area_ids
309+
310+
if disallowed_area_ids:
311+
# Get area names for error message
312+
disallowed_areas = self.env["spp.area"].sudo().browse(list(disallowed_area_ids)) # nosemgrep: odoo-sudo-without-context # noqa: E501 # fmt: skip
313+
area_names = ", ".join(disallowed_areas.mapped("draft_name"))
314+
raise ValidationError(
315+
_("Some registrants are outside your allowed areas. Disallowed areas: %s") % area_names
316+
)
317+
318+
return True

0 commit comments

Comments
 (0)