Skip to content

Commit 5b26ad7

Browse files
Merge pull request #128 from OpenSPP/ken/optimize_spp_program_phase_5
perf(spp_programs): batch create entitlements and payments
2 parents 5916d10 + 2bce58e commit 5b26ad7

File tree

8 files changed

+224
-33
lines changed

8 files changed

+224
-33
lines changed

spp_programs/README.rst

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,12 @@ Dependencies
254254
Changelog
255255
=========
256256

257+
19.0.2.0.5
258+
~~~~~~~~~~
259+
260+
- Batch create entitlements and payments instead of one-by-one ORM
261+
creates
262+
257263
19.0.2.0.4
258264
~~~~~~~~~~
259265

spp_programs/__manifest__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"name": "OpenSPP Programs",
55
"summary": "Manage cash and in-kind entitlements, integrate with inventory, and enhance program management features for comprehensive social protection and agricultural support.",
66
"category": "OpenSPP/Core",
7-
"version": "19.0.2.0.4",
7+
"version": "19.0.2.0.5",
88
"sequence": 1,
99
"author": "OpenSPP.org",
1010
"website": "https://github.com/OpenSPP/OpenSPP2",

spp_programs/models/managers/entitlement_manager_cash.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -156,13 +156,16 @@ def prepare_entitlements(self, cycle, beneficiaries):
156156
if addl_fields:
157157
new_entitlements_to_create[beneficiary_id.id].update(addl_fields)
158158

159-
# Create entitlement records
159+
# Create entitlement records in a single batch
160+
vals_list = []
160161
for ent in new_entitlements_to_create:
161162
initial_amount = new_entitlements_to_create[ent]["initial_amount"]
162163
new_entitlements_to_create[ent]["initial_amount"] = self._check_subsidy(initial_amount)
163164
# Create non-zero entitlements only
164165
if new_entitlements_to_create[ent]["initial_amount"] > 0.0:
165-
self.env["spp.entitlement"].create(new_entitlements_to_create[ent])
166+
vals_list.append(new_entitlements_to_create[ent])
167+
if vals_list:
168+
self.env["spp.entitlement"].create(vals_list)
166169

167170
def _get_addl_entitlement_fields(self, beneficiary_id):
168171
"""

spp_programs/models/managers/payment_manager.py

Lines changed: 39 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -211,42 +211,55 @@ def _prepare_payments(self, cycle, entitlements):
211211
entitlements -= tag_entitlements
212212
max_batch_size = batch_tag.max_batch_size
213213

214-
for i, entitlement_id in enumerate(tag_entitlements):
215-
payment = self.env["spp.payment"].create(
214+
if not tag_entitlements:
215+
continue
216+
217+
# Prefetch bank_ids to avoid N+1 queries
218+
tag_entitlements.mapped("partner_id.bank_ids.acc_number")
219+
220+
# Build all payment vals in one pass
221+
payment_vals_list = []
222+
for entitlement_id in tag_entitlements:
223+
account_number = None
224+
if entitlement_id.partner_id.bank_ids:
225+
account_number = entitlement_id.partner_id.bank_ids[0].acc_number
226+
payment_vals_list.append(
216227
{
217228
"name": str(uuid4()),
218229
"entitlement_id": entitlement_id.id,
219230
"cycle_id": entitlement_id.cycle_id.id,
220231
"amount_issued": entitlement_id.initial_amount,
221232
"payment_fee": entitlement_id.transfer_fee,
222233
"state": "issued",
234+
"account_number": account_number,
223235
}
224236
)
225-
if payment.partner_id.bank_ids:
226-
payment.account_number = payment.partner_id.bank_ids[0].acc_number
227-
else:
228-
payment.account_number = None
229237

230-
if not payments:
231-
payments = payment
232-
else:
233-
payments += payment
234-
if create_batch:
235-
if i % max_batch_size == 0:
236-
curr_batch = self.env["spp.payment.batch"].create(
237-
{
238-
"name": str(uuid4()),
239-
"cycle_id": cycle.id,
240-
"stats_datetime": fields.Datetime.now(),
241-
"tag_id": batch_tag.id,
242-
}
243-
)
244-
if not batches:
245-
batches = curr_batch
246-
else:
247-
batches += curr_batch
248-
curr_batch.payment_ids = [(4, payment.id)]
249-
payment.batch_id = curr_batch
238+
# Batch create all payments for this tag
239+
tag_payments = self.env["spp.payment"].create(payment_vals_list)
240+
241+
if not payments:
242+
payments = tag_payments
243+
else:
244+
payments += tag_payments
245+
246+
if create_batch:
247+
# Assign payments to batches in chunks of max_batch_size
248+
for i in range(0, len(tag_payments), max_batch_size):
249+
batch_payments = tag_payments[i : i + max_batch_size]
250+
curr_batch = self.env["spp.payment.batch"].create(
251+
{
252+
"name": str(uuid4()),
253+
"cycle_id": cycle.id,
254+
"stats_datetime": fields.Datetime.now(),
255+
"tag_id": batch_tag.id,
256+
}
257+
)
258+
batch_payments.write({"batch_id": curr_batch.id})
259+
if not batches:
260+
batches = curr_batch
261+
else:
262+
batches += curr_batch
250263
return payments, batches
251264

252265
def _prepare_payments_async(self, cycle, entitlements, entitlements_count):

spp_programs/readme/HISTORY.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
### 19.0.2.0.5
2+
3+
- Batch create entitlements and payments instead of one-by-one ORM creates
4+
15
### 19.0.2.0.4
26

37
- Fetch fund balance once per approval batch instead of per entitlement

spp_programs/static/description/index.html

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -658,26 +658,33 @@ <h2><a class="toc-backref" href="#toc-entry-1">Changelog</a></h2>
658658
</div>
659659
</div>
660660
<div class="section" id="section-1">
661+
<h1>19.0.2.0.5</h1>
662+
<ul class="simple">
663+
<li>Batch create entitlements and payments instead of one-by-one ORM
664+
creates</li>
665+
</ul>
666+
</div>
667+
<div class="section" id="section-2">
661668
<h1>19.0.2.0.4</h1>
662669
<ul class="simple">
663670
<li>Fetch fund balance once per approval batch instead of per entitlement</li>
664671
</ul>
665672
</div>
666-
<div class="section" id="section-2">
673+
<div class="section" id="section-3">
667674
<h1>19.0.2.0.3</h1>
668675
<ul class="simple">
669676
<li>Replace cycle computed fields (total_amount, entitlements_count,
670677
approval flags) with SQL aggregation queries</li>
671678
</ul>
672679
</div>
673-
<div class="section" id="section-3">
680+
<div class="section" id="section-4">
674681
<h1>19.0.2.0.2</h1>
675682
<ul class="simple">
676683
<li>Add composite indexes for frequent query patterns on entitlements and
677684
program memberships</li>
678685
</ul>
679686
</div>
680-
<div class="section" id="section-4">
687+
<div class="section" id="section-5">
681688
<h1>19.0.2.0.1</h1>
682689
<ul class="simple">
683690
<li>Replace Python-level uniqueness checks with SQL UNIQUE constraints for
@@ -686,7 +693,7 @@ <h1>19.0.2.0.1</h1>
686693
constraint creation</li>
687694
</ul>
688695
</div>
689-
<div class="section" id="section-5">
696+
<div class="section" id="section-6">
690697
<h1>19.0.2.0.0</h1>
691698
<ul class="simple">
692699
<li>Initial migration to OpenSPP2</li>

spp_programs/tests/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,4 @@
2222
from . import test_composite_indexes
2323
from . import test_cycle_computed_fields
2424
from . import test_fund_balance
25+
from . import test_batch_creation
Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
# Part of OpenSPP. See LICENSE file for full copyright and licensing details.
2+
import uuid
3+
from unittest.mock import patch
4+
5+
from odoo import fields
6+
from odoo.tests import TransactionCase
7+
8+
9+
class TestBatchEntitlementCreation(TransactionCase):
10+
"""Test that cash entitlement manager creates entitlements in a single
11+
batch call instead of one-by-one."""
12+
13+
def setUp(self):
14+
super().setUp()
15+
self.program = self.env["spp.program"].create({"name": f"Test Program {uuid.uuid4().hex[:8]}"})
16+
self.journal = self.env["account.journal"].create(
17+
{
18+
"name": "Test Journal",
19+
"type": "bank",
20+
"code": f"TJ{uuid.uuid4().hex[:4].upper()}",
21+
}
22+
)
23+
self.program.journal_id = self.journal.id
24+
25+
self.cycle = self.env["spp.cycle"].create(
26+
{
27+
"name": "Test Cycle",
28+
"program_id": self.program.id,
29+
"start_date": fields.Date.today(),
30+
"end_date": fields.Date.today(),
31+
}
32+
)
33+
self.manager = self.env["spp.program.entitlement.manager.cash"].create(
34+
{
35+
"name": "Test Cash Manager",
36+
"program_id": self.program.id,
37+
}
38+
)
39+
self.env["spp.program.entitlement.manager.cash.item"].create(
40+
{
41+
"entitlement_id": self.manager.id,
42+
"amount": 100.0,
43+
}
44+
)
45+
46+
# Create beneficiaries with cycle memberships
47+
self.registrants = self.env["res.partner"]
48+
self.memberships = self.env["spp.cycle.membership"]
49+
for i in range(5):
50+
reg = self.env["res.partner"].create({"name": f"Registrant {i}", "is_registrant": True})
51+
self.registrants |= reg
52+
self.memberships |= self.env["spp.cycle.membership"].create(
53+
{
54+
"partner_id": reg.id,
55+
"cycle_id": self.cycle.id,
56+
"state": "enrolled",
57+
}
58+
)
59+
60+
def test_cash_manager_batch_creates_entitlements(self):
61+
"""Cash entitlement manager must create entitlements for all beneficiaries
62+
using a single batch vals_list passed to create()."""
63+
self.manager.prepare_entitlements(self.cycle, self.memberships)
64+
65+
entitlements = self.env["spp.entitlement"].search([("cycle_id", "=", self.cycle.id)])
66+
self.assertEqual(
67+
len(entitlements),
68+
5,
69+
f"Expected 5 entitlements, got {len(entitlements)}",
70+
)
71+
# Verify each registrant got an entitlement
72+
entitled_partners = entitlements.mapped("partner_id")
73+
for reg in self.registrants:
74+
self.assertIn(reg, entitled_partners)
75+
76+
77+
class TestBatchPaymentCreation(TransactionCase):
78+
"""Test that payment manager creates payments in a single batch call
79+
instead of one-by-one."""
80+
81+
def setUp(self):
82+
super().setUp()
83+
self.program = self.env["spp.program"].create({"name": f"Test Program {uuid.uuid4().hex[:8]}"})
84+
self.journal = self.env["account.journal"].create(
85+
{
86+
"name": "Test Journal",
87+
"type": "bank",
88+
"code": f"TJ{uuid.uuid4().hex[:4].upper()}",
89+
}
90+
)
91+
self.program.journal_id = self.journal.id
92+
93+
self.cycle = self.env["spp.cycle"].create(
94+
{
95+
"name": "Test Cycle",
96+
"program_id": self.program.id,
97+
"start_date": fields.Date.today(),
98+
"end_date": fields.Date.today(),
99+
}
100+
)
101+
self.payment_manager = self.env["spp.program.payment.manager.default"].create(
102+
{
103+
"name": "Test Payment Manager",
104+
"program_id": self.program.id,
105+
"create_batch": False,
106+
}
107+
)
108+
109+
# Create approved entitlements
110+
self.entitlements = self.env["spp.entitlement"]
111+
for i in range(5):
112+
reg = self.env["res.partner"].create({"name": f"Registrant {i}", "is_registrant": True})
113+
self.entitlements |= self.env["spp.entitlement"].create(
114+
{
115+
"partner_id": reg.id,
116+
"cycle_id": self.cycle.id,
117+
"initial_amount": 100.0,
118+
"state": "approved",
119+
"is_cash_entitlement": True,
120+
}
121+
)
122+
123+
def test_payment_manager_batch_creates_payments(self):
124+
"""Payment manager must call create() at most once per batch tag
125+
(batch), not once per entitlement."""
126+
original_create = type(self.env["spp.payment"]).create
127+
128+
call_count = 0
129+
130+
def counting_create(self_model, vals_list):
131+
nonlocal call_count
132+
call_count += 1
133+
return original_create(self_model, vals_list)
134+
135+
# Add a batch tag so we enter the loop
136+
batch_tag = self.env["spp.payment.batch.tag"].create(
137+
{
138+
"name": "Test Tag",
139+
"order": 1,
140+
"domain": "[]",
141+
"max_batch_size": 500,
142+
}
143+
)
144+
self.payment_manager.batch_tag_ids = [(4, batch_tag.id)]
145+
146+
with patch.object(
147+
type(self.env["spp.payment"]),
148+
"create",
149+
counting_create,
150+
):
151+
self.payment_manager._prepare_payments(self.cycle, self.entitlements)
152+
153+
self.assertEqual(
154+
call_count,
155+
1,
156+
f"create() should be called once (batch), was called {call_count} times",
157+
)

0 commit comments

Comments
 (0)