Skip to content

Commit 07c11b0

Browse files
refactor: rename 'cost' to 'weight' for consistent naming with try_acquire(weight=...)
Co-Authored-By: Daryna Ishchenko <darina.ishchenko17@gmail.com>
1 parent 1b8450d commit 07c11b0

5 files changed

Lines changed: 64 additions & 64 deletions

File tree

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1832,13 +1832,12 @@ definitions:
18321832
description: The headers to match.
18331833
type: object
18341834
additionalProperties: true
1835-
cost:
1836-
title: Cost
1835+
weight:
1836+
title: Weight
18371837
description: >
1838-
The cost of a request matching this matcher in the API's rate limit cost model.
1839-
When set, this value is passed as the weight when acquiring a call from the rate limiter,
1840-
enabling cost-based rate limiting where different endpoints consume different amounts
1841-
from a shared budget. If not set, each request counts as 1.
1838+
The weight of a request matching this matcher when acquiring a call from the rate limiter.
1839+
Different endpoints can consume different amounts from a shared budget by specifying
1840+
different weights. If not set, each request counts as 1.
18421841
anyOf:
18431842
- type: integer
18441843
- type: string

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -486,10 +486,10 @@ class Config:
486486
headers: Optional[Dict[str, Any]] = Field(
487487
None, description="The headers to match.", title="Headers"
488488
)
489-
cost: Optional[Union[int, str]] = Field(
489+
weight: Optional[Union[int, str]] = Field(
490490
None,
491-
description="The cost of a request matching this matcher in the API's rate limit cost model. When set, this value is passed as the weight when acquiring a call from the rate limiter, enabling cost-based rate limiting where different endpoints consume different amounts from a shared budget. If not set, each request counts as 1.",
492-
title="Cost",
491+
description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.",
492+
title="Weight",
493493
)
494494

495495

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4387,19 +4387,19 @@ def create_rate(self, model: RateModel, config: Config, **kwargs: Any) -> Rate:
43874387
def create_http_request_matcher(
43884388
self, model: HttpRequestRegexMatcherModel, config: Config, **kwargs: Any
43894389
) -> HttpRequestRegexMatcher:
4390-
cost = model.cost
4391-
if cost is not None:
4392-
if isinstance(cost, str):
4393-
cost = int(InterpolatedString.create(cost, parameters={}).eval(config))
4390+
weight = model.weight
4391+
if weight is not None:
4392+
if isinstance(weight, str):
4393+
weight = int(InterpolatedString.create(weight, parameters={}).eval(config))
43944394
else:
4395-
cost = int(cost)
4395+
weight = int(weight)
43964396
return HttpRequestRegexMatcher(
43974397
method=model.method,
43984398
url_base=model.url_base,
43994399
url_path_pattern=model.url_path_pattern,
44004400
params=model.params,
44014401
headers=model.headers,
4402-
cost=cost,
4402+
weight=weight,
44034403
)
44044404

44054405
def set_api_budget(self, component_definition: ComponentDefinition, config: Config) -> None:

airbyte_cdk/sources/streams/call_rate.py

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -166,19 +166,20 @@ def __init__(
166166
url_path_pattern: Optional[str] = None,
167167
params: Optional[Mapping[str, Any]] = None,
168168
headers: Optional[Mapping[str, Any]] = None,
169-
cost: Optional[int] = None,
169+
weight: Optional[int] = None,
170170
):
171171
"""
172172
:param method: HTTP method (e.g. "GET", "POST"); compared case-insensitively.
173173
:param url_base: Base URL (scheme://host) that must match.
174174
:param url_path_pattern: A regex pattern that will be applied to the path portion of the URL.
175175
:param params: Dictionary of query parameters that must be present in the request.
176176
:param headers: Dictionary of headers that must be present (header keys are compared case-insensitively).
177-
:param cost: The cost (weight) of a request matching this matcher. If set, this value is used
178-
as the weight when acquiring a call from the rate limiter, enabling cost-based rate limiting.
177+
:param weight: The weight of a request matching this matcher. If set, this value is used
178+
when acquiring a call from the rate limiter, enabling cost-based rate limiting
179+
where different endpoints consume different amounts from a shared budget.
179180
If not set, each request counts as 1.
180181
"""
181-
self._cost = cost
182+
self._weight = weight
182183
self._method = method.upper() if method else None
183184

184185
# Normalize the url_base if provided: remove trailing slash.
@@ -248,15 +249,15 @@ def __call__(self, request: Any) -> bool:
248249
return True
249250

250251
@property
251-
def cost(self) -> Optional[int]:
252-
"""The cost (weight) of a request matching this matcher, or None if not set."""
253-
return self._cost
252+
def weight(self) -> Optional[int]:
253+
"""The weight of a request matching this matcher, or None if not set."""
254+
return self._weight
254255

255256
def __str__(self) -> str:
256257
regex = self._url_path_pattern.pattern if self._url_path_pattern else None
257258
return (
258259
f"HttpRequestRegexMatcher(method={self._method}, url_base={self._url_base}, "
259-
f"url_path_pattern={regex}, params={self._params}, headers={self._headers}, cost={self._cost})"
260+
f"url_path_pattern={regex}, params={self._params}, headers={self._headers}, weight={self._weight})"
260261
)
261262

262263

@@ -275,22 +276,22 @@ def matches(self, request: Any) -> bool:
275276
return True
276277
return any(matcher(request) for matcher in self._matchers)
277278

278-
def get_cost(self, request: Any) -> int:
279-
"""Get the cost (weight) for a request based on the first matching matcher.
279+
def get_weight(self, request: Any) -> int:
280+
"""Get the weight for a request based on the first matching matcher.
280281
281-
If a matcher has a cost configured, that cost is used as the weight.
282+
If a matcher has a weight configured, that weight is used.
282283
Otherwise, defaults to 1.
283284
284285
:param request: a request object
285-
:return: the cost/weight for this request
286+
:return: the weight for this request
286287
"""
287288
for matcher in self._matchers:
288289
if (
289290
matcher(request)
290291
and isinstance(matcher, HttpRequestRegexMatcher)
291-
and matcher.cost is not None
292+
and matcher.weight is not None
292293
):
293-
return matcher.cost
294+
return matcher.weight
294295
return 1
295296

296297

@@ -624,7 +625,7 @@ def _do_acquire(
624625
# sometimes we spend all budget before a second attempt, so we have a few more attempts
625626
for attempt in range(1, self._maximum_attempts_to_acquire):
626627
try:
627-
weight = policy.get_cost(request) if isinstance(policy, BaseCallRatePolicy) else 1
628+
weight = policy.get_weight(request) if isinstance(policy, BaseCallRatePolicy) else 1
628629
policy.try_acquire(request, weight=weight)
629630
return
630631
except CallRateLimitHit as exc:

unit_tests/sources/streams/test_call_rate.py

Lines changed: 34 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -140,13 +140,13 @@ def test_http_request_matching(mocker):
140140
users_policy.matches.side_effect = HttpRequestMatcher(
141141
url="http://domain/api/users", method="GET"
142142
)
143-
users_policy.get_cost.return_value = 1
143+
users_policy.get_weight.return_value = 1
144144
groups_policy.matches.side_effect = HttpRequestMatcher(
145145
url="http://domain/api/groups", method="POST"
146146
)
147-
groups_policy.get_cost.return_value = 1
147+
groups_policy.get_weight.return_value = 1
148148
root_policy.matches.side_effect = HttpRequestMatcher(method="GET")
149-
root_policy.get_cost.return_value = 1
149+
root_policy.get_weight.return_value = 1
150150
api_budget = APIBudget(
151151
policies=[
152152
users_policy,
@@ -363,64 +363,64 @@ def test_with_cache(self, mocker, requests_mock):
363363
assert MovingWindowCallRatePolicy.try_acquire.call_count == 1
364364

365365

366-
class TestCostBasedRateLimiting:
367-
"""Tests for cost-based rate limiting where different endpoints consume different amounts from a shared budget."""
366+
class TestWeightBasedRateLimiting:
367+
"""Tests for weight-based rate limiting where different endpoints consume different amounts from a shared budget."""
368368

369-
def test_matcher_cost_default_none(self):
370-
"""HttpRequestRegexMatcher cost defaults to None when not specified."""
369+
def test_matcher_weight_default_none(self):
370+
"""HttpRequestRegexMatcher weight defaults to None when not specified."""
371371
matcher = HttpRequestRegexMatcher(url_path_pattern=r"/api/test")
372-
assert matcher.cost is None
372+
assert matcher.weight is None
373373

374-
def test_matcher_cost_is_stored(self):
375-
"""HttpRequestRegexMatcher stores the cost value when provided."""
376-
matcher = HttpRequestRegexMatcher(url_path_pattern=r"/api/test", cost=60)
377-
assert matcher.cost == 60
374+
def test_matcher_weight_is_stored(self):
375+
"""HttpRequestRegexMatcher stores the weight value when provided."""
376+
matcher = HttpRequestRegexMatcher(url_path_pattern=r"/api/test", weight=60)
377+
assert matcher.weight == 60
378378

379-
def test_policy_get_cost_returns_matcher_cost(self):
380-
"""BaseCallRatePolicy.get_cost returns cost from the matching matcher."""
379+
def test_policy_get_weight_returns_matcher_weight(self):
380+
"""BaseCallRatePolicy.get_weight returns weight from the matching matcher."""
381381
policy = MovingWindowCallRatePolicy(
382-
matchers=[HttpRequestRegexMatcher(url_path_pattern=r"/api/expensive", cost=120)],
382+
matchers=[HttpRequestRegexMatcher(url_path_pattern=r"/api/expensive", weight=120)],
383383
rates=[Rate(1000, timedelta(hours=1))],
384384
)
385385
req = Request("GET", "https://example.com/api/expensive")
386-
assert policy.get_cost(req) == 120
386+
assert policy.get_weight(req) == 120
387387

388-
def test_policy_get_cost_defaults_to_1(self):
389-
"""BaseCallRatePolicy.get_cost returns 1 when no matcher has a cost set."""
388+
def test_policy_get_weight_defaults_to_1(self):
389+
"""BaseCallRatePolicy.get_weight returns 1 when no matcher has a weight set."""
390390
policy = MovingWindowCallRatePolicy(
391391
matchers=[HttpRequestRegexMatcher(url_path_pattern=r"/api/default")],
392392
rates=[Rate(1000, timedelta(hours=1))],
393393
)
394394
req = Request("GET", "https://example.com/api/default")
395-
assert policy.get_cost(req) == 1
395+
assert policy.get_weight(req) == 1
396396

397-
def test_policy_get_cost_no_matching_matcher(self):
398-
"""BaseCallRatePolicy.get_cost returns 1 when no matcher matches the request."""
397+
def test_policy_get_weight_no_matching_matcher(self):
398+
"""BaseCallRatePolicy.get_weight returns 1 when no matcher matches the request."""
399399
policy = MovingWindowCallRatePolicy(
400-
matchers=[HttpRequestRegexMatcher(url_path_pattern=r"/api/other", cost=50)],
400+
matchers=[HttpRequestRegexMatcher(url_path_pattern=r"/api/other", weight=50)],
401401
rates=[Rate(1000, timedelta(hours=1))],
402402
)
403403
req = Request("GET", "https://example.com/api/unmatched")
404-
assert policy.get_cost(req) == 1
404+
assert policy.get_weight(req) == 1
405405

406-
def test_api_budget_uses_cost_as_weight(self):
407-
"""APIBudget._do_acquire passes the matcher's cost as weight to try_acquire."""
406+
def test_api_budget_uses_weight(self):
407+
"""APIBudget._do_acquire passes the matcher's weight to try_acquire."""
408408
policy = MovingWindowCallRatePolicy(
409-
matchers=[HttpRequestRegexMatcher(url_path_pattern=r"/api/heavy", cost=10)],
409+
matchers=[HttpRequestRegexMatcher(url_path_pattern=r"/api/heavy", weight=10)],
410410
rates=[Rate(100, timedelta(hours=1))],
411411
)
412412
budget = APIBudget(policies=[policy])
413413

414-
# Make requests — each costs 10 from the budget of 100
414+
# Make requests — each weighs 10 from the budget of 100
415415
for i in range(10):
416416
budget.acquire_call(Request("GET", "https://example.com/api/heavy"), block=False)
417417

418418
# The 11th request should exceed the budget (10 * 10 = 100, one more = 110 > 100)
419419
with pytest.raises(CallRateLimitHit):
420420
budget.acquire_call(Request("GET", "https://example.com/api/heavy"), block=False)
421421

422-
def test_cost_1_backward_compatible(self):
423-
"""When cost is not set, behavior is identical to the old hardcoded weight=1."""
422+
def test_weight_1_backward_compatible(self):
423+
"""When weight is not set, behavior is identical to the old hardcoded weight=1."""
424424
policy = MovingWindowCallRatePolicy(
425425
matchers=[HttpRequestRegexMatcher(url_path_pattern=r"/api/normal")],
426426
rates=[Rate(5, timedelta(hours=1))],
@@ -433,19 +433,19 @@ def test_cost_1_backward_compatible(self):
433433
with pytest.raises(CallRateLimitHit):
434434
budget.acquire_call(Request("GET", "https://example.com/api/normal"), block=False)
435435

436-
def test_shared_budget_different_costs(self):
437-
"""Multiple matchers with different costs sharing one policy correctly consume the shared budget."""
436+
def test_shared_budget_different_weights(self):
437+
"""Multiple matchers with different weights sharing one policy correctly consume the shared budget."""
438438
# Shared policy matches both endpoints via regex
439439
policy = MovingWindowCallRatePolicy(
440440
matchers=[
441-
HttpRequestRegexMatcher(url_path_pattern=r"/api/cheap", cost=1),
442-
HttpRequestRegexMatcher(url_path_pattern=r"/api/expensive", cost=10),
441+
HttpRequestRegexMatcher(url_path_pattern=r"/api/cheap", weight=1),
442+
HttpRequestRegexMatcher(url_path_pattern=r"/api/expensive", weight=10),
443443
],
444444
rates=[Rate(20, timedelta(hours=1))],
445445
)
446446
budget = APIBudget(policies=[policy])
447447

448-
# Make 1 expensive request (costs 10) and 10 cheap requests (cost 1 each) = total 20
448+
# Make 1 expensive request (weight 10) and 10 cheap requests (weight 1 each) = total 20
449449
budget.acquire_call(Request("GET", "https://example.com/api/expensive"), block=False)
450450
for i in range(10):
451451
budget.acquire_call(Request("GET", "https://example.com/api/cheap"), block=False)

0 commit comments

Comments
 (0)