Skip to content

Commit 1444f4c

Browse files
LorisFriedelclaude
andcommitted
perf(forwarder): create scrubber, matcher, and batcher once in Forwarder.__init__
Move DatadogScrubber, DatadogMatcher, and DatadogBatcher instantiation from per-call in _forward_logs to __init__, avoiding repeated object creation on every invocation. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent b6366a3 commit 1444f4c

1 file changed

Lines changed: 23 additions & 15 deletions

File tree

aws/logs_monitoring/forwarder.py

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -36,12 +36,21 @@
3636
logger.setLevel(logging.getLevelName(os.environ.get("DD_LOG_LEVEL", "INFO").upper()))
3737

3838

39-
class Forwarder(object):
39+
class Forwarder:
4040
def __init__(self, function_prefix):
4141
self.trace_connection = TraceConnection(
4242
DD_TRACE_INTAKE_URL, DD_API_KEY, DD_SKIP_SSL_VALIDATION
4343
)
4444
self.storage = create_storage(function_prefix)
45+
self._scrubber = DatadogScrubber(SCRUBBING_RULE_CONFIGS)
46+
self._matcher = DatadogMatcher(
47+
include_pattern=INCLUDE_AT_MATCH, exclude_pattern=EXCLUDE_AT_MATCH
48+
)
49+
self._batcher = DatadogBatcher(
50+
max_item_size_bytes=512 * 1000,
51+
max_batch_size_bytes=4 * 1000 * 1000,
52+
max_items_count=400,
53+
)
4554

4655
def forward(self, logs, metrics, traces):
4756
"""
@@ -81,11 +90,6 @@ def _forward_logs(self, logs, key=None):
8190
if logger.isEnabledFor(logging.DEBUG):
8291
logger.debug(f"Forwarding {len(logs)} logs")
8392

84-
scrubber = DatadogScrubber(SCRUBBING_RULE_CONFIGS)
85-
matcher = DatadogMatcher(
86-
include_pattern=INCLUDE_AT_MATCH, exclude_pattern=EXCLUDE_AT_MATCH
87-
)
88-
8993
logs_to_forward = []
9094
for log in logs:
9195
if key:
@@ -101,20 +105,24 @@ def _forward_logs(self, logs, key=None):
101105
to_forward = dump_event(log)
102106
evaluated_log = to_forward
103107

104-
if matcher.match(evaluated_log):
108+
if self._matcher.match(evaluated_log):
105109
if to_forward is None:
106110
logs_to_forward.append(dump_event(log))
107111
else:
108112
logs_to_forward.append(to_forward)
109113

110-
batcher = DatadogBatcher(512 * 1000, 4 * 1000 * 1000, 400)
111114
cli = DatadogHTTPClient(
112-
DD_URL, DD_PORT, DD_NO_SSL, DD_SKIP_SSL_VALIDATION, DD_API_KEY, scrubber
115+
DD_URL,
116+
DD_PORT,
117+
DD_NO_SSL,
118+
DD_SKIP_SSL_VALIDATION,
119+
DD_API_KEY,
120+
self._scrubber,
113121
)
114122

115123
failed_logs = []
116124
with DatadogClient(cli) as client:
117-
for batch in batcher.batch(logs_to_forward):
125+
for batch in self._batcher.batch(logs_to_forward):
118126
try:
119127
client.send(batch)
120128
except Exception as e:
@@ -126,10 +134,10 @@ def _forward_logs(self, logs, key=None):
126134
if key:
127135
self.storage.delete_data(key)
128136

129-
if DD_STORE_FAILED_EVENTS and len(failed_logs) > 0 and not key:
137+
if DD_STORE_FAILED_EVENTS and failed_logs and not key:
130138
self.storage.store_data(RetryPrefix.LOGS, failed_logs)
131139

132-
if len(failed_logs) > 0:
140+
if failed_logs:
133141
send_event_metric("logs_failed", failed_logs)
134142

135143
send_event_metric("logs_forwarded", len(logs_to_forward) - len(failed_logs))
@@ -157,16 +165,16 @@ def _forward_metrics(self, metrics, key=None):
157165
if key:
158166
self.storage.delete_data(key)
159167

160-
if DD_STORE_FAILED_EVENTS and len(failed_metrics) > 0 and not key:
168+
if DD_STORE_FAILED_EVENTS and failed_metrics and not key:
161169
self.storage.store_data(RetryPrefix.METRICS, failed_metrics)
162170

163-
if len(failed_metrics) > 0:
171+
if failed_metrics:
164172
send_event_metric("metrics_failed", failed_metrics)
165173

166174
send_event_metric("metrics_forwarded", len(metrics) - len(failed_metrics))
167175

168176
def _forward_traces(self, traces, key=None):
169-
if not len(traces) > 0:
177+
if not traces:
170178
return
171179

172180
if logger.isEnabledFor(logging.DEBUG):

0 commit comments

Comments
 (0)