Skip to content

Commit 1041007

Browse files
cognifloydcognifloyd
authored
Merge pull request #5596 from StackStorm/rule_trace_housekeep
Add garbage collection for trace and rule_enforcement models
2 parents 2907475 + 8804acb commit 1041007

16 files changed

Lines changed: 663 additions & 0 deletions

File tree

‎CHANGELOG.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,10 @@ Added
152152

153153
Contributed by @Kami.
154154

155+
* Added garbage collection for rule_enforcement and trace models #5596
156+
Contributed by Amanda McGuinness (@amanda11 intive)
157+
158+
155159
Fixed
156160
~~~~~
157161

‎conf/st2.conf.sample‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,8 +177,12 @@ collection_interval = 600
177177
logging = /etc/st2/logging.garbagecollector.conf
178178
# Set to True to perform garbage collection on Inquiries (based on the TTL value per Inquiry)
179179
purge_inquiries = False
180+
# Rule enforcements older than this value (days) will be automatically deleted.
181+
rule_enforcement_ttl = None
180182
# How long to wait / sleep (in seconds) between collection of different object types.
181183
sleep_delay = 2
184+
# Trace objects older than this value (days) will be automatically deleted.
185+
trace_ttl = None
182186
# Trigger instances older than this value (days) will be automatically deleted.
183187
trigger_instances_ttl = None
184188

‎st2common/bin/st2-purge-rule-enforcement‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env python
2+
# Licensed to the StackStorm, Inc ('StackStorm') under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
import sys
18+
19+
from st2common.cmd.purge_rule_enforcement import main
20+
21+
if __name__ == "__main__":
22+
sys.exit(main())

‎st2common/bin/st2-purge-trace‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env python
2+
# Licensed to the StackStorm, Inc ('StackStorm') under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
import sys
18+
19+
from st2common.cmd.purge_trace import main
20+
21+
if __name__ == "__main__":
22+
sys.exit(main())

‎st2common/setup.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@
5353
"bin/st2-register-content",
5454
"bin/st2-purge-executions",
5555
"bin/st2-purge-trigger-instances",
56+
"bin/st2-purge-trace",
57+
"bin/st2-purge-rule-enforcement",
5658
"bin/st2-run-pack-tests",
5759
"bin/st2ctl",
5860
"bin/st2-generate-symmetric-crypto-key",

‎st2common/st2common/cmd/purge_rule_enforcement.py‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Copyright 2022 The StackStorm Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
"""
17+
A utility script that purges trigger instances older than certain
18+
timestamp.
19+
20+
*** RISK RISK RISK. You will lose data. Run at your own risk. ***
21+
"""
22+
23+
from __future__ import absolute_import
24+
25+
from datetime import datetime
26+
27+
import six
28+
import pytz
29+
from oslo_config import cfg
30+
31+
from st2common import config
32+
from st2common import log as logging
33+
from st2common.config import do_register_cli_opts
34+
from st2common.script_setup import setup as common_setup
35+
from st2common.script_setup import teardown as common_teardown
36+
from st2common.constants.exit_codes import SUCCESS_EXIT_CODE
37+
from st2common.constants.exit_codes import FAILURE_EXIT_CODE
38+
from st2common.garbage_collection.rule_enforcement import purge_rule_enforcement
39+
40+
__all__ = ["main"]
41+
42+
LOG = logging.getLogger(__name__)
43+
44+
45+
def _register_cli_opts():
46+
cli_opts = [
47+
cfg.StrOpt(
48+
"timestamp",
49+
default=None,
50+
help="Will delete rule_enforcement instances older than "
51+
+ "this UTC timestamp. "
52+
+ "Example value: 2015-03-13T19:01:27.255542Z",
53+
)
54+
]
55+
do_register_cli_opts(cli_opts)
56+
57+
58+
def main():
59+
_register_cli_opts()
60+
common_setup(config=config, setup_db=True, register_mq_exchanges=False)
61+
62+
# Get config values
63+
timestamp = cfg.CONF.timestamp
64+
65+
if not timestamp:
66+
LOG.error("Please supply a timestamp for purging models. Aborting.")
67+
return 1
68+
else:
69+
timestamp = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%fZ")
70+
timestamp = timestamp.replace(tzinfo=pytz.UTC)
71+
72+
# Purge models.
73+
try:
74+
purge_rule_enforcement(logger=LOG, timestamp=timestamp)
75+
except Exception as e:
76+
LOG.exception(six.text_type(e))
77+
return FAILURE_EXIT_CODE
78+
finally:
79+
common_teardown()
80+
81+
return SUCCESS_EXIT_CODE

‎st2common/st2common/cmd/purge_trace.py‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Copyright 2022 The StackStorm Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
"""
17+
A utility script that purges trigger instances older than certain
18+
timestamp.
19+
20+
*** RISK RISK RISK. You will lose data. Run at your own risk. ***
21+
"""
22+
23+
from __future__ import absolute_import
24+
25+
from datetime import datetime
26+
27+
import six
28+
import pytz
29+
from oslo_config import cfg
30+
31+
from st2common import config
32+
from st2common import log as logging
33+
from st2common.config import do_register_cli_opts
34+
from st2common.script_setup import setup as common_setup
35+
from st2common.script_setup import teardown as common_teardown
36+
from st2common.constants.exit_codes import SUCCESS_EXIT_CODE
37+
from st2common.constants.exit_codes import FAILURE_EXIT_CODE
38+
from st2common.garbage_collection.trace import purge_trace
39+
40+
__all__ = ["main"]
41+
42+
LOG = logging.getLogger(__name__)
43+
44+
45+
def _register_cli_opts():
46+
cli_opts = [
47+
cfg.StrOpt(
48+
"timestamp",
49+
default=None,
50+
help="Will delete trace instances older than "
51+
+ "this UTC timestamp. "
52+
+ "Example value: 2015-03-13T19:01:27.255542Z",
53+
)
54+
]
55+
do_register_cli_opts(cli_opts)
56+
57+
58+
def main():
59+
_register_cli_opts()
60+
common_setup(config=config, setup_db=True, register_mq_exchanges=False)
61+
62+
# Get config values
63+
timestamp = cfg.CONF.timestamp
64+
65+
if not timestamp:
66+
LOG.error("Please supply a timestamp for purging models. Aborting.")
67+
return 1
68+
else:
69+
timestamp = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%fZ")
70+
timestamp = timestamp.replace(tzinfo=pytz.UTC)
71+
72+
# Purge models.
73+
try:
74+
purge_trace(logger=LOG, timestamp=timestamp)
75+
except Exception as e:
76+
LOG.exception(six.text_type(e))
77+
return FAILURE_EXIT_CODE
78+
finally:
79+
common_teardown()
80+
81+
return SUCCESS_EXIT_CODE

‎st2common/st2common/garbage_collection/rule_enforcement.py‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# Copyright 2022 The StackStorm Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""
16+
Module with utility functions for purging old trigger instance objects.
17+
"""
18+
19+
from __future__ import absolute_import
20+
21+
import six
22+
from mongoengine.errors import InvalidQueryError
23+
24+
from st2common.persistence.rule_enforcement import RuleEnforcement
25+
from st2common.util import isotime
26+
27+
__all__ = ["purge_rule_enforcement"]
28+
29+
30+
def purge_rule_enforcement(logger, timestamp):
31+
"""
32+
:param timestamp: Rule enforcement instances older than this timestamp will be deleted.
33+
:type timestamp: ``datetime.datetime
34+
"""
35+
if not timestamp:
36+
raise ValueError("Specify a valid timestamp to purge.")
37+
38+
logger.info(
39+
"Purging rule enforcements older than timestamp: %s"
40+
% timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
41+
)
42+
43+
query_filters = {"enforced_at__lt": isotime.parse(timestamp)}
44+
45+
try:
46+
deleted_count = RuleEnforcement.delete_by_query(**query_filters)
47+
except InvalidQueryError as e:
48+
msg = (
49+
"Bad query (%s) used to delete rule enforcements: %s"
50+
"Please contact support."
51+
% (
52+
query_filters,
53+
six.text_type(e),
54+
)
55+
)
56+
raise InvalidQueryError(msg)
57+
except:
58+
logger.exception(
59+
"Deleting rule enforcements using query_filters %s failed.", query_filters
60+
)
61+
else:
62+
logger.info("Deleted %s rule enforcement objects" % (deleted_count))
63+
64+
# Print stats
65+
logger.info(
66+
"All rule enforcement models older than timestamp %s were deleted.", timestamp
67+
)

‎st2common/st2common/garbage_collection/trace.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# Copyright 2022 The StackStorm Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""
16+
Module with utility functions for purging old trigger instance objects.
17+
"""
18+
19+
from __future__ import absolute_import
20+
21+
import six
22+
from mongoengine.errors import InvalidQueryError
23+
24+
from st2common.persistence.trace import Trace
25+
from st2common.util import isotime
26+
27+
__all__ = ["purge_trace"]
28+
29+
30+
def purge_trace(logger, timestamp):
31+
"""
32+
:param timestamp: Trace instances older than this timestamp will be deleted.
33+
:type timestamp: ``datetime.datetime
34+
"""
35+
if not timestamp:
36+
raise ValueError("Specify a valid timestamp to purge.")
37+
38+
logger.info(
39+
"Purging trace instances older than timestamp: %s"
40+
% timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
41+
)
42+
43+
query_filters = {"start_timestamp__lt": isotime.parse(timestamp)}
44+
45+
try:
46+
deleted_count = Trace.delete_by_query(**query_filters)
47+
except InvalidQueryError as e:
48+
msg = (
49+
"Bad query (%s) used to delete trace instances: %s"
50+
"Please contact support."
51+
% (
52+
query_filters,
53+
six.text_type(e),
54+
)
55+
)
56+
raise InvalidQueryError(msg)
57+
except:
58+
logger.exception(
59+
"Deleting trace instances using query_filters %s failed.", query_filters
60+
)
61+
else:
62+
logger.info("Deleted %s trace objects" % (deleted_count))
63+
64+
# Print stats
65+
logger.info("All trace models older than timestamp %s were deleted.", timestamp)

‎st2common/st2common/persistence/rule_enforcement.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,7 @@ class RuleEnforcement(Access):
2424
@classmethod
2525
def _get_impl(cls):
2626
return cls.impl
27+
28+
@classmethod
29+
def delete_by_query(cls, *args, **query):
30+
return cls._get_impl().delete_by_query(*args, **query)

0 commit comments

Comments
 (0)