Skip to content

Commit d21d5ef

Browse files
cognifloydcognifloyd
committed
add redis to uses_services plugin
1 parent 8a92560 commit d21d5ef

6 files changed

Lines changed: 309 additions & 0 deletions

File tree

‎contrib/runners/orquesta_runner/tests/integration/BUILD‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,5 @@ __defaults__(
55

66
python_tests(
77
name="tests",
8+
uses=["redis"],
89
)

‎pants-plugins/uses_services/BUILD‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,6 @@ python_tests(
1616
# overrides={
1717
# "mongo_rules_test.py": {"uses": ["mongo"]},
1818
# "rabbitmq_rules_test.py": {"uses": ["rabbitmq"]},
19+
# "redis_rules_test.py": {"uses": ["redis"]},
1920
# },
2021
)

‎pants-plugins/uses_services/redis_rules.py‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
# Copyright 2023 The StackStorm Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
from __future__ import annotations
15+
16+
import os
17+
18+
from dataclasses import dataclass
19+
from textwrap import dedent
20+
21+
from pants.backend.python.goals.pytest_runner import (
22+
PytestPluginSetupRequest,
23+
PytestPluginSetup,
24+
)
25+
from pants.backend.python.util_rules.pex import (
26+
PexRequest,
27+
PexRequirements,
28+
VenvPex,
29+
VenvPexProcess,
30+
rules as pex_rules,
31+
)
32+
from pants.engine.fs import CreateDigest, Digest, FileContent
33+
from pants.engine.rules import collect_rules, Get, MultiGet, rule
34+
from pants.engine.process import FallibleProcessResult, ProcessCacheScope
35+
from pants.engine.target import Target
36+
from pants.engine.unions import UnionRule
37+
from pants.util.logging import LogLevel
38+
39+
from uses_services.exceptions import ServiceMissingError
40+
from uses_services.platform_rules import Platform
41+
from uses_services.scripts.is_redis_running import (
42+
__file__ as is_redis_running_full_path,
43+
)
44+
from uses_services.target_types import UsesServicesField
45+
46+
47+
@dataclass(frozen=True)
48+
class UsesRedisRequest:
49+
"""One or more targets need a running redis service using these settings.
50+
51+
The coord_* attributes represent the coordination settings from st2.conf.
52+
In st2 code, they come from:
53+
oslo_config.cfg.CONF.coordination.url
54+
"""
55+
56+
# These config opts for integration tests are in:
57+
# conf/st2.dev.conf (copied to conf/st2.ci.conf)
58+
# TODO: for int tests: set url by either modifying st2.{dev,ci}.conf on the fly or via env vars.
59+
60+
# with our version of oslo.config (newer are slower) we can't directly override opts w/ environment variables.
61+
62+
coord_url: str = "redis://127.0.0.1:6379"
63+
64+
65+
@dataclass(frozen=True)
66+
class RedisIsRunning:
67+
pass
68+
69+
70+
class PytestUsesRedisRequest(PytestPluginSetupRequest):
71+
@classmethod
72+
def is_applicable(cls, target: Target) -> bool:
73+
if not target.has_field(UsesServicesField):
74+
return False
75+
uses = target.get(UsesServicesField).value
76+
return uses is not None and "redis" in uses
77+
78+
79+
@rule(
80+
desc="Ensure redis is running and accessible before running tests.",
81+
level=LogLevel.DEBUG,
82+
)
83+
async def redis_is_running_for_pytest(
84+
request: PytestUsesRedisRequest,
85+
) -> PytestPluginSetup:
86+
# this will raise an error if redis is not running
87+
_ = await Get(RedisIsRunning, UsesRedisRequest())
88+
89+
return PytestPluginSetup()
90+
91+
92+
@rule(
93+
desc="Test to see if redis is running and accessible.",
94+
level=LogLevel.DEBUG,
95+
)
96+
async def redis_is_running(
97+
request: UsesRedisRequest, platform: Platform
98+
) -> RedisIsRunning:
99+
script_path = "./is_redis_running.py"
100+
101+
# pants is already watching this directory as it is under a source root.
102+
# So, we don't need to double watch with PathGlobs, just open it.
103+
with open(is_redis_running_full_path, "rb") as script_file:
104+
script_contents = script_file.read()
105+
106+
script_digest, kombu_pex = await MultiGet(
107+
Get(Digest, CreateDigest([FileContent(script_path, script_contents)])),
108+
Get(
109+
VenvPex,
110+
PexRequest(
111+
output_filename="kombu.pex",
112+
internal_only=True,
113+
requirements=PexRequirements({"kombu",}),
114+
),
115+
),
116+
)
117+
118+
result = await Get(
119+
FallibleProcessResult,
120+
VenvPexProcess(
121+
kombu_pex,
122+
argv=(
123+
script_path,
124+
"amqp://guest:guest@127.0.0.1:5672/",
125+
),
126+
input_digest=script_digest,
127+
description="Checking to see if Redis is up and accessible.",
128+
# this can change from run to run, so don't cache results.
129+
cache_scope=ProcessCacheScope.PER_SESSION,
130+
level=LogLevel.DEBUG,
131+
),
132+
)
133+
is_running = result.exit_code == 0
134+
135+
if is_running:
136+
return RedisIsRunning()
137+
138+
# redis is not running, so raise an error with instructions.
139+
raise ServiceMissingError(
140+
platform,
141+
ServiceSpecificMessages(
142+
service="redis",
143+
service_start_cmd_el_7="service redis start",
144+
service_start_cmd_el="systemctl start redis",
145+
not_installed_clause_el="this is one way to install it:",
146+
install_instructions_el=dedent(
147+
"""\
148+
sudo yum -y install redis
149+
# Don't forget to start redis.
150+
"""
151+
),
152+
service_start_cmd_deb="systemctl start redis",
153+
not_installed_clause_deb="this is one way to install it:",
154+
install_instructions_deb=dedent(
155+
"""\
156+
sudo apt-get install -y mongodb redis
157+
# Don't forget to start redis.
158+
"""
159+
),
160+
service_start_cmd_generic="systemctl start redis",
161+
),
162+
)
163+
164+
165+
def rules():
166+
return [
167+
*collect_rules(),
168+
UnionRule(PytestPluginSetupRequest, PytestUsesRedisRequest),
169+
*pex_rules(),
170+
]

‎pants-plugins/uses_services/redis_rules_test.py‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
# Copyright 2023 The StackStorm Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
from __future__ import annotations
15+
16+
import pytest
17+
18+
from pants.engine.internals.scheduler import ExecutionError
19+
from pants.engine.target import Target
20+
from pants.testutil.rule_runner import QueryRule, RuleRunner
21+
22+
from .data_fixtures import platform, platform_samples
23+
from .exceptions import ServiceMissingError
24+
from .redis_rules import (
25+
RedisIsRunning,
26+
UsesRedisRequest,
27+
rules as redis_rules,
28+
)
29+
from .platform_rules import Platform
30+
31+
32+
@pytest.fixture
33+
def rule_runner() -> RuleRunner:
34+
return RuleRunner(
35+
rules=[
36+
*redis_rules(),
37+
QueryRule(RedisIsRunning, (UsesRedisRequest, Platform)),
38+
],
39+
target_types=[],
40+
)
41+
42+
43+
def run_redis_is_running(
44+
rule_runner: RuleRunner,
45+
uses_redis_request: UsesRedisRequest,
46+
mock_platform: Platform,
47+
*,
48+
extra_args: list[str] | None = None,
49+
) -> RedisIsRunning:
50+
rule_runner.set_options(
51+
[
52+
"--backend-packages=uses_services",
53+
*(extra_args or ()),
54+
],
55+
env_inherit={"PATH", "PYENV_ROOT", "HOME"},
56+
)
57+
result = rule_runner.request(
58+
RedisIsRunning,
59+
[uses_redis_request, mock_platform],
60+
)
61+
return result
62+
63+
64+
# Warning this requires that redis be running
65+
def test_redis_is_running(rule_runner: RuleRunner) -> None:
66+
request = UsesRedisRequest()
67+
mock_platform = platform()
68+
69+
# we are asserting that this does not raise an exception
70+
is_running = run_redis_is_running(rule_runner, request, mock_platform)
71+
assert is_running
72+
73+
74+
@pytest.mark.parametrize("mock_platform", platform_samples)
75+
def test_redis_not_running(rule_runner: RuleRunner, mock_platform: Platform) -> None:
76+
request = UsesRedisRequest(
77+
coord_url="redis://127.100.20.7:10", # 10 is an unassigned port, unlikely to be used
78+
)
79+
80+
with pytest.raises(ExecutionError) as exception_info:
81+
run_redis_is_running(rule_runner, request, mock_platform)
82+
83+
execution_error = exception_info.value
84+
assert len(execution_error.wrapped_exceptions) == 1
85+
86+
exc = execution_error.wrapped_exceptions[0]
87+
assert isinstance(exc, ServiceMissingError)
88+
89+
assert exc.service == "redis"
90+
assert "The redis service does not seem to be running" in str(exc)
91+
assert exc.instructions != ""

‎pants-plugins/uses_services/scripts/is_redis_running.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Copyright 2023 The StackStorm Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
from __future__ import annotations
15+
16+
import sys
17+
18+
19+
def _is_redis_running(coord_url: str) -> bool:
20+
"""Connect to redis with connection logic that mirrors the st2 code.
21+
22+
In particular, this is based on:
23+
- st2common.services.coordination.coordinator_setup()
24+
25+
This should not import the st2 code as it should be self-contained.
26+
"""
27+
# late import so that __file__ can be imported in the pants plugin without these imports
28+
from tooz import coordination
29+
30+
coordinator = coordination.get_coordinator(url)
31+
coordinator.start(start_heart=False)
32+
#return False
33+
return True
34+
35+
36+
if __name__ == "__main__":
37+
args = dict((k, v) for k, v in enumerate(sys.argv))
38+
39+
# unit tests do not use redis, they use use an in-memory coordinator: "zake://"
40+
# integration tests use this url with a conf file derived from conf/st2.dev.conf
41+
coord_url = args.get(1, "redis://127.0.0.1:6379")
42+
43+
is_running = _is_redis_running(coord_url)
44+
exit_code = 0 if is_running else 1
45+
sys.exit(exit_code)

‎st2tests/integration/orquesta/BUILD‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@ python_sources()
22

33
python_tests(
44
name="tests",
5+
uses=["redis"],
56
)

0 commit comments

Comments
 (0)