Skip to content

Commit 6985a42

Browse files
TimPansinomergify[bot]
TimPansino
and
mergify[bot]
authored
Clean Up Custom CI Images (#1746)
* Move firestore to non-hacky solution * Codify rediscluster setup and build at runtime * [MegaLinter] Apply linters fixes * Fix redis cluster networking --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
1 parent e67be67 commit 6985a42

4 files changed

Lines changed: 205 additions & 74 deletions

File tree

‎.github/containers/firestore/docker-compose.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Copyright 2010 New Relic, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
services:
16+
firestore:
17+
image: gcr.io/google.com/cloudsdktool/google-cloud-cli:437.0.1-emulators
18+
command:
19+
[
20+
"/bin/bash",
21+
"-c",
22+
"gcloud emulators firestore start --host-port=0.0.0.0:8080",
23+
]
24+
ports:
25+
- 8080:8080
26+
healthcheck:
27+
test: ["CMD-SHELL", "curl -fsS http://localhost:8080/ || exit 1"]
28+
interval: 5s
29+
timeout: 3s
30+
retries: 12
31+
start_period: 10s

‎.github/containers/rediscluster/Dockerfile‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# syntax=docker/dockerfile:1.4
2+
# Copyright 2010 New Relic, Inc.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
FROM redis:7.0.12
17+
18+
COPY <<"EOF" /etc/redis.conf
19+
bind 0.0.0.0
20+
cluster-enabled yes
21+
cluster-config-file nodes.conf
22+
cluster-node-timeout 5000
23+
cluster-preferred-endpoint-type hostname
24+
appendonly yes
25+
EOF
26+
27+
ENV REDIS_PORT=6379 \
28+
REDIS_ANNOUNCE_HOSTNAME=localhost
29+
30+
CMD ["sh", "-c", "exec redis-server /etc/redis.conf --port \"$REDIS_PORT\" --cluster-announce-hostname \"$REDIS_ANNOUNCE_HOSTNAME\" --cluster-announce-port \"$REDIS_PORT\""]

‎.github/containers/rediscluster/docker-compose.yml‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
# Copyright 2010 New Relic, Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
x-redis-node: &redis-node
16+
build: .
17+
image: redis-cluster-node:local
18+
tmpfs:
19+
- /data
20+
healthcheck:
21+
test: ["CMD-SHELL", "redis-cli -p $$REDIS_PORT ping"]
22+
interval: 2s
23+
timeout: 3s
24+
retries: 15
25+
26+
services:
27+
redis1:
28+
<<: *redis-node
29+
environment:
30+
REDIS_PORT: "6379"
31+
REDIS_ANNOUNCE_HOSTNAME: ${REDIS_ANNOUNCE_HOSTNAME:-localhost}
32+
ports:
33+
- 6379:6379
34+
35+
redis2:
36+
<<: *redis-node
37+
environment:
38+
REDIS_PORT: "6380"
39+
REDIS_ANNOUNCE_HOSTNAME: ${REDIS_ANNOUNCE_HOSTNAME:-localhost}
40+
ports:
41+
- 6380:6380
42+
43+
redis3:
44+
<<: *redis-node
45+
environment:
46+
REDIS_PORT: "6381"
47+
REDIS_ANNOUNCE_HOSTNAME: ${REDIS_ANNOUNCE_HOSTNAME:-localhost}
48+
ports:
49+
- 6381:6381
50+
51+
redis4:
52+
<<: *redis-node
53+
environment:
54+
REDIS_PORT: "6382"
55+
REDIS_ANNOUNCE_HOSTNAME: ${REDIS_ANNOUNCE_HOSTNAME:-localhost}
56+
ports:
57+
- 6382:6382
58+
59+
redis5:
60+
<<: *redis-node
61+
environment:
62+
REDIS_PORT: "6383"
63+
REDIS_ANNOUNCE_HOSTNAME: ${REDIS_ANNOUNCE_HOSTNAME:-localhost}
64+
ports:
65+
- 6383:6383
66+
67+
redis6:
68+
<<: *redis-node
69+
environment:
70+
REDIS_PORT: "6384"
71+
REDIS_ANNOUNCE_HOSTNAME: ${REDIS_ANNOUNCE_HOSTNAME:-localhost}
72+
ports:
73+
- 6384:6384
74+
75+
cluster-setup:
76+
image: redis-cluster-node:local
77+
restart: "no"
78+
command:
79+
- bash
80+
- -c
81+
- >-
82+
redis-cli --cluster create
83+
redis1:6379 redis2:6380 redis3:6381 redis4:6382 redis5:6383 redis6:6384
84+
--cluster-replicas 1
85+
--cluster-yes &&
86+
exec sleep infinity
87+
healthcheck:
88+
test:
89+
[
90+
"CMD-SHELL",
91+
"redis-cli -h redis1 cluster info | grep -q cluster_state:ok",
92+
]
93+
interval: 2s
94+
timeout: 3s
95+
retries: 30
96+
start_period: 30s
97+
depends_on:
98+
redis1: { condition: service_healthy }
99+
redis2: { condition: service_healthy }
100+
redis3: { condition: service_healthy }
101+
redis4: { condition: service_healthy }
102+
redis5: { condition: service_healthy }
103+
redis6: { condition: service_healthy }

‎.github/workflows/tests.yml‎

Lines changed: 41 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -751,26 +751,6 @@ jobs:
751751
options: >-
752752
--add-host=host.docker.internal:host-gateway
753753
timeout-minutes: 30
754-
services:
755-
firestore:
756-
# Image set here MUST be repeated down below in options. See comment below.
757-
image: gcr.io/google.com/cloudsdktool/google-cloud-cli:437.0.1-emulators
758-
ports:
759-
- 8080:8080
760-
# Set health checks to wait until container has started
761-
options: >-
762-
--health-cmd "echo success"
763-
--health-interval 10s
764-
--health-timeout 5s
765-
--health-retries 5
766-
--health-start-period 5s
767-
gcr.io/google.com/cloudsdktool/google-cloud-cli:437.0.1-emulators /bin/bash -c "gcloud emulators firestore start --host-port=0.0.0.0:8080" ||
768-
# This is a very hacky solution. GitHub Actions doesn't provide APIs for setting commands on services, but allows adding arbitrary options.
769-
# --entrypoint won't work as it only accepts an executable and not the [] syntax.
770-
# Instead, we specify the image again the command afterwards like a call to docker create. The result is a few environment variables
771-
# and the original command being appended to our hijacked docker create command. We can avoid any issues by adding || to prevent that
772-
# from every being executed as bash commands.
773-
774754
steps:
775755
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # 6.0.3
776756

@@ -784,6 +764,18 @@ jobs:
784764
mkdir -p /github/home/.cache/pip
785765
chown -R "$(whoami)" /github/home/.cache/pip
786766
767+
- name: Start firestore
768+
run: |
769+
docker compose \
770+
-f .github/containers/firestore/docker-compose.yml \
771+
up -d \
772+
--wait \
773+
--wait-timeout 60 \
774+
&& exit 0
775+
echo "firestore did not become healthy in time"
776+
docker compose -f .github/containers/firestore/docker-compose.yml logs
777+
exit 1
778+
787779
- name: Get Environments
788780
id: get-envs
789781
run: |
@@ -821,6 +813,11 @@ jobs:
821813
if-no-files-found: error
822814
retention-days: 1
823815

816+
- name: Stop firestore
817+
if: always()
818+
run: |
819+
docker compose -f .github/containers/firestore/docker-compose.yml down
820+
824821
grpc:
825822
env:
826823
TOTAL_GROUPS: 1
@@ -1874,60 +1871,6 @@ jobs:
18741871
options: >-
18751872
--add-host=host.docker.internal:host-gateway
18761873
timeout-minutes: 30
1877-
services:
1878-
redis1:
1879-
image: hmstepanek/redis-cluster-node:1.0.0
1880-
ports:
1881-
- 6379:6379
1882-
- 16379:16379
1883-
options: >-
1884-
--add-host=host.docker.internal:host-gateway
1885-
1886-
redis2:
1887-
image: hmstepanek/redis-cluster-node:1.0.0
1888-
ports:
1889-
- 6380:6379
1890-
- 16380:16379
1891-
options: >-
1892-
--add-host=host.docker.internal:host-gateway
1893-
1894-
redis3:
1895-
image: hmstepanek/redis-cluster-node:1.0.0
1896-
ports:
1897-
- 6381:6379
1898-
- 16381:16379
1899-
options: >-
1900-
--add-host=host.docker.internal:host-gateway
1901-
1902-
redis4:
1903-
image: hmstepanek/redis-cluster-node:1.0.0
1904-
ports:
1905-
- 6382:6379
1906-
- 16382:16379
1907-
options: >-
1908-
--add-host=host.docker.internal:host-gateway
1909-
1910-
redis5:
1911-
image: hmstepanek/redis-cluster-node:1.0.0
1912-
ports:
1913-
- 6383:6379
1914-
- 16383:16379
1915-
options: >-
1916-
--add-host=host.docker.internal:host-gateway
1917-
1918-
redis6:
1919-
image: hmstepanek/redis-cluster-node:1.0.0
1920-
ports:
1921-
- 6384:6379
1922-
- 16384:16379
1923-
options: >-
1924-
--add-host=host.docker.internal:host-gateway
1925-
1926-
cluster-setup:
1927-
image: hmstepanek/redis-cluster:1.0.0
1928-
options: >-
1929-
--add-host=host.docker.internal:host-gateway
1930-
19311874
steps:
19321875
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # 6.0.3
19331876

@@ -1941,6 +1884,25 @@ jobs:
19411884
mkdir -p /github/home/.cache/pip
19421885
chown -R "$(whoami)" /github/home/.cache/pip
19431886
1887+
- name: Start rediscluster
1888+
env:
1889+
REDIS_ANNOUNCE_HOSTNAME: host.docker.internal
1890+
run: |
1891+
# Build only the redis1 image to prevent fighting, then start the entire cluster
1892+
docker compose \
1893+
-f .github/containers/rediscluster/docker-compose.yml \
1894+
build \
1895+
redis1 && \
1896+
docker compose \
1897+
-f .github/containers/rediscluster/docker-compose.yml \
1898+
up -d \
1899+
--wait \
1900+
--wait-timeout 120 \
1901+
&& exit 0
1902+
echo "rediscluster did not become healthy in time"
1903+
docker compose -f .github/containers/rediscluster/docker-compose.yml logs
1904+
exit 1
1905+
19441906
- name: Get Environments
19451907
id: get-envs
19461908
run: |
@@ -1978,6 +1940,11 @@ jobs:
19781940
if-no-files-found: error
19791941
retention-days: 1
19801942

1943+
- name: Stop rediscluster
1944+
if: always()
1945+
run: |
1946+
docker compose -f .github/containers/rediscluster/docker-compose.yml down
1947+
19811948
solr:
19821949
env:
19831950
TOTAL_GROUPS: 1

0 commit comments

Comments
 (0)