Skip to content

Commit b8cb6a3

Browse files
blagblag
authored
Merge pull request #76 from StackStorm-Exchange/long-strings
Change Makefile to handle reeeeeally long lists of files
2 parents 3cc2e69 + 5b818a9 commit b8cb6a3

6 files changed

Lines changed: 96 additions & 39 deletions

File tree

‎.circle/Makefile‎

Lines changed: 95 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
ROOT_DIR ?= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
2+
ifndef FORCE_CHECK_ALL_FILES
23
CHANGED_FILES := $(shell $(CI_DIR)/utils/git-changes files)
3-
CHANGED_DIRECTORIES := $(shell $(CI_DIR)/utils/git-changes directories)
44
CHANGED_PY := $(shell ${CI_DIR}/utils/git-changes py)
55
CHANGED_YAML := $(shell $(CI_DIR)/utils/git-changes yaml)
66
CHANGED_JSON := $(shell $(CI_DIR)/utils/git-changes json)
7+
endif
8+
CHANGED_DIRECTORIES := $(shell $(CI_DIR)/utils/git-changes directories)
79
VIRTUALENV_DIR ?= virtualenv
810
ST2_REPO_PATH ?= /tmp/st2
911
ST2_REPO_BRANCH ?= master
@@ -65,54 +67,137 @@ compile:
6567
@echo
6668
@echo "==================== flake8 ===================="
6769
@echo
68-
. $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_PY}" ]; then echo "No files have changed, skipping run..."; fi; for file in ${CHANGED_PY}; do if [ -n "$$file" ]; then flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$file || exit 1; fi; done
70+
. $(VIRTUALENV_DIR)/bin/activate; \
71+
if [ "$${FORCE_CHECK_ALL_FILES}" ]; then \
72+
find ./* -name "*.py" | while read py_file; do \
73+
flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$py_file || exit 1; \
74+
done; \
75+
elif [ -n "${CHANGED_PY}" ]; then \
76+
for file in ${CHANGED_PY}; do \
77+
if [ -n "$$file" ]; then \
78+
flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$file || exit 1; \
79+
fi; \
80+
done; \
81+
else \
82+
echo "No files have changed, skipping run..."; \
83+
fi;
6984

7085
.PHONY: .pylint
7186
.pylint:
7287
@echo
7388
@echo "==================== pylint ===================="
7489
@echo
75-
. $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_PY}" ]; then echo "No files have changed, skipping run..."; else (REQUIREMENTS_DIR=$(CI_DIR)/.circle/ CONFIG_DIR=$(CI_DIR)/lint-configs/ st2-check-pylint-pack $(ROOT_DIR) || exit 1); fi;
90+
. $(VIRTUALENV_DIR)/bin/activate; \
91+
if [ "$${FORCE_CHECK_ALL_FILES}" ] || [ -n "${CHANGED_PY}" ]; then \
92+
REQUIREMENTS_DIR=$(CI_DIR)/.circle/ \
93+
CONFIG_DIR=$(CI_DIR)/lint-configs/ \
94+
st2-check-pylint-pack $(ROOT_DIR) || exit 1; \
95+
else \
96+
echo "No files have changed, skipping run..."; \
97+
fi;
7698

7799
.PHONY: .configs-check
78100
.configs-check:
79101
@echo
80102
@echo "==================== configs-check ===================="
81103
@echo
82-
. $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_YAML}" ]; then echo "No files have changed, skipping run..."; fi; for file in $(CHANGED_YAML); do if [ -n "$$file" ]; then st2-check-validate-yaml-file $$file || exit 1 ; fi; done
83-
. $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_JSON}" ]; then echo "No files have changed, skipping run..."; fi; for file in $(CHANGED_JSON); do if [ -n "$$file" ]; then st2-check-validate-json-file $$file || exit 1 ; fi; done
104+
@# The number of changed files in the AWS pack exceeds the limits of Bash,
105+
@# leading to CI failures like this:
106+
@# https://circleci.com/gh/StackStorm-Exchange/stackstorm-aws/320
107+
@# Instead of passing the entire list into a Bash for loop, we convert the
108+
@# make variable to a Bash string, convert that to a Bash array, and then
109+
@# iterate through each element of the array
110+
. $(VIRTUALENV_DIR)/bin/activate; \
111+
if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \
112+
find $(CI_DIR)/* -name "*.yaml" -o -name "*.yml" | while read yaml_file; do \
113+
st2-check-validate-yaml-file "$$yaml_file" || exit 1 ; \
114+
done; \
115+
elif [ -n "${CHANGED_YAML}" ]; then \
116+
for file in $(CHANGED_YAML); do \
117+
if [ -n "$$file" ]; then \
118+
st2-check-validate-yaml-file $$file || exit 1 ; \
119+
fi; \
120+
done; \
121+
else \
122+
echo "No files have changed, skipping run..."; \
123+
fi
124+
@#
125+
. $(VIRTUALENV_DIR)/bin/activate; \
126+
if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \
127+
find $(CI_DIR)/* -name "*.json" | while read json_file; do \
128+
st2-check-validate-json-file "$$json_file" || exit 1 ; \
129+
done; \
130+
elif [ -n "${CHANGED_JSON}" ]; then \
131+
for file in $(CHANGED_JSON); do \
132+
if [ -n "$$file" ]; then \
133+
echo "file: $$file"; \
134+
st2-check-validate-json-file $$file || exit 1 ; \
135+
fi; \
136+
done; \
137+
else \
138+
echo "No files have changed, skipping run..."; \
139+
fi
140+
@#
84141
@echo
85142
@echo "==================== example config check ===================="
86143
@echo
87-
. $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_FILES}" ]; then echo "No files have changed, skipping run..."; else st2-check-validate-pack-example-config /tmp/packs/$(PACK_NAME) || exit 1; fi;
144+
. $(VIRTUALENV_DIR)/bin/activate; \
145+
if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \
146+
find ./* | while read file; do \
147+
st2-check-validate-pack-example-config $$file || exit 1; \
148+
done; \
149+
elif [ -n "${CHANGED_FILES}" ]; then \
150+
st2-check-validate-pack-example-config /tmp/packs/$(PACK_NAME) || exit 1; \
151+
else \
152+
echo "No files have changed, skipping run..."; \
153+
fi;
88154

89155
.PHONY: .metadata-check
90156
.metadata-check:
91157
@echo
92158
@echo "==================== metadata-check ===================="
93159
@echo
94-
. $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_YAML}" ]; then echo "No files have changed, skipping run..."; else (st2-check-validate-pack-metadata-exists $(ROOT_DIR) || exit 1); fi;
160+
. $(VIRTUALENV_DIR)/bin/activate; \
161+
if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ -n "${CHANGED_YAML}" ]; then \
162+
st2-check-validate-pack-metadata-exists $(ROOT_DIR) || exit 1; \
163+
else \
164+
echo "No files have changed, skipping run..."; \
165+
fi;
95166

96167
.PHONY: .packs-resource-register
97168
.packs-resource-register:
98169
@echo
99170
@echo "==================== packs-resource-register ===================="
100171
@echo
101-
. $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_FILES}" ]; then echo "No files have changed, skipping run..."; else st2-check-register-pack-resources /tmp/packs/$(PACK_NAME) || exit 1; fi;
172+
. $(VIRTUALENV_DIR)/bin/activate; \
173+
if [ ! "${CHANGED_FILES}" ]; then \
174+
echo "No files have changed, skipping run..."; \
175+
else \
176+
st2-check-register-pack-resources /tmp/packs/$(PACK_NAME) || exit 1; \
177+
fi;
102178

103179
.PHONY: .packs-tests
104180
.packs-tests:
105181
@echo
106182
@echo "==================== packs-tests ===================="
107183
@echo
108-
. $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_FILES}" ]; then echo "No files have changed, skipping run..."; else ($(ST2_REPO_PATH)/st2common/bin/st2-run-pack-tests -c -t -x -j -p $(ROOT_DIR) || exit 1); fi;
184+
. $(VIRTUALENV_DIR)/bin/activate; \
185+
if [ ! "${CHANGED_FILES}" ]; then \
186+
echo "No files have changed, skipping run..."; \
187+
else \
188+
$(ST2_REPO_PATH)/st2common/bin/st2-run-pack-tests -c -t -x -j -p $(ROOT_DIR) || exit 1; \
189+
fi;
109190

110191
.PHONY: .packs-missing-tests
111192
.packs-missing-tests:
112193
@echo
113194
@echo "==================== pack-missing-tests ===================="
114195
@echo
115-
if [ ! "${CHANGED_FILES}" ]; then echo "No files have changed, skipping run..."; else (st2-check-print-pack-tests-coverage $(ROOT_DIR) || exit 1); fi;
196+
if [ ! "${CHANGED_FILES}" ]; then \
197+
echo "No files have changed, skipping run..."; \
198+
else \
199+
st2-check-print-pack-tests-coverage $(ROOT_DIR) || exit 1; \
200+
fi;
116201

117202
.PHONY: .clone_st2_repo
118203
.clone_st2_repo: /tmp/st2

‎utils/git-changes-files‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,6 @@
22
ROOT_DIRECTORY=$1
33
BASE_BRANCH="origin/master"
44

5-
# If this environment variable if specified we will return all the files so the
6-
# checks run on all the files not only changed ones.
7-
if [ "${FORCE_CHECK_ALL_FILES}" = "true" ]; then
8-
echo $(find ${ROOT_DIRECTORY}/*)
9-
exit 0
10-
fi
11-
125
cd ${ROOT_DIRECTORY}
136
# Note: We include all but deleted files ("D" flag)
147
echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH})

‎utils/git-changes-json‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,6 @@
22
ROOT_DIRECTORY=$1
33
BASE_BRANCH="origin/master"
44

5-
# If this environment variable if specified we will return all the files so the
6-
# checks run on all the files not only changed ones.
7-
if [ "${FORCE_CHECK_ALL_FILES}" = "true" ]; then
8-
echo $(find ${ROOT_DIRECTORY}/* -name "*.json")
9-
exit 0
10-
fi
11-
125
cd ${ROOT_DIRECTORY}
136
# Note: We include all but deleted files ("D" flag)
147
echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH} -- '*.json')

‎utils/git-changes-packs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ BASE_BRANCH="origin/master"
55
# If this environment variable if specified we will return all the files so the
66
# checks run on all the files not only changed ones.
77
if [ "${FORCE_CHECK_ALL_FILES}" = "true" ]; then
8-
echo $(find ${ROOT_DIRECTORY}/* -maxdepth 0 -type d | grep -v linux | grep -v openstack)
8+
find ${ROOT_DIRECTORY}/* -maxdepth 0 -type d | grep -v linux | grep -v openstack
99
exit 0
1010
fi
1111

‎utils/git-changes-py‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,6 @@
22
ROOT_DIRECTORY=$1
33
BASE_BRANCH="origin/master"
44

5-
# If this environment variable if specified we will return all the files so the
6-
# checks run on all the files not only changed ones.
7-
if [ "${FORCE_CHECK_ALL_FILES}" = "true" ]; then
8-
echo $(find ${ROOT_DIRECTORY}/* -name "*.py")
9-
exit 0
10-
fi
11-
125
cd ${ROOT_DIRECTORY}
136
# Note: We include all but deleted files ("D" flag)
147
echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH} -- '*.py')

‎utils/git-changes-yaml‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,6 @@
22
ROOT_DIRECTORY=$1
33
BASE_BRANCH="origin/master"
44

5-
# If this environment variable if specified we will return all the files so the
6-
# checks run on all the files not only changed ones.
7-
if [ "${FORCE_CHECK_ALL_FILES}" = "true" ]; then
8-
echo $(find ${ROOT_DIRECTORY}/* -name "*.yaml")
9-
exit 0
10-
fi
11-
125
cd ${ROOT_DIRECTORY}
136
# Note: We include all but deleted files ("D" flag)
147
echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH} -- '*.yaml' '*.yml')

0 commit comments

Comments
 (0)