Skip to content

Commit 165dbb3

Browse files
mmwinthermmwinther
authored
Further options for filtering catalog services (#532)
1 parent b0783ab commit 165dbb3

6 files changed

Lines changed: 227 additions & 21 deletions

File tree

‎onyxia-api/src/main/java/fr/insee/onyxia/api/configuration/CatalogWrapper.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@
88
import fr.insee.onyxia.model.helm.Repository;
99
import io.swagger.v3.oas.annotations.media.Schema;
1010
import java.util.ArrayList;
11+
import java.util.HashMap;
1112
import java.util.List;
13+
import java.util.Map;
1214
import java.util.regex.Pattern;
1315

1416
@JsonIgnoreProperties(ignoreUnknown = true)
@@ -54,6 +56,15 @@ public class CatalogWrapper {
5456
@Schema(description = "only include charts with one or more of the given keywords")
5557
private List<String> includeKeywords = new ArrayList<>();
5658

59+
@Schema(description = "exclude any charts which have one or more of the given keywords")
60+
private List<String> excludeKeywords = new ArrayList<>();
61+
62+
@Schema(description = "only include charts with one or more of the given annotations")
63+
private Map<String, String> includeAnnotations = new HashMap<>();
64+
65+
@Schema(description = "exclude any charts which have one or more of the given annotations")
66+
private Map<String, String> excludeAnnotations = new HashMap<>();
67+
5768
@Schema(description = "Skip tls certificate checks for the repository")
5869
private boolean skipTlsVerify;
5970

@@ -205,6 +216,30 @@ public void setIncludeKeywords(List<String> includeKeywords) {
205216
this.includeKeywords = includeKeywords;
206217
}
207218

219+
public void setExcludeKeywords(List<String> excludeKeywords) {
220+
this.excludeKeywords = excludeKeywords;
221+
}
222+
223+
public List<String> getExcludeKeywords() {
224+
return excludeKeywords;
225+
}
226+
227+
public Map<String, String> getIncludeAnnotations() {
228+
return includeAnnotations;
229+
}
230+
231+
public void setIncludeAnnotations(Map<String, String> includeAnnotations) {
232+
this.includeAnnotations = includeAnnotations;
233+
}
234+
235+
public Map<String, String> getExcludeAnnotations() {
236+
return excludeAnnotations;
237+
}
238+
239+
public void setExcludeAnnotations(Map<String, String> excludeAnnotations) {
240+
this.excludeAnnotations = excludeAnnotations;
241+
}
242+
208243
public boolean getSkipTlsVerify() {
209244
return skipTlsVerify;
210245
}

‎onyxia-api/src/main/java/fr/insee/onyxia/api/dao/universe/CatalogLoader.java‎

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.springframework.core.io.Resource;
3030
import org.springframework.core.io.ResourceLoader;
3131
import org.springframework.stereotype.Service;
32+
import org.springframework.util.CollectionUtils;
3233
import org.springframework.util.StringUtils;
3334

3435
@Service
@@ -76,19 +77,33 @@ private void updateHelmRepository(CatalogWrapper cw) {
7677
excludedChart.equalsIgnoreCase(
7778
entry.getKey())))
7879
.filter(
79-
// If includeKeywords is defined, include only services where
80-
// the latest version has the desired keyword.
8180
entry ->
82-
cw.getIncludeKeywords() == null
83-
|| cw.getIncludeKeywords().isEmpty()
84-
|| cw.getIncludeKeywords().stream()
85-
.anyMatch(
86-
include ->
87-
entry.getValue()
88-
.getFirst()
89-
.getKeywords()
90-
.contains(
91-
include)))
81+
(CollectionUtils.isEmpty(cw.getIncludeKeywords())
82+
|| entry.getValue()
83+
.getFirst()
84+
.hasKeywords(
85+
cw
86+
.getIncludeKeywords()))
87+
&& (CollectionUtils.isEmpty(
88+
cw.getIncludeAnnotations())
89+
|| entry.getValue()
90+
.getFirst()
91+
.hasAnnotations(
92+
cw
93+
.getIncludeAnnotations())))
94+
.filter(
95+
entry ->
96+
CollectionUtils.isEmpty(cw.getExcludeKeywords())
97+
|| !entry.getValue()
98+
.getFirst()
99+
.hasKeywords(cw.getExcludeKeywords()))
100+
.filter(
101+
entry ->
102+
CollectionUtils.isEmpty(cw.getExcludeAnnotations())
103+
|| !entry.getValue()
104+
.getFirst()
105+
.hasAnnotations(
106+
cw.getExcludeAnnotations()))
92107
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
93108
// For each service, filter the multiple versions if needed then refresh remaining
94109
// versions

‎onyxia-api/src/test/java/fr/insee/onyxia/api/dao/universe/CatalogLoaderTest.java‎

Lines changed: 132 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
import static org.hamcrest.Matchers.containsString;
55
import static org.hamcrest.Matchers.is;
66
import static org.junit.jupiter.api.Assertions.*;
7-
import static org.junit.jupiter.params.provider.Arguments.arguments;
7+
import static org.junit.jupiter.params.provider.Arguments.argumentSet;
88

99
import fr.insee.onyxia.api.configuration.CatalogWrapper;
1010
import fr.insee.onyxia.api.configuration.CustomObjectMapper;
@@ -13,6 +13,7 @@
1313
import fr.insee.onyxia.api.util.TestUtils;
1414
import fr.insee.onyxia.model.helm.Chart;
1515
import java.util.List;
16+
import java.util.Map;
1617
import java.util.Set;
1718
import java.util.stream.Collectors;
1819
import java.util.stream.Stream;
@@ -161,21 +162,144 @@ void packageOnClassPathNotFound() {
161162

162163
@ParameterizedTest
163164
@MethodSource("includeKeywords")
164-
void filterIncludeKeywordsTest(List<String> includeKeywords, Set<String> expectedServices) {
165+
@MethodSource("excludeKeywords")
166+
@MethodSource("includeAnnotations")
167+
@MethodSource("excludeAnnotations")
168+
void filterServicesTest(
169+
List<String> includeKeywords,
170+
List<String> excludeKeywords,
171+
Map<String, String> includeAnnotations,
172+
Map<String, String> excludeAnnotations,
173+
Set<String> expectedServices) {
165174
CatalogWrapper cw = new CatalogWrapper();
166175
cw.setType("helm");
167-
cw.setLocation("classpath:/catalog-loader-test-with-keywords");
176+
cw.setLocation("classpath:/catalog-loader-test-with-keywords-and-annotations");
168177
cw.setIncludeKeywords(includeKeywords);
178+
cw.setExcludeKeywords(excludeKeywords);
179+
cw.setIncludeAnnotations(includeAnnotations);
180+
cw.setExcludeAnnotations(excludeAnnotations);
169181
catalogLoader.updateCatalog(cw);
170182
assertEquals(expectedServices, cw.getCatalog().getEntries().keySet());
171183
}
172184

173185
private static Stream<Arguments> includeKeywords() {
174186
return Stream.of(
175-
arguments(List.of("CD"), Set.of("keepme")),
176-
arguments(List.of("CD", "Experimental"), Set.of("keepme", "excludeme")),
177-
arguments(List.of(), Set.of("keepme", "excludeme")),
178-
arguments(null, Set.of("keepme", "excludeme")),
179-
arguments(List.of("no one knows"), Set.of()));
187+
argumentSet(
188+
"One keyword to include",
189+
List.of("CD"),
190+
null,
191+
null,
192+
null,
193+
Set.of("keepme")),
194+
argumentSet(
195+
"Two keywords to include",
196+
List.of("CD", "Experimental"),
197+
null,
198+
null,
199+
null,
200+
Set.of("keepme", "excludeme")),
201+
argumentSet(
202+
"Empty list of keywords to include",
203+
List.of(),
204+
null,
205+
null,
206+
null,
207+
Set.of("keepme", "excludeme")),
208+
argumentSet(
209+
"null for all filters",
210+
null,
211+
null,
212+
null,
213+
null,
214+
Set.of("keepme", "excludeme")),
215+
argumentSet(
216+
"Unknown keyword to include",
217+
List.of("no one knows"),
218+
null,
219+
null,
220+
null,
221+
Set.of()));
222+
}
223+
224+
private static Stream<Arguments> excludeKeywords() {
225+
return Stream.of(
226+
argumentSet(
227+
"One keyword to exclude",
228+
null,
229+
List.of("Experimental"),
230+
null,
231+
null,
232+
Set.of("keepme")),
233+
argumentSet(
234+
"Exclusive keywords to include and exclude",
235+
List.of("CD"),
236+
List.of("Experimental"),
237+
null,
238+
null,
239+
Set.of("keepme")),
240+
argumentSet(
241+
"Keyword to exclude takes precedence",
242+
List.of("Experimental"),
243+
List.of("Experimental"),
244+
null,
245+
null,
246+
Set.of()),
247+
argumentSet(
248+
"Two keywords to exclude",
249+
List.of("CD"),
250+
List.of("CD", "Experimental"),
251+
null,
252+
null,
253+
Set.of()),
254+
argumentSet(
255+
"Empty lists of keywords to include and exclude",
256+
List.of(),
257+
List.of(),
258+
null,
259+
null,
260+
Set.of("keepme", "excludeme")),
261+
argumentSet(
262+
"Unknown keyword to exclude",
263+
null,
264+
List.of("no one knows"),
265+
null,
266+
null,
267+
Set.of("keepme", "excludeme")));
268+
}
269+
270+
private static Stream<Arguments> includeAnnotations() {
271+
return Stream.of(
272+
argumentSet(
273+
"One annotation to include",
274+
null,
275+
null,
276+
Map.of("lifecycle", "production"),
277+
null,
278+
Set.of("keepme")),
279+
argumentSet(
280+
"Exclude keyword takes precedence",
281+
null,
282+
List.of("CD"),
283+
Map.of("lifecycle", "production"),
284+
null,
285+
Set.of()));
286+
}
287+
288+
private static Stream<Arguments> excludeAnnotations() {
289+
return Stream.of(
290+
argumentSet(
291+
"One annotation to exclude",
292+
null,
293+
null,
294+
null,
295+
Map.of("lifecycle", "production"),
296+
Set.of("excludeme")),
297+
argumentSet(
298+
"Exclude annotation takes precedence",
299+
null,
300+
null,
301+
Map.of("lifecycle", "production"),
302+
Map.of("lifecycle", "production"),
303+
Set.of()));
180304
}
181305
}

onyxia-api/src/test/resources/catalog-loader-test-with-keywords/catalogs.json5 renamed to onyxia-api/src/test/resources/catalog-loader-test-with-keywords-and-annotations/catalogs.json5

File renamed without changes.

onyxia-api/src/test/resources/catalog-loader-test-with-keywords/index.yaml renamed to onyxia-api/src/test/resources/catalog-loader-test-with-keywords-and-annotations/index.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ entries:
2828
maintainers:
2929
- email: test@example.com
3030
name: test
31+
annotations:
32+
lifecycle: production
3133
- apiVersion: v2
3234
appVersion: "1"
3335
created: "2022-10-03T11:53:20.589754116Z"
@@ -55,6 +57,8 @@ entries:
5557
maintainers:
5658
- email: test@example.com
5759
name: test
60+
annotations:
61+
lifecycle: production
5862
- apiVersion: v2
5963
appVersion: "2"
6064
created: "2022-10-03T11:53:20.589754116Z"
@@ -79,6 +83,8 @@ entries:
7983
urls:
8084
- keepeme2.gz
8185
version: 2.4.0
86+
annotations:
87+
lifecycle: experimental
8288
excludeme:
8389
- apiVersion: v2
8490
appVersion: "1"
@@ -129,3 +135,5 @@ entries:
129135
urls:
130136
- excludeme2.gz
131137
version: 2.4.0
138+
annotations:
139+
lifecycle: experimental

‎onyxia-model/src/main/java/fr/insee/onyxia/model/helm/Chart.java‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@
2222
"name",
2323
"sources",
2424
"urls",
25-
"version"
25+
"version",
26+
"annotations",
2627
})
2728
@Schema(description = "")
2829
public class Chart extends Pkg {
@@ -277,6 +278,29 @@ public void setAdditionalProperty(String name, Object value) {
277278
this.additionalProperties.put(name, value);
278279
}
279280

281+
/**
282+
* Does the chart have any of the given keywords?
283+
*
284+
* @param keywordsToCheck The list of keywords we're interested in.
285+
* @return true if any of the given keywords appear in the keywords on the chart.
286+
*/
287+
public Boolean hasKeywords(List<String> keywordsToCheck) {
288+
return getKeywords() != null
289+
&& keywordsToCheck.stream().anyMatch(keyword -> getKeywords().contains(keyword));
290+
}
291+
292+
/**
293+
* Does the chart have any of the given annotations?
294+
*
295+
* @param annotationsToCheck The map of annotations we're interested in.
296+
* @return true if any of the given annotations appear in the annotations on the chart.
297+
*/
298+
public Boolean hasAnnotations(Map<String, String> annotationsToCheck) {
299+
return getAnnotations() != null
300+
&& annotationsToCheck.entrySet().stream()
301+
.anyMatch(annotation -> getAnnotations().entrySet().contains(annotation));
302+
}
303+
280304
public static class Maintainer {
281305
@JsonProperty("email")
282306
private String email;

0 commit comments

Comments
 (0)