Skip to content

Commit 6e0de3d

Browse files
stevenwinshipstevenwinship
committed
Adding filtering and pagination to collectionList APIs
1 parent 199057d commit 6e0de3d

11 files changed

Lines changed: 142 additions & 43 deletions

File tree

‎doc/release-notes/12423-collection-list-search-and-pagination.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
## Feature ##
2+
3+
Updated the following APIs to add ability to filter using `searchTerm` and added optional pagination parameters `offset` and `pageSize` to limit the results with each GET.
4+
5+
GET `/api/users/$USERNAME/allowedCollections/$PERMISSION?pageSize=10&offset=0&searchTerm=bio`
6+
7+
GET `/api/mydata/retrieve/collectionList?userIdentifier=anotherUser&pageSize=10&offset=11&searchTerm=bio`

‎doc/sphinx-guides/source/api/native-api.rst‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8465,13 +8465,20 @@ The ``$identifier`` is the username of the requested user.
84658465
The ``$permission`` is the permission (tied to the roles) that gives the user access to the collection.
84668466
Passing ``$permission`` as 'any' will return the collection as long as the user has any access/permission on the collection
84678467
8468+
**For filtering and pagination these query parameters can be used:**
8469+
8470+
- ``searchTerm``: To filter the results.
8471+
- ``offset``: Starting row ('nextOffset' or 'prevOffset' from the Json output can be used to get the next or previous page).
8472+
- ``pageSize``: Number of items to limit the output.
8473+
84688474
.. code-block:: bash
84698475
84708476
export SERVER_URL=https://demo.dataverse.org
84718477
export $USERNAME=jsmith
84728478
export PERMISSION=PublishDataverse
84738479
84748480
curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/users/$USERNAME/allowedCollections/$PERMISSION"
8481+
curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/users/$USERNAME/allowedCollections/$PERMISSION?pageSize=10&offset=0&searchTerm=bio"
84758482
84768483
Show Role Assignee
84778484
~~~~~~~~~~~~~~~~~~
@@ -9059,6 +9066,12 @@ MyData Collection List
90599066
The MyData Collection List API is used to get a list of the collections an authenticated user can create a Dataset in.
90609067
Param userIdentifier={userName} is used by a superuser to get the collections for a specific user.
90619068
9069+
**For filtering and pagination these query parameters can be used:**
9070+
9071+
- ``searchTerm``: To filter the results.
9072+
- ``offset``: Starting row ('nextOffset' or 'prevOffset' from the Json output can be used to get the next or previous page).
9073+
- ``pageSize``: Number of items to limit the output.
9074+
90629075
A curl example listing collections:
90639076
90649077
.. code-block:: bash
@@ -9068,4 +9081,5 @@ A curl example listing collections:
90689081
90699082
curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/mydata/retrieve/collectionList"
90709083
curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/mydata/retrieve/collectionList?userIdentifier=anotherUser"
9084+
curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/mydata/retrieve/collectionList?userIdentifier=anotherUser&pageSize=10&offset=11&searchTerm=bio"
90719085

‎src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java‎

Lines changed: 41 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
11
package edu.harvard.iq.dataverse;
22

3-
import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
43
import edu.harvard.iq.dataverse.authorization.DataverseRole;
54
import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IPv4Address;
65
import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IPv6Address;
76
import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress;
8-
import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean;
97
import edu.harvard.iq.dataverse.authorization.users.GuestUser;
108
import edu.harvard.iq.dataverse.authorization.Permission;
119
import edu.harvard.iq.dataverse.authorization.RoleAssignee;
@@ -16,6 +14,8 @@
1614

1715
import java.util.*;
1816
import java.util.logging.Logger;
17+
18+
import edu.harvard.iq.dataverse.mydata.Pager;
1919
import jakarta.ejb.EJB;
2020
import jakarta.ejb.Stateless;
2121
import jakarta.inject.Inject;
@@ -59,20 +59,11 @@ public class PermissionServiceBean {
5959
.filter(Permission::requiresAuthenticatedUser)
6060
.collect(Collectors.toList()));
6161

62-
@EJB
63-
BuiltinUserServiceBean userService;
64-
65-
@EJB
66-
AuthenticationServiceBean authenticationService;
67-
6862
@EJB
6963
DataverseRoleServiceBean roleService;
7064

7165
@EJB
7266
RoleAssigneeServiceBean roleAssigneeService;
73-
74-
@EJB
75-
DataverseServiceBean dataverseService;
7667

7768
@EJB
7869
DvObjectServiceBean dvObjectServiceBean;
@@ -86,26 +77,23 @@ public class PermissionServiceBean {
8677
@EJB
8778
GroupServiceBean groupService;
8879

89-
@Inject
90-
DataverseSession session;
91-
9280
@Inject
9381
DataverseRequestServiceBean dvRequestService;
9482

9583
@Inject
9684
DatasetVersionFilesServiceBean datasetVersionFilesServiceBean;
9785

9886
private static final String LIST_ALL_DATAVERSES_SUPERUSER_HAS_PERMISSION = """
99-
SELECT id, name, alias FROM DATAVERSE dv
87+
SELECT id, name, alias, ROW_NUMBER() OVER (ORDER BY id) AS row_num FROM DATAVERSE dv
10088
""";
10189

10290
private static final String LIST_ALL_DATAVERSES_USER_HAS_PERMISSION = """
10391
WITH grouplist AS (
10492
SELECT explicitgroup_authenticateduser.explicitgroup_id as id FROM explicitgroup_authenticateduser
10593
WHERE explicitgroup_authenticateduser.containedauthenticatedusers_id = @USERID
10694
)
107-
108-
SELECT * FROM DATAVERSE dv WHERE id IN (
95+
96+
SELECT *, ROW_NUMBER() OVER (ORDER BY id) AS row_num FROM DATAVERSE dv WHERE id IN (
10997
SELECT definitionpoint_id
11098
FROM roleassignment
11199
WHERE roleassignment.assigneeidentifier IN (
@@ -179,6 +167,11 @@ AND EXISTS (SELECT id FROM dataverserole WHERE dataverserole.id = roleassignment
179167
or (LOWER(dv.name) LIKE ?))))
180168
""";
181169

170+
private static final String PAGE_PARAMS = """
171+
select * from ( @SQL@ )
172+
where row_num BETWEEN @START AND @END
173+
""";
174+
182175
/**
183176
* A request-level permission query (e.g includes IP ras).
184177
*/
@@ -939,33 +932,51 @@ private boolean hasUnrestrictedReleasedFiles(DatasetVersion targetDatasetVersion
939932
}
940933

941934
public List<Dataverse> findPermittedCollections(DataverseRequest request, AuthenticatedUser user, Permission permission) {
942-
return findPermittedCollections(request, user, 1 << permission.ordinal(), "");
935+
return findPermittedCollections(request, user, 1 << permission.ordinal(), "", null);
943936
}
944937

945938
public List<Dataverse> findPermittedCollections(DataverseRequest request, AuthenticatedUser user, Permission permission, String searchTerm) {
946-
return findPermittedCollections(request, user, 1 << permission.ordinal(), searchTerm);
939+
return findPermittedCollections(request, user, 1 << permission.ordinal(), searchTerm, null);
947940
}
948941

949942
public List<Dataverse> findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit) {
950-
return findPermittedCollections(request, user, permissionBit, "");
943+
return findPermittedCollections(request, user, permissionBit, "", null);
951944
}
952945

953946

954-
public List<Dataverse> findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit, String searchTerm) {
947+
public List<Dataverse> findPermittedCollections(DataverseRequest request, AuthenticatedUser user, int permissionBit, String searchTerm, Pager pager) {
955948
if (user != null) {
949+
List<Dataverse> dataverses = new ArrayList<>();
956950
var sqlCode = getBaseQueryForAllPermittedDataverses(request, user, permissionBit);
957-
if (searchTerm == null || searchTerm.isEmpty()) {
958-
return em.createNativeQuery(sqlCode, Dataverse.class).getResultList();
959-
} else if (user.isSuperuser()) {
960-
Query query = em.createNativeQuery(sqlCode.concat(WHERE).concat(SEARCH_PARAMS), Dataverse.class);
961-
setSearchParamValues(searchTerm, query);
962-
return query.getResultList();
951+
if (searchTerm != null && !searchTerm.isEmpty()) {
952+
if (user.isSuperuser()) {
953+
sqlCode = sqlCode.concat(WHERE).concat(SEARCH_PARAMS);
954+
} else {
955+
sqlCode = sqlCode.concat(AND).concat(SEARCH_PARAMS);
956+
}
963957
}
964-
else {
965-
Query query = em.createNativeQuery(sqlCode.concat(AND).concat(SEARCH_PARAMS), Dataverse.class);
958+
if (pager != null) {
959+
// Add a pagination wrapper around the sqlCode
960+
int pageSize = pager.getDocsPerPage();
961+
int pageStart = (pager.getSelectedPageNumber()-1) * pageSize + 1;
962+
int pageEnd = pageStart + pageSize - 1;
963+
sqlCode = PAGE_PARAMS
964+
.replace("@START", String.valueOf(pageStart))
965+
.replace("@END", String.valueOf(pageEnd))
966+
.replace("@SQL@", sqlCode);
967+
}
968+
969+
Query query = em.createNativeQuery(sqlCode, Dataverse.class);
970+
if (searchTerm != null && !searchTerm.isEmpty()) {
966971
setSearchParamValues(searchTerm, query);
967-
return query.getResultList();
968972
}
973+
974+
List resultList = query.getResultList();
975+
if (pager != null) {
976+
pager.setNumResults((pager.getSelectedPageNumber()-1) * pager.getDocsPerPage() + resultList.size());
977+
}
978+
dataverses.addAll(resultList);
979+
return dataverses;
969980
}
970981
return null;
971982
}

‎src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import edu.harvard.iq.dataverse.externaltools.ExternalToolServiceBean;
2626
import edu.harvard.iq.dataverse.license.LicenseServiceBean;
2727
import edu.harvard.iq.dataverse.makedatacount.DatasetMetricsServiceBean;
28+
import edu.harvard.iq.dataverse.mydata.Pager;
2829
import edu.harvard.iq.dataverse.pidproviders.FailedPIDResolutionLoggingServiceBean;
2930
import edu.harvard.iq.dataverse.pidproviders.PidUtil;
3031
import edu.harvard.iq.dataverse.pidproviders.FailedPIDResolutionLoggingServiceBean.FailedPIDResolutionEntry;
@@ -299,6 +300,19 @@ protected boolean parseBooleanOrDie( String input ) throws WrappedResponse {
299300
}
300301
}
301302

303+
// Get a Pager object for adding pagination to a list result
304+
protected Pager getPager(Integer pageSize, Integer start) {
305+
if (pageSize != null || start != null) {
306+
int maxPageSize = pageSize != null ? Math.max(pageSize, 10) : 10;
307+
int offset = start != null ? start : 0;
308+
int selectedPageNumber = offset / maxPageSize + 1;
309+
// Since a new Pager is created for each API call we need to default some values to make things work
310+
int numResults = selectedPageNumber * maxPageSize;
311+
return new Pager(numResults, maxPageSize, selectedPageNumber);
312+
}
313+
return null;
314+
}
315+
302316
/**
303317
* Returns the {@code key} query parameter from the current request, or {@code null} if
304318
* the request has no such parameter.

‎src/main/java/edu/harvard/iq/dataverse/api/Users.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import edu.harvard.iq.dataverse.authorization.users.GuestUser;
1313
import edu.harvard.iq.dataverse.authorization.users.User;
1414
import edu.harvard.iq.dataverse.engine.command.impl.*;
15+
import edu.harvard.iq.dataverse.mydata.Pager;
1516
import edu.harvard.iq.dataverse.settings.FeatureFlags;
1617
import edu.harvard.iq.dataverse.util.BundleUtil;
1718
import edu.harvard.iq.dataverse.util.FileUtil;
@@ -276,7 +277,11 @@ public Response getTracesElement(@Context ContainerRequestContext crc, @Context
276277
@AuthRequired
277278
@Path("{identifier}/allowedCollections/{permission}")
278279
@Produces("application/json")
279-
public Response getUserPermittedCollections(@Context ContainerRequestContext crc, @Context Request req, @PathParam("identifier") String identifier, @PathParam("permission") String permission) {
280+
public Response getUserPermittedCollections(@Context ContainerRequestContext crc, @Context Request req,
281+
@PathParam("identifier") String identifier,
282+
@PathParam("permission") String permission,
283+
@QueryParam("offset") Integer start,
284+
@QueryParam("pageSize") Integer pageSize) {
280285
AuthenticatedUser authenticatedUser = null;
281286
try {
282287
authenticatedUser = getRequestAuthenticatedUserOrDie(crc);
@@ -288,8 +293,9 @@ public Response getUserPermittedCollections(@Context ContainerRequestContext crc
288293
}
289294
try {
290295
AuthenticatedUser userToQuery = authSvc.getAuthenticatedUser(identifier);
291-
List<Dataverse> collections = execCommand(new GetUserPermittedCollectionsCommand(createDataverseRequest(getRequestUser(crc)), userToQuery, permission));
292-
return ok(JsonPrinter.jsonArray(collections));
296+
Pager pager = getPager(pageSize, start);
297+
List<Dataverse> collections = execCommand(new GetUserPermittedCollectionsCommand(createDataverseRequest(getRequestUser(crc)), userToQuery, permission, null, pager));
298+
return ok(JsonPrinter.jsonArray(collections, pager));
293299
} catch (WrappedResponse ex) {
294300
return ex.getResponse();
295301
}

‎src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GetUserPermittedCollectionsCommand.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
1111
import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
1212
import edu.harvard.iq.dataverse.engine.command.exception.InvalidCommandArgumentsException;
13+
import edu.harvard.iq.dataverse.mydata.Pager;
1314
import edu.harvard.iq.dataverse.util.BundleUtil;
1415

1516
import java.util.List;
@@ -40,12 +41,19 @@ public class GetUserPermittedCollectionsCommand extends AbstractCommand<List<Dat
4041
private final DataverseRequest request;
4142
private final AuthenticatedUser user;
4243
private final String permission;
44+
private String searchTerm;
45+
private final Pager pager;
4346

4447
public GetUserPermittedCollectionsCommand(DataverseRequest request, AuthenticatedUser user, String permission) {
48+
this(request, user, permission, null, null);
49+
}
50+
public GetUserPermittedCollectionsCommand(DataverseRequest request, AuthenticatedUser user, String permission, String searchTerm, Pager pager) {
4551
super(request, (DvObject) null);
4652
this.request = request;
4753
this.user = user;
4854
this.permission = permission;
55+
this.searchTerm = searchTerm;
56+
this.pager = pager;
4957
}
5058

5159
@Override
@@ -59,6 +67,6 @@ public List<Dataverse> execute(CommandContext ctxt) throws CommandException {
5967
} catch (IllegalArgumentException e) {
6068
throw new InvalidCommandArgumentsException(BundleUtil.getStringFromBundle("getUserPermittedCollectionsCommand.errors.permissionNotValid"), this);
6169
}
62-
return ctxt.permissions().findPermittedCollections(request, user, permissionBit);
70+
return ctxt.permissions().findPermittedCollections(request, user, permissionBit, searchTerm, pager);
6371
}
6472
}

‎src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -327,11 +327,17 @@ private JsonObjectBuilder myDataAsJson(String message, Pager pager, RoleTagRetri
327327
@AuthRequired
328328
@Path(retrieveDataPartialAPIPath + "/collectionList")
329329
@Produces("application/json")
330-
public Response retrieveMyCollectionList(@Context ContainerRequestContext crc, @QueryParam("userIdentifier") String userIdentifier) {
330+
public Response retrieveMyCollectionList(@Context ContainerRequestContext crc,
331+
@QueryParam("userIdentifier") String userIdentifier,
332+
@QueryParam("searchTerm") String searchTerm,
333+
@QueryParam("offset") Integer start,
334+
@QueryParam("pageSize") Integer pageSize) {
331335
try {
332336
verifyAuth(crc, userIdentifier);
333-
List<Dataverse> collections = execCommand(new GetUserPermittedCollectionsCommand(createDataverseRequest(getRequestUser(crc)), searchUser, Permission.AddDataset.name()));
334-
return ok(JsonPrinter.jsonArray(collections));
337+
Pager pager = getPager(pageSize, start);
338+
List<Dataverse> collections = execCommand(new GetUserPermittedCollectionsCommand(createDataverseRequest(getRequestUser(crc)), searchUser, Permission.AddDataset.name(),
339+
searchTerm, pager));
340+
return ok(JsonPrinter.jsonArray(collections, pager));
335341
} catch (WrappedResponse wr) {
336342
return wr.getResponse();
337343
}

‎src/main/java/edu/harvard/iq/dataverse/mydata/Pager.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,7 @@ public int getNumResults(){
155155
*/
156156
public void setNumResults(int numResults){
157157
this.numResults = numResults;
158+
makePageStats();
158159
}
159160

160161

@@ -482,4 +483,4 @@ private void msgt(String s){
482483
msg(s);
483484
msg("-------------------------------");
484485
}
485-
}
486+
}

‎src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import edu.harvard.iq.dataverse.globus.FileDetailsHolder;
2727
import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
2828
import edu.harvard.iq.dataverse.license.License;
29+
import edu.harvard.iq.dataverse.mydata.Pager;
2930
import edu.harvard.iq.dataverse.privateurl.PrivateUrl;
3031
import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
3132
import edu.harvard.iq.dataverse.util.BundleUtil;
@@ -384,8 +385,12 @@ public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean re
384385
}
385386

386387
public static JsonObjectBuilder jsonArray(List<Dataverse> dataverses) {
388+
return jsonArray(dataverses, null);
389+
}
390+
public static JsonObjectBuilder jsonArray(List<Dataverse> dataverses, Pager pager) {
387391
JsonObjectBuilder job = Json.createObjectBuilder();
388-
job.add("count", dataverses.size());
392+
int count = dataverses.size();
393+
job.add("count", count);
389394
JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder();
390395
for (Dataverse dataverse : dataverses) {
391396
NullSafeJsonBuilder jsonObject = NullSafeJsonBuilder.jsonObjectBuilder();
@@ -395,6 +400,17 @@ public static JsonObjectBuilder jsonArray(List<Dataverse> dataverses) {
395400
jsonArrayBuilder.add(jsonObject);
396401
}
397402
job.add("items", jsonArrayBuilder);
403+
if (pager != null) {
404+
job.add("pageSize", pager.getDocsPerPage());
405+
int nextOffset = pager.getSelectedPageNumber() * pager.getDocsPerPage() + 1;
406+
int prevOffset = nextOffset - (2 * pager.getDocsPerPage());
407+
if (count >= pager.getDocsPerPage()) {
408+
job.add("nextOffset", nextOffset);
409+
}
410+
if (prevOffset > 0) {
411+
job.add("prevOffset", prevOffset);
412+
}
413+
}
398414
return job;
399415
}
400416

‎src/test/java/edu/harvard/iq/dataverse/api/DataRetrieverApiIT.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -167,12 +167,18 @@ public void testRetrieveMyDataCollections() throws InterruptedException {
167167
// Sleep for indexing
168168
Thread.sleep(4000);
169169

170-
// User1 gets the list of Dataverses/Collections it has access to
171-
retrieveMyCollectionListResponse = UtilIT.retrieveMyCollectionList(User1ApiToken, null);
170+
// User1 gets the list of Dataverses/Collections it has access to (with pagination)
171+
// Get the first page
172+
retrieveMyCollectionListResponse = UtilIT.retrieveMyCollectionList(User1ApiToken, null, null, 10);
173+
retrieveMyCollectionListResponse.prettyPrint();
174+
int count = retrieveMyCollectionListResponse.getBody().jsonPath().getInt("data.count");
175+
// get the second page
176+
int offset = retrieveMyCollectionListResponse.getBody().jsonPath().getInt("data.nextOffset");
177+
retrieveMyCollectionListResponse = UtilIT.retrieveMyCollectionList(User1ApiToken, null, offset, 10);
172178
retrieveMyCollectionListResponse.prettyPrint();
179+
count = count + retrieveMyCollectionListResponse.getBody().jsonPath().getInt("data.count");
173180
// The count should show the list size to be User1's + Root Dataverse count
174-
items = retrieveMyCollectionListResponse.getBody().jsonPath().getList("data.items");
175-
assertEquals(rootCount + user1DataverseCount, items.size());
181+
assertEquals(rootCount + user1DataverseCount, count);
176182

177183
// User2 gets the list of Dataverses/Collections it has access to
178184
retrieveMyCollectionListResponse = UtilIT.retrieveMyCollectionList(User2ApiToken, null);

0 commit comments

Comments
 (0)