Skip to content

Commit bf84302

Browse files
author
Brendan Benner
committed
Update log for null item to warn
Signed-off-by: Brendan Benner <bbenner@amazon.com>
1 parent b0b0f09 commit bf84302

2 files changed

Lines changed: 4 additions & 3 deletions

File tree

data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/TokenPaginationCrawler.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public Instant crawl(LeaderPartition leaderPartition,
6464
final ItemInfo nextItem = itemInfoIterator.next();
6565
if (nextItem == null) {
6666
//we don't expect null items, but just in case, we'll skip them
67-
log.info("Unexpected encounter of a null item.");
67+
log.warn("Unexpected encounter of a null item while processing batch with last item ID " + lastToken);
6868
invalidPaginationItemsCounter.increment();
6969
continue;
7070
}

data-prepper-plugins/saas-source-plugins/source-crawler/src/test/java/org/opensearch/dataprepper/plugins/source/source_crawler/base/TokenPaginationCrawlerTest.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727

2828
import static org.junit.jupiter.api.Assertions.assertEquals;
2929
import static org.junit.jupiter.api.Assertions.assertNotNull;
30+
import static org.mockito.ArgumentMatchers.anyString;
3031
import static org.mockito.ArgumentMatchers.any;
3132
import static org.mockito.Mockito.never;
3233
import static org.mockito.Mockito.reset;
@@ -104,7 +105,7 @@ void testCrawlWithMultiplePartitions() {
104105
}
105106

106107
@Test
107-
void testBatchSize() {
108+
void testBatchSize() {
108109
List<ItemInfo> itemInfoList = new ArrayList<>();
109110
int maxItemsPerPage = DEFAULT_BATCH_SIZE;
110111
for (int i = 0; i < maxItemsPerPage; i++) {
@@ -119,7 +120,7 @@ void testBatchSize() {
119120
for (int i = 0; i < maxItemsPerPage * 2; i++) {
120121
itemInfoList2.add(new TestItemInfo("testId" + i));
121122
}
122-
when(client.listItems(any())).thenReturn(itemInfoList2.iterator());
123+
when(client.listItems(anyString())).thenReturn(itemInfoList2.iterator());
123124
crawler.crawl(leaderPartition, coordinator);
124125
expectedNumberOfInvocations += 2;
125126
verify(coordinator, times(expectedNumberOfInvocations)).createPartition(any(SaasSourcePartition.class));

0 commit comments

Comments
 (0)