Skip to content

Commit 9ab7d50

Browse files
committed
respond to comments
1 parent 18f7042 commit 9ab7d50

3 files changed

Lines changed: 6 additions & 21 deletions

File tree

runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingModeExecutionContext.java

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -723,13 +723,8 @@ private void validateCommitRequestSize() {
723723
// so, we're purposefully dropping them here
724724
Windmill.WorkItemCommitRequest.Builder truncationBuilder =
725725
buildWorkItemTruncationRequestBuilder(currentWork, estimatedCommitSize);
726-
for (int i = 0; i < outputBuilders.size(); i++) {
727-
if (outputBuilders.get(i) == currentBuilder) {
728-
outputBuilders.set(i, truncationBuilder);
729-
break;
730-
}
731-
}
732-
this.outputBuilder = truncationBuilder;
726+
this.outputBuilder.clear();
727+
this.outputBuilder.mergeFrom(truncationBuilder);
733728
}
734729

735730
private Windmill.WorkItemCommitRequest.Builder buildWorkItemTruncationRequestBuilder(

runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/streaming/KeyCommitTooLargeException.java

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -22,19 +22,6 @@
2222

2323
public final class KeyCommitTooLargeException extends Exception {
2424

25-
public static KeyCommitTooLargeException causedBy(
26-
String stageName, long byteLimit, Windmill.WorkItemCommitRequest request) {
27-
return causedBy(stageName, byteLimit, request, null, false);
28-
}
29-
30-
public static KeyCommitTooLargeException causedBy(
31-
String stageName,
32-
long byteLimit,
33-
Windmill.WorkItemCommitRequest request,
34-
boolean hotKeyLoggingEnabled) {
35-
return causedBy(stageName, byteLimit, request, null, hotKeyLoggingEnabled);
36-
}
37-
3825
public static KeyCommitTooLargeException causedBy(
3926
String stageName,
4027
long byteLimit,

runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorkerTest.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1401,15 +1401,18 @@ private void runKeyCommitTooLargeExceptionTest(
14011401
1, "large_key", DEFAULT_SHARDING_KEY, largeCommit.getEstimatedWorkItemCommitBytes())
14021402
.build(),
14031403
removeDynamicFields(largeCommit));
1404-
1404+
// Check this explicitly since the estimated commit bytes weren't actuallyExpand commentComment on line L1340
1405+
// checked against an expected value in the previous step
14051406
assertTrue(largeCommit.getEstimatedWorkItemCommitBytes() > 1000);
14061407

1408+
// Spam worker updates a few times.
14071409
int maxTries = 10;
14081410
while (--maxTries > 0) {
14091411
worker.reportPeriodicWorkerUpdatesForTest();
14101412
Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
14111413
}
14121414

1415+
// We should see an exception reported for the large commit but not the small one.
14131416
ArgumentCaptor<WorkItemStatus> workItemStatusCaptor =
14141417
ArgumentCaptor.forClass(WorkItemStatus.class);
14151418
verify(mockWorkUnitClient, atLeast(2)).reportWorkItemStatus(workItemStatusCaptor.capture());

0 commit comments

Comments
 (0)