Skip to content

Commit 4b5f166

Browse files
committed
address comments
1 parent 2169a2e commit 4b5f166

2 files changed

Lines changed: 9 additions & 12 deletions

File tree

runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowStreamingPipelineOptions.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ public interface DataflowStreamingPipelineOptions extends PipelineOptions {
201201
@Default.InstanceFactory(CommitWorkStreamRetryTimeoutMillisFactory.class)
202202
long getCommitWorkStreamRetryTimeoutMillis();
203203

204-
void getCommitWorkStreamRetryTimeoutMillis(long value);
204+
void setCommitWorkStreamRetryTimeoutMillis(long value);
205205

206206
@Description(
207207
"Period for sending 'global get config' requests to the service. The duration is "
@@ -351,7 +351,6 @@ public Boolean create(PipelineOptions options) {
351351
}
352352
}
353353

354-
/** defaults to false unless one of the experiment is set. */
355354
class CommitWorkStreamRetryTimeoutMillisFactory implements DefaultValueFactory<Long> {
356355
@Override
357356
public Long create(PipelineOptions options) {

runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/windmill/client/grpc/GrpcCommitWorkStream.java

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -243,27 +243,25 @@ public boolean hasPendingRequests() {
243243
public void onDone(Status status) {
244244
if (maxRetryDuration.compareTo(Duration.ZERO) > 0) {
245245
// Remove the requests that have exceeded the retry time so they are not retried.
246-
long startTimeRetryThresholdMsec = System.currentTimeMillis() - maxRetryDuration.toMillis();
246+
long startTimeRetryThresholdNanos = System.nanoTime() - maxRetryDuration.toNanos();
247247
Iterator<Map.Entry<Long, StreamAndRequest>> iterator = pending.entrySet().iterator();
248248
int keptRequests = 0, removedRequests = 0;
249249
while (iterator.hasNext()) {
250250
StreamAndRequest streamAndRequest = checkNotNull(iterator.next().getValue());
251251
PendingRequest pendingRequest = streamAndRequest.request;
252252
if (!belongsToThisHandler(streamAndRequest)
253-
|| pendingRequest.getStartTimeMillis() > startTimeRetryThresholdMsec) {
253+
|| pendingRequest.getStartTimeNanos() > startTimeRetryThresholdNanos) {
254254
++keptRequests;
255255
continue;
256256
}
257257
++removedRequests;
258258
try {
259259
pendingRequest.completeWithStatus(CommitStatus.ABORTED);
260+
iterator.remove();
260261
} catch (RuntimeException e) {
261-
// Catch possible exceptions to ensure that an exception for one commit does not prevent
262-
// other commits from being processed. Aggregate all the failures to throw after
263-
// processing the response if they exist.
262+
// Ignore exceptions and retry the commit.
264263
LOG.warn("Exception while aborting commit due to retry timeout.", e);
265264
}
266-
iterator.remove();
267265
}
268266
if (removedRequests > 0) {
269267
LOG.info(
@@ -420,14 +418,14 @@ private static class PendingRequest {
420418
private final String computationId;
421419
private final WorkItemCommitRequest request;
422420
private final Consumer<CommitStatus> onDone;
423-
private final long startTimeMillis;
421+
private final long startTimeNanos; // System.nanoTime() of when request began.
424422

425423
private PendingRequest(
426424
String computationId, WorkItemCommitRequest request, Consumer<CommitStatus> onDone) {
427425
this.computationId = computationId;
428426
this.request = request;
429427
this.onDone = onDone;
430-
this.startTimeMillis = System.currentTimeMillis();
428+
this.startTimeNanos = System.nanoTime();
431429
}
432430

433431
String getComputationId() {
@@ -438,8 +436,8 @@ WorkItemCommitRequest getRequest() {
438436
return request;
439437
}
440438

441-
long getStartTimeMillis() {
442-
return startTimeMillis;
439+
long getStartTimeNanos() {
440+
return startTimeNanos;
443441
}
444442

445443
private long getBytes() {

0 commit comments

Comments
 (0)