Skip to content

Commit 2d8a2bb

Browse files
committed
Stabilize FileIOTest updated-file timestamp assertions
1 parent 8ade6a7 commit 2d8a2bb

1 file changed

Lines changed: 16 additions & 2 deletions

File tree

sdks/java/core/src/test/java/org/apache/beam/sdk/io/FileIOTest.java

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -232,9 +232,10 @@ public void testMatchAllDisallowEmptyNonWildcard() throws IOException {
232232
/** DoFn that copy test files from source to watch path. */
233233
private static class CopyFilesFn
234234
extends DoFn<KV<String, MatchResult.Metadata>, MatchResult.Metadata> {
235-
public CopyFilesFn(Path sourcePath, Path watchPath) {
235+
public CopyFilesFn(Path sourcePath, Path watchPath, long baseTimestampMillis) {
236236
this.sourcePathStr = sourcePath.toString();
237237
this.watchPathStr = watchPath.toString();
238+
this.baseTimestampMillis = baseTimestampMillis;
238239
}
239240

240241
@StateId("count")
@@ -257,10 +258,16 @@ public void processElement(ProcessContext context, @StateId("count") ValueState<
257258

258259
if (0 == current) {
259260
Thread.sleep(100);
261+
// Ensure overwrite updates get a distinct mtime even when COPY_ATTRIBUTES is enabled.
262+
Files.setLastModifiedTime(
263+
sourcePath.resolve("first"), FileTime.fromMillis(baseTimestampMillis + 2000));
260264
Files.copy(sourcePath.resolve("first"), watchPath.resolve("first"), updOptions);
261265
Files.copy(sourcePath.resolve("second"), watchPath.resolve("second"), cpOptions);
262266
} else if (1 == current) {
263267
Thread.sleep(100);
268+
FileTime updateTime = FileTime.fromMillis(baseTimestampMillis + 4000);
269+
Files.setLastModifiedTime(sourcePath.resolve("first"), updateTime);
270+
Files.setLastModifiedTime(sourcePath.resolve("second"), updateTime);
264271
Files.copy(sourcePath.resolve("first"), watchPath.resolve("first"), updOptions);
265272
Files.copy(sourcePath.resolve("second"), watchPath.resolve("second"), updOptions);
266273
Files.copy(sourcePath.resolve("third"), watchPath.resolve("third"), cpOptions);
@@ -271,6 +278,7 @@ public void processElement(ProcessContext context, @StateId("count") ValueState<
271278
// Member variables need to be serializable.
272279
private final String sourcePathStr;
273280
private final String watchPathStr;
281+
private final long baseTimestampMillis;
274282
}
275283

276284
@Test
@@ -282,6 +290,12 @@ public void testMatchWatchForNewFiles() throws IOException, InterruptedException
282290
Files.write(sourcePath.resolve("first"), new byte[42]);
283291
Files.write(sourcePath.resolve("second"), new byte[37]);
284292
Files.write(sourcePath.resolve("third"), new byte[99]);
293+
// Keep controlled mtimes in the past so updates are distinct without future timestamps.
294+
long baseTimestampMillis = System.currentTimeMillis() - Duration.standardMinutes(1).getMillis();
295+
FileTime baseTimestamp = FileTime.fromMillis(baseTimestampMillis);
296+
Files.setLastModifiedTime(sourcePath.resolve("first"), baseTimestamp);
297+
Files.setLastModifiedTime(sourcePath.resolve("second"), baseTimestamp);
298+
Files.setLastModifiedTime(sourcePath.resolve("third"), baseTimestamp);
285299

286300
// Create a "watch" directory that the pipeline will copy files into.
287301
final Path watchPath = tmpFolder.getRoot().toPath().resolve("watch");
@@ -336,7 +350,7 @@ public void testMatchWatchForNewFiles() throws IOException, InterruptedException
336350
TypeDescriptors.strings(),
337351
TypeDescriptor.of(MatchResult.Metadata.class)))
338352
.via((metadata) -> KV.of("dumb key", metadata)))
339-
.apply(ParDo.of(new CopyFilesFn(sourcePath, watchPath)));
353+
.apply(ParDo.of(new CopyFilesFn(sourcePath, watchPath, baseTimestampMillis)));
340354

341355
assertEquals(PCollection.IsBounded.UNBOUNDED, matchMetadata.isBounded());
342356
assertEquals(PCollection.IsBounded.UNBOUNDED, matchAllMetadata.isBounded());

0 commit comments

Comments
 (0)