Skip to content

Commit e717dc4

Browse files
committed
Fix upload_graph condition regression and formatting issues
1 parent 6e70264 commit e717dc4

3 files changed

Lines changed: 12 additions & 8 deletions

File tree

runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1244,7 +1244,8 @@ public DataflowPipelineJob run(Pipeline pipeline) {
12441244
// Multi-language pipelines and pipelines that include upgrades should automatically be upgraded
12451245
// to Runner v2.
12461246
if (DataflowRunner.isMultiLanguagePipeline(pipeline) || includesTransformUpgrades(pipeline)) {
1247-
if (!firstNonNull(options.getExperiments(), Collections.emptyList()).contains("use_runner_v2")) {
1247+
if (!firstNonNull(options.getExperiments(), Collections.emptyList())
1248+
.contains("use_runner_v2")) {
12481249
LOG.info(
12491250
"Automatically enabling Dataflow Runner v2 since the pipeline used cross-language"
12501251
+ " transforms or pipeline needed a transform upgrade.");
@@ -1290,7 +1291,8 @@ public DataflowPipelineJob run(Pipeline pipeline) {
12901291
{
12911292
// Experiment marking that the harness supports tag encoding v2
12921293
// Backend will enable tag encoding v2 only if the harness supports it.
1293-
ExperimentalOptions.addExperiment(options, "streaming_engine_state_tag_encoding_v2_supported");
1294+
ExperimentalOptions.addExperiment(
1295+
options, "streaming_engine_state_tag_encoding_v2_supported");
12941296
}
12951297

12961298
if (useUnifiedWorker(options)) {
@@ -1454,7 +1456,8 @@ public DataflowPipelineJob run(Pipeline pipeline) {
14541456
.collect(Collectors.toList());
14551457

14561458
if (minCpuFlags.isEmpty()) {
1457-
ExperimentalOptions.addExperiment(dataflowOptions, "min_cpu_platform=" + dataflowOptions.getMinCpuPlatform());
1459+
ExperimentalOptions.addExperiment(
1460+
dataflowOptions, "min_cpu_platform=" + dataflowOptions.getMinCpuPlatform());
14581461
} else {
14591462
LOG.warn(
14601463
"Flag min_cpu_platform is defined in both top level PipelineOption, "
@@ -1490,8 +1493,7 @@ public DataflowPipelineJob run(Pipeline pipeline) {
14901493
// enable upload_graph when the graph is too large
14911494
byte[] jobGraphBytes = DataflowPipelineTranslator.jobToString(newJob).getBytes(UTF_8);
14921495
int jobGraphByteSize = jobGraphBytes.length;
1493-
if (jobGraphByteSize >= CREATE_JOB_REQUEST_LIMIT_BYTES
1494-
&& !useUnifiedWorker(options)) {
1496+
if (jobGraphByteSize >= CREATE_JOB_REQUEST_LIMIT_BYTES && !useUnifiedWorker(options)) {
14951497
ExperimentalOptions.addExperiment(options, "upload_graph");
14961498
LOG.info(
14971499
"The job graph size ({} in bytes) is larger than {}. Automatically add "
@@ -1500,7 +1502,7 @@ public DataflowPipelineJob run(Pipeline pipeline) {
15001502
CREATE_JOB_REQUEST_LIMIT_BYTES);
15011503
}
15021504

1503-
if (useUnifiedWorker(options)) {
1505+
if (hasExperiment(options, "upload_graph") && useUnifiedWorker(options)) {
15041506
ArrayList<String> experiments = new ArrayList<>(options.getExperiments());
15051507
while (experiments.remove("upload_graph")) {}
15061508
options.setExperiments(experiments);

sdks/python/apache_beam/io/external/xlang_parquetio_test.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@ def test_xlang_parquetio_write(self):
5151
address = 'localhost:%s' % port
5252
try:
5353
with TestPipeline() as p:
54-
p.get_pipeline_options().view_as(DebugOptions).add_experiment('jar_packages=' + expansion_jar)
54+
p.get_pipeline_options().view_as(DebugOptions).add_experiment(
55+
'jar_packages=' + expansion_jar)
5556
p.not_use_test_runner_api = True
5657
_ = p \
5758
| beam.Create([

sdks/python/apache_beam/runners/dataflow/internal/apiclient.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,8 @@ def __init__(
150150
# add the flag if 'no_use_multiple_sdk_containers' is present.
151151
# TODO: Cleanup use_multiple_sdk_containers once we deprecate Python SDK
152152
# till version 2.4.
153-
if ('no_use_multiple_sdk_containers' not in (self.debug_options.experiments or [])):
153+
if ('no_use_multiple_sdk_containers'
154+
not in (self.debug_options.experiments or [])):
154155
self.debug_options.add_experiment('use_multiple_sdk_containers')
155156
# FlexRS
156157
if self.google_cloud_options.flexrs_goal == 'COST_OPTIMIZED':

0 commit comments

Comments
 (0)