Skip to content

Commit f46213e

Browse files
committed
Fix upload_graph condition regression and formatting issues
1 parent 25ea087 commit f46213e

3 files changed

Lines changed: 12 additions & 8 deletions

File tree

runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1244,7 +1244,8 @@ public DataflowPipelineJob run(Pipeline pipeline) {
12441244
// Multi-language pipelines and pipelines that include upgrades should automatically be upgraded
12451245
// to Dataflow Portable Runner.
12461246
if (DataflowRunner.isMultiLanguagePipeline(pipeline) || includesTransformUpgrades(pipeline)) {
1247-
if (!firstNonNull(options.getExperiments(), Collections.emptyList()).contains("use_runner_v2")) {
1247+
if (!firstNonNull(options.getExperiments(), Collections.emptyList())
1248+
.contains("use_runner_v2")) {
12481249
LOG.info(
12491250
"Automatically enabling Dataflow Portable Runner since the pipeline used cross-language"
12501251
+ " transforms or pipeline needed a transform upgrade.");
@@ -1292,7 +1293,8 @@ public DataflowPipelineJob run(Pipeline pipeline) {
12921293
{
12931294
// Experiment marking that the harness supports tag encoding v2
12941295
// Backend will enable tag encoding v2 only if the harness supports it.
1295-
ExperimentalOptions.addExperiment(options, "streaming_engine_state_tag_encoding_v2_supported");
1296+
ExperimentalOptions.addExperiment(
1297+
options, "streaming_engine_state_tag_encoding_v2_supported");
12961298
}
12971299

12981300
if (useUnifiedWorker(options)) {
@@ -1462,7 +1464,8 @@ public DataflowPipelineJob run(Pipeline pipeline) {
14621464
.collect(Collectors.toList());
14631465

14641466
if (minCpuFlags.isEmpty()) {
1465-
ExperimentalOptions.addExperiment(dataflowOptions, "min_cpu_platform=" + dataflowOptions.getMinCpuPlatform());
1467+
ExperimentalOptions.addExperiment(
1468+
dataflowOptions, "min_cpu_platform=" + dataflowOptions.getMinCpuPlatform());
14661469
} else {
14671470
LOG.warn(
14681471
"Flag min_cpu_platform is defined in both top level PipelineOption, "
@@ -1498,8 +1501,7 @@ public DataflowPipelineJob run(Pipeline pipeline) {
14981501
// enable upload_graph when the graph is too large
14991502
byte[] jobGraphBytes = DataflowPipelineTranslator.jobToString(newJob).getBytes(UTF_8);
15001503
int jobGraphByteSize = jobGraphBytes.length;
1501-
if (jobGraphByteSize >= CREATE_JOB_REQUEST_LIMIT_BYTES
1502-
&& !useUnifiedWorker(options)) {
1504+
if (jobGraphByteSize >= CREATE_JOB_REQUEST_LIMIT_BYTES && !useUnifiedWorker(options)) {
15031505
ExperimentalOptions.addExperiment(options, "upload_graph");
15041506
LOG.info(
15051507
"The job graph size ({} in bytes) is larger than {}. Automatically add "
@@ -1508,7 +1510,7 @@ public DataflowPipelineJob run(Pipeline pipeline) {
15081510
CREATE_JOB_REQUEST_LIMIT_BYTES);
15091511
}
15101512

1511-
if (useUnifiedWorker(options)) {
1513+
if (hasExperiment(options, "upload_graph") && useUnifiedWorker(options)) {
15121514
ArrayList<String> experiments = new ArrayList<>(options.getExperiments());
15131515
while (experiments.remove("upload_graph")) {}
15141516
options.setExperiments(experiments);

sdks/python/apache_beam/io/external/xlang_parquetio_test.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@ def test_xlang_parquetio_write(self):
5151
address = 'localhost:%s' % port
5252
try:
5353
with TestPipeline() as p:
54-
p.get_pipeline_options().view_as(DebugOptions).add_experiment('jar_packages=' + expansion_jar)
54+
p.get_pipeline_options().view_as(DebugOptions).add_experiment(
55+
'jar_packages=' + expansion_jar)
5556
p.not_use_test_runner_api = True
5657
_ = p \
5758
| beam.Create([

sdks/python/apache_beam/runners/dataflow/internal/apiclient.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,8 @@ def __init__(
151151
# add the flag if 'no_use_multiple_sdk_containers' is present.
152152
# TODO: Cleanup use_multiple_sdk_containers once we deprecate Python SDK
153153
# till version 2.4.
154-
if ('no_use_multiple_sdk_containers' not in (self.debug_options.experiments or [])):
154+
if ('no_use_multiple_sdk_containers'
155+
not in (self.debug_options.experiments or [])):
155156
self.debug_options.add_experiment('use_multiple_sdk_containers')
156157
# FlexRS
157158
if self.google_cloud_options.flexrs_goal == 'COST_OPTIMIZED':

0 commit comments

Comments
 (0)