Update Protobuf Bazel Workspace #11994

Workflow file for this run

	# GitHub Actions CI definition for TensorBoard.
	#
	# YAML schema for GitHub Actions:
	# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions
	#
	# Helpful YAML parser to clarify YAML syntax:
	# https://yaml-online-parser.appspot.com/

	name: CI

	on:
	push:
	branches:
	- master
	- '[0-9]+.*'
	- 'ci-*'
	tags:
	- '*'
	pull_request: {}
	workflow_call: {}

	permissions:
	contents: read

	env:
	# Keep this Bazel version in sync with the `versions.check` directive
	# in our WORKSPACE file.
	BAZEL_VERSION: '7.7.0'
	BAZEL_SHA256SUM: 'fe7e799cbc9140f986b063e06800a3d4c790525075c877d00a7112669824acbf'
	BUILDTOOLS_VERSION: '3.0.0'
	BUILDIFIER_SHA256SUM: 'e92a6793c7134c5431c58fbc34700664f101e5c9b1c1fcd93b97978e8b7f88db'
	BUILDOZER_SHA256SUM: '3d58a0b6972e4535718cdd6c12778170ea7382de7c75bc3728f5719437ffb84d'
	TENSORFLOW_VERSION: 'tf-nightly'

	defaults:
	run:
	shell: bash

	jobs:
	build:
	# Self-hosted runner since the ones provided by GH were running out of space.
	# Other workflows are likely ok to use standard GH runners for now.
	# Googlers, see b/460874304.
	runs-on: linux-x86-n2-32
	container: us-docker.pkg.dev/ml-oss-artifacts-published/ml-public-container/ml-build:latest
	needs: lint-python-flake8 # fail fast in case of "undefined variable" errors
	strategy:
	fail-fast: false
	matrix:
	tf_version_id: ['tf', 'notf']
	python_version: ['3.10']
	steps:
	- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
	- uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
	with:
	python-version: ${{ matrix.python_version }}
	architecture: 'x64'
	- name: 'Set up Bazel'
	run: \|
	ci/download_bazel.sh "${BAZEL_VERSION}" "${BAZEL_SHA256SUM}" ~/bazel
	sudo mv ~/bazel /usr/local/bin/bazel
	sudo chmod +x /usr/local/bin/bazel
	cp ./ci/bazelrc ~/.bazelrc
	- name: 'Install TensorFlow'
	run: \|
	python -m pip install -U pip
	pip install "${TENSORFLOW_VERSION}"
	# tf-nightly currently pulls in tb-nightly as a dependency. Bazel's
	# source-tree tests must import TensorBoard from runfiles rather than
	# from site-packages, otherwise tests that rely on local-only modules
	# like `tensorboard.test` resolve against the installed wheel and fail.
	pip uninstall -y tensorboard tb-nightly \|\| true
	if: matrix.tf_version_id != 'notf'
	- name: 'Install Python dependencies'
	run: \|
	python -m pip install -U pip
	pip install \
	-r ./tensorboard/pip_package/requirements.txt \
	-r ./tensorboard/pip_package/requirements_dev.txt \
	;
	- name: 'Install system dependencies'
	run: \|
	py_abi="python${{ matrix.python_version }}"
	sudo apt-get update
	sudo apt-get install -y "${py_abi}-dev" libgbm-dev libxss1 libasound2
	# Bazel's system_python repository resolves headers relative to the
	# interpreter installed by actions/setup-python. The self-hosted
	# ml-build container does not provide Python.h under that prefix, so
	# copy the system headers into the managed interpreter include dir.
	# A real directory copy is more robust here than a symlink because
	# Bazel's repository setup may materialize or traverse this path in
	# ways that do not preserve symlink behavior.
	sudo mkdir -p "${pythonLocation}/include"
	sudo rm -rf "${pythonLocation}/include/${py_abi}"
	sudo mkdir -p "${pythonLocation}/include/${py_abi}"
	sudo cp -a "/usr/include/${py_abi}/." "${pythonLocation}/include/${py_abi}/"
	includepy="$(python - <<'PY'
	import sysconfig
	print(sysconfig.get_config_var("INCLUDEPY"))
	PY
	)"
	# upb's system_python repository rule shells out to `python3` and
	# uses sysconfig.get_config_var("INCLUDEPY"), which under
	# actions/setup-python inside this container points at
	# /opt/hostedtoolcache/... rather than ${pythonLocation}. Materialize
	# that INCLUDEPY path too so Bazel's @system_python repo can symlink
	# real headers into external/system_python/python.
	sudo mkdir -p "$(dirname "${includepy}")"
	sudo rm -rf "${includepy}"
	sudo cp -a "${pythonLocation}/include/${py_abi}" "${includepy}"
	test -f "${pythonLocation}/include/${py_abi}/Python.h"
	test -f "${includepy}/Python.h"
	- name: 'Debug Python toolchain'
	run: \|
	py_abi="python${{ matrix.python_version }}"
	echo "pythonLocation=${pythonLocation}"
	echo "PATH=${PATH}"
	which python
	python --version
	command -v python3.10-config \|\| true
	python3.10-config --includes \|\| true
	python - <<'PY'
	import os
	import pathlib
	import sys
	import sysconfig

	print("sys.executable =", sys.executable)
	for key in ("include", "platinclude", "stdlib", "platstdlib", "scripts", "data"):
	print(f"{key} =", sysconfig.get_path(key))
	for key in ("INCLUDEPY", "CONFINCLUDEPY", "LIBDIR", "LDLIBRARY", "MULTIARCH"):
	print(f"{key} =", sysconfig.get_config_var(key))
	include_dir = pathlib.Path(sysconfig.get_path("include"))
	print("include_dir exists =", include_dir.exists(), include_dir)
	print("Python.h exists =", (include_dir / "Python.h").exists())
	print("pythonLocation =", os.environ.get("pythonLocation"))
	PY
	ls -la "${pythonLocation}" \|\| true
	ls -la "${pythonLocation}/include" \|\| true
	ls -la "${pythonLocation}/include/${py_abi}" \|\| true
	- name: 'Check Pip state'
	run: pip freeze --all
	- name: 'Bazel: fetch'
	run: bazel fetch //tensorboard/...
	- name: 'Debug Bazel system_python repo'
	run: \|
	output_base="$(bazel info output_base)"
	echo "output_base=${output_base}"
	find "${output_base}/external/system_python" -maxdepth 3 \
	$ -name 'Python.h' -o -name 'BUILD.bazel' -o -name 'WORKSPACE' $ \
	-print \| sort \|\| true
	if [ -f "${output_base}/external/system_python/BUILD.bazel" ]; then
	sed -n '1,200p' "${output_base}/external/system_python/BUILD.bazel"
	fi
	if [ -d "${output_base}/external/system_python/python" ]; then
	ls -la "${output_base}/external/system_python/python"
	else
	echo "system_python/python directory not found"
	fi
	- name: 'Bazel: build'
	# Note we suppress a flood of warnings from the proto compiler.
	# Googlers see b/222706811 & b/182876485 discussion and preconditions
	# for removing this kludge.
	run: \|
	( set -o pipefail;
	bazel build //tensorboard/... \|&\
	grep -v 'external/com_google_protobuf/python: warning: directory' \|&\
	grep -v 'INFO: From ProtoCompile ' )
	- name: 'Bazel: test (with TensorFlow support)'
	run: bazel test //tensorboard/...
	if: matrix.tf_version_id != 'notf'
	- name: 'Bazel: test (non-TensorFlow only)'
	run: bazel test //tensorboard/... --test_tag_filters="support_notf"
	if: matrix.tf_version_id == 'notf'
	- name: 'Bazel: run Pip package test (with TensorFlow support)'
	run: \|
	# `bazel run` has been flaky under this self-hosted container runner
	# even when the smoke test itself completes successfully. Build the
	# launcher target, invoke the generated binary directly, then shut
	# down the Bazel server so the step exits cleanly.
	bazel build //tensorboard/pip_package:test_pip_package
	./bazel-bin/tensorboard/pip_package/test_pip_package \
	--tf-version "${TENSORFLOW_VERSION}"
	bazel shutdown
	if: matrix.tf_version_id != 'notf'
	- name: 'Bazel: run Pip package test (non-TensorFlow only)'
	run: \|
	bazel build //tensorboard/pip_package:test_pip_package
	./bazel-bin/tensorboard/pip_package/test_pip_package \
	--tf-version notf
	bazel shutdown
	if: matrix.tf_version_id == 'notf'
	- name: 'Bazel: run manual tests'
	run: \|
	bazel test //tensorboard/compat/tensorflow_stub:gfile_s3_test &&
	bazel test //tensorboard/summary/writer:event_file_writer_s3_test &&
	bazel test //tensorboard/compat/tensorflow_stub:gfile_fsspec_test &&
	bazel test //tensorboard/summary/writer:event_file_writer_fsspec_test
	- name: 'Bazel: build Pip package (master branch TF build only)'
	# This pip package is only needed if we will upload it, so run this step
	# only under the same conditions as the next step (see comment below).
	if: matrix.tf_version_id == 'tf' && github.repository == 'tensorflow/tensorboard' && github.ref == 'refs/heads/master'
	run: \|
	./tensorboard/tools/set_nightly_version.sh
	rm -rf /tmp/tb_nightly_pip_package && mkdir /tmp/tb_nightly_pip_package
	bazel run //tensorboard/pip_package:build_pip_package -- /tmp/tb_nightly_pip_package
	- name: 'Upload Pip package as an artifact (master branch TF build only)'
	# Prevent uploads when running on forks or non-master branch.
	#
	# Note that upload-artifact GH action, starting in v4, requires the name of the uploaded
	# file(s) to be unique per workflow run, so make sure that the name is unique for each
	# "matrix" combination for which this is executed.
	if: matrix.tf_version_id == 'tf' && github.repository == 'tensorflow/tensorboard' && github.ref == 'refs/heads/master'
	uses: actions/upload-artifact@v4
	with:
	name: tb-nightly_py${{ matrix.python_version }}
	path: /tmp/tb_nightly_pip_package/*

	build-data-server-pip:
	runs-on: ${{ matrix.platform }}
	strategy:
	fail-fast: false
	matrix:
	mode: ['native']
	platform: ['ubuntu-22.04']
	rust_version: ['1.65.0']
	include:
	- mode: 'universal'
	platform: 'ubuntu-22.04'
	rust_version: '1.65.0'
	steps:
	- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
	- uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
	with:
	python-version: '3.9'
	architecture: 'x64'
	- name: 'Cache Cargo artifacts'
	if: matrix.mode == 'native'
	uses: actions/cache@v4
	with:
	path: \|
	tensorboard/data/server/target/
	# https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
	~/.cargo/bin/
	~/.cargo/registry/index/
	~/.cargo/registry/cache/
	~/.cargo/git/db/
	# Needed if we `cargo install` binaries (at time of writing, this
	# job doesn't but the files are tiny, and this will mitigate
	# confusion if we need to `cargo install` later)
	~/.cargo/.crates.toml
	~/.cargo/.crates2.json
	key: build-data-server-pip-${{ runner.os }}-cargo-${{ matrix.rust_version }}-${{ hashFiles('**/Cargo.lock', '.github/workflows/ci.yml') }}
	- name: 'Install Rust toolchain'
	if: matrix.mode == 'native'
	uses: dtolnay/rust-toolchain@5f2e2a7aff63d8cbdacb4832218a30fa7a37ee6e # current HEAD as of 1/19/2023
	with:
	toolchain: ${{ matrix.rust_version }}
	components: rustfmt
	- name: 'Install Python packaging deps'
	run: \|
	python -m pip install -U pip
	pip install \
	-c ./tensorboard/pip_package/requirements.txt \
	-c ./tensorboard/pip_package/requirements_dev.txt \
	setuptools wheel
	- name: 'Build'
	if: matrix.mode == 'native'
	run: cd tensorboard/data/server/ && cargo build --release
	- name: 'Test'
	if: matrix.mode == 'native'
	run: cd tensorboard/data/server/ && cargo test --release
	- name: 'Package (native)'
	if: matrix.mode == 'native'
	run: \|
	rm -rf /tmp/pip_package && mkdir /tmp/pip_package
	python tensorboard/data/server/pip_package/build.py \
	--server-binary tensorboard/data/server/target/release/rustboard \
	--out-dir /tmp/pip_package \
	;
	- name: 'Package (universal)'
	if: matrix.mode == 'universal'
	run: \|
	rm -rf /tmp/pip_package && mkdir /tmp/pip_package
	python tensorboard/data/server/pip_package/build.py \
	--universal \
	--out-dir /tmp/pip_package \
	;
	- name: 'Upload'
	uses: actions/upload-artifact@v4
	with:
	name: tensorboard-data-server_${{ matrix.mode }}_${{ matrix.platform }}_${{ matrix.rust_version }}
	path: /tmp/pip_package/*

	lint-python-flake8:
	runs-on: ubuntu-22.04
	strategy:
	fail-fast: false
	matrix:
	# flake8 should run on each Python version that we target,
	# because the errors and warnings can differ due to language
	# changes, and we want to catch them all.
	python_version: ['3.9', '3.10', '3.11']
	steps:
	- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
	- uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
	with:
	python-version: ${{ matrix.python_version }}
	architecture: 'x64'
	- name: 'Install flake8'
	run: \|
	python -m pip install -U pip
	pip install flake8 -c ./tensorboard/pip_package/requirements_dev.txt
	- run: pip freeze --all
	- name: 'Lint Python code for errors with flake8'
	# See: http://flake8.pycqa.org/en/3.7.8/user/error-codes.html
	# Use the comment '# noqa: <error code>' to suppress.
	run: flake8 . --count --select=E9,F63,F7,F82,F401 --show-source --statistics

	lint-python-yaml-docs:
	runs-on: ubuntu-22.04
	steps:
	- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
	- uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
	with:
	python-version: '3.10'
	architecture: 'x64'
	- name: 'Install black, yamllint, and the TensorFlow docs notebook tools'
	run: \|
	python -m pip install -U pip
	nbfmt_version="84bd7bea0c468667dd0ec273d426ed50f23e1a15"
	pip install black yamllint -c ./tensorboard/pip_package/requirements_dev.txt
	pip install -U git+https://github.com/tensorflow/docs@${nbfmt_version}
	# Workaround tensorflow incompatibility with protobuf>4.
	# See: https://github.com/tensorflow/tensorboard/issues/5708.
	# See: https://github.com/tensorflow/tensorflow/issues/53234.
	- name: 'Downgrade to compatible protobuf version'
	run: pip install -U "protobuf<3.20"
	- run: pip freeze --all
	- name: 'Lint Python code for style with Black'
	# You can run `black .` to fix all Black complaints.
	run: black --check --diff .
	- name: 'Lint YAML for gotchas with yamllint'
	# Use '# yamllint disable-line rule:foo' to suppress.
	run: yamllint -c docs/.yamllint docs docs/.yamllint
	- name: 'Lint Colab notebooks for formatting with nbfmt'
	run: git ls-files -z '*.ipynb' \| xargs -0 python3 -m tensorflow_docs.tools.nbfmt --test

	lint-rust:
	runs-on: ubuntu-22.04
	strategy:
	matrix:
	rust_version: ['1.65.0']
	cargo_raze_version: ['0.16.1']
	steps:
	- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
	- name: 'Cache Cargo artifacts'
	uses: actions/cache@v4
	with:
	path: \|
	tensorboard/data/server/target/
	# https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
	~/.cargo/bin/
	~/.cargo/registry/index/
	~/.cargo/registry/cache/
	~/.cargo/git/db/
	# Needed for installing binaries (`cargo-raze`) with cache
	~/.cargo/.crates.toml
	~/.cargo/.crates2.json
	key: lint-rust-${{ runner.os }}-cargo-${{ matrix.rust_version }}-${{ matrix.cargo_raze_version }}-${{ hashFiles('**/Cargo.lock', '.github/workflows/ci.yml') }}
	- name: 'Install Rust toolchain'
	uses: dtolnay/rust-toolchain@5f2e2a7aff63d8cbdacb4832218a30fa7a37ee6e # current HEAD as of 1/19/2023
	with:
	toolchain: ${{ matrix.rust_version }}
	components: rustfmt, clippy
	- name: 'Install cargo-raze'
	run: cargo install cargo-raze --version ${{ matrix.cargo_raze_version }} --locked
	- name: 'Run Rustfmt'
	run: (cd tensorboard/data/server/ && cargo fmt -- --check)
	- name: 'Run Clippy'
	# You can run `cargo clippy --all-targets --manifest-path tensorboard/data/server/Cargo.toml --fix` to fix all Clippy complaints.
	# This will only apply `MachineApplicable` fixes (https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint_defs/enum.Applicability.html), so some modifications may need to be done manually.
	run: cargo clippy --all-targets --manifest-path tensorboard/data/server/Cargo.toml -- -D warnings
	- name: 'Check cargo-raze freshness'
	run: \|
	rm -rf third_party/rust/
	(cd tensorboard/data/server/ && cargo fetch && cargo raze)
	git add .
	git diff --staged --exit-code

	lint-frontend:
	runs-on: ubuntu-22.04
	steps:
	- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
	- uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
	with:
	node-version: 16
	- run: yarn install --ignore-engines
	# You can run `yarn fix-lint` to fix all Prettier complaints, although at this point this will try to fix too many things.
	# To fix only the files changed in this PR, see the command below.
	- run: yarn lint \|\| (( echo 'Try running `$ prettier --write $(git diff-tree --no-commit-id --name-only <latest commit> <parent branch commit> -r)`.' && false ))
	# Make sure no tests are skipped with "focused" tests.
	- run: \|
	! git grep -E 'f(it\|describe)\(' 'tensorboard/*_test.ts'
	# Make sure no one depends on Angular material and CDK directly. Please
	# import the indirection in //tensorboard/webapp/angular.
	- run: \|
	! git grep -E '"@npm//@angular/material"\|"@npm//@angular/cdk"' 'tensorboard/*/BUILD' ':!tensorboard/webapp/BUILD' ':!tensorboard/webapp/angular/BUILD'
	# Cannot directly depend on d3 in webapp. Must depend on
	# `//tensorboard/webapp/third_party:d3` instead.
	- run: \|
	! git grep -E '"@npm//d3"\|"@npm//@types/d3"' 'tensorboard/webapp/*/BUILD' ':!tensorboard/webapp/third_party/**'
	- run: \|
	! git grep -E '"@npm_angular_bazel//:index.bzl"' 'tensorboard/**/BUILD'
	- run: \|
	! git grep -E 'mat-color\|$mat-' 'tensorboard/*/.scss'
	# Make sure no one depends on the numeric library directly. Please
	# import the indirection in //tensorboard/webapp/third_party.
	- run: \|
	! git grep -E '@npm//numeric' 'tensorboard//BUILD' ':!tensorboard/webapp/third_party/*'

	lint-misc: # build, protos, etc.
	runs-on: ubuntu-22.04
	steps:
	- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
	- name: 'Set up Buildifier'
	run: \|
	ci/download_buildifier.sh "${BUILDTOOLS_VERSION}" "${BUILDIFIER_SHA256SUM}" ~/buildifier
	sudo mv ~/buildifier /usr/local/bin/buildifier
	- name: 'Set up Buildozer'
	run: \|
	ci/download_buildozer.sh "${BUILDTOOLS_VERSION}" "${BUILDOZER_SHA256SUM}" ~/buildozer
	sudo mv ~/buildozer /usr/local/bin/buildozer
	- name: 'Lint BUILD files'
	# TODO(tensorboard-team): address all lint warnings and remove the exemption.
	run:
	git ls-files -z '*BUILD' third_party/js.bzl third_party/workspace.bzl WORKSPACE \| xargs -0 buildifier --mode=check --lint=warn
	--warnings=-native-py,-native-java
	- run: ./tensorboard/tools/mirror_urls_test.sh
	- name: 'Lint for no py2 BUILD targets'
	# Use \| to start a literal so YAML doesn't complain about the '!' character.
	run: \|
	! git grep 'python_version = "PY2"' '*BUILD'
	- name: 'No comments on licenses rule'
	# Assert buildozer error code for 'success, when no changes were made'.
	# https://github.com/bazelbuild/buildtools/blob/master/buildozer/README.md#error-code
	run: \|
	buildozer '//tensorboard/...:%licenses' remove_comment && false \|\| test $? = 3
	- name: clang-format lint
	uses: DoozyX/clang-format-lint-action@0138140a2adaafd3032a3ff37f66366fd7dc88e0
	with:
	source: ./tensorboard
	# Exclude tensorboard/compat because the source of truth is TensorFlow.
	exclude: ./tensorboard/compat/proto
	extensions: 'proto'
	clangFormatVersion: 9
	- run: ./tensorboard/tools/do_not_submit_test.sh
	- run: ./tensorboard/tools/license_test.sh
	- run: ./tensorboard/tools/whitespace_hygiene_test.py

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Update Protobuf Bazel Workspace #11994

Workflow file

Update Protobuf Bazel Workspace #11994

Uh oh!

Workflow file for this run