Merge branch 'master' into decouple-routing-primitives-benchmark

Author: jeffreywang88

.buildkite/data.rayci.yml

@@ -40,13 +40,6 @@ steps:
       PYTHON: "3.10"
     tags: cibase
 
-  - name: datatfxbslbuild-multipy
-    label: "wanda: datatfxbslbuild-py3.11"
-    wanda: ci/docker/datatfxbsl.build.wanda.yaml
-    env:
-      PYTHON: "3.11"
-    tags: cibase
-
   - name: datatfdsbuild-multipy
     label: "wanda: datatfdsbuild-py3.12"
     wanda: ci/docker/datatfds.build.wanda.yaml
@@ -239,16 +232,6 @@ steps:
         --only-tags dask
     depends_on: datalbuild-multipy(python=3.12)
 
-  - label: ":database: data: TFRecords (tfx-bsl) tests"
-    tags:
-      - data
-    instance_type: medium
-    commands:
-      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... data
-        --build-name datatfxbslbuild-py3.11 --python-version 3.11
-        --only-tags tfxbsl
-    depends_on: datatfxbslbuild-multipy
-
   - label: ":database: data: doc tests"
     tags:
       - data

.buildkite/serve.rayci.yml

@@ -224,13 +224,14 @@ steps:
       - python
     instance_type: large
     commands:
+      # Exit 42 skips whole-step retry (per @kouroshHakha); per-test retry is enough.
       - bazel run //ci/ray_ci:test_in_docker -- $(cat ci/ray_ci/serve_hap_test_names.txt) serve
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}"
         --build-name servebuild-py3.10 --python-version 3.10
         --test-env=RAY_SERVE_ENABLE_HA_PROXY=1
         --test-env=RAY_SERVE_ENABLE_DIRECT_INGRESS=0
         --test-env=RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S=0.01
-        --test-env=SERVE_SOCKET_REUSE_PORT_ENABLED=1
+        --test-env=SERVE_SOCKET_REUSE_PORT_ENABLED=1 || exit 42
     depends_on: servebuild-multipy(python=3.10)
 
   - label: ":ray-serve: serve: HAProxy tests (pip)"
@@ -240,14 +241,15 @@ steps:
       - python
     instance_type: large
     commands:
+      # Exit 42 skips whole-step retry (per @kouroshHakha); per-test retry is enough.
       - bazel run //ci/ray_ci:test_in_docker -- $(cat ci/ray_ci/serve_hap_test_names.txt) serve
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}"
         --build-name servebuild-py3.10 --python-version 3.10
         --test-env=RAY_SERVE_ENABLE_HA_PROXY=1
         --test-env=RAY_SERVE_EXPERIMENTAL_PIP_HAPROXY=1
         --test-env=RAY_SERVE_ENABLE_DIRECT_INGRESS=0
         --test-env=RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S=0.01
-        --test-env=SERVE_SOCKET_REUSE_PORT_ENABLED=1
+        --test-env=SERVE_SOCKET_REUSE_PORT_ENABLED=1 || exit 42
     depends_on: servebuild-multipy(python=3.10)
 
   - label: ":ray-serve: serve: direct ingress tests"

ci/docker/data-tfxbsl.build.Dockerfile

@@ -965,7 +965,7 @@ python/ray/data/_internal/datasource/tfrecords_datasource.py
     DOC001: Method `__init__` Potential formatting errors in docstring. Error message: No specification for "Args": ""
     DOC001: Function/method `__init__`: Potential formatting errors in docstring. Error message: No specification for "Args": "" (Note: DOC001 could trigger other unrelated violations under this function/method too. Please fix the docstring formatting first.)
     DOC101: Method `TFRecordDatasource.__init__`: Docstring contains fewer arguments than in function signature.
-    DOC103: Method `TFRecordDatasource.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [**file_based_datasource_kwargs: , paths: Union[str, List[str]], tf_schema: Optional['schema_pb2.Schema'], tfx_read_options: Optional['TFXReadOptions']].
+    DOC103: Method `TFRecordDatasource.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [**file_based_datasource_kwargs: , paths: Union[str, List[str]], tf_schema: Optional['schema_pb2.Schema']].
 --------------------
 python/ray/data/_internal/datasource/webdataset_datasource.py
     DOC201: Function `_valid_sample` does not have a return section in docstring

ci/docker/datatfxbsl.build.wanda.yaml

@@ -117,60 +117,3 @@ depsets:
       - --unsafe-package ray
     build_arg_sets:
       - py310
-
-  - name: relaxed_data_tfxbsl_ci_depset_${PYTHON_SHORT}
-    operation: relax
-    source_depset: data_base_ci_depset_${PYTHON_SHORT}
-    packages:
-    # these packages are relaxed for alternate fixed-pyarrow depsets
-      - datasets
-      - delta-sharing
-      - tensorflow-metadata
-      - tensorflow
-      - tf-keras
-      - pyarrow
-      - pandas
-      - protobuf
-      - hudi
-      - modin
-      - opentelemetry-proto
-      - pylance
-      - numpy
-      - cloudpickle
-      - dill
-      - dask
-      - distributed
-      - multiprocess
-      - tensorboard
-      - ml-dtypes
-      - cudf-cu12
-      - pymars
-      # tfx-bsl up through 1.17.1 caps absl-py<2.0.0, but the py3.13 lock has
-      # absl-py==2.4.0 from transitive resolution. Relax so uv picks an absl-py
-      # 1.x version that tfx-bsl accepts.
-      - absl-py
-      # grpcio-status / grpcio-tools >=1.72 require protobuf>=6, but tfx-bsl 1.16.1
-      # caps protobuf<6. Relax so uv can drop to 1.71.x (grpcio itself has no
-      # protobuf pin but typically moves in lockstep).
-      - grpcio-status
-      # contourpy 1.3.3 (gated py>=3.11 in the lock) needs numpy>=1.25, but
-      # apache-beam 2.53.0 caps numpy<1.25.
-      - contourpy
-      - scipy
-    output: python/deplocks/ci/relaxed_data_tfxbsl-ci_depset_py${PYTHON_VERSION}.lock
-    build_arg_sets:
-      - py311
-
-  - name: data_tfxbsl_depset_${PYTHON_SHORT}
-    operation: expand
-    depsets:
-      - relaxed_data_tfxbsl_ci_depset_${PYTHON_SHORT}
-    requirements:
-      - python/requirements/data/tfxbsl.txt
-    output: python/deplocks/ci/data-tfxbsl-ci_depset_py${PYTHON_VERSION}.lock
-    append_flags:
-      - --python-version=${PYTHON_VERSION}
-      - --python-platform=linux
-      - --unsafe-package ray
-    build_arg_sets:
-      - py311

ci/lint/pydoclint-baseline.txt

@@ -297,7 +297,6 @@ TFRecords
    :toctree: doc/
 
    read_tfrecords
-   TFXReadOptions
 
 TensorFlow
 ^^^^^^^^^^

ci/raydepsets/configs/ci_data.depsets.yaml

@@ -81,6 +81,18 @@ Ray exports a number of system metrics, which provide introspection into the sta
    * - `ray_node_mem_total`
      - `instance`
      - The amount of physical memory available per node, in bytes.
+   * - `ray_node_mem_used_host`
+     - `instance`
+     - The host (OS-level) physical memory used per node, in bytes.
+   * - `ray_node_mem_total_host`
+     - `instance`
+     - The host (OS-level) total physical memory per node, in bytes.
+   * - `ray_node_cgroup_mem_used`
+     - `instance`
+     - The container memory usage per node (from cgroup), in bytes. Only emitted when cgroup memory limits are present.
+   * - `ray_node_cgroup_mem_total`
+     - `instance`
+     - The container memory limit per node (from cgroup), in bytes. Only emitted when cgroup memory limits are present.
    * - `ray_component_uss_mb`
      - `Component`, `instance`
      - The measured unique set size in megabytes, broken down by logical Ray component. Ray components consist of system components (e.g., raylet, gcs, dashboard, or agent) and the method names of running tasks/actors.

doc/source/data/api/loading_data.rst

@@ -54,7 +54,6 @@ ignore-missing-imports = [
     "tensorflow_datasets.*",
     "tensorflow_metadata.*",
     "testcontainers.*",
-    "tfx_bsl.*",
     "torch.*",
     "torchvision.*",
     "tqdm.*",