From 6043492dd617baddf3b110abbb6cba013c631b22 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 7 Sep 2021 11:40:07 -0400 Subject: [PATCH 001/199] Add readme --- README.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..82e5bc3 --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ + + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +# The RAPIDS-Triton Linear Example + +This repository offers an annotated example of how to create a custom Triton +backend using the RAPIDS-Triton library. From fde4f80a0d7f19f5aff4e63710ff02530d453033 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 7 Sep 2021 11:45:01 -0400 Subject: [PATCH 002/199] Add README --- README.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..05c9934 --- /dev/null +++ b/README.md @@ -0,0 +1,31 @@ + + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +# The RAPIDS-Triton Library From b4fb8da586ee27645e8fe031bb9a7233ad8538b9 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 17 Aug 2021 16:06:49 -0400 Subject: [PATCH 003/199] Move basic parts into new repo --- include/rapids_triton/build_control.hpp | 30 +++ include/rapids_triton/exceptions.hpp | 86 ++++++ include/rapids_triton/memory/buffer.hpp | 251 ++++++++++++++++++ .../rapids_triton/memory/detail/allocate.hpp | 47 ++++ include/rapids_triton/memory/detail/copy.hpp | 79 ++++++ include/rapids_triton/memory/types.hpp | 24 ++ include/rapids_triton/model/model.hpp | 55 ++++ include/rapids_triton/tensor/dtype.hpp | 152 +++++++++++ include/rapids_triton/tensor/tensor.hpp | 106 ++++++++ include/rapids_triton/utils/narrow.hpp | 36 +++ src/impl/names.h | 19 ++ src/interface/api.cc | 161 +++++++++++ src/interface/model_state.h | 34 +++ 13 files changed, 1080 insertions(+) create mode 100644 include/rapids_triton/build_control.hpp create mode 100644 include/rapids_triton/exceptions.hpp create mode 100644 include/rapids_triton/memory/buffer.hpp create mode 100644 include/rapids_triton/memory/detail/allocate.hpp create mode 100644 include/rapids_triton/memory/detail/copy.hpp create mode 100644 include/rapids_triton/memory/types.hpp create mode 100644 include/rapids_triton/model/model.hpp create mode 100644 include/rapids_triton/tensor/dtype.hpp create mode 100644 include/rapids_triton/tensor/tensor.hpp create mode 100644 include/rapids_triton/utils/narrow.hpp create mode 100644 src/impl/names.h create mode 100644 src/interface/api.cc create mode 100644 src/interface/model_state.h diff --git a/include/rapids_triton/build_control.hpp b/include/rapids_triton/build_control.hpp new file mode 100644 index 0000000..ef1b3d1 --- /dev/null +++ b/include/rapids_triton/build_control.hpp @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { namespace backend { namespace rapids { + +#ifdef TRITON_ENABLE_GPU +auto constexpr IS_GPU_BUILD = true; +#else +auto constexpr IS_GPU_BUILD = false; +using cudaStream_t = void*; +inline void cudaStreamSynchronize(void *){} +#endif + +}}} // namespace triton::backend::rapids diff --git a/include/rapids_triton/exceptions.hpp b/include/rapids_triton/exceptions.hpp new file mode 100644 index 0000000..f3f2d88 --- /dev/null +++ b/include/rapids_triton/exceptions.hpp @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include + +namespace triton { namespace backend { namespace rapids { + +using ErrorCode = TRITONSERVER_Error_Code; + +namespace Error { + using Unknown = ErrorCode::TRITONSERVER_ERROR_UNKNOWN; + using Internal = ErrorCode::TRITONSERVER_ERROR_INTERNAL; + using NotFound = ErrorCode::TRITONSERVER_ERROR_NOT_FOUND; + using InvalidArg = ErrorCode::TRITONSERVER_ERROR_INVALID_ARG; + using Unavailable = ErrorCode::TRITONSERVER_ERROR_UNAVAILABLE; + using Unsupported = ErrorCode::TRITONSERVER_ERROR_UNSUPPORTED; + using AlreadyExists = ErrorCode::TRITONSERVER_ERROR_ALREADY_EXISTS; +} + +/** + * @brief Exception thrown if processing cannot continue for a request + * + * This exception should be thrown whenever a condition is encountered that (if + * it is not appropriately handled by some other exception handler) SHOULD + * result in Triton reporting an error for the request being processed. It + * signals that (absent any other fallbacks), this request cannot be fulfilled + * but that the server may still be in a state to continue handling other + * requests, including requests to other models. + */ +struct TritonException : std::exception { + + public: + TritonException() + : error_(TRITONSERVER_ErrorNew(Error::Unknown, + "encountered unknown error")) + { + } + + TritonException(ErrorCode code, const std::string& msg) + : error_(TRITONSERVER_ErrorNew(code, msg.c_str())) + { + } + + TritonException(ErrorCode code, const char* msg) + : error_{TRITONSERVER_ErrorNew(code, msg)} + { + } + + // Exists only for triton_check; should not be used elsewhere + TritonException(TRITONSERVER_Error* prev_error) : error_(prev_error) {} + + const char* what() const noexcept + { + return TRITONSERVER_ErrorMessage(error_); + } + + TRITONSERVER_Error* error() { return error_; } + + private: + TRITONSERVER_Error* error_; +}; + +inline void triton_check(TRITONSERVER_Error* err) { + if (err != nullptr) { + throw TritonException(err); + } +} + +}}} // namespace triton::backend::rapids + diff --git a/include/rapids_triton/memory/buffer.hpp b/include/rapids_triton/memory/buffer.hpp new file mode 100644 index 0000000..b76deb6 --- /dev/null +++ b/include/rapids_triton/memory/buffer.hpp @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace triton { namespace backend { namespace rapids { + template + struct Buffer { + using size_type = std::size_t; + using value_type = T; + + using h_ptr = T*; + using d_ptr = T*; + using owned_h_ptr = std::unique_ptr; + using owned_d_ptr = std::unique_ptr{}>; + using data_ptr = std::variant; + + Buffer() noexcept : data_{std::in_place_index<0>, nullptr}, size_{}, stream_{} {} + + /** + * @brief Construct buffer of given size in given memory location (either + * on host or on device) + * A buffer constructed in this way is owning and will release allocated + * resources on deletion + */ + Buffer(size_type size, MemoryType memory_type=DeviceMemory, cudaStream_t + stream=0) : + data_{allocate(size, memory_type)}, size_{size}, stream_{stream} {} + + /** + * @brief Construct buffer from given source in given memory location (either + * on host or on device) + * A buffer constructed in this way is non-owning; the caller is + * responsible for freeing any resources associated with the input pointer + */ + Buffer(T* input_data, size_type size, MemoryType memory_type=DeviceMemory, + cudaStream_t stream=0) : + data_{allocate(size, memory_type)}, size_{size}, stream_{stream} {} + + /** + * @brief Construct one buffer from another in the given memory location + * (either on host or on device) + * A buffer constructed in this way is owning and will copy the data from + * the original location + */ + Buffer(Buffer const& other, MemoryType memory_type) : data_([&other](){ + auto result = allocate(other.size_, memory_type); + copy(result, other.data_, other.size_, other.stream_); + return result; + }()), size_{other.size_}, stream_{other.stream_} {} + + /** + * @brief Create owning copy of existing buffer + * The memory type of this new buffer will be the same as the original + */ + Buffer(Buffer const& other) : Buffer(other, other.mem_type()) {} + + Buffer(Buffer&& other, MemoryType memory_type) : data_{[&other, memory_type](){ + data_ptr result; + if(memory_type == other.mem_type()) { + result = std::move(other.data_); + } else { + result = allocate(other.size_, memory_type); + copy(result, other.data_, other.size_, other.stream_); + } + return result; + }()}, size_{other.size_}, stream{other.stream_} {} + + Buffer(Buffer&& other) noexcept : data_{std::move{other.data_}}, size_{other.size_}, + stream_{other.stream_} {} + + ~Buffer() = default; + + /** + * @brief Return where memory for this buffer is located (host or device) + */ + auto mem_type() const noexcept { + return data_ptr.index() % 2 == 0 ? HostMemory, DeviceMemory; + } + + /** + * @brief Return number of elements in buffer + */ + auto size() const noexcept { + return size_; + } + + /** + * @brief Return pointer to data stored in buffer + */ + auto* data() noexcept { + return get_raw_ptr(data_); + } + + /** + * @brief Return CUDA stream associated with this buffer + */ + auto stream() const noexcept { + return stream_; + } + + /** + * @brief Set CUDA stream for this buffer to new value + * + * @warning This method calls cudaStreamSynchronize on the old stream + * before updating. Be aware of performance implications and try to avoid + * interactions between buffers on different streams where possible. + */ + void set_stream(cudaStream_t new_stream) { + if constexpr (IS_GPU_BUILD) { + cudaStreamSynchronize(stream_); + } + stream_ = new_stream; + } + + private: + data_ptr data_; + size_type size_; + cudaStream_t stream_; + + // Helper function for accessing raw pointer to underlying data of data_ptr + static auto get_raw_ptr(data_ptr ptr) noexcept { + /* Switch statement is an optimization relative to std::visit to avoid + * vtable overhead for a small number of alternatives */ + T* result; + switch (ptr.index()) { + case 0: + result = std::get<0>(ptr); + break; + case 1: + result = std::get<1>(ptr); + break; + case 2: + result = std::get<2>(ptr).get(); + break; + case 3: + result = std::get<3>(ptr).get(); + break; + } + return result; + } + + // Helper function for allocating memory in constructors + static auto allocate(size_type size, MemoryType memory_type=DeviceMemory) { + data_ptr result; + if (memory_type == DeviceMemory) { + if constexpr (IS_GPU_BUILD) { + result = data_ptr{owned_d_ptr{detail::dev_allocate(size)}}; + } else { + throw TritonException( + Error::Internal, + "DeviceMemory requested in CPU-only build of FIL backend" + ); + } + } else { + result = std::make_unique(size); + } + return result; + } + + // Helper function for copying memory in constructors, where there are + // stronger guarantees on conditions that would otherwise need to be + // checked + static void copy( + data_ptr dst, data_ptr src, size_type len, cudaStream_t stream) { + auto raw_dst = get_raw_ptr(dst); + auto raw_src = get_raw_ptr(src); + + auto dst_mem_type = dst.index() % 2 == 0 ? HostMemory : DeviceMemory; + auto src_mem_type = src.index() % 2 == 0 ? HostMemory : DeviceMemory; + + detail::copy(raw_dst, raw_src, len, dst_mem_type, src_mem_type); + } + + }; + + /** + * @brief Copy data from one Buffer to another + * + * @param dst The destination buffer + * @param src The source buffer + * @param dst_begin The offset from the beginning of the destination buffer + * at which to begin copying to. + * @param src_begin The offset from the beginning of the source buffer + * at which to begin copying from. + * @param src_end The offset from the beginning of the source buffer + * before which to end copying from. + * + * @warning This method is NOT thread-safe. If the stream of the src buffer + * changes while a copy is in progress, dst may receive incorrect data from + * src. Avoid interactions between buffers on different streams, *especially* + * when those buffers may be modified on different host threads as well. + */ + template + copy(Buffer dst&&, Buffer src&&, Buffer::size_type dst_begin, + Buffer::size_type src_begin, Buffer::size_type src_end) { + if(dst.stream() != src.stream()) { + dst.set_stream(src.stream()); + } + auto len = src_end - src_begin; + if (len < 0 || src_end > src.size() || len > dst.size() - dst_begin) { + throw TritonException(Error::Internal, "bad copy between buffers"); + } + + auto raw_dst = dst.data() + dst_begin; + auto raw_src = src.data() + src_begin; + + detail::copy(raw_dst, raw_src, len, dst.stream(), dst.mem_type(), + src.mem_type()); + } + + template + void copy(Buffer dst&&, Buffer src&&) { + copy(dst, src, dst.begin(), src.begin(), src.begin() + src.size()); + } + + template + void copy(Buffer dst&&, Buffer src&&, Buffer::size_type dst_begin) { + copy(dst, src, dst_begin, src.begin(), src.begin() + src.size()); + } + + template + void copy(Buffer dst&&, Buffer src&&, Buffer::size_type src_begin, + Buffer::size_type src_end) { + copy(dst, src, dst_begin, src.begin(), src.begin() + src_end); + } +}}} // namespace triton::backend::rapids diff --git a/include/rapids_triton/memory/detail/allocate.hpp b/include/rapids_triton/memory/detail/allocate.hpp new file mode 100644 index 0000000..5e83266 --- /dev/null +++ b/include/rapids_triton/memory/detail/allocate.hpp @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +#include + +#include +#include +#include + +namespace triton { namespace backend { namespace rapids { namespace detail { +template +struct dev_deallocater { + void operator(T* d_ptr) { + cudaFree(reinterpret_cast(d_ptr)); + } +} + +/** + * @brief Allocate given number of elements on GPU and return device pointer + */ +template +T* +dev_allocate(std::size_t count, cudaStream_t stream) +{ + auto* ptr_d = + static_cast(rmm::mr::get_current_device_resource()->allocate( + sizeof(T) * count, stream)); + return ptr_d; +} + +}}}} // namespace triton::backend::rapids::detail diff --git a/include/rapids_triton/memory/detail/copy.hpp b/include/rapids_triton/memory/detail/copy.hpp new file mode 100644 index 0000000..9298e57 --- /dev/null +++ b/include/rapids_triton/memory/detail/copy.hpp @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +#include + +#include +#include + + +namespace triton { namespace backend { namespace rapids { namespace detail { + +/** + * @brief Copy given number of elements from one place to another, with either + * source or destination on device + */ +template +void +dev_copy(T* dst, T const* src, std::size_t len, cudaStream_t stream) +{ + if constexpr (IS_GPU_BUILD) { + try { + raft::copy(raw_dst, raw_src, len, stream); + } catch (const raft::cuda_error& err) { + throw TritonException(Error::Internal, err.what()); + } + } else { + throw TritonException( + Error::Internal, + "copy to or from device memory cannot be used in CPU-only builds" + ); + } +} + +/** + * @brief Copy given number of elements from one place to another, with either + * source or destination on device + */ +template +void +host_copy(T* dst, T const* src, std::size_t len) +{ + std::memcpy(dst, src, len * sizeof(T)); +} + +template +void +copy(T* dst, T const* src, std::size_t len, cudaStream_t stream, MemoryType + dst_type, MemoryType src_type) { + if (dst_type == DeviceMemory || src_type == DeviceMemory) { + if constexpr (IS_GPU_BUILD) { + dev_copy(dst, src, len, stream); + } else { + throw TritonException( + Error::Internal, + "DeviceMemory copy cannot be used in CPU-only builds" + ); + } + } else { + host_copy(dst, src, len); + } +} + +}}}} // namespace triton::backend::rapids::detail diff --git a/include/rapids_triton/memory/types.hpp b/include/rapids_triton/memory/types.hpp new file mode 100644 index 0000000..38d5b86 --- /dev/null +++ b/include/rapids_triton/memory/types.hpp @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { namespace backend { namespace rapids { + using MemoryType = TRITONSERVER_MemoryType; + using DeviceMemory = TRITONSERVER_MEMORY_GPU; + using HostMemory = TRITONSERVER_MEMORY_CPU; +}}} // namespace triton::backend::rapids diff --git a/include/rapids_triton/model/model.hpp b/include/rapids_triton/model/model.hpp new file mode 100644 index 0000000..9493bf1 --- /dev/null +++ b/include/rapids_triton/model/model.hpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include + +namespace triton { namespace backend { namespace rapids { + /** + * @brief Stores shared state for multiple instances of the same model + */ + struct SharedModelState { + virtual void load() { + } + }; + + template + struct Model { + + virtual void predict(std::vector>&& inputs, std::vector>&& outputs); + + void predict() { + predict(get_inputs(), get_outputs()); + } + + private: + std::shared_ptr shared_state; + + auto get_inputs() { + // TODO(wphicks) + return std::vector>(); + } + + auto get_outputs() { + // TODO(wphicks) + return std::vector>(); + } + }; +}}} // namespace triton::backend::rapids diff --git a/include/rapids_triton/tensor/dtype.hpp b/include/rapids_triton/tensor/dtype.hpp new file mode 100644 index 0000000..0a05f96 --- /dev/null +++ b/include/rapids_triton/tensor/dtype.hpp @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +using DType = TRITONSERVER_DataType; +using DTypeBool = TRITONSERVER_TYPE_BOOL; +using DTypeUint8 = TRITONSERVER_TYPE_UINT8; +using DTypeUint16 = TRITONSERVER_TYPE_UINT16; +using DTypeUint32 = TRITONSERVER_TYPE_UINT32; +using DTypeUint64 = TRITONSERVER_TYPE_UINT64; +using DTypeInt8 = TRITONSERVER_TYPE_INT8; +using DTypeInt16 = TRITONSERVER_TYPE_INT16; +using DTypeInt32 = TRITONSERVER_TYPE_INT32; +using DTypeInt64 = TRITONSERVER_TYPE_INT64; +using DTypeFloat32 = TRITONSERVER_TYPE_FP32; +using DTypeFloat64 = TRITONSERVER_TYPE_FP64; + + +template +struct TritonType { +}; + +template +struct TritonDtype { +}; + +template <> +struct TritonType { + typedef bool type; +}; + +template <> +struct TritonDtype { + static constexpr DType value = DTypeBool; +}; + +template <> +struct TritonType { + typedef uint8_t type; +}; + +template <> +struct TritonDtype { + static constexpr DType value = DTypeUint8; +}; + +template <> +struct TritonType { + typedef uint16_t type; +}; + +template <> +struct TritonDtype { + static constexpr DType value = DTypeUint16; +}; + +template <> +struct TritonType { + typedef uint32_t type; +}; + +template <> +struct TritonDtype { + static constexpr DType value = DTypeUint32; +}; + +template <> +struct TritonType { + typedef uint64_t type; +}; + +template <> +struct TritonDtype { + static constexpr DType value = DTypeUint64; +}; + +template <> +struct TritonType { + typedef int8_t type; +}; + +template <> +struct TritonDtype { + static constexpr DType value = DTypeInt8; +}; + +template <> +struct TritonType { + typedef int16_t type; +}; + +template <> +struct TritonDtype { + static constexpr DType value = DTypeInt16; +}; + +template <> +struct TritonType { + typedef int32_t type; +}; + +template <> +struct TritonDtype { + static constexpr DType value = DTypeInt32; +}; + +template <> +struct TritonType { + typedef int64_t type; +}; + +template <> +struct TritonDtype { + static constexpr DType value = DTypeInt64; +}; + +template <> +struct TritonType { + typedef float type; +}; + +template <> +struct TritonDtype { + static constexpr DType value = DTypeFloat32; +}; + +template <> +struct TritonType { + typedef double type; +}; + +template <> +struct TritonDtype { + static constexpr DType value = DTypeFloat64; +}; + +// TODO(whicks): Correctly handle const versions of types diff --git a/include/rapids_triton/tensor/tensor.hpp b/include/rapids_triton/tensor/tensor.hpp new file mode 100644 index 0000000..f9edb5f --- /dev/null +++ b/include/rapids_triton/tensor/tensor.hpp @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include + +#include + +#include +#include + +namespace triton { namespace backend { namespace rapids { + template + struct Tensor { + using size_type = typename Buffer::size_type; + + Tensor() : shape_{}, buffer_{} {} + + /** + * @brief Construct a Tensor from a collection of buffers + * + * Given a collection of buffers, collate them all into one buffer stored in + * a new Tensor + */ + template + Tensor(std::vector const& shape, Iter begin, Iter end, MemoryType mem_type, cudaStream_t stream=0) : + shape_(shape), + buffer_([&begin, &end, &mem_type, &stream] () { + auto total_size = std::transform_reduce( + begin, end, std::plus<>{}, [](auto&& buffer) { return buffer.size(); } + ); + + auto result = Buffer(total_size, mem_type, stream); + + std::reduce(begin, end, size_type{0}, [&result](auto&& buffer, auto offset) { + copy(result, buffer, offset); + return offset + buffer.size(); + }); + return result; + }()) {} + + auto const& shape() const { return shape_; } + auto data() { return buffer_.data(); } + auto& buffer() { return buffer_; } + + auto constexpr dtype() const { return TritonDtype::value; } + auto mem_type() const { return data.mem_type(); } + auto stream() const { return data.stream(); } + + private: + std::vector shape_; + Buffer buffer_; + }; + + template + void copy(Tensor> dst, Tensor src) { + copy(dst.buffer(), src.buffer()); + } + + /** + * @brief Copy data from src Tensor into buffers indicated by iterators + * + * This method is provided to assist with distributing data from a single + * Tensor into many smaller buffers which have been set up to receive a part + * of the data from the src Tensor + */ + template + copy(Iter begin, Iter end, Tensor src) { + std::reduce( + begin, + end, + decltype(*begin)::size_type{0}, + [&src] (auto&& buffer, auto offset) { + auto end_offset = offset + buffer.size(); + copy(buffer, src.buffer(), offset, end_offset); + return end_offset; + } + } + + /** + * @brief A simple struct containing only a string used to name a Tensor and + * the Tensor itself + */ + template + struct NamedTensor { + std::string name; + Tensor tensor; + }; +}}} // namespace triton::backend::rapids diff --git a/include/rapids_triton/utils/narrow.hpp b/include/rapids_triton/utils/narrow.hpp new file mode 100644 index 0000000..5e7a590 --- /dev/null +++ b/include/rapids_triton/utils/narrow.hpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include + +namespace triton { namespace backend { namespace fil { + +template +auto +narrow(F from) +{ + auto to = static_cast(from); + if (static_cast(to) != from || + (std::is_signed::value == std::is_signed::value && + ((to < T{}) != (from < F{})))) { + throw TritonException(Error::Internal, "invalid narrowing"); + } + return to; +} + +}}} // namespace triton::backend::fil diff --git a/src/impl/names.h b/src/impl/names.h new file mode 100644 index 0000000..215430d --- /dev/null +++ b/src/impl/names.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#define NAMESPACE rapids_identity diff --git a/src/interface/api.cc b/src/interface/api.cc new file mode 100644 index 0000000..5b51d9e --- /dev/null +++ b/src/interface/api.cc @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "triton/backend/backend_common.h" +#include "triton/backend/backend_model.h" +#include "triton/backend/backend_model_instance.h" + +namespace triton { namespace backend { namespace rapids_identity { + +extern "C" { + +TRITONSERVER_Error* +TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) +{ + // TODO (wphicks): Move helpers + try { + std::string name = get_backend_name(*backend); + + log_info( + __FILE__, __LINE__, + (std::string("TRITONBACKEND_Initialize: ") + name).c_str()); + + if (!check_backend_version(*backend)) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_UNSUPPORTED, + "triton backend API version does not support this backend"); + } + } + catch (TritonException& err) { + return err.error(); + } + return nullptr; // success +} + +TRITONSERVER_Error* +TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) +{ + try { + std::string name = get_model_name(*model); + + auto version = get_model_version(*model); + + log_info( + __FILE__, __LINE__, + (std::string("TRITONBACKEND_ModelInitialize: ") + name + " (version " + + std::to_string(version) + ")") + .c_str()); + + // TODO (wphicks): Replace + set_model_state(*model, ModelState::Create(*model)); + } + catch (TritonException& err) { + return err.error(); + } + + return nullptr; // success +} + +TRITONSERVER_Error* +TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) +{ + try { + auto model_state = get_model_state(*model); + if (model_state != nullptr) { + // TODO (wphicks): Replace + model_state->UnloadModel(); + } + + log_info( + __FILE__, __LINE__, "TRITONBACKEND_ModelFinalize: delete model state"); + + // TODO (wphicks) Necessary? + delete model_state; + } + catch (TritonException& err) { + return err.error(); + } + + return nullptr; // success +} + +TRITONSERVER_Error* +TRITONBACKEND_ModelInstanceInitialize(TRITONBACKEND_ModelInstance* instance) +{ + // TODO (wphicks): Replace + try { + std::string name = get_model_instance_name(*instance); + int32_t device_id = get_device_id(*instance); + TRITONSERVER_InstanceGroupKind kind = get_instance_kind(*instance); + + log_info( + __FILE__, __LINE__, + (std::string("TRITONBACKEND_ModelInstanceInitialize: ") + name + " (" + + TRITONSERVER_InstanceGroupKindString(kind) + " device " + + std::to_string(device_id) + ")") + .c_str()); + + ModelState* model_state = get_model_state(*instance); + + // WH + set_instance_state( + *instance, ModelInstanceState::Create(model_state, instance)); + } + catch (TritonException& err) { + return err.error(); + } + return nullptr; // success +} + +TRITONSERVER_Error* +TRITONBACKEND_ModelInstanceFinalize(TRITONBACKEND_ModelInstance* instance) +{ + try { + void* vstate; + triton_check(TRITONBACKEND_ModelInstanceState(instance, &vstate)); + ModelInstanceState* instance_state = + reinterpret_cast(vstate); + + if (instance_state != nullptr) { + // WH + instance_state->UnloadFILModel(); + + log_info( + __FILE__, __LINE__, + "TRITONBACKEND_ModelInstanceFinalize: delete instance state"); + + delete instance_state; + } + } + catch (TritonException& err) { + return err.error(); + } + + return nullptr; // success +} + +TRITONSERVER_Error* +TRITONBACKEND_ModelInstanceExecute( + TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** raw_requests, + const uint32_t request_count) +{ + // TODO (wphicks) + // model.predict(); +} + +} // extern "C" + +}}} // namespace triton::backend::rapids_identity diff --git a/src/interface/model_state.h b/src/interface/model_state.h new file mode 100644 index 0000000..7652980 --- /dev/null +++ b/src/interface/model_state.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +namespace triton { namespace backend { namespace NAMESPACE { +struct ModelState : public BackendModel { + static std::unique_ptr Create(TRITONBACKEND_Model& triton_model); + + ModelState( + TRITONBACKEND_Model* triton_model, const char* name, + const uint64_t version); + + private: + RapidsModel model; +}; + +}}} // namespace triton::backend::NAMESPACE From 124b1929199a689db4cb3c723d61dea26ce2da51 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 17 Aug 2021 16:12:10 -0400 Subject: [PATCH 004/199] Add license and readme --- LICENSE | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 5 ++ 2 files changed, 206 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b360c42 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2021 NVIDIA CORPORATION + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 05c9934..82c36c3 100644 --- a/README.md +++ b/README.md @@ -29,3 +29,8 @@ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) # The RAPIDS-Triton Library + +This project is designed to make it easy to integrate any C++-based algorithm +into the NVIDIA Triton Inference Server. Originally developed to assist with +the integration of RAPIDS algorithms, this library can be used by anyone to +quickly get up and running with a custom backend for Triton. From f0e1691a4dde2d9bb301a77b4e214e44c0d193be Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 17 Aug 2021 19:24:13 -0400 Subject: [PATCH 005/199] Add Triton ModelState interface --- include/rapids_triton/build_control.hpp | 2 - include/rapids_triton/exceptions.hpp | 9 +-- include/rapids_triton/model/model.hpp | 18 +---- include/rapids_triton/triton/backend.hpp | 74 +++++++++++++++++++ include/rapids_triton/triton/logging.hpp | 94 ++++++++++++++++++++++++ include/rapids_triton/triton/model.hpp | 93 +++++++++++++++++++++++ src/interface/api.cc | 44 ++++++----- src/interface/model_state.h | 4 +- 8 files changed, 297 insertions(+), 41 deletions(-) create mode 100644 include/rapids_triton/triton/backend.hpp create mode 100644 include/rapids_triton/triton/logging.hpp create mode 100644 include/rapids_triton/triton/model.hpp diff --git a/include/rapids_triton/build_control.hpp b/include/rapids_triton/build_control.hpp index ef1b3d1..e05dc12 100644 --- a/include/rapids_triton/build_control.hpp +++ b/include/rapids_triton/build_control.hpp @@ -23,8 +23,6 @@ namespace triton { namespace backend { namespace rapids { auto constexpr IS_GPU_BUILD = true; #else auto constexpr IS_GPU_BUILD = false; -using cudaStream_t = void*; -inline void cudaStreamSynchronize(void *){} #endif }}} // namespace triton::backend::rapids diff --git a/include/rapids_triton/exceptions.hpp b/include/rapids_triton/exceptions.hpp index f3f2d88..8db8198 100644 --- a/include/rapids_triton/exceptions.hpp +++ b/include/rapids_triton/exceptions.hpp @@ -52,25 +52,24 @@ struct TritonException : std::exception { { } - TritonException(ErrorCode code, const std::string& msg) + TritonException(ErrorCode code, std::string const & msg) : error_(TRITONSERVER_ErrorNew(code, msg.c_str())) { } - TritonException(ErrorCode code, const char* msg) + TritonException(ErrorCode code, char const* msg) : error_{TRITONSERVER_ErrorNew(code, msg)} { } - // Exists only for triton_check; should not be used elsewhere TritonException(TRITONSERVER_Error* prev_error) : error_(prev_error) {} - const char* what() const noexcept + auto* what() const noexcept { return TRITONSERVER_ErrorMessage(error_); } - TRITONSERVER_Error* error() { return error_; } + auto* error() { return error_; } private: TRITONSERVER_Error* error_; diff --git a/include/rapids_triton/model/model.hpp b/include/rapids_triton/model/model.hpp index 9493bf1..143517a 100644 --- a/include/rapids_triton/model/model.hpp +++ b/include/rapids_triton/model/model.hpp @@ -28,28 +28,16 @@ namespace triton { namespace backend { namespace rapids { struct SharedModelState { virtual void load() { } + virtual void unload() { + } }; template struct Model { - virtual void predict(std::vector>&& inputs, std::vector>&& outputs); - - void predict() { - predict(get_inputs(), get_outputs()); - } + virtual void predict(); private: std::shared_ptr shared_state; - - auto get_inputs() { - // TODO(wphicks) - return std::vector>(); - } - - auto get_outputs() { - // TODO(wphicks) - return std::vector>(); - } }; }}} // namespace triton::backend::rapids diff --git a/include/rapids_triton/triton/backend.hpp b/include/rapids_triton/triton/backend.hpp new file mode 100644 index 0000000..0adcb13 --- /dev/null +++ b/include/rapids_triton/triton/backend.hpp @@ -0,0 +1,74 @@ +// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include +#include +#include + +namespace triton { namespace backend { namespace rapids { + inline auto + get_backend_name(TRITONBACKEND_Backend& backend) + { + const char* cname; + triton_check(TRITONBACKEND_BackendName(&backend, &cname)); + return std::string(cname); + } + + namespace { + struct backend_version { + std::uint32_t major; + std::uint32_t minor; + }; + } + + inline auto + check_backend_version(TRITONBACKEND_Backend& backend) + { + auto version = backend_version{}; + triton_check(TRITONBACKEND_ApiVersion(&version.major, &version.minor)); + + log_info( + (std::string("Triton TRITONBACKEND API version: ") + + std::to_string(version.major) + "." + std::to_string(version.minor)) + .c_str()); + + auto name = get_backend_name(backend); + + log_info( + (std::string("'") + name + "' TRITONBACKEND API version: " + + std::to_string(TRITONBACKEND_API_VERSION_MAJOR) + "." + + std::to_string(TRITONBACKEND_API_VERSION_MINOR)) + .c_str()); + + return ( + (version.major == TRITONBACKEND_API_VERSION_MAJOR) && + (version.minor >= TRITONBACKEND_API_VERSION_MINOR)); + } +}}} // namespace triton::backend::rapids diff --git a/include/rapids_triton/triton/logging.hpp b/include/rapids_triton/triton/logging.hpp new file mode 100644 index 0000000..dbc7c26 --- /dev/null +++ b/include/rapids_triton/triton/logging.hpp @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include + +#include + +namespace triton { namespace backend { namespace rapids { + + namespace { + /** Log message at indicated level */ + inline void log( + TRITONSERVER_LogLevel level, const char* filename, const int line, + const char* message) { + triton_check(TRITONSERVER_LogMessage(level, filename, line, message)); + } + } + + + /** Log message at INFO level */ + inline void log_info(const char* filename, const int line, const char* message) { + log(TRITONSERVER_LOG_INFO, filename, line, message); + } + inline void log_info(const char* filename, const int line, std::string const& message) { + log_info(filename, line, message.c_str()); + } + inline void log_info(const char* message) { + log_info(__FILE__, __LINE__, message); + } + inline void log_info(std::string const& message) { + log_info(__FILE__, __LINE__, message.c_str()); + } + + + /** Log message at WARN level */ + inline void log_warn(const char* filename, const int line, const char* message) { + log(TRITONSERVER_LOG_WARN, filename, line, message); + } + inline void log_warn(const char* filename, const int line, std::string const& message) { + log_warn(filename, line, message.c_str()); + } + inline void log_warn(const char* message) { + log_warn(__FILE__, __LINE__, message); + } + inline void log_warn(std::string const& message) { + log_warn(__FILE__, __LINE__, message.c_str()); + } + + + /** Log message at ERROR level */ + inline void log_error(const char* filename, const int line, const char* message) { + log(TRITONSERVER_LOG_ERROR, filename, line, message); + } + inline void log_error(const char* filename, const int line, std::string const& message) { + log_error(filename, line, message.c_str()); + } + inline void log_error(const char* message) { + log_error(__FILE__, __LINE__, message); + } + inline void log_error(std::string const& message) { + log_error(__FILE__, __LINE__, message.c_str()); + } + + + /** Log message at VERBOSE level */ + inline void log_debug(const char* filename, const int line, const char* message) { + log(TRITONSERVER_LOG_VERBOSE, filename, line, message); + } + inline void log_debug(const char* filename, const int line, std::string const& message) { + log_debug(filename, line, message.c_str()); + } + inline void log_debug(const char* message) { + log_debug(__FILE__, __LINE__, message); + } + inline void log_debug(std::string const& message) { + log_debug(__FILE__, __LINE__, message.c_str()); + } + +}}} // namespace triton::backend::rapids diff --git a/include/rapids_triton/triton/model.hpp b/include/rapids_triton/triton/model.hpp new file mode 100644 index 0000000..b4fbc34 --- /dev/null +++ b/include/rapids_triton/triton/model.hpp @@ -0,0 +1,93 @@ +// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once +#include +#include +#include +#include + +namespace triton { namespace backend { namespace rapids { + +inline auto +get_model_version(TRITONBACKEND_Model& model) +{ + auto version = std::uint64_t{}; + triton_check(TRITONBACKEND_ModelVersion(&model, &version)); + return version; +} + +inline auto +get_model_name(TRITONBACKEND_Model& model) +{ + auto* cname = static_cast(nullptr); + triton_check(TRITONBACKEND_ModelName(&model, &cname)); + return std::string(cname); +} + +inline +auto +get_model_config(TRITONBACKEND_Model& model) +{ + TRITONSERVER_Message* config_message = nullptr; + triton_check(TRITONBACKEND_ModelConfig(&model, 1, &config_message)); + + auto* buffer = static_cast(nullptr); + auto byte_size = std::size_t{}; + triton_check( + TRITONSERVER_MessageSerializeToJson(config_message, &buffer, &byte_size)); + + auto model_config = std::make_unique(); + auto* err = model_config->Parse(buffer, byte_size); + auto* result = TRITONSERVER_MessageDelete(config_message); + if (err != nullptr) { + throw(TritonException(err)); + } + if (result != nullptr) { + throw(TritonException(result)); + } + return model_config; +} + +/** + * @brief Set model state (as used by Triton) to given object + * + * This function accepts a unique_ptr to an object derived from a Triton + * BackendModel object and sets it as the stored state for a model in the + * Triton server. Note that this object is not the same as a RAPIDS-Triton + * "SharedModelState" object. The object that Triton expects must wrap this + * SharedModelState and provide additional interface compatibility. + */ +template +void +set_model_state( + TRITONBACKEND_Model& model, std::unique_ptr&& model_state) +{ + triton_check(TRITONBACKEND_ModelSetState( + &model, reinterpret_cast(model_state.release()))); +} + +}}} // namespace triton::backend::rapids diff --git a/src/interface/api.cc b/src/interface/api.cc index 5b51d9e..6821183 100644 --- a/src/interface/api.cc +++ b/src/interface/api.cc @@ -14,29 +14,35 @@ * limitations under the License. */ -#include "triton/backend/backend_common.h" -#include "triton/backend/backend_model.h" -#include "triton/backend/backend_model_instance.h" +#include +#include +#include +#include +#include +#include -namespace triton { namespace backend { namespace rapids_identity { +namespace triton { +namespace backend { +namespace NAMESPACE { extern "C" { +/** Confirm that backend is compatible with Triton's backend API version + */ TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { - // TODO (wphicks): Move helpers try { - std::string name = get_backend_name(*backend); + auto name = rapids::get_backend_name(*backend); - log_info( + rapids::log_info( __FILE__, __LINE__, (std::string("TRITONBACKEND_Initialize: ") + name).c_str()); - if (!check_backend_version(*backend)) { - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_UNSUPPORTED, - "triton backend API version does not support this backend"); + if (!rapids::check_backend_version(*backend)) { + throw rapids::TritonException{ + rapids::Error::Unsupported, + "triton backend API version does not support this backend"}; } } catch (TritonException& err) { @@ -49,7 +55,7 @@ TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { try { - std::string name = get_model_name(*model); + auto name = get_model_name(*model); auto version = get_model_version(*model); @@ -59,7 +65,6 @@ TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) std::to_string(version) + ")") .c_str()); - // TODO (wphicks): Replace set_model_state(*model, ModelState::Create(*model)); } catch (TritonException& err) { @@ -75,14 +80,12 @@ TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) try { auto model_state = get_model_state(*model); if (model_state != nullptr) { - // TODO (wphicks): Replace - model_state->UnloadModel(); + model_state->get_shared_state()->unload(); } log_info( __FILE__, __LINE__, "TRITONBACKEND_ModelFinalize: delete model state"); - // TODO (wphicks) Necessary? delete model_state; } catch (TritonException& err) { @@ -153,9 +156,14 @@ TRITONBACKEND_ModelInstanceExecute( const uint32_t request_count) { // TODO (wphicks) - // model.predict(); + // auto model = ...; + // auto batch_input = ...; + // auto batch_output = ...; + // model.predict(batch_input, batch_output); } } // extern "C" -}}} // namespace triton::backend::rapids_identity +} // namespace NAMESPACE +} // namespace backend +} // namespace triton diff --git a/src/interface/model_state.h b/src/interface/model_state.h index 7652980..92e4c38 100644 --- a/src/interface/model_state.h +++ b/src/interface/model_state.h @@ -27,8 +27,10 @@ struct ModelState : public BackendModel { TRITONBACKEND_Model* triton_model, const char* name, const uint64_t version); + auto get_shared_state() { return state_; } + private: - RapidsModel model; + std::shared_ptr state_; }; }}} // namespace triton::backend::NAMESPACE From c20fa757ba539a5979c45e8fe194b58488614160 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 18 Aug 2021 12:53:13 -0400 Subject: [PATCH 006/199] Provide API interface for everything except execution --- include/rapids_triton/model/model.hpp | 7 +- include/rapids_triton/tensor/dtype.hpp | 6 + include/rapids_triton/triton/deployment.hpp | 27 ++++ include/rapids_triton/triton/model.hpp | 13 ++ .../rapids_triton/triton/model_instance.hpp | 108 ++++++++++++++ src/interface/api.cc | 132 ++++++++---------- src/interface/model_instance_state.h | 43 ++++++ 7 files changed, 262 insertions(+), 74 deletions(-) create mode 100644 include/rapids_triton/triton/deployment.hpp create mode 100644 include/rapids_triton/triton/model_instance.hpp create mode 100644 src/interface/model_instance_state.h diff --git a/include/rapids_triton/model/model.hpp b/include/rapids_triton/model/model.hpp index 143517a..7fa1862 100644 --- a/include/rapids_triton/model/model.hpp +++ b/include/rapids_triton/model/model.hpp @@ -32,10 +32,13 @@ namespace triton { namespace backend { namespace rapids { } }; - template + template struct Model { - virtual void predict(); + /* virtual void predict(Batch batch) { + * Fetch tensors in order and feed them to predict overload + * }; + */ private: std::shared_ptr shared_state; diff --git a/include/rapids_triton/tensor/dtype.hpp b/include/rapids_triton/tensor/dtype.hpp index 0a05f96..06ec163 100644 --- a/include/rapids_triton/tensor/dtype.hpp +++ b/include/rapids_triton/tensor/dtype.hpp @@ -17,9 +17,13 @@ #pragma once #include +namespace triton { namespace backend { namespace rapids { + using DType = TRITONSERVER_DataType; using DTypeBool = TRITONSERVER_TYPE_BOOL; using DTypeUint8 = TRITONSERVER_TYPE_UINT8; +using DTypeChar = DTypeUint8; +using DTypeByte = DTypeUint8; using DTypeUint16 = TRITONSERVER_TYPE_UINT16; using DTypeUint32 = TRITONSERVER_TYPE_UINT32; using DTypeUint64 = TRITONSERVER_TYPE_UINT64; @@ -149,4 +153,6 @@ struct TritonDtype { static constexpr DType value = DTypeFloat64; }; +}}} + // TODO(whicks): Correctly handle const versions of types diff --git a/include/rapids_triton/triton/deployment.hpp b/include/rapids_triton/triton/deployment.hpp new file mode 100644 index 0000000..5ae3776 --- /dev/null +++ b/include/rapids_triton/triton/deployment.hpp @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { namespace backend { namespace rapids { + using DeploymentType = TRITONSERVER_InstanceGroupKind; + using GPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_GPU; + using CPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_CPU; + // Note (wphicks): We currently are not including "Auto" or "Model" because I + // am not sure exactly how those would be used in context. If there is a + // demand, they can be added. +}}} diff --git a/include/rapids_triton/triton/model.hpp b/include/rapids_triton/triton/model.hpp index b4fbc34..a019335 100644 --- a/include/rapids_triton/triton/model.hpp +++ b/include/rapids_triton/triton/model.hpp @@ -90,4 +90,17 @@ set_model_state( &model, reinterpret_cast(model_state.release()))); } +/** Given a model, return its associated ModelState object */ +template +auto* +get_model_state(TRITONBACKEND_Model& model) +{ + auto vstate = static_cast(nullptr); + triton_check(TRITONBACKEND_ModelState(&model, &vstate)); + + auto* model_state = reinterpret_cast(vstate); + + return model_state; +} + }}} // namespace triton::backend::rapids diff --git a/include/rapids_triton/triton/model_instance.hpp b/include/rapids_triton/triton/model_instance.hpp new file mode 100644 index 0000000..61f0fbc --- /dev/null +++ b/include/rapids_triton/triton/model_instance.hpp @@ -0,0 +1,108 @@ +// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once +#include +#include +#include + +namespace triton { namespace backend { namespace rapids { + /** Get the name of a Triton model instance from the instance itself */ + inline auto + get_model_instance_name(TRITONBACKEND_ModelInstance& instance) + { + auto cname = static_cast(nullptr); + triton_check(TRITONBACKEND_ModelInstanceName(&instance, &cname)); + return std::string(cname); + } + + /** Get the device on which a Triton model instance is loaded + * + * If this instance is loaded on the host, 0 will be returned. Otherwise the + * GPU device id will be returned.*/ + inline auto + inline auto + get_device_id(TRITONBACKEND_ModelInstance& instance) + { + auto device_id = std::int32_t{}; + triton_check(TRITONBACKEND_ModelInstanceDeviceId(&instance, &device_id)); + return device_id; + } + + /** Determine how a Triton model instance is deployed + * + * Returns enum value indicating whether the instance is deployed on device + * or on the host + */ + inline auto + get_deployment_type(TRITONBACKEND_ModelInstance& instance) + { + auto kind = GPUDeployment; + triton_check(TRITONBACKEND_ModelInstanceKind(&instance, &kind)); + return kind; + } + + /** Return the Triton model from one of its instances + */ + inline auto* + get_model_from_instance(TRITONBACKEND_ModelInstance& instance) + { + auto* model = static_cast(nullptr); + triton_check(TRITONBACKEND_ModelInstanceModel(&instance, &model)); + return model; + } + + /** + * @brief Set Triton model instance state to given object + * + * This function accepts a unique_ptr to an object derived from a Triton + * BackendModelInstance object and sets it as the stored state for a model in the + * Triton server. Note that this object is not the same as a RAPIDS-Triton + * "Model" object. The object that Triton expects must wrap this Model and + * provide additional interface compatibility. + */ + template + void + set_instance_state( + TRITONBACKEND_ModelInstance& instance, + std::unique_ptr&& model_instance_state) + { + triton_check(TRITONBACKEND_ModelInstanceSetState( + &instance, reinterpret_cast(model_instance_state.release()))); + } + + /** Get model instance state from instance */ + template + auto* + get_instance_state(TRITONBACKEND_ModelInstance& instance) + { + auto instance_state = static_cast(nullptr); + triton_check(TRITONBACKEND_ModelInstanceState( + &instance, reinterpret_cast(&instance_state))); + return instance_state; + } + +}}} // namespace triton::backend::rapids diff --git a/src/interface/api.cc b/src/interface/api.cc index 6821183..fb2ae6c 100644 --- a/src/interface/api.cc +++ b/src/interface/api.cc @@ -15,9 +15,11 @@ */ #include +#include #include #include #include + #include #include @@ -29,12 +31,12 @@ extern "C" { /** Confirm that backend is compatible with Triton's backend API version */ -TRITONSERVER_Error* -TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) -{ +auto* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { + auto* result = static_cast(nullptr); try { auto name = rapids::get_backend_name(*backend); + // TODO (wphicks) rapids::log_info( __FILE__, __LINE__, (std::string("TRITONBACKEND_Initialize: ") + name).c_str()); @@ -44,117 +46,103 @@ TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) rapids::Error::Unsupported, "triton backend API version does not support this backend"}; } + } catch (rapids::TritonException& err) { + result = err.error(); } - catch (TritonException& err) { - return err.error(); - } - return nullptr; // success + return result; } -TRITONSERVER_Error* -TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) -{ +auto* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { + auto* result = static_cast(nullptr); try { - auto name = get_model_name(*model); + auto name = rapids::get_model_name(*model); - auto version = get_model_version(*model); + auto version = rapids::get_model_version(*model); - log_info( - __FILE__, __LINE__, - (std::string("TRITONBACKEND_ModelInitialize: ") + name + " (version " + - std::to_string(version) + ")") - .c_str()); + // TODO (wphicks) + rapids::log_info(__FILE__, __LINE__, + (std::string("TRITONBACKEND_ModelInitialize: ") + name + + " (version " + std::to_string(version) + ")") + .c_str()); - set_model_state(*model, ModelState::Create(*model)); - } - catch (TritonException& err) { - return err.error(); + rapids::set_model_state(*model, ModelState::Create(*model)); + } catch (rapids::TritonException& err) { + result = err.error(); } - return nullptr; // success + return result; } -TRITONSERVER_Error* -TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) -{ +auto* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { + auto* result = static_cast(nullptr); try { - auto model_state = get_model_state(*model); + auto model_state = rapids::get_model_state(*model); if (model_state != nullptr) { model_state->get_shared_state()->unload(); } - log_info( - __FILE__, __LINE__, "TRITONBACKEND_ModelFinalize: delete model state"); + rapids::log_info(__FILE__, __LINE__, + "TRITONBACKEND_ModelFinalize: delete model state"); delete model_state; - } - catch (TritonException& err) { - return err.error(); + } catch (rapids::TritonException& err) { + result = err.error(); } - return nullptr; // success + return result; } -TRITONSERVER_Error* -TRITONBACKEND_ModelInstanceInitialize(TRITONBACKEND_ModelInstance* instance) -{ - // TODO (wphicks): Replace +auto* TRITONBACKEND_ModelInstanceInitialize( + TRITONBACKEND_ModelInstance* instance) { + auto* result = static_cast(nullptr); try { - std::string name = get_model_instance_name(*instance); - int32_t device_id = get_device_id(*instance); - TRITONSERVER_InstanceGroupKind kind = get_instance_kind(*instance); + auto name = rapids::get_model_instance_name(*instance); + auto device_id = rapids::get_device_id(*instance); + auto deployment_type = rapids::get_deployment_type(*instance); - log_info( - __FILE__, __LINE__, - (std::string("TRITONBACKEND_ModelInstanceInitialize: ") + name + " (" + - TRITONSERVER_InstanceGroupKindString(kind) + " device " + - std::to_string(device_id) + ")") - .c_str()); + // TODO (wphicks) + // TODO (wphicks): Replace InstanceGroupKindString call + rapids::log_info(__FILE__, __LINE__, + (std::string("TRITONBACKEND_ModelInstanceInitialize: ") + + name + " (" + TRITONSERVER_InstanceGroupKindString(kind) + + " device " + std::to_string(device_id) + ")") + .c_str()); - ModelState* model_state = get_model_state(*instance); + auto* model_state = rapids::get_model_state(*instance); - // WH - set_instance_state( + rapids::set_instance_state( *instance, ModelInstanceState::Create(model_state, instance)); + } catch (rapids::TritonException& err) { + result = err.error(); } - catch (TritonException& err) { - return err.error(); - } - return nullptr; // success + return result; } -TRITONSERVER_Error* -TRITONBACKEND_ModelInstanceFinalize(TRITONBACKEND_ModelInstance* instance) -{ +auto* TRITONBACKEND_ModelInstanceFinalize( + TRITONBACKEND_ModelInstance* instance) { + auto* result = static_cast(nullptr); try { - void* vstate; - triton_check(TRITONBACKEND_ModelInstanceState(instance, &vstate)); - ModelInstanceState* instance_state = - reinterpret_cast(vstate); - + auto* instance_state = + rapids::get_instance_state(*instance); if (instance_state != nullptr) { - // WH - instance_state->UnloadFILModel(); + instance_state->get_model().unload(); - log_info( + rapids::log_info( __FILE__, __LINE__, "TRITONBACKEND_ModelInstanceFinalize: delete instance state"); delete instance_state; } - } - catch (TritonException& err) { - return err.error(); + } catch (rapids::TritonException& err) { + result = err.error(); } - return nullptr; // success + return result; } -TRITONSERVER_Error* -TRITONBACKEND_ModelInstanceExecute( - TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** raw_requests, - const uint32_t request_count) -{ +auto* TRITONBACKEND_ModelInstanceExecute(TRITONBACKEND_ModelInstance* instance, + TRITONBACKEND_Request** raw_requests, + uint32_t const request_count) { // TODO (wphicks) // auto model = ...; // auto batch_input = ...; diff --git a/src/interface/model_instance_state.h b/src/interface/model_instance_state.h new file mode 100644 index 0000000..0aa5518 --- /dev/null +++ b/src/interface/model_instance_state.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include + +namespace triton { +namespace backend { +namespace NAMESPACE { +struct ModelInstanceState : public BackendModelInstance { + static std::unique_ptr Create( + ModelState* model_state, + TRITONBACKEND_ModelInstance* triton_model_instance); + + ModelInstanceState(ModelState* model_state, + TRITONBACKEND_ModelInstance* triton_model_instance, + char const* name, std::int32_t device_id); + auto& get_model() { return model_; } + + private: + RapidsModel model_; +}; + +} // namespace NAMESPACE +} // namespace backend +} // namespace triton From c340158bea9e2bed942ef9e7dcacdcc1729dd27d Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 18 Aug 2021 14:57:06 -0400 Subject: [PATCH 007/199] Begin batch implementation --- include/rapids_triton/batch/batch.hpp | 43 +++++++++++++++++ include/rapids_triton/model/model.hpp | 42 +++++++++++++++- include/rapids_triton/triton/device.hpp | 24 ++++++++++ .../rapids_triton/triton/model_instance.hpp | 4 +- include/rapids_triton/triton/responses.hpp | 48 +++++++++++++++++++ 5 files changed, 157 insertions(+), 4 deletions(-) create mode 100644 include/rapids_triton/batch/batch.hpp create mode 100644 include/rapids_triton/triton/device.hpp create mode 100644 include/rapids_triton/triton/responses.hpp diff --git a/include/rapids_triton/batch/batch.hpp b/include/rapids_triton/batch/batch.hpp new file mode 100644 index 0000000..683df27 --- /dev/null +++ b/include/rapids_triton/batch/batch.hpp @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace triton { namespace backend { namespace rapids { + struct Batch { + template + auto get_input(std::string const& name, MemoryType memory_type, device_id_t device_id, cudaStream_t stream) { + // TODO(wphicks) + } + + template + auto get_output(std::string name, MemoryType memory_type, device_id_t device_id, cudaStream_t stream) { + // TODO(wphicks) + } + + private: + BackendInputCollector collector; + BackendOutputResponder responder; + } +}}} // namespace triton::backend::rapids + diff --git a/include/rapids_triton/model/model.hpp b/include/rapids_triton/model/model.hpp index 7fa1862..50e792f 100644 --- a/include/rapids_triton/model/model.hpp +++ b/include/rapids_triton/model/model.hpp @@ -16,9 +16,12 @@ #pragma once #include +#include #include #include +#include #include +#include #include namespace triton { namespace backend { namespace rapids { @@ -35,12 +38,47 @@ namespace triton { namespace backend { namespace rapids { template struct Model { - /* virtual void predict(Batch batch) { + /* virtual void predict(Batch& batch) { * Fetch tensors in order and feed them to predict overload * }; */ + /** + * @brief Return the preferred memory type in which to store data for this + * batch or std::nullopt to accept whatever Triton returns + * + * The base implementation of this method will require data on-host if the + * model itself is deployed on the host OR if this backend has not been + * compiled with GPU support. Otherwise, models deployed on device will + * receive memory on device. Overriding this method will allow derived + * model classes to select a preferred memory location based on properties + * of the batch or to simply return std::nullopt if device memory or host + * memory will do equally well. + */ + virtual std::optional preferred_mem_type(Batch& batch) const { + return (IS_GPU_BUILD && deployment_type_ == GPUDeployment) ? DeviceMemory : HostMemory; + } + + /** + * @brief Get input tensor of a particular named input for an entire batch + */ + template + auto get_input(Batch& batch, std::string const& name, std::optional> const& mem_type, cudaStream_t stream) const { + return batch.get_input(name, mem_type, device_id_, stream); + } + template + auto get_input(Batch& batch, std::string const& name, std::optional const& mem_type) const { + return get_input(name, mem_type, device_id_, default_stream_); + } + template + auto get_input(Batch& batch, std::string const& name) const { + return get_input(name, preferred_mem_type(batch), device_id_, default_stream_); + } + private: - std::shared_ptr shared_state; + std::shared_ptr shared_state_; + device_id_t device_id_; + cudaStream_t default_stream_; + DeploymentType deployment_type_; }; }}} // namespace triton::backend::rapids diff --git a/include/rapids_triton/triton/device.hpp b/include/rapids_triton/triton/device.hpp new file mode 100644 index 0000000..257a6ef --- /dev/null +++ b/include/rapids_triton/triton/device.hpp @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { namespace backend { namespace rapids { + using device_id_t = std::int32_t; +}}} // namespace triton::backend::rapids + + diff --git a/include/rapids_triton/triton/model_instance.hpp b/include/rapids_triton/triton/model_instance.hpp index 61f0fbc..fb65aa6 100644 --- a/include/rapids_triton/triton/model_instance.hpp +++ b/include/rapids_triton/triton/model_instance.hpp @@ -28,6 +28,7 @@ #include #include #include +#include namespace triton { namespace backend { namespace rapids { /** Get the name of a Triton model instance from the instance itself */ @@ -44,10 +45,9 @@ namespace triton { namespace backend { namespace rapids { * If this instance is loaded on the host, 0 will be returned. Otherwise the * GPU device id will be returned.*/ inline auto - inline auto get_device_id(TRITONBACKEND_ModelInstance& instance) { - auto device_id = std::int32_t{}; + auto device_id = device_id_t{}; triton_check(TRITONBACKEND_ModelInstanceDeviceId(&instance, &device_id)); return device_id; } diff --git a/include/rapids_triton/triton/responses.hpp b/include/rapids_triton/triton/responses.hpp new file mode 100644 index 0000000..af23c91 --- /dev/null +++ b/include/rapids_triton/triton/responses.hpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include + +namespace triton { namespace backend { namespace rapids { + +template +auto construct_responses(Iter requests_begin, Iter requests_end) +{ + auto responses = std::vector{}; + + auto requests_size = std::distance(requests_begin, requests_end); + if (!requests > 0) { + throw TritonException(Error::Internal, + "Invalid iterators for requests when constructing responses"); + } + responses.reserve(requests_size); + + std::transform( + requests_begin, requests_end, std::back_inserter(responses), + [](auto* request) { + auto* response = static_cast(nullptr); + triton_check(TRITONBACKEND_ResponseNew(&response, request)); + return response; + }); + return responses; +} + +}}} // namespace triton::backend::rapids + + From f4afbbb69c1424091b575c235a24e7787beaf7c5 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 18 Aug 2021 16:57:55 -0400 Subject: [PATCH 008/199] Use collector and responder for Batch --- include/rapids_triton/batch/batch.hpp | 28 ++++++++++++++++++++-- include/rapids_triton/triton/requests.hpp | 25 +++++++++++++++++++ include/rapids_triton/triton/responses.hpp | 4 +++- 3 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 include/rapids_triton/triton/requests.hpp diff --git a/include/rapids_triton/batch/batch.hpp b/include/rapids_triton/batch/batch.hpp index 683df27..829bfaa 100644 --- a/include/rapids_triton/batch/batch.hpp +++ b/include/rapids_triton/batch/batch.hpp @@ -20,11 +20,33 @@ #include #include #include +#include #include #include namespace triton { namespace backend { namespace rapids { struct Batch { + Batch(TRITONBACKEND_Request** raw_requests, request_size_t count, ModelState const& model_state, cudaStream_t stream) : + requests_(raw_requests, count), + responses_(construct_responses(requests_.begin(), requests_.end())), + collector_{ + raw_requests, + count, + &responses_, + model_state.TritonMemoryManager(), + model_state.EnablePinnedInput(), + stream + }, + responder_{ + raw_requests, + count, + &responses_, + model_state.MaxBatchSize(), + model_state.EnablePinnedInput(), + stream + } {} + + template auto get_input(std::string const& name, MemoryType memory_type, device_id_t device_id, cudaStream_t stream) { // TODO(wphicks) @@ -36,8 +58,10 @@ namespace triton { namespace backend { namespace rapids { } private: - BackendInputCollector collector; - BackendOutputResponder responder; + std::vector requests_; + std::vector responses_; + BackendInputCollector collector_; + BackendOutputResponder responder_; } }}} // namespace triton::backend::rapids diff --git a/include/rapids_triton/triton/requests.hpp b/include/rapids_triton/triton/requests.hpp new file mode 100644 index 0000000..4839486 --- /dev/null +++ b/include/rapids_triton/triton/requests.hpp @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { namespace backend { namespace rapids { + using request_size_t = uint32_t; +}}} // namespace triton::backend::rapids + + + diff --git a/include/rapids_triton/triton/responses.hpp b/include/rapids_triton/triton/responses.hpp index af23c91..7d6743c 100644 --- a/include/rapids_triton/triton/responses.hpp +++ b/include/rapids_triton/triton/responses.hpp @@ -15,8 +15,10 @@ */ #pragma once +#include #include #include +#include #include namespace triton { namespace backend { namespace rapids { @@ -27,7 +29,7 @@ auto construct_responses(Iter requests_begin, Iter requests_end) auto responses = std::vector{}; auto requests_size = std::distance(requests_begin, requests_end); - if (!requests > 0) { + if (!requests_size > 0) { throw TritonException(Error::Internal, "Invalid iterators for requests when constructing responses"); } From ce3424482c4d067ebcfeed44d0b68ca39e0ed242 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 24 Aug 2021 09:50:21 -0400 Subject: [PATCH 009/199] Add configuration reading to shared state --- include/rapids_triton/batch/batch.hpp | 12 ++- include/rapids_triton/model/model.hpp | 7 ++ include/rapids_triton/model/shared_state.hpp | 98 ++++++++++++++++++++ 3 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 include/rapids_triton/model/shared_state.hpp diff --git a/include/rapids_triton/batch/batch.hpp b/include/rapids_triton/batch/batch.hpp index 829bfaa..e30a4b8 100644 --- a/include/rapids_triton/batch/batch.hpp +++ b/include/rapids_triton/batch/batch.hpp @@ -49,7 +49,12 @@ namespace triton { namespace backend { namespace rapids { template auto get_input(std::string const& name, MemoryType memory_type, device_id_t device_id, cudaStream_t stream) { - // TODO(wphicks) + collector_.ProcessTensor( + name.c_str(), + nullptr // Return data without copy if possible + // TODO + ); + } template @@ -62,6 +67,9 @@ namespace triton { namespace backend { namespace rapids { std::vector responses_; BackendInputCollector collector_; BackendOutputResponder responder_; - } + + auto get_shape() { + } + }; }}} // namespace triton::backend::rapids diff --git a/include/rapids_triton/model/model.hpp b/include/rapids_triton/model/model.hpp index 50e792f..9064f4b 100644 --- a/include/rapids_triton/model/model.hpp +++ b/include/rapids_triton/model/model.hpp @@ -59,6 +59,13 @@ namespace triton { namespace backend { namespace rapids { return (IS_GPU_BUILD && deployment_type_ == GPUDeployment) ? DeviceMemory : HostMemory; } + /** + * @brief Whether or not pinned memory should be used for I/O with this model + */ + virtual bool enable_pinned() const { + return false; + } + /** * @brief Get input tensor of a particular named input for an entire batch */ diff --git a/include/rapids_triton/model/shared_state.hpp b/include/rapids_triton/model/shared_state.hpp new file mode 100644 index 0000000..3ca2ff9 --- /dev/null +++ b/include/rapids_triton/model/shared_state.hpp @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace triton { namespace backend { namespace rapids { + /** + * @brief Stores shared state for multiple instances of the same model + */ + struct SharedModelState { + // TODO(wphicks): + virtual void load() { + } + virtual void unload() { + } + + explicit SharedModelState(common::TritonJSON::Value&& config) : config_{std::move(config)} {} + + template + auto get_config_param(std::string const& name) { + return get_config_param(name, std::optional{}); + } + + template + auto get_config_param(std::string const& name, T default_value) { + return get_config_param(name, std::make_optional(default_value)); + } + + private: + common::TritonJSON::Value config_; + + template + auto get_config_param(std::string const& name, std::optional const& default_value) { + auto result = T{}; + auto json_value = common::TritonJson::Value{}; + if (config_.Find(name.c_str(), &json_value) { + auto string_repr = std::string{}; + triton_check(json_value.MemberAsString("string_value", &string_repr)); + + auto input_stream = std::istringstream{string_repr}; + + if (std::is_same::value) { + input_stream >> std::boolalpha >> result; + } else { + input_stream >> result; + } + + if (input_stream.fail()) { + if (default_value) { + result = *default_value; + } else { + throw TritonException( + Error::InvalidArg, + std::string("Bad input for parameter ") + name + ); + } + } + } else { + if (default_value) { + result = *default_value; + } else { + throw TritonException( + Error::InvalidArg, + std::string("Required parameter ") + name + std::string(" not found in config") + ); + } + } + + return result; + } + }; +}}} // namespace triton::backend::rapids + From 7369b4ee0a16e4a110e3ba18ede028d62c7b9702 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 25 Aug 2021 10:31:00 -0400 Subject: [PATCH 010/199] Provide basics of API functions for models --- include/rapids_triton/batch/batch.hpp | 12 ++-- include/rapids_triton/model/model.hpp | 38 +++++++---- include/rapids_triton/model/shared_state.hpp | 19 ++++-- include/rapids_triton/triton/config.hpp | 32 +++++++++ src/impl/model.h | 71 ++++++++++++++++++++ src/impl/names.h | 10 +++ src/impl/shared_state.h | 34 ++++++++++ src/interface/api.cc | 32 ++++++--- src/interface/model_instance_state.h | 21 +++--- src/interface/model_state.h | 10 +-- 10 files changed, 233 insertions(+), 46 deletions(-) create mode 100644 include/rapids_triton/triton/config.hpp create mode 100644 src/impl/model.h create mode 100644 src/impl/shared_state.h diff --git a/include/rapids_triton/batch/batch.hpp b/include/rapids_triton/batch/batch.hpp index e30a4b8..03760af 100644 --- a/include/rapids_triton/batch/batch.hpp +++ b/include/rapids_triton/batch/batch.hpp @@ -26,23 +26,25 @@ namespace triton { namespace backend { namespace rapids { struct Batch { - Batch(TRITONBACKEND_Request** raw_requests, request_size_t count, ModelState const& model_state, cudaStream_t stream) : + using size_type = std::size_t; + + Batch(TRITONBACKEND_Request** raw_requests, request_size_t count, TRITONBACKEND_MemoryManager& triton_mem_manager, bool use_pinned_input, bool use_pinned_output, size_type max_batch_size, cudaStream_t stream) : requests_(raw_requests, count), responses_(construct_responses(requests_.begin(), requests_.end())), collector_{ raw_requests, count, &responses_, - model_state.TritonMemoryManager(), - model_state.EnablePinnedInput(), + &triton_mem_manager, + use_pinned_input, stream }, responder_{ raw_requests, count, &responses_, - model_state.MaxBatchSize(), - model_state.EnablePinnedInput(), + max_batch_size, + use_pinned_output, stream } {} diff --git a/include/rapids_triton/model/model.hpp b/include/rapids_triton/model/model.hpp index 9064f4b..59f55b6 100644 --- a/include/rapids_triton/model/model.hpp +++ b/include/rapids_triton/model/model.hpp @@ -20,28 +20,17 @@ #include #include #include +#include #include #include +#include #include namespace triton { namespace backend { namespace rapids { - /** - * @brief Stores shared state for multiple instances of the same model - */ - struct SharedModelState { - virtual void load() { - } - virtual void unload() { - } - }; - template struct Model { - /* virtual void predict(Batch& batch) { - * Fetch tensors in order and feed them to predict overload - * }; - */ + virtual void predict(Batch& batch) = 0; /** * @brief Return the preferred memory type in which to store data for this @@ -58,6 +47,12 @@ namespace triton { namespace backend { namespace rapids { virtual std::optional preferred_mem_type(Batch& batch) const { return (IS_GPU_BUILD && deployment_type_ == GPUDeployment) ? DeviceMemory : HostMemory; } + virtual std::optional preferred_mem_type_in(Batch& batch) const { + return preferred_mem_type(batch); + } + virtual std::optional preferred_mem_type_out(Batch& batch) const { + return preferred_mem_type(batch); + } /** * @brief Whether or not pinned memory should be used for I/O with this model @@ -82,6 +77,21 @@ namespace triton { namespace backend { namespace rapids { return get_input(name, preferred_mem_type(batch), device_id_, default_stream_); } + /** + * @brief Retrieve value of configuration parameter + */ + template + auto get_config_param(std::string const& name) { + return shared_state_->get_config_param(name); + } + template + auto get_config_param(std::string const& name, T default_value) { + return shared_state_->get_config_param(name, default_value); + } + + Model(std::shared_ptr shared_state, device_id_t device_id, cudaStream_t default_stream, DeploymentType deployment_type) : + shared_state_{shared_state}, device_id_{device_id}, default_stream_{default_stream}, deployment_type_{deployment_type} {} + private: std::shared_ptr shared_state_; device_id_t device_id_; diff --git a/include/rapids_triton/model/shared_state.hpp b/include/rapids_triton/model/shared_state.hpp index 3ca2ff9..6f4845e 100644 --- a/include/rapids_triton/model/shared_state.hpp +++ b/include/rapids_triton/model/shared_state.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,7 @@ #include #include #include +#include #include #include @@ -33,13 +35,13 @@ namespace triton { namespace backend { namespace rapids { * @brief Stores shared state for multiple instances of the same model */ struct SharedModelState { - // TODO(wphicks): - virtual void load() { - } - virtual void unload() { - } - explicit SharedModelState(common::TritonJSON::Value&& config) : config_{std::move(config)} {} + virtual void load() {} + virtual void unload() {} + + explicit SharedModelState( + common::TritonJSON::Value config) : config_{config}, + max_batch_size_(get_max_batch_size(config)) {} template auto get_config_param(std::string const& name) { @@ -53,10 +55,15 @@ namespace triton { namespace backend { namespace rapids { private: common::TritonJSON::Value config_; + Batch::size_type max_batch_size_; template auto get_config_param(std::string const& name, std::optional const& default_value) { auto result = T{}; + if (name == std::string("max_batch_size")) { + result = max_batch_size_; + return result; + } auto json_value = common::TritonJson::Value{}; if (config_.Find(name.c_str(), &json_value) { auto string_repr = std::string{}; diff --git a/include/rapids_triton/triton/config.hpp b/include/rapids_triton/triton/config.hpp new file mode 100644 index 0000000..3f1c3ed --- /dev/null +++ b/include/rapids_triton/triton/config.hpp @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include +#include +#include + +namespace triton { namespace backend { namespace rapids { + inline auto get_max_batch_size(common::TritonJSON::Value& config) { + auto reported = int64_t{}; + triton_check(config.MemberAsInt("max_batch_size", &reported)); + return narrow(reported); + } +}}} // namespace triton::backend::rapids diff --git a/src/impl/model.h b/src/impl/model.h new file mode 100644 index 0000000..4b94fa7 --- /dev/null +++ b/src/impl/model.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include // rapids::Batch +#include // rapids::MemoryType +#include // rapids::Model + +namespace triton { +namespace backend { +namespace NAMESPACE { + +struct RapidsModel : rapids::Model { + void load() {} + void unload() {} + void predict(rapids::Batch& batch) { + // TODO(wphicks) + } + + /*************************************************************************** + * ADVANCED FEATURES * + * *********************************************************************** * + * None of the following methods are required to be implemented in order to + * create a valid model, but they are presented here for those who require + * the additional functionality they provide. + **************************************************************************/ + + /*************************************************************************** + * preferred_mem_type / preferred_mem_type_in / preferred_mem_type_out * + * *********************************************************************** * + * If implemented, `preferred_mem_type` allows for control over when input + * and output data are provided on the host versus on device. In the case + * that a model prefers to receive its input on-host but return output + * on-device (or vice versa), `preferred_mem_type_in` and + * `preferred_mem_type_out` can be used for even more precise control. + * + * In this example, we simply return `std::nullopt` to indicate that the + * model has no preference on its input/output data locations. Note that the + * Batch being processed is taken as input to this function to facilitate + * implementations that may switch their preferred memory location based on + * e.g. the size of the batch. + * + * Valid MemoryType options to return are rapids::HostMemory and + * rapids::DeviceMemory. + **************************************************************************/ + std::optional preferred_mem_type( + rapids::Batch& batch) const { + return std::nullopt; + } +}; + +} // namespace NAMESPACE +} // namespace backend +} // namespace triton diff --git a/src/impl/names.h b/src/impl/names.h index 215430d..33d6fe0 100644 --- a/src/impl/names.h +++ b/src/impl/names.h @@ -16,4 +16,14 @@ #pragma once +/* Triton expects certain definitions within its backend libraries to follow + * specific naming conventions. Specifically, for a backend named + * "rapids_identity," most definitions should appear within a namespace called + * triton::backend::rapids_identity. + * + * In order to facilitate this with minimal effort on the part of backend + * developers, we ask that you put the name of your backend here. This macro is + * then used to propagate the correct namespace name wherever it is needed in + * the impl and interface code. */ + #define NAMESPACE rapids_identity diff --git a/src/impl/shared_state.h b/src/impl/shared_state.h new file mode 100644 index 0000000..0a292a0 --- /dev/null +++ b/src/impl/shared_state.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include + +namespace triton { +namespace backend { +namespace NAMESPACE { + +struct RapidsSharedState : rapids::SharedState { + void load() {} + void unload() {} +}; + +} // namespace NAMESPACE +} // namespace backend +} // namespace triton diff --git a/src/interface/api.cc b/src/interface/api.cc index fb2ae6c..9dd56bb 100644 --- a/src/interface/api.cc +++ b/src/interface/api.cc @@ -65,7 +65,10 @@ auto* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { " (version " + std::to_string(version) + ")") .c_str()); - rapids::set_model_state(*model, ModelState::Create(*model)); + auto rapids_model_state = std::make_unique(*model); + rapids_model_state->load(); + + rapids::set_model_state(*model, std::move(rapids_model_state)); } catch (rapids::TritonException& err) { result = err.error(); } @@ -110,8 +113,10 @@ auto* TRITONBACKEND_ModelInstanceInitialize( auto* model_state = rapids::get_model_state(*instance); - rapids::set_instance_state( - *instance, ModelInstanceState::Create(model_state, instance)); + auto rapids_model = + std::make_unique(model_state, instance) + + rapids::set_instance_state(*instance, std::move(rapids_model)); } catch (rapids::TritonException& err) { result = err.error(); } @@ -125,7 +130,7 @@ auto* TRITONBACKEND_ModelInstanceFinalize( auto* instance_state = rapids::get_instance_state(*instance); if (instance_state != nullptr) { - instance_state->get_model().unload(); + instance_state->unload(); rapids::log_info( __FILE__, __LINE__, @@ -143,11 +148,20 @@ auto* TRITONBACKEND_ModelInstanceFinalize( auto* TRITONBACKEND_ModelInstanceExecute(TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** raw_requests, uint32_t const request_count) { - // TODO (wphicks) - // auto model = ...; - // auto batch_input = ...; - // auto batch_output = ...; - // model.predict(batch_input, batch_output); + auto* model_state = rapids::get_model_state(*instance); + auto* instance_state = + rapids::get_instance_state(*instance); + auto& model = instance_state->get_model(); + auto max_batch_size = model.get_config_param("max_batch_size"); + auto batch = Batch{raw_requests, + request_count, + model_state->TritonMemoryManager(), + model_state->EnablePinnedInput(), + model_state->EnablePinnedOutput(), + max_batch_size, + model.get_default_stream()}; + + model.predict(batch); } } // extern "C" diff --git a/src/interface/model_instance_state.h b/src/interface/model_instance_state.h index 0aa5518..b6f6dd4 100644 --- a/src/interface/model_instance_state.h +++ b/src/interface/model_instance_state.h @@ -17,22 +17,27 @@ #pragma once #include +#include #include #include +#include namespace triton { namespace backend { namespace NAMESPACE { struct ModelInstanceState : public BackendModelInstance { - static std::unique_ptr Create( - ModelState* model_state, - TRITONBACKEND_ModelInstance* triton_model_instance); - - ModelInstanceState(ModelState* model_state, - TRITONBACKEND_ModelInstance* triton_model_instance, - char const* name, std::int32_t device_id); - auto& get_model() { return model_; } + ModelInstanceState(ModelState& model_state, + TRITONBACKEND_ModelInstance* triton_model_instance) + : BackendModelInstance(&model_state, triton_model_instance), + model_(model_state.get_shared_state(), + rapids::get_device_id(*triton_model_instance), CudaStream(), + Kind()) {} + + auto& get_model() const { return model_; } + + void load() { model_->load(); } + void unload() { model_->unload(); } private: RapidsModel model_; diff --git a/src/interface/model_state.h b/src/interface/model_state.h index 92e4c38..093bfba 100644 --- a/src/interface/model_state.h +++ b/src/interface/model_state.h @@ -21,11 +21,13 @@ namespace triton { namespace backend { namespace NAMESPACE { struct ModelState : public BackendModel { - static std::unique_ptr Create(TRITONBACKEND_Model& triton_model); - ModelState( - TRITONBACKEND_Model* triton_model, const char* name, - const uint64_t version); + ModelState(TRITONBACKEND_Model& triton_model) + : state_{std::make_shared( + get_model_config(triton_model))} {} + + void load() { state_->load(); } + void unload() { state_->unload(); } auto get_shared_state() { return state_; } From 995b70bb23a5d15a5c89252c6baa2726e12fba94 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 26 Aug 2021 12:51:28 -0400 Subject: [PATCH 011/199] Set up output synchronization --- include/rapids_triton/batch/batch.hpp | 64 +++++++++++++++++--- include/rapids_triton/exceptions.hpp | 7 +++ include/rapids_triton/memory/buffer.hpp | 27 +++++---- include/rapids_triton/model/model.hpp | 50 +++++++++++++--- include/rapids_triton/tensor/tensor.hpp | 77 +++++++++++++++++++----- include/rapids_triton/triton/input.hpp | 78 +++++++++++++++++++++++++ src/impl/model.h | 5 +- src/interface/api.cc | 37 +++++++----- src/interface/model_instance_state.h | 4 +- 9 files changed, 291 insertions(+), 58 deletions(-) create mode 100644 include/rapids_triton/triton/input.hpp diff --git a/include/rapids_triton/batch/batch.hpp b/include/rapids_triton/batch/batch.hpp index 03760af..86bdbe3 100644 --- a/include/rapids_triton/batch/batch.hpp +++ b/include/rapids_triton/batch/batch.hpp @@ -39,38 +39,84 @@ namespace triton { namespace backend { namespace rapids { use_pinned_input, stream }, - responder_{ + responder_{std::make_shared( raw_requests, count, &responses_, max_batch_size, use_pinned_output, stream - } {} + )}, + stream_{stream} {} template - auto get_input(std::string const& name, MemoryType memory_type, device_id_t device_id, cudaStream_t stream) { + auto get_input(std::string const& name, MemoryType memory_type, device_id_t device_id) { + auto shape = get_input_shape(requests_.begin(), requests_.end(), name); + auto size_bytes = sizeof(T) * std::reduce(shape.begin(), shape.end(), std::size_t{1}, std::multiplies<>()); + + auto const* raw_buffer = static_cast(nullptr); + auto reported_bytes = std::size_t{}; + auto reported_mem_type = memory_type; + auto reported_device_id = device_id; + collector_.ProcessTensor( name.c_str(), - nullptr // Return data without copy if possible - // TODO + nullptr, // Return data without copy if possible + size_bytes, + {{memory_type, device_id}}, + &raw_buffer, + &reported_bytes, + &reported_memory_type, + &reported_device_id + ); + + auto buffer = Buffer( + reinterpret_cast(raw_buffer), + reported_bytes, + reported_memory_type, + stream_ ); + if (reported_memory_type != memory_type || reported_device_id != device_id) { + throw(TritonException(Error::Internal, "data collected in wrong location"); + } + + return Tensor(std::move(shape), std::move(buffer)); } template - auto get_output(std::string name, MemoryType memory_type, device_id_t device_id, cudaStream_t stream) { - // TODO(wphicks) + auto get_output(std::string const& name, MemoryType memory_type, device_id_t device_id) { + auto shape = get_output_shape(requests_.begin(), requests_.end(), name); // TODO(wphicks) + // TODO(wphicks): construct buffer + return OutputTensor(std::move(shape), std::move(buffer), responder_, name); + } + + auto stream() const { + return stream_; + } + + void finalize() { + // TODO(wphicks): report statistics + if (responder_->Finalize()) { + cuda_check(cudaStreamSynchronize(stream_)); + } + //TODO(wphicks): Send responses } private: std::vector requests_; std::vector responses_; BackendInputCollector collector_; - BackendOutputResponder responder_; + std::shared_pointer responder_; + cudaStream_t stream_; - auto get_shape() { + auto get_input_shape(std::string const& name) { + auto result = decltype(get_triton_input_shape(*std::begin(requests_))){}; + if(!requests_.empty()) { + result = get_triton_input_shape(*std::begin(requests_)) + } + return result; } }; }}} // namespace triton::backend::rapids diff --git a/include/rapids_triton/exceptions.hpp b/include/rapids_triton/exceptions.hpp index 8db8198..122c8e7 100644 --- a/include/rapids_triton/exceptions.hpp +++ b/include/rapids_triton/exceptions.hpp @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include @@ -81,5 +82,11 @@ inline void triton_check(TRITONSERVER_Error* err) { } } +inline void cuda_check(cudaError_t const& err) { + if (err != cudaSuccess) { + throw TritonException(Error::Internal, cudaGetErrorString(err)); + } +} + }}} // namespace triton::backend::rapids diff --git a/include/rapids_triton/memory/buffer.hpp b/include/rapids_triton/memory/buffer.hpp index b76deb6..0089c35 100644 --- a/include/rapids_triton/memory/buffer.hpp +++ b/include/rapids_triton/memory/buffer.hpp @@ -26,6 +26,7 @@ #include #include #include +#include namespace triton { namespace backend { namespace rapids { template @@ -39,7 +40,7 @@ namespace triton { namespace backend { namespace rapids { using owned_d_ptr = std::unique_ptr{}>; using data_ptr = std::variant; - Buffer() noexcept : data_{std::in_place_index<0>, nullptr}, size_{}, stream_{} {} + Buffer() noexcept : device_{}, data_{std::in_place_index<0>, nullptr}, size_{}, stream_{} {} /** * @brief Construct buffer of given size in given memory location (either @@ -47,9 +48,9 @@ namespace triton { namespace backend { namespace rapids { * A buffer constructed in this way is owning and will release allocated * resources on deletion */ - Buffer(size_type size, MemoryType memory_type=DeviceMemory, cudaStream_t + Buffer(size_type size, MemoryType memory_type=DeviceMemory, device_id_t device=0, cudaStream_t stream=0) : - data_{allocate(size, memory_type)}, size_{size}, stream_{stream} {} + device_{device}, data_{allocate(size, memory_type)}, size_{size}, stream_{stream} {} /** * @brief Construct buffer from given source in given memory location (either @@ -58,8 +59,8 @@ namespace triton { namespace backend { namespace rapids { * responsible for freeing any resources associated with the input pointer */ Buffer(T* input_data, size_type size, MemoryType memory_type=DeviceMemory, - cudaStream_t stream=0) : - data_{allocate(size, memory_type)}, size_{size}, stream_{stream} {} + device_id_t device=0, cudaStream_t stream=0) : + device_{device}, data_{input_data}, size_{size}, stream_{stream} {} /** * @brief Construct one buffer from another in the given memory location @@ -67,7 +68,7 @@ namespace triton { namespace backend { namespace rapids { * A buffer constructed in this way is owning and will copy the data from * the original location */ - Buffer(Buffer const& other, MemoryType memory_type) : data_([&other](){ + Buffer(Buffer const& other, MemoryType memory_type, device_id_t device=0) : device_{device}, data_([&other](){ auto result = allocate(other.size_, memory_type); copy(result, other.data_, other.size_, other.stream_); return result; @@ -77,9 +78,9 @@ namespace triton { namespace backend { namespace rapids { * @brief Create owning copy of existing buffer * The memory type of this new buffer will be the same as the original */ - Buffer(Buffer const& other) : Buffer(other, other.mem_type()) {} + Buffer(Buffer const& other) : Buffer(other, other.mem_type(), other.device(), other.stream()) {} - Buffer(Buffer&& other, MemoryType memory_type) : data_{[&other, memory_type](){ + Buffer(Buffer&& other, MemoryType memory_type) : device_{other.device()}, data_{[&other, memory_type](){ data_ptr result; if(memory_type == other.mem_type()) { result = std::move(other.data_); @@ -90,7 +91,7 @@ namespace triton { namespace backend { namespace rapids { return result; }()}, size_{other.size_}, stream{other.stream_} {} - Buffer(Buffer&& other) noexcept : data_{std::move{other.data_}}, size_{other.size_}, + Buffer(Buffer&& other) noexcept : device_{other.device_}, data_{std::move{other.data_}}, size_{other.size_}, stream_{other.stream_} {} ~Buffer() = default; @@ -116,6 +117,10 @@ namespace triton { namespace backend { namespace rapids { return get_raw_ptr(data_); } + auto device() const noexcept { + return device_; + } + /** * @brief Return CUDA stream associated with this buffer */ @@ -132,12 +137,13 @@ namespace triton { namespace backend { namespace rapids { */ void set_stream(cudaStream_t new_stream) { if constexpr (IS_GPU_BUILD) { - cudaStreamSynchronize(stream_); + cuda_check(cudaStreamSynchronize(stream_)); } stream_ = new_stream; } private: + device_id_t device_; data_ptr data_; size_type size_; cudaStream_t stream_; @@ -169,6 +175,7 @@ namespace triton { namespace backend { namespace rapids { data_ptr result; if (memory_type == DeviceMemory) { if constexpr (IS_GPU_BUILD) { + cuda_check(cudaSetDevice(device_)); result = data_ptr{owned_d_ptr{detail::dev_allocate(size)}}; } else { throw TritonException( diff --git a/include/rapids_triton/model/model.hpp b/include/rapids_triton/model/model.hpp index 59f55b6..1552fae 100644 --- a/include/rapids_triton/model/model.hpp +++ b/include/rapids_triton/model/model.hpp @@ -32,6 +32,9 @@ namespace triton { namespace backend { namespace rapids { virtual void predict(Batch& batch) = 0; + virtual void load() {} + virtual void unload() {} + /** * @brief Return the preferred memory type in which to store data for this * batch or std::nullopt to accept whatever Triton returns @@ -55,15 +58,24 @@ namespace triton { namespace backend { namespace rapids { } /** - * @brief Whether or not pinned memory should be used for I/O with this model + * @brief Retrieve a stream used to set up batches for this model + * + * The base implementation of this method simply returns the default stream + * provided by Triton for use with this model. Child classes may choose to + * override this in order to provide different streams for use with + * successive incoming batches. For instance, one might cycle through + * several streams in order to distribute batches across them, but care + * should be taken to ensure proper synchronization in this case. It is + * recommended that this method be overridden only when strictly necessary. */ - virtual bool enable_pinned() const { - return false; + virtual cudaStream_t get_stream() { + return default_stream_; } /** * @brief Get input tensor of a particular named input for an entire batch */ + //TODO(wphicks): Handle const for input type template auto get_input(Batch& batch, std::string const& name, std::optional> const& mem_type, cudaStream_t stream) const { return batch.get_input(name, mem_type, device_id_, stream); @@ -77,25 +89,49 @@ namespace triton { namespace backend { namespace rapids { return get_input(name, preferred_mem_type(batch), device_id_, default_stream_); } + /** + * @brief Get output tensor of a particular named output for an entire batch + */ + template + auto get_output(Batch& batch, std::string const& name, std::optional> const& mem_type, cudaStream_t stream) const { + return batch.get_output(name, mem_type, device_id_, stream); + } + template + auto get_output(Batch& batch, std::string const& name, std::optional const& mem_type) const { + return get_output(name, mem_type, device_id_, default_stream_); + } + template + auto get_output(Batch& batch, std::string const& name) const { + return get_output(name, preferred_mem_type(batch), device_id_, default_stream_); + } + /** * @brief Retrieve value of configuration parameter */ template - auto get_config_param(std::string const& name) { + auto get_config_param(std::string const& name) const { return shared_state_->get_config_param(name); } template - auto get_config_param(std::string const& name, T default_value) { + auto get_config_param(std::string const& name, T default_value) const { return shared_state_->get_config_param(name, default_value); } - Model(std::shared_ptr shared_state, device_id_t device_id, cudaStream_t default_stream, DeploymentType deployment_type) : - shared_state_{shared_state}, device_id_{device_id}, default_stream_{default_stream}, deployment_type_{deployment_type} {} + Model(std::shared_ptr shared_state, device_id_t device_id, cudaStream_t default_stream, DeploymentType deployment_type, std::string const& filepath) : + shared_state_{shared_state}, device_id_{device_id}, default_stream_{default_stream}, deployment_type_{deployment_type} filepath_{filepath} {} + + auto get_device_id() const { return device_id_; } + auto get_deployment_type() const { return deployment_type_; } + auto const& get_filepath() const { return filepath_; } + + protected: + auto get_shared_state() const { return shared_state_; } private: std::shared_ptr shared_state_; device_id_t device_id_; cudaStream_t default_stream_; DeploymentType deployment_type_; + std::string filepath_; }; }}} // namespace triton::backend::rapids diff --git a/include/rapids_triton/tensor/tensor.hpp b/include/rapids_triton/tensor/tensor.hpp index f9edb5f..a26580f 100644 --- a/include/rapids_triton/tensor/tensor.hpp +++ b/include/rapids_triton/tensor/tensor.hpp @@ -23,24 +23,29 @@ #include +#include +#include #include #include namespace triton { namespace backend { namespace rapids { template - struct Tensor { + struct BaseTensor { using size_type = typename Buffer::size_type; - Tensor() : shape_{}, buffer_{} {} + BaseTensor() : shape_{}, buffer_{} {} + BaseTensor(std::vector&& shape, Buffer&& buffer) : shape_(std::move(shape)), buffer_{std::move(buffer)} {} + + virtual ~BaseTensor() = 0; /** - * @brief Construct a Tensor from a collection of buffers + * @brief Construct a BaseTensor from a collection of buffers * * Given a collection of buffers, collate them all into one buffer stored in - * a new Tensor + * a new BaseTensor */ template - Tensor(std::vector const& shape, Iter begin, Iter end, MemoryType mem_type, cudaStream_t stream=0) : + BaseTensor(std::vector const& shape, Iter begin, Iter end, MemoryType mem_type, cudaStream_t stream=0) : shape_(shape), buffer_([&begin, &end, &mem_type, &stream] () { auto total_size = std::transform_reduce( @@ -60,9 +65,18 @@ namespace triton { namespace backend { namespace rapids { auto data() { return buffer_.data(); } auto& buffer() { return buffer_; } - auto constexpr dtype() const { return TritonDtype::value; } + auto dtype() constexpr { return TritonDtype::value; } auto mem_type() const { return data.mem_type(); } auto stream() const { return data.stream(); } + auto device() const { return data.device(); } + + void stream_synchronize() const { + if constexpr (IS_GPU_BUILD) { + if (mem_type() == DeviceMemory) { + cuda_check(cudaStreamSynchronize(stream()); + } + } + } private: std::vector shape_; @@ -70,7 +84,7 @@ namespace triton { namespace backend { namespace rapids { }; template - void copy(Tensor> dst, Tensor src) { + void copy(BaseTensor> dst, BaseTensor src) { copy(dst.buffer(), src.buffer()); } @@ -82,7 +96,7 @@ namespace triton { namespace backend { namespace rapids { * of the data from the src Tensor */ template - copy(Iter begin, Iter end, Tensor src) { + copy(Iter begin, Iter end, BaseTensor src) { std::reduce( begin, end, @@ -94,13 +108,44 @@ namespace triton { namespace backend { namespace rapids { } } - /** - * @brief A simple struct containing only a string used to name a Tensor and - * the Tensor itself - */ - template - struct NamedTensor { - std::string name; - Tensor tensor; + template + struct Tensor final : BaseTensor { + }; + + template + struct OutputTensor final : BaseTensor { + OutputTensor(std::vector&& shape, Buffer&& buffer, + std::string const& name, std::shared_pointer responder) : + BaseTensor(std::move(shape), std::move(buffer)), name_{name}, responder_{responder} + {} + /** + * @brief Prepare final output data from this tensor for responding to + * request + * + * This method *must* be called by rapids_triton backends on all of their + * output tensors before returning from their `predict` methods. Because we + * cannot known a priori what names backends might have for their tensors + * and what types will be stored in those tensors, the rapids_triton + * library cannot store references to those tensors that might otherwise be + * used to finalize them. + */ + void finalize() { + // Must call the following because BackendOutputResponder does not expose + // its stream, so we cannot be certain that our data is not being + // processed on another stream. + stream_synchronize(); + responder_->ProcessTensor( + name_.c_str(), + TritonDtype::value, + shape(), + data(), + mem_type(), + device() + ); + } + + private: + std::shared_pointer responder_; + std::string name_; }; }}} // namespace triton::backend::rapids diff --git a/include/rapids_triton/triton/input.hpp b/include/rapids_triton/triton/input.hpp new file mode 100644 index 0000000..751276a --- /dev/null +++ b/include/rapids_triton/triton/input.hpp @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace triton { namespace backend { namespace rapids { + inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) { + auto result = static_cast(nullptr); + triton_check( + TRITONBACKEND_RequestInput(request, name.c_str(), &result)); + return result; + } + + template + auto get_triton_input_shape(Iter requests_begin, Iter requests_end, std::string const& name) { + auto result = std::vector{}; + + auto reported_dtype = DType{}; + auto const* input_shape = static_cast(nullptr); + auto input_dims = uint32_t{}; + + auto batch_dim = std::reduce( + requests_begin, + requests_end, + int64_t{}, + [&reported_dtype, &input_shape, &input_dims, &name](auto& request, auto total) { + auto* input = get_triton_input(request, name); + triton_check( + TRITONBACKEND_InputProperties( + input, nullptr, &reported_dtype, &input_shape, + &input_dims, nullptr, nullptr)); + + if (reported_dtype != TritonDtype::value) { + throw(TritonException(Error::Internal, "incorrect type for requested input")); + } + + if (input_dims != 0) { + total += *input_shape; + } + return total; + }); + + result.reserve(input_dims); + std::transform( + input_shape, + input_shape + input_dims, + std::back_inserter(result), + [](auto& val) { return narrow(val); } + ); + + if (!result.empty()) { + result[0] = narrow(batch_dim); + } + + return result; + } +}}} diff --git a/src/impl/model.h b/src/impl/model.h index 4b94fa7..5b6c016 100644 --- a/src/impl/model.h +++ b/src/impl/model.h @@ -31,7 +31,10 @@ struct RapidsModel : rapids::Model { void load() {} void unload() {} void predict(rapids::Batch& batch) { - // TODO(wphicks) + auto input = model.get_input(batch, "input__0"); + auto output = model.get_output(batch, "output__0"); + copy(output, input); + // TODO(wphicks): Read config and make this interesting } /*************************************************************************** diff --git a/src/interface/api.cc b/src/interface/api.cc index 9dd56bb..9bb369e 100644 --- a/src/interface/api.cc +++ b/src/interface/api.cc @@ -148,20 +148,29 @@ auto* TRITONBACKEND_ModelInstanceFinalize( auto* TRITONBACKEND_ModelInstanceExecute(TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** raw_requests, uint32_t const request_count) { - auto* model_state = rapids::get_model_state(*instance); - auto* instance_state = - rapids::get_instance_state(*instance); - auto& model = instance_state->get_model(); - auto max_batch_size = model.get_config_param("max_batch_size"); - auto batch = Batch{raw_requests, - request_count, - model_state->TritonMemoryManager(), - model_state->EnablePinnedInput(), - model_state->EnablePinnedOutput(), - max_batch_size, - model.get_default_stream()}; - - model.predict(batch); + auto* result = static_cast(nullptr); + + try { + auto* model_state = rapids::get_model_state(*instance); + auto* instance_state = + rapids::get_instance_state(*instance); + auto& model = instance_state->get_model(); + auto max_batch_size = model.get_config_param("max_batch_size"); + auto batch = Batch{raw_requests, + request_count, + model_state->TritonMemoryManager(), + model_state->EnablePinnedInput(), + model_state->EnablePinnedOutput(), + max_batch_size, + model.get_stream()}; + + model.predict(batch); + batch.finalize(); + } catch (rapids::TritonException& err) { + result = err.error(); + } + + return result; } } // extern "C" diff --git a/src/interface/model_instance_state.h b/src/interface/model_instance_state.h index b6f6dd4..19c69cc 100644 --- a/src/interface/model_instance_state.h +++ b/src/interface/model_instance_state.h @@ -32,7 +32,9 @@ struct ModelInstanceState : public BackendModelInstance { : BackendModelInstance(&model_state, triton_model_instance), model_(model_state.get_shared_state(), rapids::get_device_id(*triton_model_instance), CudaStream(), - Kind()) {} + Kind(), + JoinPath({RepositoryPath(), std::to_string(Version()), + ArtifactFilename()})) {} auto& get_model() const { return model_; } From 27e9b8a4e1cdf47ec3fa5b1ed537f64fea5c3934 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 26 Aug 2021 13:59:19 -0400 Subject: [PATCH 012/199] Set up CMake infrastructure --- cpp/CMakeLists.txt | 176 ++++++++++++++++++ cpp/cmake/doxygen.cmake | 33 ++++ cpp/cmake/modules/ConfigureCUDA.cmake | 43 +++++ cpp/cmake/thirdparty/get_gtest.cmake | 43 +++++ cpp/cmake/thirdparty/get_raft.cmake | 48 +++++ cpp/cmake/thirdparty/get_rmm.cmake | 47 +++++ cpp/cmake/thirdparty/get_thrust.cmake | 30 +++ .../include}/rapids_triton/batch/batch.hpp | 0 .../include}/rapids_triton/build_control.hpp | 0 .../include}/rapids_triton/exceptions.hpp | 0 .../include}/rapids_triton/memory/buffer.hpp | 0 .../rapids_triton/memory/detail/allocate.hpp | 0 .../rapids_triton/memory/detail/copy.hpp | 0 .../include}/rapids_triton/memory/types.hpp | 0 .../include}/rapids_triton/model/model.hpp | 0 .../rapids_triton/model/shared_state.hpp | 0 .../include}/rapids_triton/tensor/dtype.hpp | 0 .../include}/rapids_triton/tensor/tensor.hpp | 0 .../include}/rapids_triton/triton/backend.hpp | 0 .../include}/rapids_triton/triton/config.hpp | 0 .../rapids_triton/triton/deployment.hpp | 0 .../include}/rapids_triton/triton/device.hpp | 0 .../include}/rapids_triton/triton/input.hpp | 0 .../include}/rapids_triton/triton/logging.hpp | 0 .../include}/rapids_triton/triton/model.hpp | 0 .../rapids_triton/triton/model_instance.hpp | 0 .../rapids_triton/triton/requests.hpp | 0 .../rapids_triton/triton/responses.hpp | 0 .../include}/rapids_triton/utils/narrow.hpp | 0 {src => cpp/src}/impl/model.h | 0 {src => cpp/src}/impl/names.h | 0 {src => cpp/src}/impl/shared_state.h | 0 {src => cpp/src}/interface/api.cc | 0 .../src}/interface/model_instance_state.h | 0 {src => cpp/src}/interface/model_state.h | 0 cpp/test/CMakeLists.txt | 50 +++++ 36 files changed, 470 insertions(+) create mode 100644 cpp/CMakeLists.txt create mode 100644 cpp/cmake/doxygen.cmake create mode 100644 cpp/cmake/modules/ConfigureCUDA.cmake create mode 100644 cpp/cmake/thirdparty/get_gtest.cmake create mode 100644 cpp/cmake/thirdparty/get_raft.cmake create mode 100644 cpp/cmake/thirdparty/get_rmm.cmake create mode 100644 cpp/cmake/thirdparty/get_thrust.cmake rename {include => cpp/include}/rapids_triton/batch/batch.hpp (100%) rename {include => cpp/include}/rapids_triton/build_control.hpp (100%) rename {include => cpp/include}/rapids_triton/exceptions.hpp (100%) rename {include => cpp/include}/rapids_triton/memory/buffer.hpp (100%) rename {include => cpp/include}/rapids_triton/memory/detail/allocate.hpp (100%) rename {include => cpp/include}/rapids_triton/memory/detail/copy.hpp (100%) rename {include => cpp/include}/rapids_triton/memory/types.hpp (100%) rename {include => cpp/include}/rapids_triton/model/model.hpp (100%) rename {include => cpp/include}/rapids_triton/model/shared_state.hpp (100%) rename {include => cpp/include}/rapids_triton/tensor/dtype.hpp (100%) rename {include => cpp/include}/rapids_triton/tensor/tensor.hpp (100%) rename {include => cpp/include}/rapids_triton/triton/backend.hpp (100%) rename {include => cpp/include}/rapids_triton/triton/config.hpp (100%) rename {include => cpp/include}/rapids_triton/triton/deployment.hpp (100%) rename {include => cpp/include}/rapids_triton/triton/device.hpp (100%) rename {include => cpp/include}/rapids_triton/triton/input.hpp (100%) rename {include => cpp/include}/rapids_triton/triton/logging.hpp (100%) rename {include => cpp/include}/rapids_triton/triton/model.hpp (100%) rename {include => cpp/include}/rapids_triton/triton/model_instance.hpp (100%) rename {include => cpp/include}/rapids_triton/triton/requests.hpp (100%) rename {include => cpp/include}/rapids_triton/triton/responses.hpp (100%) rename {include => cpp/include}/rapids_triton/utils/narrow.hpp (100%) rename {src => cpp/src}/impl/model.h (100%) rename {src => cpp/src}/impl/names.h (100%) rename {src => cpp/src}/impl/shared_state.h (100%) rename {src => cpp/src}/interface/api.cc (100%) rename {src => cpp/src}/interface/model_instance_state.h (100%) rename {src => cpp/src}/interface/model_state.h (100%) create mode 100644 cpp/test/CMakeLists.txt diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt new file mode 100644 index 0000000..6b65328 --- /dev/null +++ b/cpp/CMakeLists.txt @@ -0,0 +1,176 @@ +#============================================================================= +# Copyright (c) 2020-2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +cmake_minimum_required(VERSION 3.20.1 FATAL_ERROR) +file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-21.10/RAPIDS.cmake + ${CMAKE_BINARY_DIR}/RAPIDS.cmake) +include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) +include(rapids-cmake) +include(rapids-cpm) +include(rapids-cuda) +include(rapids-export) +include(rapids-find) + +rapids_cuda_init_architectures(RAPIDS_TRITON) + +project(RAPIDS_TRITON VERSION 21.10.00 LANGUAGES CXX CUDA) + +############################################################################## +# - build type --------------------------------------------------------------- + +# Set a default build type if none was specified +rapids_cmake_build_type(Release) + +# this is needed for clang-tidy runs +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +############################################################################## +# - User Options ------------------------------------------------------------ + +option(BUILD_TESTS "Build rapids_triton unit-tests" ON) +option(CUDA_ENABLE_KERNELINFO "Enable kernel resource usage info" OFF) +option(CUDA_ENABLE_LINEINFO "Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler)" OFF) +option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF) +option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" ON) +option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) +option(NVTX "Enable nvtx markers" OFF) + +message(VERBOSE "RAPIDS_TRITON: Build RAPIDS_TRITON unit-tests: ${BUILD_TESTS}") +message(VERBOSE "RAPIDS_TRITON: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") +message(VERBOSE "RAPIDS_TRITON: Disable depreaction warnings " ${DISABLE_DEPRECATION_WARNINGS}) +message(VERBOSE "RAPIDS_TRITON: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELINFO}") +message(VERBOSE "RAPIDS_TRITON: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") +message(VERBOSE "RAPIDS_TRITON: Enable nvtx markers: ${NVTX}") +message(VERBOSE "RAPIDS_TRITON: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") + +# Set RMM logging level +set(RMM_LOGGING_LEVEL "INFO" CACHE STRING "Choose the logging level.") +set_property(CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF") +message(VERBOSE "RAPIDS_TRITON: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") + +############################################################################## +# - Conda environment detection ---------------------------------------------- + +if(DETECT_CONDA_ENV) + rapids_cmake_support_conda_env( conda_env MODIFY_PREFIX_PATH ) + if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED ENV{CONDA_PREFIX}) + message(STATUS "RAPIDS_TRITON: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}") + set(CMAKE_INSTALL_PREFIX "$ENV{CONDA_PREFIX}") + endif() +endif() + +############################################################################## +# - compiler options --------------------------------------------------------- + +# * find CUDAToolkit package +# * determine GPU architectures +# * enable the CMake CUDA language +# * set other CUDA compilation flags +rapids_find_package(CUDAToolkit REQUIRED + BUILD_EXPORT_SET rapids_triton-exports + INSTALL_EXPORT_SET rapids_triton-exports + ) +include(cmake/modules/ConfigureCUDA.cmake) + +############################################################################## +# - Requirements ------------------------------------------------------------- + +# add third party dependencies using CPM +rapids_cpm_init() + +include(cmake/thirdparty/get_thrust.cmake) +include(cmake/thirdparty/get_rmm.cmake) +include(cmake/thirdparty/get_raft.cmake) + +if(BUILD_TESTS) + include(cmake/thirdparty/get_gtest.cmake) +endif() + +############################################################################## +# - install targets----------------------------------------------------------- + +add_library(rapids_triton INTERFACE) +add_library(rapids_triton::rapids_triton ALIAS rapids_triton) +target_include_directories(rapids_triton INTERFACE "$" + "$") + +target_link_libraries(rapids_triton +INTERFACE + rmm::rmm + raft::raft + ) + +target_compile_features(rapids_triton INTERFACE cxx_std_17 $) + +install(TARGETS rapids_triton + DESTINATION lib + EXPORT rapids_triton-exports + ) + +include(GNUInstallDirs) +install(DIRECTORY include/rapids_triton/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rapids_triton + ) + +# Temporary install of rapids_triton.hpp while the file is removed +install(FILES include/rapids_triton.hpp + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rapids_triton + ) + +############################################################################## +# - install export ----------------------------------------------------------- +set(doc_string +[=[ +Provide targets for RAPIDS_TRITON. + +RAPIDS_TRITON is a header-only library designed to make it easier and faster +to integrate RAPIDS algorithms as Triton backends. + +]=]) + + rapids_export(INSTALL rapids_triton + EXPORT_SET rapids_triton-exports + GLOBAL_TARGETS rapids_triton # since we can't hook into EXPORT SETS + NAMESPACE rapids_triton:: + DOCUMENTATION doc_string + ) + +############################################################################## +# - build export ------------------------------------------------------------- + +rapids_export(BUILD rapids_triton + EXPORT_SET rapids_triton-exports + GLOBAL_TARGETS rapids_triton # since we can't hook into EXPORT SETS + LANGUAGES CUDA + DOCUMENTATION doc_string + NAMESPACE rapids_triton:: + ) + +############################################################################## +# - build test executable ---------------------------------------------------- + +if(BUILD_TESTS) + include(test/CMakeLists.txt) +endif() + +############################################################################## +# - doxygen targets ---------------------------------------------------------- + +# TODO(wphicks) +# include(cmake/doxygen.cmake) +# add_doxygen_target(IN_DOXYFILE Doxyfile.in +# OUT_DOXYFILE ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile +# CWD ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/cpp/cmake/doxygen.cmake b/cpp/cmake/doxygen.cmake new file mode 100644 index 0000000..061981f --- /dev/null +++ b/cpp/cmake/doxygen.cmake @@ -0,0 +1,33 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +find_package(Doxygen 1.8.11) + +function(add_doxygen_target) + if(Doxygen_FOUND) + set(options "") + set(oneValueArgs IN_DOXYFILE OUT_DOXYFILE CWD) + set(multiValueArgs "") + cmake_parse_arguments(dox "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + configure_file(${dox_IN_DOXYFILE} ${dox_OUT_DOXYFILE} @ONLY) + add_custom_target(doc + ${DOXYGEN_EXECUTABLE} ${dox_OUT_DOXYFILE} + WORKING_DIRECTORY ${dox_CWD} + VERBATIM + COMMENT "Generate doxygen docs") + else() + message("add_doxygen_target: doxygen exe not found") + endif() +endfunction(add_doxygen_target) diff --git a/cpp/cmake/modules/ConfigureCUDA.cmake b/cpp/cmake/modules/ConfigureCUDA.cmake new file mode 100644 index 0000000..f170102 --- /dev/null +++ b/cpp/cmake/modules/ConfigureCUDA.cmake @@ -0,0 +1,43 @@ +#============================================================================= +# Copyright (c) 2018-2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +if(DISABLE_DEPRECATION_WARNINGS) + list(APPEND RAPIDS_TRITON_CXX_FLAGS -Wno-deprecated-declarations) + list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Xcompiler=-Wno-deprecated-declarations) +endif() + +if(CMAKE_COMPILER_IS_GNUCXX) + list(APPEND RAPIDS_TRITON_CXX_FLAGS -Wall -Werror -Wno-unknown-pragmas -Wno-error=deprecated-declarations) +endif() + +list(APPEND RAPIDS_TRITON_CUDA_FLAGS --expt-extended-lambda --expt-relaxed-constexpr) + +# set warnings as errors +if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 11.2.0) + list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Werror=all-warnings) +endif() +list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Xcompiler=-Wall,-Werror,-Wno-error=deprecated-declarations) + +# Option to enable line info in CUDA device compilation to allow introspection when profiling / memchecking +if(CUDA_ENABLE_LINEINFO) + list(APPEND RAPIDS_TRITON_CUDA_FLAGS -lineinfo) +endif() + +# Debug options +if(CMAKE_BUILD_TYPE MATCHES Debug) + message(VERBOSE "RAPIDS_TRITON: Building with debugging flags") + list(APPEND RAPIDS_TRITON_CUDA_FLAGS -G -Xcompiler=-rdynamic) +endif() diff --git a/cpp/cmake/thirdparty/get_gtest.cmake b/cpp/cmake/thirdparty/get_gtest.cmake new file mode 100644 index 0000000..4cd11da --- /dev/null +++ b/cpp/cmake/thirdparty/get_gtest.cmake @@ -0,0 +1,43 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_gtest VERSION) + + if(TARGET GTest::gtest) + return() + endif() + + rapids_cpm_find(GTest ${VERSION} + GLOBAL_TARGETS gest gtest_main GTest::gtest GTest::gtest_main + CPM_ARGS + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG release-${VERSION} + GIT_SHALLOW TRUE + OPTIONS "INSTALL_GTEST OFF" + # googletest >= 1.10.0 provides a cmake config file -- use it if it exists + FIND_PACKAGE_ARGUMENTS "CONFIG" + ) + + if(NOT TARGET GTest::gtest) + add_library(GTest::gtest ALIAS gtest) + add_library(GTest::gtest_main ALIAS gtest_main) + endif() + +endfunction() + +set(RAFT_MIN_VERSION_gtest 1.10.0) + +find_and_configure_gtest(${RAFT_MIN_VERSION_gtest}) diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake new file mode 100644 index 0000000..1dc0876 --- /dev/null +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -0,0 +1,48 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_raft) + + set(oneValueArgs VERSION FORK PINNED_TAG) + cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN} ) + + rapids_cpm_find(raft ${PKG_VERSION} + GLOBAL_TARGETS raft::raft + BUILD_EXPORT_SET rapids_triton-exports + INSTALL_EXPORT_SET rapids_triton-exports + CPM_ARGS + GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git + GIT_TAG ${PKG_PINNED_TAG} + SOURCE_SUBDIR cpp + OPTIONS + "BUILD_TESTS OFF" + ) + + message(VERBOSE "RAPIDS_TRITON: Using RAFT located in ${raft_SOURCE_DIR}") + +endfunction() + +set(RAPIDS_TRITON_MIN_VERSION_raft "${RAPIDS_TRITON_VERSION_MAJOR}.${RAPIDS_TRITON_VERSION_MINOR}.00") +set(RAPIDS_TRITON_BRANCH_VERSION_raft "${RAPIDS_TRITON_VERSION_MAJOR}.${RAPIDS_TRITON_VERSION_MINOR}") + +# Change pinned tag here to test a commit in CI +# To use a different RAFT locally, set the CMake variable +# CPM_raft_SOURCE=/path/to/local/raft +find_and_configure_raft(VERSION ${RAPIDS_TRITON_MIN_VERSION_raft} + FORK rapidsai + PINNED_TAG branch-${RAPIDS_TRITON_BRANCH_VERSION_raft} + ) diff --git a/cpp/cmake/thirdparty/get_rmm.cmake b/cpp/cmake/thirdparty/get_rmm.cmake new file mode 100644 index 0000000..b4a6782 --- /dev/null +++ b/cpp/cmake/thirdparty/get_rmm.cmake @@ -0,0 +1,47 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_rmm VERSION) + + if(TARGET rmm::rmm) + return() + endif() + + if(${VERSION} MATCHES [=[([0-9]+)\.([0-9]+)\.([0-9]+)]=]) + set(MAJOR_AND_MINOR "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}") + else() + set(MAJOR_AND_MINOR "${VERSION}") + endif() + + rapids_cpm_find(rmm ${VERSION} + GLOBAL_TARGETS rmm::rmm + BUILD_EXPORT_SET rapids_triton-exports + INSTALL_EXPORT_SET rapids_triton-exports + CPM_ARGS + GIT_REPOSITORY https://github.com/rapidsai/rmm.git + GIT_TAG 23bbe745af1d988224b5498f7b8e3fe3720532d4 + GIT_SHALLOW FALSE + OPTIONS "BUILD_TESTS OFF" + "BUILD_BENCHMARKS OFF" + "CUDA_STATIC_RUNTIME ${CUDA_STATIC_RUNTIME}" + "DISABLE_DEPRECATION_WARNING ${DISABLE_DEPRECATION_WARNING}" + ) + +endfunction() + +set(RAPIDS_TRITON_MIN_VERSION_rmm "${RAPIDS_TRITON_VERSION_MAJOR}.${RAPIDS_TRITON_VERSION_MINOR}.00") + +find_and_configure_rmm(${RAPIDS_TRITON_MIN_VERSION_rmm}) diff --git a/cpp/cmake/thirdparty/get_thrust.cmake b/cpp/cmake/thirdparty/get_thrust.cmake new file mode 100644 index 0000000..3764192 --- /dev/null +++ b/cpp/cmake/thirdparty/get_thrust.cmake @@ -0,0 +1,30 @@ +# ============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +# Use CPM to find or clone thrust +function(find_and_configure_thrust VERSION) + + rapids_cpm_find( + Thrust ${VERSION} + BUILD_EXPORT_SET rapids_triton-exports + INSTALL_EXPORT_SET rapids_triton-exports + CPM_ARGS + GIT_REPOSITORY https://github.com/NVIDIA/thrust.git + GIT_TAG ${VERSION} + GIT_SHALLOW TRUE + OPTIONS "THRUST_INSTALL OFF") + +endfunction() + +find_and_configure_thrust(1.12.0) diff --git a/include/rapids_triton/batch/batch.hpp b/cpp/include/rapids_triton/batch/batch.hpp similarity index 100% rename from include/rapids_triton/batch/batch.hpp rename to cpp/include/rapids_triton/batch/batch.hpp diff --git a/include/rapids_triton/build_control.hpp b/cpp/include/rapids_triton/build_control.hpp similarity index 100% rename from include/rapids_triton/build_control.hpp rename to cpp/include/rapids_triton/build_control.hpp diff --git a/include/rapids_triton/exceptions.hpp b/cpp/include/rapids_triton/exceptions.hpp similarity index 100% rename from include/rapids_triton/exceptions.hpp rename to cpp/include/rapids_triton/exceptions.hpp diff --git a/include/rapids_triton/memory/buffer.hpp b/cpp/include/rapids_triton/memory/buffer.hpp similarity index 100% rename from include/rapids_triton/memory/buffer.hpp rename to cpp/include/rapids_triton/memory/buffer.hpp diff --git a/include/rapids_triton/memory/detail/allocate.hpp b/cpp/include/rapids_triton/memory/detail/allocate.hpp similarity index 100% rename from include/rapids_triton/memory/detail/allocate.hpp rename to cpp/include/rapids_triton/memory/detail/allocate.hpp diff --git a/include/rapids_triton/memory/detail/copy.hpp b/cpp/include/rapids_triton/memory/detail/copy.hpp similarity index 100% rename from include/rapids_triton/memory/detail/copy.hpp rename to cpp/include/rapids_triton/memory/detail/copy.hpp diff --git a/include/rapids_triton/memory/types.hpp b/cpp/include/rapids_triton/memory/types.hpp similarity index 100% rename from include/rapids_triton/memory/types.hpp rename to cpp/include/rapids_triton/memory/types.hpp diff --git a/include/rapids_triton/model/model.hpp b/cpp/include/rapids_triton/model/model.hpp similarity index 100% rename from include/rapids_triton/model/model.hpp rename to cpp/include/rapids_triton/model/model.hpp diff --git a/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp similarity index 100% rename from include/rapids_triton/model/shared_state.hpp rename to cpp/include/rapids_triton/model/shared_state.hpp diff --git a/include/rapids_triton/tensor/dtype.hpp b/cpp/include/rapids_triton/tensor/dtype.hpp similarity index 100% rename from include/rapids_triton/tensor/dtype.hpp rename to cpp/include/rapids_triton/tensor/dtype.hpp diff --git a/include/rapids_triton/tensor/tensor.hpp b/cpp/include/rapids_triton/tensor/tensor.hpp similarity index 100% rename from include/rapids_triton/tensor/tensor.hpp rename to cpp/include/rapids_triton/tensor/tensor.hpp diff --git a/include/rapids_triton/triton/backend.hpp b/cpp/include/rapids_triton/triton/backend.hpp similarity index 100% rename from include/rapids_triton/triton/backend.hpp rename to cpp/include/rapids_triton/triton/backend.hpp diff --git a/include/rapids_triton/triton/config.hpp b/cpp/include/rapids_triton/triton/config.hpp similarity index 100% rename from include/rapids_triton/triton/config.hpp rename to cpp/include/rapids_triton/triton/config.hpp diff --git a/include/rapids_triton/triton/deployment.hpp b/cpp/include/rapids_triton/triton/deployment.hpp similarity index 100% rename from include/rapids_triton/triton/deployment.hpp rename to cpp/include/rapids_triton/triton/deployment.hpp diff --git a/include/rapids_triton/triton/device.hpp b/cpp/include/rapids_triton/triton/device.hpp similarity index 100% rename from include/rapids_triton/triton/device.hpp rename to cpp/include/rapids_triton/triton/device.hpp diff --git a/include/rapids_triton/triton/input.hpp b/cpp/include/rapids_triton/triton/input.hpp similarity index 100% rename from include/rapids_triton/triton/input.hpp rename to cpp/include/rapids_triton/triton/input.hpp diff --git a/include/rapids_triton/triton/logging.hpp b/cpp/include/rapids_triton/triton/logging.hpp similarity index 100% rename from include/rapids_triton/triton/logging.hpp rename to cpp/include/rapids_triton/triton/logging.hpp diff --git a/include/rapids_triton/triton/model.hpp b/cpp/include/rapids_triton/triton/model.hpp similarity index 100% rename from include/rapids_triton/triton/model.hpp rename to cpp/include/rapids_triton/triton/model.hpp diff --git a/include/rapids_triton/triton/model_instance.hpp b/cpp/include/rapids_triton/triton/model_instance.hpp similarity index 100% rename from include/rapids_triton/triton/model_instance.hpp rename to cpp/include/rapids_triton/triton/model_instance.hpp diff --git a/include/rapids_triton/triton/requests.hpp b/cpp/include/rapids_triton/triton/requests.hpp similarity index 100% rename from include/rapids_triton/triton/requests.hpp rename to cpp/include/rapids_triton/triton/requests.hpp diff --git a/include/rapids_triton/triton/responses.hpp b/cpp/include/rapids_triton/triton/responses.hpp similarity index 100% rename from include/rapids_triton/triton/responses.hpp rename to cpp/include/rapids_triton/triton/responses.hpp diff --git a/include/rapids_triton/utils/narrow.hpp b/cpp/include/rapids_triton/utils/narrow.hpp similarity index 100% rename from include/rapids_triton/utils/narrow.hpp rename to cpp/include/rapids_triton/utils/narrow.hpp diff --git a/src/impl/model.h b/cpp/src/impl/model.h similarity index 100% rename from src/impl/model.h rename to cpp/src/impl/model.h diff --git a/src/impl/names.h b/cpp/src/impl/names.h similarity index 100% rename from src/impl/names.h rename to cpp/src/impl/names.h diff --git a/src/impl/shared_state.h b/cpp/src/impl/shared_state.h similarity index 100% rename from src/impl/shared_state.h rename to cpp/src/impl/shared_state.h diff --git a/src/interface/api.cc b/cpp/src/interface/api.cc similarity index 100% rename from src/interface/api.cc rename to cpp/src/interface/api.cc diff --git a/src/interface/model_instance_state.h b/cpp/src/interface/model_instance_state.h similarity index 100% rename from src/interface/model_instance_state.h rename to cpp/src/interface/model_instance_state.h diff --git a/src/interface/model_state.h b/cpp/src/interface/model_state.h similarity index 100% rename from src/interface/model_state.h rename to cpp/src/interface/model_state.h diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt new file mode 100644 index 0000000..5a6315a --- /dev/null +++ b/cpp/test/CMakeLists.txt @@ -0,0 +1,50 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +# keep the files in alphabetical order! +add_executable(test_rapids_triton +) + +set_target_properties(test_rapids_triton +PROPERTIES BUILD_RPATH "\$ORIGIN" + # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON +) + +target_compile_options(test_rapids_triton + PRIVATE "$<$:${RAPIDS_TRITON_CXX_FLAGS}>" + "$<$:${RAPIDS_TRITON_CUDA_FLAGS}>" +) + +target_include_directories(test_rapids_triton + PUBLIC "$" + "$" +) + + +target_link_libraries(test_rapids_triton +PRIVATE + rmm::rmm + raft::raft + GTest::gtest + GTest::gtest_main + $ +) From 4f29cc741a354b4eca5bf62820f6e6920b745b69 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 26 Aug 2021 14:41:17 -0400 Subject: [PATCH 013/199] Add basic test --- .gitignore | 1 + .../rapids_triton_dev_cuda11.2.yml | 8 +++++ cpp/CMakeLists.txt | 2 ++ cpp/include/rapids_triton.hpp | 29 +++++++++++++++++++ cpp/test/CMakeLists.txt | 1 + cpp/test/test.cpp | 25 ++++++++++++++++ 6 files changed, 66 insertions(+) create mode 100644 .gitignore create mode 100644 conda/environments/rapids_triton_dev_cuda11.2.yml create mode 100644 cpp/include/rapids_triton.hpp create mode 100644 cpp/test/test.cpp diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a2d3658 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +cpp/build diff --git a/conda/environments/rapids_triton_dev_cuda11.2.yml b/conda/environments/rapids_triton_dev_cuda11.2.yml new file mode 100644 index 0000000..aa99579 --- /dev/null +++ b/conda/environments/rapids_triton_dev_cuda11.2.yml @@ -0,0 +1,8 @@ +--- +name: rapids_triton_dev +channels: + - nvidia + - conda-forge +dependencies: + - cudatoolkit=11.4 + - cmake=3.20.1 diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 6b65328..45c4955 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -47,6 +47,8 @@ option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF) option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" ON) option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(NVTX "Enable nvtx markers" OFF) +option(TRITON_ENABLE_GPU "Enable GPU support in Triton" ON) +option(TRITON_ENABLE_STATS "Enable statistics collection in Triton" ON) message(VERBOSE "RAPIDS_TRITON: Build RAPIDS_TRITON unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAPIDS_TRITON: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") diff --git a/cpp/include/rapids_triton.hpp b/cpp/include/rapids_triton.hpp new file mode 100644 index 0000000..de73942 --- /dev/null +++ b/cpp/include/rapids_triton.hpp @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { namespace backend { namespace rapids { + +/* Function for testing rapids_triton include + * + * @return message indicating rapids_triton has been included succesfully*/ +inline auto test_install() { + return std::string("rapids_triton set up successfully"); +} + +}}} diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 5a6315a..59e3280 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -16,6 +16,7 @@ # keep the files in alphabetical order! add_executable(test_rapids_triton + test/test.cpp ) set_target_properties(test_rapids_triton diff --git a/cpp/test/test.cpp b/cpp/test/test.cpp new file mode 100644 index 0000000..26e60e3 --- /dev/null +++ b/cpp/test/test.cpp @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +namespace triton { namespace backend { namespace rapids { + +TEST(RapidsTriton, installed) { std::cout << test_install() << "\n"; } +}}} + From 6581f456ca90962ae5333c53e9221ca9d2a0afcf Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 26 Aug 2021 14:47:31 -0400 Subject: [PATCH 014/199] Add build control test --- cpp/test/CMakeLists.txt | 1 + cpp/test/build_control.cpp | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 cpp/test/build_control.cpp diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 59e3280..8e857a8 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -16,6 +16,7 @@ # keep the files in alphabetical order! add_executable(test_rapids_triton + test/build_control.cpp test/test.cpp ) diff --git a/cpp/test/build_control.cpp b/cpp/test/build_control.cpp new file mode 100644 index 0000000..4696fad --- /dev/null +++ b/cpp/test/build_control.cpp @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +namespace triton { +namespace backend { +namespace rapids { + +TEST(RapidsTriton, build_control) { +#ifdef TRITON_ENABLE_GPU + ASSERT_EQ(IS_GPU_BUILD, true) << "IS_GPU_BUILD constant has wrong value\n"; +#else + ASSERT_EQ(IS_GPU_BUILD, false) << "IS_GPU_BUILD constant has wrong value\n"; +#endif +} +} // namespace rapids +} // namespace backend +} // namespace triton From 347fb1d36c41d382520740ad66e2332956bfb395 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 26 Aug 2021 18:57:54 -0400 Subject: [PATCH 015/199] Bring in Triton dependencies --- cpp/CMakeLists.txt | 7 +++++ cpp/cmake/thirdparty/get_rapidjson.cmake | 37 ++++++++++++++++++++++++ cpp/cmake/thirdparty/get_triton.cmake | 36 +++++++++++++++++++++++ cpp/include/rapids_triton/exceptions.hpp | 16 +++++----- cpp/test/CMakeLists.txt | 3 ++ cpp/test/exceptions.cpp | 27 +++++++++++++++++ 6 files changed, 118 insertions(+), 8 deletions(-) create mode 100644 cpp/cmake/thirdparty/get_rapidjson.cmake create mode 100644 cpp/cmake/thirdparty/get_triton.cmake create mode 100644 cpp/test/exceptions.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 45c4955..8ee3748 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -49,6 +49,9 @@ option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(NVTX "Enable nvtx markers" OFF) option(TRITON_ENABLE_GPU "Enable GPU support in Triton" ON) option(TRITON_ENABLE_STATS "Enable statistics collection in Triton" ON) +set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo") +set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo") +set(TRITON_BACKEND_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/backend repo") message(VERBOSE "RAPIDS_TRITON: Build RAPIDS_TRITON unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAPIDS_TRITON: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") @@ -96,6 +99,8 @@ rapids_cpm_init() include(cmake/thirdparty/get_thrust.cmake) include(cmake/thirdparty/get_rmm.cmake) include(cmake/thirdparty/get_raft.cmake) +include(cmake/thirdparty/get_rapidjson.cmake) +include(cmake/thirdparty/get_triton.cmake) if(BUILD_TESTS) include(cmake/thirdparty/get_gtest.cmake) @@ -113,6 +118,8 @@ target_link_libraries(rapids_triton INTERFACE rmm::rmm raft::raft + triton-core-serverstub + triton-backend-utils ) target_compile_features(rapids_triton INTERFACE cxx_std_17 $) diff --git a/cpp/cmake/thirdparty/get_rapidjson.cmake b/cpp/cmake/thirdparty/get_rapidjson.cmake new file mode 100644 index 0000000..e9051c6 --- /dev/null +++ b/cpp/cmake/thirdparty/get_rapidjson.cmake @@ -0,0 +1,37 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +# TODO(wphicks): Pass in version +function(find_and_configure_rapidjson VERSION) + + rapids_cpm_find(rapidjson ${VERSION} + GLOBAL_TARGETS rapidjson::rapidjson + BUILD_EXPORT_SET rapids_triton-exports + INSTALL_EXPORT_SET rapids_triton-exports + CPM_ARGS + GIT_REPOSITORY https://github.com/Tencent/rapidjson + GIT_TAG "v${VERSION}" + GIT_SHALLOW ON + OPTIONS + "RAPIDJSON_BUILD_DOC OFF" + "RAPIDJSON_BUILD_EXAMPLES OFF" + "RAPIDJSON_BUILD_TESTS OFF" + "RAPIDJSON_BUILD_THIRDPARTY_GTEST OFF" + ) + +endfunction() + +find_and_configure_rapidjson("1.1.0") diff --git a/cpp/cmake/thirdparty/get_triton.cmake b/cpp/cmake/thirdparty/get_triton.cmake new file mode 100644 index 0000000..e5353ec --- /dev/null +++ b/cpp/cmake/thirdparty/get_triton.cmake @@ -0,0 +1,36 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= +include(FetchContent) + +FetchContent_Declare( + repo-common + GIT_REPOSITORY https://github.com/triton-inference-server/common.git + GIT_TAG ${TRITON_COMMON_REPO_TAG} + GIT_SHALLOW ON +) +FetchContent_Declare( + repo-core + GIT_REPOSITORY https://github.com/triton-inference-server/core.git + GIT_TAG ${TRITON_CORE_REPO_TAG} + GIT_SHALLOW ON +) +FetchContent_Declare( + repo-backend + GIT_REPOSITORY https://github.com/triton-inference-server/backend.git + GIT_TAG ${TRITON_BACKEND_REPO_TAG} + GIT_SHALLOW ON +) +FetchContent_MakeAvailable(repo-common repo-core repo-backend) diff --git a/cpp/include/rapids_triton/exceptions.hpp b/cpp/include/rapids_triton/exceptions.hpp index 122c8e7..4ad55db 100644 --- a/cpp/include/rapids_triton/exceptions.hpp +++ b/cpp/include/rapids_triton/exceptions.hpp @@ -25,13 +25,13 @@ namespace triton { namespace backend { namespace rapids { using ErrorCode = TRITONSERVER_Error_Code; namespace Error { - using Unknown = ErrorCode::TRITONSERVER_ERROR_UNKNOWN; - using Internal = ErrorCode::TRITONSERVER_ERROR_INTERNAL; - using NotFound = ErrorCode::TRITONSERVER_ERROR_NOT_FOUND; - using InvalidArg = ErrorCode::TRITONSERVER_ERROR_INVALID_ARG; - using Unavailable = ErrorCode::TRITONSERVER_ERROR_UNAVAILABLE; - using Unsupported = ErrorCode::TRITONSERVER_ERROR_UNSUPPORTED; - using AlreadyExists = ErrorCode::TRITONSERVER_ERROR_ALREADY_EXISTS; + auto constexpr Unknown = ErrorCode::TRITONSERVER_ERROR_UNKNOWN; + auto constexpr Internal = ErrorCode::TRITONSERVER_ERROR_INTERNAL; + auto constexpr NotFound = ErrorCode::TRITONSERVER_ERROR_NOT_FOUND; + auto constexpr InvalidArg = ErrorCode::TRITONSERVER_ERROR_INVALID_ARG; + auto constexpr Unavailable = ErrorCode::TRITONSERVER_ERROR_UNAVAILABLE; + auto constexpr Unsupported = ErrorCode::TRITONSERVER_ERROR_UNSUPPORTED; + auto constexpr AlreadyExists = ErrorCode::TRITONSERVER_ERROR_ALREADY_EXISTS; } /** @@ -65,7 +65,7 @@ struct TritonException : std::exception { TritonException(TRITONSERVER_Error* prev_error) : error_(prev_error) {} - auto* what() const noexcept + virtual char const* what() const noexcept { return TRITONSERVER_ErrorMessage(error_); } diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 8e857a8..5453ade 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -17,6 +17,7 @@ # keep the files in alphabetical order! add_executable(test_rapids_triton test/build_control.cpp + test/exceptions.cpp test/test.cpp ) @@ -46,6 +47,8 @@ target_link_libraries(test_rapids_triton PRIVATE rmm::rmm raft::raft + triton-core-serverstub + triton-backend-utils GTest::gtest GTest::gtest_main $ diff --git a/cpp/test/exceptions.cpp b/cpp/test/exceptions.cpp new file mode 100644 index 0000000..7f9e5f1 --- /dev/null +++ b/cpp/test/exceptions.cpp @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +namespace triton { +namespace backend { +namespace rapids { + +} // namespace rapids +} // namespace backend +} // namespace triton From 58ab75793c7ff5f811396fb446f83dad32160bcd Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 26 Aug 2021 20:01:18 -0400 Subject: [PATCH 016/199] Add tests for exceptions --- .dockerignore | 1 + Dockerfile | 53 +++++++++++++++++++ ...1.2.yml => rapids_triton_dev_cuda11.4.yml} | 4 +- cpp/include/rapids_triton/exceptions.hpp | 2 +- cpp/test/CMakeLists.txt | 7 +++ cpp/test/exceptions.cpp | 48 +++++++++++++++++ 6 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile rename conda/environments/{rapids_triton_dev_cuda11.2.yml => rapids_triton_dev_cuda11.4.yml} (82%) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a2d3658 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +cpp/build diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3b9f11f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,53 @@ +ARG TRITON_VERSION=21.07 +ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 + +FROM ${BASE_IMAGE} as base + +ENV PATH="/root/miniconda3/bin:${PATH}" + +RUN apt-get update \ + && apt-get install --no-install-recommends -y wget patchelf \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +ENV PYTHONDONTWRITEBYTECODE=true + +RUN wget \ + https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && mkdir /root/.conda \ + && bash Miniconda3-latest-Linux-x86_64.sh -b \ + && rm -f Miniconda3-latest-Linux-x86_64.sh + +COPY ./conda/environments/rapids_triton_dev_cuda11.4.yml /environment.yml + +RUN conda env update -f /environment.yml \ + && rm /environment.yml \ + && conda clean -afy \ + && find /root/miniconda3/ -follow -type f -name '*.pyc' -delete \ + && find /root/miniconda3/ -follow -type f -name '*.js.map' -delete + +ENV PYTHONDONTWRITEBYTECODE=false + +COPY ./cpp /rapids_triton + +WORKDIR /rapids_triton + +SHELL ["conda", "run", "--no-capture-output", "-n", "rapids_triton_dev", "/bin/bash", "-c"] + +FROM base as build-stage + +ARG TRITON_VERSION +ENV TRITON_VERSION=$TRITON_VERSION + +ARG BUILD_TYPE=Release +ENV BUILD_TYPE=$BUILD_TYPE + +RUN mkdir /rapids_triton/build + +WORKDIR /rapids_triton/build + +RUN cmake -GNinja .. + +RUN ninja + +CMD ["/rapids_triton/build/test_rapids_triton"] diff --git a/conda/environments/rapids_triton_dev_cuda11.2.yml b/conda/environments/rapids_triton_dev_cuda11.4.yml similarity index 82% rename from conda/environments/rapids_triton_dev_cuda11.2.yml rename to conda/environments/rapids_triton_dev_cuda11.4.yml index aa99579..18d47e6 100644 --- a/conda/environments/rapids_triton_dev_cuda11.2.yml +++ b/conda/environments/rapids_triton_dev_cuda11.4.yml @@ -4,5 +4,7 @@ channels: - nvidia - conda-forge dependencies: - - cudatoolkit=11.4 - cmake=3.20.1 + - cudatoolkit=11.4 + - ninja + - rapidjson diff --git a/cpp/include/rapids_triton/exceptions.hpp b/cpp/include/rapids_triton/exceptions.hpp index 4ad55db..da7614d 100644 --- a/cpp/include/rapids_triton/exceptions.hpp +++ b/cpp/include/rapids_triton/exceptions.hpp @@ -70,7 +70,7 @@ struct TritonException : std::exception { return TRITONSERVER_ErrorMessage(error_); } - auto* error() { return error_; } + auto* error() const { return error_; } private: TRITONSERVER_Error* error_; diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 5453ade..427ce8f 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -43,6 +43,12 @@ target_include_directories(test_rapids_triton ) +find_library( + TRITONSERVER_LIB + tritonserver + PATHS /opt/tritonserver/lib +) + target_link_libraries(test_rapids_triton PRIVATE rmm::rmm @@ -51,5 +57,6 @@ PRIVATE triton-backend-utils GTest::gtest GTest::gtest_main + "${TRITONSERVER_LIB}" $ ) diff --git a/cpp/test/exceptions.cpp b/cpp/test/exceptions.cpp index 7f9e5f1..09ad631 100644 --- a/cpp/test/exceptions.cpp +++ b/cpp/test/exceptions.cpp @@ -14,14 +14,62 @@ * limitations under the License. */ +#include #include #include +#include namespace triton { namespace backend { namespace rapids { +TEST(RapidsTriton, default_except) { + try { + throw TritonException(); + } catch (TritonException const& err) { + EXPECT_EQ(std::string(err.what()), + std::string("encountered unknown error")); + } +} + +TEST(RapidsTriton, msg_except) { + auto msg = std::string("TEST ERROR MESSAGE"); + try { + throw TritonException(Error::Internal, msg); + } catch (TritonException const& err) { + EXPECT_EQ(std::string(err.what()), msg); + } + try { + throw TritonException(Error::Internal, msg.c_str()); + } catch (TritonException const& err) { + EXPECT_EQ(std::string(err.what()), msg); + } + try { + throw TritonException(Error::Internal, msg); + } catch (TritonException const& err) { + try { + throw(TritonException(err.error())); + } catch (TritonException const& err2) { + EXPECT_EQ(std::string(err2.what()), msg); + } + } +} + +TEST(RapidsTriton, triton_check) { + auto msg = std::string("TEST ERROR MESSAGE"); + EXPECT_THROW( + triton_check(TRITONSERVER_ErrorNew(Error::Internal, msg.c_str())), + TritonException); + triton_check(nullptr); +} + +TEST(RapidsTriton, cuda_check) { + EXPECT_THROW(cuda_check(cudaError::cudaErrorMissingConfiguration), + TritonException); + cuda_check(cudaError::cudaSuccess); +} + } // namespace rapids } // namespace backend } // namespace triton From 2e318df141940b72de66860a88415a1de302b5b8 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 26 Aug 2021 20:27:17 -0400 Subject: [PATCH 017/199] Test narrow --- cpp/include/rapids_triton/utils/narrow.hpp | 5 ++- cpp/test/CMakeLists.txt | 1 + cpp/test/utils/narrow.cpp | 36 ++++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 cpp/test/utils/narrow.cpp diff --git a/cpp/include/rapids_triton/utils/narrow.hpp b/cpp/include/rapids_triton/utils/narrow.hpp index 5e7a590..0d74cff 100644 --- a/cpp/include/rapids_triton/utils/narrow.hpp +++ b/cpp/include/rapids_triton/utils/narrow.hpp @@ -18,14 +18,17 @@ #include #include -namespace triton { namespace backend { namespace fil { +namespace triton { namespace backend { namespace rapids { template auto narrow(F from) { auto to = static_cast(from); + if (static_cast(to) != from || + (std::is_signed::value && !std::is_signed::value && from < F{}) || + (std::is_signed::value && !std::is_signed::value && to < T{}) || (std::is_signed::value == std::is_signed::value && ((to < T{}) != (from < F{})))) { throw TritonException(Error::Internal, "invalid narrowing"); diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 427ce8f..7dd8b89 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -19,6 +19,7 @@ add_executable(test_rapids_triton test/build_control.cpp test/exceptions.cpp test/test.cpp + test/utils/narrow.cpp ) set_target_properties(test_rapids_triton diff --git a/cpp/test/utils/narrow.cpp b/cpp/test/utils/narrow.cpp new file mode 100644 index 0000000..e0543d4 --- /dev/null +++ b/cpp/test/utils/narrow.cpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +TEST(RapidsTriton, narrow) { + EXPECT_THROW(narrow(-1), TritonException); + narrow(int{5}); + EXPECT_THROW(narrow(std::numeric_limits::max()), + TritonException); + narrow(std::size_t{5}); +} + +} // namespace rapids +} // namespace backend +} // namespace triton From ee6c09e98805c28a8fdeabd87951c2daeca3960a Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 27 Aug 2021 10:03:17 -0400 Subject: [PATCH 018/199] Test memory allocation functions --- cpp/cmake/thirdparty/get_gtest.cmake | 2 +- .../rapids_triton/memory/detail/allocate.hpp | 20 ++++++-- cpp/test/CMakeLists.txt | 3 ++ cpp/test/memory/detail/allocate.cpp | 47 +++++++++++++++++++ 4 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 cpp/test/memory/detail/allocate.cpp diff --git a/cpp/cmake/thirdparty/get_gtest.cmake b/cpp/cmake/thirdparty/get_gtest.cmake index 4cd11da..02cf9cc 100644 --- a/cpp/cmake/thirdparty/get_gtest.cmake +++ b/cpp/cmake/thirdparty/get_gtest.cmake @@ -21,7 +21,7 @@ function(find_and_configure_gtest VERSION) endif() rapids_cpm_find(GTest ${VERSION} - GLOBAL_TARGETS gest gtest_main GTest::gtest GTest::gtest_main + GLOBAL_TARGETS gtest gtest_main GTest::gtest GTest::gtest_main gmock gmock_main CPM_ARGS GIT_REPOSITORY https://github.com/google/googletest.git GIT_TAG release-${VERSION} diff --git a/cpp/include/rapids_triton/memory/detail/allocate.hpp b/cpp/include/rapids_triton/memory/detail/allocate.hpp index 5e83266..3f3f8e4 100644 --- a/cpp/include/rapids_triton/memory/detail/allocate.hpp +++ b/cpp/include/rapids_triton/memory/detail/allocate.hpp @@ -21,23 +21,35 @@ #include #include +#include #include namespace triton { namespace backend { namespace rapids { namespace detail { template struct dev_deallocater { - void operator(T* d_ptr) { - cudaFree(reinterpret_cast(d_ptr)); + void operator()(T* d_ptr) { + if constexpr (IS_GPU_BUILD) { + cudaFree(reinterpret_cast(d_ptr)); + } else { + log_error( + __FILE__, + __LINE__, + "ERROR: device deallocation cannot be performed in non-GPU build!" + ); + } } -} +}; /** * @brief Allocate given number of elements on GPU and return device pointer */ template -T* +[[nodiscard]] T* dev_allocate(std::size_t count, cudaStream_t stream) { + if constexpr (!IS_GPU_BUILD) { + throw TritonException(Error::Internal, "device allocation attempted in non-GPU build"); + } auto* ptr_d = static_cast(rmm::mr::get_current_device_resource()->allocate( sizeof(T) * count, stream)); diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 7dd8b89..9969109 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -18,6 +18,7 @@ add_executable(test_rapids_triton test/build_control.cpp test/exceptions.cpp + test/memory/detail/allocate.cpp test/test.cpp test/utils/narrow.cpp ) @@ -56,6 +57,8 @@ PRIVATE raft::raft triton-core-serverstub triton-backend-utils + gmock + gmock_main GTest::gtest GTest::gtest_main "${TRITONSERVER_LIB}" diff --git a/cpp/test/memory/detail/allocate.cpp b/cpp/test/memory/detail/allocate.cpp new file mode 100644 index 0000000..b0014d5 --- /dev/null +++ b/cpp/test/memory/detail/allocate.cpp @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +TEST(RapidsTriton, dev_allocate) { + auto data = std::vector{1, 2, 3}; + if constexpr (IS_GPU_BUILD) { + auto ptr_d = detail::dev_allocate(data.size(), 0); + auto data_out = std::vector(data.size()); + cudaMemcpy(static_cast(ptr_d), static_cast(data.data()), + sizeof(int) * data.size(), cudaMemcpyHostToDevice); + cudaMemcpy(static_cast(data_out.data()), static_cast(ptr_d), + sizeof(int) * data.size(), cudaMemcpyDeviceToHost); + ASSERT_THAT(data_out, ::testing::ElementsAreArray(data)); + } else { + EXPECT_THROW( + [&data]() { return detail::dev_allocate(data.size(), 0); }(), + TritonException); + } +} + +} // namespace rapids +} // namespace backend +} // namespace triton From f05351fde08220266be8c34fbbc419442f456ec1 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 27 Aug 2021 10:38:27 -0400 Subject: [PATCH 019/199] Test memory copy functions --- .../rapids_triton/memory/detail/copy.hpp | 3 +- cpp/include/rapids_triton/memory/types.hpp | 4 +- cpp/test/CMakeLists.txt | 1 + cpp/test/memory/detail/allocate.cpp | 4 +- cpp/test/memory/detail/copy.cpp | 84 +++++++++++++++++++ 5 files changed, 92 insertions(+), 4 deletions(-) create mode 100644 cpp/test/memory/detail/copy.cpp diff --git a/cpp/include/rapids_triton/memory/detail/copy.hpp b/cpp/include/rapids_triton/memory/detail/copy.hpp index 9298e57..6774656 100644 --- a/cpp/include/rapids_triton/memory/detail/copy.hpp +++ b/cpp/include/rapids_triton/memory/detail/copy.hpp @@ -16,6 +16,7 @@ #pragma once #include +#include #include @@ -35,7 +36,7 @@ dev_copy(T* dst, T const* src, std::size_t len, cudaStream_t stream) { if constexpr (IS_GPU_BUILD) { try { - raft::copy(raw_dst, raw_src, len, stream); + raft::copy(dst, src, len, stream); } catch (const raft::cuda_error& err) { throw TritonException(Error::Internal, err.what()); } diff --git a/cpp/include/rapids_triton/memory/types.hpp b/cpp/include/rapids_triton/memory/types.hpp index 38d5b86..8540775 100644 --- a/cpp/include/rapids_triton/memory/types.hpp +++ b/cpp/include/rapids_triton/memory/types.hpp @@ -19,6 +19,6 @@ namespace triton { namespace backend { namespace rapids { using MemoryType = TRITONSERVER_MemoryType; - using DeviceMemory = TRITONSERVER_MEMORY_GPU; - using HostMemory = TRITONSERVER_MEMORY_CPU; + auto constexpr DeviceMemory = TRITONSERVER_MEMORY_GPU; + auto constexpr HostMemory = TRITONSERVER_MEMORY_CPU; }}} // namespace triton::backend::rapids diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 9969109..89426d4 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -19,6 +19,7 @@ add_executable(test_rapids_triton test/build_control.cpp test/exceptions.cpp test/memory/detail/allocate.cpp + test/memory/detail/copy.cpp test/test.cpp test/utils/narrow.cpp ) diff --git a/cpp/test/memory/detail/allocate.cpp b/cpp/test/memory/detail/allocate.cpp index b0014d5..dcfce83 100644 --- a/cpp/test/memory/detail/allocate.cpp +++ b/cpp/test/memory/detail/allocate.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -34,7 +35,8 @@ TEST(RapidsTriton, dev_allocate) { sizeof(int) * data.size(), cudaMemcpyHostToDevice); cudaMemcpy(static_cast(data_out.data()), static_cast(ptr_d), sizeof(int) * data.size(), cudaMemcpyDeviceToHost); - ASSERT_THAT(data_out, ::testing::ElementsAreArray(data)); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + detail::dev_deallocater{}(ptr_d); } else { EXPECT_THROW( [&data]() { return detail::dev_allocate(data.size(), 0); }(), diff --git a/cpp/test/memory/detail/copy.cpp b/cpp/test/memory/detail/copy.cpp new file mode 100644 index 0000000..beb2fe2 --- /dev/null +++ b/cpp/test/memory/detail/copy.cpp @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +TEST(RapidsTriton, dev_copy) { + auto data = std::vector{1, 2, 3}; + auto data_out = std::vector(data.size()); + if constexpr (IS_GPU_BUILD) { + auto* ptr_d = static_cast(nullptr); + cudaMalloc(reinterpret_cast(&ptr_d), sizeof(int) * data.size()); + detail::dev_copy(ptr_d, data.data(), data.size(), 0); + + cudaMemcpy(static_cast(data_out.data()), static_cast(ptr_d), + sizeof(int) * data.size(), cudaMemcpyDeviceToHost); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + cudaFree(reinterpret_cast(ptr_d)); + } else { + ASSERT_THROW(detail::dev_copy(data_out.data(), data.data(), data.size(), 0), + TritonException); + } +} + +TEST(RapidsTriton, host_copy) { + auto data = std::vector{1, 2, 3}; + auto data_out = std::vector(data.size()); + detail::host_copy(data_out.data(), data.data(), data.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +TEST(RapidsTriton, copy) { + auto data = std::vector{1, 2, 3}; + auto data_out = std::vector(data.size()); + detail::copy(data_out.data(), data.data(), data.size(), 0, HostMemory, + HostMemory); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + + data_out = std::vector(data.size()); + if constexpr (IS_GPU_BUILD) { + auto* ptr_d = static_cast(nullptr); + cudaMalloc(reinterpret_cast(&ptr_d), sizeof(int) * data.size()); + detail::copy(ptr_d, data.data(), data.size(), 0, DeviceMemory, HostMemory); + + cudaMemcpy(static_cast(data_out.data()), static_cast(ptr_d), + sizeof(int) * data.size(), cudaMemcpyDeviceToHost); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + cudaFree(reinterpret_cast(ptr_d)); + } else { + EXPECT_THROW(detail::copy(data_out.data(), data.data(), data.size(), 0, + HostMemory, DeviceMemory), + TritonException); + EXPECT_THROW(detail::copy(data_out.data(), data.data(), data.size(), 0, + DeviceMemory, HostMemory), + TritonException); + } +} + +} // namespace rapids +} // namespace backend +} // namespace triton From 28cccdd9b2f73f034ec8c041c1f5484a6aa03df4 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 27 Aug 2021 12:34:20 -0400 Subject: [PATCH 020/199] Add buffer tests --- cpp/include/rapids_triton/exceptions.hpp | 1 + cpp/include/rapids_triton/memory/buffer.hpp | 60 ++++---- cpp/test/CMakeLists.txt | 3 + cpp/test/memory/buffer.cpp | 154 ++++++++++++++++++++ cpp/test/memory/types.cpp | 17 +++ cpp/test/triton/device.cpp | 17 +++ 6 files changed, 227 insertions(+), 25 deletions(-) create mode 100644 cpp/test/memory/buffer.cpp create mode 100644 cpp/test/memory/types.cpp create mode 100644 cpp/test/triton/device.cpp diff --git a/cpp/include/rapids_triton/exceptions.hpp b/cpp/include/rapids_triton/exceptions.hpp index da7614d..76040f0 100644 --- a/cpp/include/rapids_triton/exceptions.hpp +++ b/cpp/include/rapids_triton/exceptions.hpp @@ -84,6 +84,7 @@ inline void triton_check(TRITONSERVER_Error* err) { inline void cuda_check(cudaError_t const& err) { if (err != cudaSuccess) { + cudaGetLastError(); throw TritonException(Error::Internal, cudaGetErrorString(err)); } } diff --git a/cpp/include/rapids_triton/memory/buffer.hpp b/cpp/include/rapids_triton/memory/buffer.hpp index 0089c35..b117dc2 100644 --- a/cpp/include/rapids_triton/memory/buffer.hpp +++ b/cpp/include/rapids_triton/memory/buffer.hpp @@ -37,7 +37,7 @@ namespace triton { namespace backend { namespace rapids { using h_ptr = T*; using d_ptr = T*; using owned_h_ptr = std::unique_ptr; - using owned_d_ptr = std::unique_ptr{}>; + using owned_d_ptr = std::unique_ptr>; using data_ptr = std::variant; Buffer() noexcept : device_{}, data_{std::in_place_index<0>, nullptr}, size_{}, stream_{} {} @@ -50,7 +50,7 @@ namespace triton { namespace backend { namespace rapids { */ Buffer(size_type size, MemoryType memory_type=DeviceMemory, device_id_t device=0, cudaStream_t stream=0) : - device_{device}, data_{allocate(size, memory_type)}, size_{size}, stream_{stream} {} + device_{device}, data_{allocate(size, device, memory_type, stream)}, size_{size}, stream_{stream} {} /** * @brief Construct buffer from given source in given memory location (either @@ -60,7 +60,17 @@ namespace triton { namespace backend { namespace rapids { */ Buffer(T* input_data, size_type size, MemoryType memory_type=DeviceMemory, device_id_t device=0, cudaStream_t stream=0) : - device_{device}, data_{input_data}, size_{size}, stream_{stream} {} + device_{device}, data_{ + [&memory_type, &input_data](){ + auto result = data_ptr{}; + if(memory_type == HostMemory) { + result = data_ptr{std::in_place_index<0>, input_data}; + } else { + result = data_ptr{std::in_place_index<1>, input_data}; + } + return result; + }() + }, size_{size}, stream_{stream} {} /** * @brief Construct one buffer from another in the given memory location @@ -68,8 +78,8 @@ namespace triton { namespace backend { namespace rapids { * A buffer constructed in this way is owning and will copy the data from * the original location */ - Buffer(Buffer const& other, MemoryType memory_type, device_id_t device=0) : device_{device}, data_([&other](){ - auto result = allocate(other.size_, memory_type); + Buffer(Buffer const& other, MemoryType memory_type, device_id_t device=0) : device_{device}, data_([&other, &memory_type, &device](){ + auto result = allocate(other.size_, device, memory_type, other.stream_); copy(result, other.data_, other.size_, other.stream_); return result; }()), size_{other.size_}, stream_{other.stream_} {} @@ -78,20 +88,20 @@ namespace triton { namespace backend { namespace rapids { * @brief Create owning copy of existing buffer * The memory type of this new buffer will be the same as the original */ - Buffer(Buffer const& other) : Buffer(other, other.mem_type(), other.device(), other.stream()) {} + Buffer(Buffer const& other) : Buffer(other, other.mem_type(), other.device()) {} Buffer(Buffer&& other, MemoryType memory_type) : device_{other.device()}, data_{[&other, memory_type](){ data_ptr result; if(memory_type == other.mem_type()) { result = std::move(other.data_); } else { - result = allocate(other.size_, memory_type); + result = allocate(other.size_, memory_type, other.device(), other.stream()); copy(result, other.data_, other.size_, other.stream_); } return result; - }()}, size_{other.size_}, stream{other.stream_} {} + }()}, size_{other.size_}, stream_{other.stream_} {} - Buffer(Buffer&& other) noexcept : device_{other.device_}, data_{std::move{other.data_}}, size_{other.size_}, + Buffer(Buffer&& other) noexcept : device_{other.device_}, data_{std::move(other.data_)}, size_{other.size_}, stream_{other.stream_} {} ~Buffer() = default; @@ -100,7 +110,7 @@ namespace triton { namespace backend { namespace rapids { * @brief Return where memory for this buffer is located (host or device) */ auto mem_type() const noexcept { - return data_ptr.index() % 2 == 0 ? HostMemory, DeviceMemory; + return data_.index() % 2 == 0 ? HostMemory : DeviceMemory; } /** @@ -149,10 +159,10 @@ namespace triton { namespace backend { namespace rapids { cudaStream_t stream_; // Helper function for accessing raw pointer to underlying data of data_ptr - static auto get_raw_ptr(data_ptr ptr) noexcept { + static auto* get_raw_ptr(data_ptr const& ptr) noexcept { /* Switch statement is an optimization relative to std::visit to avoid * vtable overhead for a small number of alternatives */ - T* result; + auto* result = static_cast(nullptr); switch (ptr.index()) { case 0: result = std::get<0>(ptr); @@ -171,12 +181,12 @@ namespace triton { namespace backend { namespace rapids { } // Helper function for allocating memory in constructors - static auto allocate(size_type size, MemoryType memory_type=DeviceMemory) { - data_ptr result; + static auto allocate(size_type size, device_id_t device=0, MemoryType memory_type=DeviceMemory, cudaStream_t stream=0) { + auto result = data_ptr{}; if (memory_type == DeviceMemory) { if constexpr (IS_GPU_BUILD) { - cuda_check(cudaSetDevice(device_)); - result = data_ptr{owned_d_ptr{detail::dev_allocate(size)}}; + cuda_check(cudaSetDevice(device)); + result = data_ptr{owned_d_ptr{detail::dev_allocate(size, stream)}}; } else { throw TritonException( Error::Internal, @@ -193,14 +203,14 @@ namespace triton { namespace backend { namespace rapids { // stronger guarantees on conditions that would otherwise need to be // checked static void copy( - data_ptr dst, data_ptr src, size_type len, cudaStream_t stream) { + data_ptr const& dst, data_ptr const& src, size_type len, cudaStream_t stream) { auto raw_dst = get_raw_ptr(dst); auto raw_src = get_raw_ptr(src); auto dst_mem_type = dst.index() % 2 == 0 ? HostMemory : DeviceMemory; auto src_mem_type = src.index() % 2 == 0 ? HostMemory : DeviceMemory; - detail::copy(raw_dst, raw_src, len, dst_mem_type, src_mem_type); + detail::copy(raw_dst, raw_src, len, stream, dst_mem_type, src_mem_type); } }; @@ -223,8 +233,8 @@ namespace triton { namespace backend { namespace rapids { * when those buffers may be modified on different host threads as well. */ template - copy(Buffer dst&&, Buffer src&&, Buffer::size_type dst_begin, - Buffer::size_type src_begin, Buffer::size_type src_end) { + void copy(Buffer&& dst, Buffer&& src, typename Buffer::size_type dst_begin, + typename Buffer::size_type src_begin, typename Buffer::size_type src_end) { if(dst.stream() != src.stream()) { dst.set_stream(src.stream()); } @@ -241,18 +251,18 @@ namespace triton { namespace backend { namespace rapids { } template - void copy(Buffer dst&&, Buffer src&&) { + void copy(Buffer&& dst, Buffer&& src) { copy(dst, src, dst.begin(), src.begin(), src.begin() + src.size()); } template - void copy(Buffer dst&&, Buffer src&&, Buffer::size_type dst_begin) { + void copy(Buffer&& dst, Buffer&& src, typename Buffer::size_type dst_begin) { copy(dst, src, dst_begin, src.begin(), src.begin() + src.size()); } template - void copy(Buffer dst&&, Buffer src&&, Buffer::size_type src_begin, - Buffer::size_type src_end) { - copy(dst, src, dst_begin, src.begin(), src.begin() + src_end); + void copy(Buffer&& dst, Buffer&& src, typename Buffer::size_type src_begin, + typename Buffer::size_type src_end) { + copy(dst, src, dst.begin(), src.begin(), src.begin() + src_end); } }}} // namespace triton::backend::rapids diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 89426d4..cd9fd72 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -18,8 +18,11 @@ add_executable(test_rapids_triton test/build_control.cpp test/exceptions.cpp + test/memory/buffer.cpp test/memory/detail/allocate.cpp test/memory/detail/copy.cpp + test/memory/types.cpp + test/triton/device.cpp test/test.cpp test/utils/narrow.cpp ) diff --git a/cpp/test/memory/buffer.cpp b/cpp/test/memory/buffer.cpp new file mode 100644 index 0000000..164cbf6 --- /dev/null +++ b/cpp/test/memory/buffer.cpp @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +TEST(RapidsTriton, default_buffer) { + auto buffer = Buffer(); + EXPECT_EQ(buffer.mem_type(), HostMemory); + EXPECT_EQ(buffer.size(), 0); + EXPECT_EQ(buffer.data(), nullptr); + EXPECT_EQ(buffer.device(), 0); + EXPECT_EQ(buffer.stream(), cudaStream_t{}); + if constexpr (IS_GPU_BUILD) { + auto stream = cudaStream_t{}; + cudaStreamCreate(&stream); + buffer.set_stream(stream); + EXPECT_EQ(buffer.stream(), stream); + cudaStreamDestroy(stream); + } +} + +TEST(RapidsTriton, device_buffer) { + auto data = std::vector{1, 2, 3}; + if (IS_GPU_BUILD) { + auto buffer = Buffer(data.size(), DeviceMemory, 0, 0); + + ASSERT_EQ(buffer.mem_type(), DeviceMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_NE(buffer.data(), nullptr); + + auto data_out = std::vector(data.size()); + cudaMemcpy(static_cast(buffer.data()), + static_cast(data.data()), sizeof(int) * data.size(), + cudaMemcpyHostToDevice); + cudaMemcpy(static_cast(data_out.data()), + static_cast(buffer.data()), sizeof(int) * data.size(), + cudaMemcpyDeviceToHost); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + + } else { + EXPECT_THROW(Buffer(data.size(), DeviceMemory, 0, 0), TritonException); + } +} + +TEST(RapidsTriton, non_owning_device_buffer) { + auto data = std::vector{1, 2, 3}; + if constexpr (IS_GPU_BUILD) { + auto* ptr_d = static_cast(nullptr); + cudaMalloc(reinterpret_cast(&ptr_d), sizeof(int) * data.size()); + cudaMemcpy(static_cast(ptr_d), static_cast(data.data()), + sizeof(int) * data.size(), cudaMemcpyHostToDevice); + auto buffer = Buffer(ptr_d, data.size(), DeviceMemory); + + ASSERT_EQ(buffer.mem_type(), DeviceMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_EQ(buffer.data(), ptr_d); + + auto data_out = std::vector(data.size()); + cudaMemcpy(static_cast(data_out.data()), + static_cast(buffer.data()), sizeof(int) * data.size(), + cudaMemcpyDeviceToHost); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + + cudaFree(reinterpret_cast(ptr_d)); + } else { + ASSERT_THROW(Buffer(data.data(), data.size(), DeviceMemory), + TritonException); + } +} + +TEST(RapidsTriton, host_buffer) { + auto data = std::vector{1, 2, 3}; + auto buffer = Buffer(data.size(), HostMemory, 0, 0); + + ASSERT_EQ(buffer.mem_type(), HostMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_NE(buffer.data(), nullptr); + + std::memcpy(static_cast(buffer.data()), + static_cast(data.data()), data.size() * sizeof(int)); + + auto data_out = + std::vector(buffer.data(), buffer.data() + buffer.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +TEST(RapidsTriton, non_owning_host_buffer) { + auto data = std::vector{1, 2, 3}; + auto buffer = Buffer(data.data(), data.size(), HostMemory); + + ASSERT_EQ(buffer.mem_type(), HostMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_EQ(buffer.data(), data.data()); + + auto data_out = + std::vector(buffer.data(), buffer.data() + buffer.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +TEST(RapidsTriton, copy_buffer) { + auto data = std::vector{1, 2, 3}; + auto orig_buffer = Buffer(data.data(), data.size(), HostMemory); + auto buffer = Buffer(orig_buffer); + + ASSERT_EQ(buffer.mem_type(), HostMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_NE(buffer.data(), orig_buffer.data()); + + auto data_out = + std::vector(buffer.data(), buffer.data() + buffer.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +TEST(RapidsTriton, move_buffer) { + auto data = std::vector{1, 2, 3}; + auto buffer = Buffer(Buffer(data.data(), data.size(), HostMemory)); + + ASSERT_EQ(buffer.mem_type(), HostMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_EQ(buffer.data(), data.data()); + + auto data_out = + std::vector(buffer.data(), buffer.data() + buffer.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/test/memory/types.cpp b/cpp/test/memory/types.cpp new file mode 100644 index 0000000..65eb386 --- /dev/null +++ b/cpp/test/memory/types.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/cpp/test/triton/device.cpp b/cpp/test/triton/device.cpp new file mode 100644 index 0000000..53e49e0 --- /dev/null +++ b/cpp/test/triton/device.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include From b3300f66b05775f2bc92c712dbe041905b0d8618 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 27 Aug 2021 15:39:26 -0400 Subject: [PATCH 021/199] Test dtype --- cpp/include/rapids_triton/tensor/dtype.hpp | 91 ++++++++++--------- .../rapids_triton/utils/const_agnostic.hpp | 25 +++++ cpp/test/CMakeLists.txt | 2 + cpp/test/tensor/dtype.cpp | 48 ++++++++++ cpp/test/utils/const_agnostic.cpp | 33 +++++++ 5 files changed, 154 insertions(+), 45 deletions(-) create mode 100644 cpp/include/rapids_triton/utils/const_agnostic.hpp create mode 100644 cpp/test/tensor/dtype.cpp create mode 100644 cpp/test/utils/const_agnostic.cpp diff --git a/cpp/include/rapids_triton/tensor/dtype.hpp b/cpp/include/rapids_triton/tensor/dtype.hpp index 06ec163..01f8601 100644 --- a/cpp/include/rapids_triton/tensor/dtype.hpp +++ b/cpp/include/rapids_triton/tensor/dtype.hpp @@ -15,31 +15,32 @@ */ #pragma once +#include +#include #include namespace triton { namespace backend { namespace rapids { using DType = TRITONSERVER_DataType; -using DTypeBool = TRITONSERVER_TYPE_BOOL; -using DTypeUint8 = TRITONSERVER_TYPE_UINT8; -using DTypeChar = DTypeUint8; -using DTypeByte = DTypeUint8; -using DTypeUint16 = TRITONSERVER_TYPE_UINT16; -using DTypeUint32 = TRITONSERVER_TYPE_UINT32; -using DTypeUint64 = TRITONSERVER_TYPE_UINT64; -using DTypeInt8 = TRITONSERVER_TYPE_INT8; -using DTypeInt16 = TRITONSERVER_TYPE_INT16; -using DTypeInt32 = TRITONSERVER_TYPE_INT32; -using DTypeInt64 = TRITONSERVER_TYPE_INT64; -using DTypeFloat32 = TRITONSERVER_TYPE_FP32; -using DTypeFloat64 = TRITONSERVER_TYPE_FP64; - +auto constexpr DTypeBool = TRITONSERVER_TYPE_BOOL; +auto constexpr DTypeUint8 = TRITONSERVER_TYPE_UINT8; +auto constexpr DTypeChar = DTypeUint8; +auto constexpr DTypeByte = DTypeUint8; +auto constexpr DTypeUint16 = TRITONSERVER_TYPE_UINT16; +auto constexpr DTypeUint32 = TRITONSERVER_TYPE_UINT32; +auto constexpr DTypeUint64 = TRITONSERVER_TYPE_UINT64; +auto constexpr DTypeInt8 = TRITONSERVER_TYPE_INT8; +auto constexpr DTypeInt16 = TRITONSERVER_TYPE_INT16; +auto constexpr DTypeInt32 = TRITONSERVER_TYPE_INT32; +auto constexpr DTypeInt64 = TRITONSERVER_TYPE_INT64; +auto constexpr DTypeFloat32 = TRITONSERVER_TYPE_FP32; +auto constexpr DTypeFloat64 = TRITONSERVER_TYPE_FP64; template struct TritonType { }; -template +template struct TritonDtype { }; @@ -48,88 +49,88 @@ struct TritonType { typedef bool type; }; -template <> -struct TritonDtype { +template +struct TritonDtype> { static constexpr DType value = DTypeBool; }; template <> struct TritonType { - typedef uint8_t type; + typedef std::uint8_t type; }; -template <> -struct TritonDtype { +template +struct TritonDtype> { static constexpr DType value = DTypeUint8; }; template <> struct TritonType { - typedef uint16_t type; + typedef std::uint16_t type; }; -template <> -struct TritonDtype { +template +struct TritonDtype> { static constexpr DType value = DTypeUint16; }; template <> struct TritonType { - typedef uint32_t type; + typedef std::uint32_t type; }; -template <> -struct TritonDtype { +template +struct TritonDtype> { static constexpr DType value = DTypeUint32; }; template <> struct TritonType { - typedef uint64_t type; + typedef std::uint64_t type; }; -template <> -struct TritonDtype { +template +struct TritonDtype> { static constexpr DType value = DTypeUint64; }; template <> struct TritonType { - typedef int8_t type; + typedef std::int8_t type; }; -template <> -struct TritonDtype { +template +struct TritonDtype> { static constexpr DType value = DTypeInt8; }; template <> struct TritonType { - typedef int16_t type; + typedef std::int16_t type; }; -template <> -struct TritonDtype { +template +struct TritonDtype> { static constexpr DType value = DTypeInt16; }; template <> struct TritonType { - typedef int32_t type; + typedef std::int32_t type; }; -template <> -struct TritonDtype { +template +struct TritonDtype> { static constexpr DType value = DTypeInt32; }; template <> struct TritonType { - typedef int64_t type; + typedef std::int64_t type; }; -template <> -struct TritonDtype { +template +struct TritonDtype> { static constexpr DType value = DTypeInt64; }; @@ -138,8 +139,8 @@ struct TritonType { typedef float type; }; -template <> -struct TritonDtype { +template +struct TritonDtype> { static constexpr DType value = DTypeFloat32; }; @@ -148,8 +149,8 @@ struct TritonType { typedef double type; }; -template <> -struct TritonDtype { +template +struct TritonDtype> { static constexpr DType value = DTypeFloat64; }; diff --git a/cpp/include/rapids_triton/utils/const_agnostic.hpp b/cpp/include/rapids_triton/utils/const_agnostic.hpp new file mode 100644 index 0000000..da3acad --- /dev/null +++ b/cpp/include/rapids_triton/utils/const_agnostic.hpp @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { namespace backend { namespace rapids { + +template +using const_agnostic_same_t = std::enable_if_t, std::remove_const_t>::value>; + +}}} // namespace triton::backend::rapids diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index cd9fd72..74d6ca3 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -22,8 +22,10 @@ add_executable(test_rapids_triton test/memory/detail/allocate.cpp test/memory/detail/copy.cpp test/memory/types.cpp + test/tensor/dtype.cpp test/triton/device.cpp test/test.cpp + test/utils/const_agnostic.cpp test/utils/narrow.cpp ) diff --git a/cpp/test/tensor/dtype.cpp b/cpp/test/tensor/dtype.cpp new file mode 100644 index 0000000..46c9aaa --- /dev/null +++ b/cpp/test/tensor/dtype.cpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include + +namespace triton { +namespace backend { +namespace rapids { +template +void check_dtype_conversion() { + EXPECT_EQ(D, TritonDtype::type>::value); + EXPECT_EQ(D, TritonDtype::type const>::value); +} +TEST(RapidsTriton, dtype) { + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/test/utils/const_agnostic.cpp b/cpp/test/utils/const_agnostic.cpp new file mode 100644 index 0000000..592ecc6 --- /dev/null +++ b/cpp/test/utils/const_agnostic.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +TEST(RapidsTriton, const_agnostic) { + static_assert( + std::is_same, void>::value); + static_assert(std::is_same, void>::value); +} + +} // namespace rapids +} // namespace backend +} // namespace triton From a46a5034c0e136a79c10d3e29594d5af8bb25d07 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 30 Aug 2021 10:13:02 -0400 Subject: [PATCH 022/199] Test Tensor objects --- cpp/include/rapids_triton/memory/buffer.hpp | 26 ++++--- cpp/include/rapids_triton/tensor/tensor.hpp | 71 +++++++++++-------- cpp/test/CMakeLists.txt | 1 + cpp/test/tensor/tensor.cpp | 77 +++++++++++++++++++++ 4 files changed, 137 insertions(+), 38 deletions(-) create mode 100644 cpp/test/tensor/tensor.cpp diff --git a/cpp/include/rapids_triton/memory/buffer.hpp b/cpp/include/rapids_triton/memory/buffer.hpp index b117dc2..5de2b69 100644 --- a/cpp/include/rapids_triton/memory/buffer.hpp +++ b/cpp/include/rapids_triton/memory/buffer.hpp @@ -123,7 +123,7 @@ namespace triton { namespace backend { namespace rapids { /** * @brief Return pointer to data stored in buffer */ - auto* data() noexcept { + auto* data() const noexcept { return get_raw_ptr(data_); } @@ -138,6 +138,12 @@ namespace triton { namespace backend { namespace rapids { return stream_; } + void stream_synchronize() const { + if constexpr (IS_GPU_BUILD) { + cuda_check(cudaStreamSynchronize(stream_)); + } + } + /** * @brief Set CUDA stream for this buffer to new value * @@ -146,9 +152,7 @@ namespace triton { namespace backend { namespace rapids { * interactions between buffers on different streams where possible. */ void set_stream(cudaStream_t new_stream) { - if constexpr (IS_GPU_BUILD) { - cuda_check(cudaStreamSynchronize(stream_)); - } + stream_synchronize(); stream_ = new_stream; } @@ -233,7 +237,7 @@ namespace triton { namespace backend { namespace rapids { * when those buffers may be modified on different host threads as well. */ template - void copy(Buffer&& dst, Buffer&& src, typename Buffer::size_type dst_begin, + void copy(Buffer& dst, Buffer const& src, typename Buffer::size_type dst_begin, typename Buffer::size_type src_begin, typename Buffer::size_type src_end) { if(dst.stream() != src.stream()) { dst.set_stream(src.stream()); @@ -251,18 +255,18 @@ namespace triton { namespace backend { namespace rapids { } template - void copy(Buffer&& dst, Buffer&& src) { - copy(dst, src, dst.begin(), src.begin(), src.begin() + src.size()); + void copy(Buffer& dst, Buffer const& src) { + copy(dst, src, 0, 0, src.size()); } template - void copy(Buffer&& dst, Buffer&& src, typename Buffer::size_type dst_begin) { - copy(dst, src, dst_begin, src.begin(), src.begin() + src.size()); + void copy(Buffer& dst, Buffer const& src, typename Buffer::size_type dst_begin) { + copy(dst, src, dst_begin, 0, src.size()); } template - void copy(Buffer&& dst, Buffer&& src, typename Buffer::size_type src_begin, + void copy(Buffer& dst, Buffer const& src, typename Buffer::size_type src_begin, typename Buffer::size_type src_end) { - copy(dst, src, dst.begin(), src.begin(), src.begin() + src_end); + copy(dst, src, 0, 0, src.size()); } }}} // namespace triton::backend::rapids diff --git a/cpp/include/rapids_triton/tensor/tensor.hpp b/cpp/include/rapids_triton/tensor/tensor.hpp index a26580f..0ebc7f7 100644 --- a/cpp/include/rapids_triton/tensor/tensor.hpp +++ b/cpp/include/rapids_triton/tensor/tensor.hpp @@ -16,9 +16,11 @@ #pragma once #include +#include #include #include #include +#include #include #include @@ -27,6 +29,8 @@ #include #include #include +#include +#include namespace triton { namespace backend { namespace rapids { template @@ -34,7 +38,7 @@ namespace triton { namespace backend { namespace rapids { using size_type = typename Buffer::size_type; BaseTensor() : shape_{}, buffer_{} {} - BaseTensor(std::vector&& shape, Buffer&& buffer) : shape_(std::move(shape)), buffer_{std::move(buffer)} {} + BaseTensor(std::vector const& shape, Buffer&& buffer) : shape_(shape), buffer_{std::move(buffer)} {} virtual ~BaseTensor() = 0; @@ -45,16 +49,16 @@ namespace triton { namespace backend { namespace rapids { * a new BaseTensor */ template - BaseTensor(std::vector const& shape, Iter begin, Iter end, MemoryType mem_type, cudaStream_t stream=0) : + BaseTensor(std::vector const& shape, Iter begin, Iter end, MemoryType mem_type, device_id_t device, cudaStream_t stream) : shape_(shape), - buffer_([&begin, &end, &mem_type, &stream] () { + buffer_([&begin, &end, &mem_type, &device, &stream] () { auto total_size = std::transform_reduce( - begin, end, std::plus<>{}, [](auto&& buffer) { return buffer.size(); } + begin, end, size_type{}, std::plus<>{}, [](auto&& buffer) { return buffer.size(); } ); - auto result = Buffer(total_size, mem_type, stream); + auto result = Buffer(total_size, mem_type, device, stream); - std::reduce(begin, end, size_type{0}, [&result](auto&& buffer, auto offset) { + std::accumulate(begin, end, size_type{}, [&result](auto offset, auto& buffer) { copy(result, buffer, offset); return offset + buffer.size(); }); @@ -62,27 +66,33 @@ namespace triton { namespace backend { namespace rapids { }()) {} auto const& shape() const { return shape_; } - auto data() { return buffer_.data(); } + auto size() const { return buffer_.size(); } + auto data() const { return buffer_.data(); } auto& buffer() { return buffer_; } - auto dtype() constexpr { return TritonDtype::value; } - auto mem_type() const { return data.mem_type(); } - auto stream() const { return data.stream(); } - auto device() const { return data.device(); } + auto constexpr dtype() { return TritonDtype::value; } + auto mem_type() const { return buffer_.mem_type(); } + auto stream() const { return buffer_.stream(); } + auto device() const { return buffer_.device(); } void stream_synchronize() const { - if constexpr (IS_GPU_BUILD) { - if (mem_type() == DeviceMemory) { - cuda_check(cudaStreamSynchronize(stream()); - } + if (mem_type() == DeviceMemory) { + buffer_.stream_synchronize(); } } + void set_stream(cudaStream_t new_stream) { + buffer_.set_stream(new_stream); + } + private: std::vector shape_; Buffer buffer_; }; + template + BaseTensor::~BaseTensor() {} + template void copy(BaseTensor> dst, BaseTensor src) { copy(dst.buffer(), src.buffer()); @@ -96,26 +106,33 @@ namespace triton { namespace backend { namespace rapids { * of the data from the src Tensor */ template - copy(Iter begin, Iter end, BaseTensor src) { - std::reduce( + void copy(Iter begin, Iter end, BaseTensor src) { + std::accumulate( begin, end, - decltype(*begin)::size_type{0}, - [&src] (auto&& buffer, auto offset) { + BaseTensor::size_type(0), + [&src] (auto offset, auto& buffer) { auto end_offset = offset + buffer.size(); copy(buffer, src.buffer(), offset, end_offset); return end_offset; } + ); } template struct Tensor final : BaseTensor { + Tensor() : BaseTensor{} {} + Tensor(std::vector::size_type> const& shape, Buffer&& buffer) : BaseTensor(shape, std::move(buffer)) {} + + template + Tensor(std::vector::size_type> const& shape, Iter begin, Iter end, MemoryType mem_type, device_id_t device, cudaStream_t stream) : BaseTensor(shape, begin, end, mem_type, device, stream) {} + }; template struct OutputTensor final : BaseTensor { - OutputTensor(std::vector&& shape, Buffer&& buffer, - std::string const& name, std::shared_pointer responder) : + OutputTensor(std::vector::size_type>&& shape, Buffer&& buffer, + std::string const& name, std::shared_ptr responder) : BaseTensor(std::move(shape), std::move(buffer)), name_{name}, responder_{responder} {} /** @@ -133,19 +150,19 @@ namespace triton { namespace backend { namespace rapids { // Must call the following because BackendOutputResponder does not expose // its stream, so we cannot be certain that our data is not being // processed on another stream. - stream_synchronize(); + BaseTensor::stream_synchronize(); responder_->ProcessTensor( name_.c_str(), TritonDtype::value, - shape(), - data(), - mem_type(), - device() + BaseTensor::shape(), + BaseTensor::data(), + BaseTensor::mem_type(), + BaseTensor::device() ); } private: - std::shared_pointer responder_; + std::shared_ptr responder_; std::string name_; }; }}} // namespace triton::backend::rapids diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 74d6ca3..e765cf7 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -23,6 +23,7 @@ add_executable(test_rapids_triton test/memory/detail/copy.cpp test/memory/types.cpp test/tensor/dtype.cpp + test/tensor/tensor.cpp test/triton/device.cpp test/test.cpp test/utils/const_agnostic.cpp diff --git a/cpp/test/tensor/tensor.cpp b/cpp/test/tensor/tensor.cpp new file mode 100644 index 0000000..49ea793 --- /dev/null +++ b/cpp/test/tensor/tensor.cpp @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { + +TEST(RapidsTriton, default_tensor) { + auto tensor = Tensor(); + EXPECT_EQ(tensor.buffer().size(), 0); + EXPECT_EQ(tensor.shape().size(), 0); +} + +TEST(RapidsTriton, move_buffer_tensor) { + auto shape = std::vector{2, 2}; + auto data = std::vector{1, 2, 3, 4}; + auto tensor = + Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); + + EXPECT_EQ(data.data(), tensor.data()); + EXPECT_EQ(data.size(), tensor.size()); + EXPECT_THAT(tensor.shape(), ::testing::ElementsAreArray(shape)); + + EXPECT_EQ(tensor.dtype(), DTypeInt32); + EXPECT_EQ(tensor.mem_type(), HostMemory); + EXPECT_EQ(tensor.stream(), cudaStream_t{}); + EXPECT_EQ(tensor.device(), 0); + + auto data_out = + std::vector(tensor.data(), tensor.data() + tensor.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +TEST(RapidsTriton, multi_buffer_tensor) { + auto shape = std::vector{2, 2}; + auto data = std::vector{1, 2, 3, 4}; + + auto all_buffers = std::vector>{}; + all_buffers.reserve(data.size()); + std::transform(data.begin(), data.end(), std::back_inserter(all_buffers), + [](auto& elem) { + return Buffer{&elem, 1, DeviceMemory}; + }); + auto tensor = Tensor(shape, all_buffers.begin(), all_buffers.end(), + DeviceMemory, 0, cudaStream_t{}); + + auto data_out = std::vector(data.size()); + cudaMemcpy(static_cast(data_out.data()), + static_cast(tensor.data()), sizeof(int) * tensor.size(), + cudaMemcpyDeviceToHost); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +} // namespace rapids +} // namespace backend +} // namespace triton From 56de9669f329632c53750263f2889bf01882e9eb Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 30 Aug 2021 13:02:00 -0400 Subject: [PATCH 023/199] Add test skeleton for all Triton headers --- cpp/include/rapids_triton/triton/config.hpp | 4 ++-- cpp/include/rapids_triton/triton/deployment.hpp | 4 ++-- cpp/include/rapids_triton/triton/input.hpp | 2 ++ cpp/include/rapids_triton/triton/model.hpp | 3 +++ .../rapids_triton/triton/model_instance.hpp | 2 ++ cpp/include/rapids_triton/triton/responses.hpp | 2 +- cpp/test/CMakeLists.txt | 8 ++++++++ cpp/test/triton/backend.cpp | 17 +++++++++++++++++ cpp/test/triton/config.cpp | 17 +++++++++++++++++ cpp/test/triton/deployment.cpp | 17 +++++++++++++++++ cpp/test/triton/input.cpp | 17 +++++++++++++++++ cpp/test/triton/logging.cpp | 17 +++++++++++++++++ cpp/test/triton/model.cpp | 17 +++++++++++++++++ cpp/test/triton/model_instance.cpp | 17 +++++++++++++++++ cpp/test/triton/requests.cpp | 17 +++++++++++++++++ cpp/test/triton/responses.cpp | 17 +++++++++++++++++ 16 files changed, 173 insertions(+), 5 deletions(-) create mode 100644 cpp/test/triton/backend.cpp create mode 100644 cpp/test/triton/config.cpp create mode 100644 cpp/test/triton/deployment.cpp create mode 100644 cpp/test/triton/input.cpp create mode 100644 cpp/test/triton/logging.cpp create mode 100644 cpp/test/triton/model.cpp create mode 100644 cpp/test/triton/model_instance.cpp create mode 100644 cpp/test/triton/requests.cpp create mode 100644 cpp/test/triton/responses.cpp diff --git a/cpp/include/rapids_triton/triton/config.hpp b/cpp/include/rapids_triton/triton/config.hpp index 3f1c3ed..393599b 100644 --- a/cpp/include/rapids_triton/triton/config.hpp +++ b/cpp/include/rapids_triton/triton/config.hpp @@ -21,10 +21,10 @@ #include #include -#include +#include namespace triton { namespace backend { namespace rapids { - inline auto get_max_batch_size(common::TritonJSON::Value& config) { + inline auto get_max_batch_size(common::TritonJson::Value& config) { auto reported = int64_t{}; triton_check(config.MemberAsInt("max_batch_size", &reported)); return narrow(reported); diff --git a/cpp/include/rapids_triton/triton/deployment.hpp b/cpp/include/rapids_triton/triton/deployment.hpp index 5ae3776..48aa4d5 100644 --- a/cpp/include/rapids_triton/triton/deployment.hpp +++ b/cpp/include/rapids_triton/triton/deployment.hpp @@ -19,8 +19,8 @@ namespace triton { namespace backend { namespace rapids { using DeploymentType = TRITONSERVER_InstanceGroupKind; - using GPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_GPU; - using CPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_CPU; + auto constexpr GPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_GPU; + auto constexpr CPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_CPU; // Note (wphicks): We currently are not including "Auto" or "Model" because I // am not sure exactly how those would be used in context. If there is a // demand, they can be added. diff --git a/cpp/include/rapids_triton/triton/input.hpp b/cpp/include/rapids_triton/triton/input.hpp index 751276a..fe31af8 100644 --- a/cpp/include/rapids_triton/triton/input.hpp +++ b/cpp/include/rapids_triton/triton/input.hpp @@ -18,11 +18,13 @@ #include #include +#include #include #include #include #include #include +#include namespace triton { namespace backend { namespace rapids { inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) { diff --git a/cpp/include/rapids_triton/triton/model.hpp b/cpp/include/rapids_triton/triton/model.hpp index a019335..25e06a2 100644 --- a/cpp/include/rapids_triton/triton/model.hpp +++ b/cpp/include/rapids_triton/triton/model.hpp @@ -27,8 +27,11 @@ #pragma once #include #include +#include #include #include +#include +#include namespace triton { namespace backend { namespace rapids { diff --git a/cpp/include/rapids_triton/triton/model_instance.hpp b/cpp/include/rapids_triton/triton/model_instance.hpp index fb65aa6..01e7727 100644 --- a/cpp/include/rapids_triton/triton/model_instance.hpp +++ b/cpp/include/rapids_triton/triton/model_instance.hpp @@ -25,10 +25,12 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once +#include #include #include #include #include +#include namespace triton { namespace backend { namespace rapids { /** Get the name of a Triton model instance from the instance itself */ diff --git a/cpp/include/rapids_triton/triton/responses.hpp b/cpp/include/rapids_triton/triton/responses.hpp index 7d6743c..5693bcb 100644 --- a/cpp/include/rapids_triton/triton/responses.hpp +++ b/cpp/include/rapids_triton/triton/responses.hpp @@ -29,7 +29,7 @@ auto construct_responses(Iter requests_begin, Iter requests_end) auto responses = std::vector{}; auto requests_size = std::distance(requests_begin, requests_end); - if (!requests_size > 0) { + if (!(requests_size > 0)) { throw TritonException(Error::Internal, "Invalid iterators for requests when constructing responses"); } diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index e765cf7..21964ad 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -24,7 +24,15 @@ add_executable(test_rapids_triton test/memory/types.cpp test/tensor/dtype.cpp test/tensor/tensor.cpp + test/triton/backend.cpp + test/triton/config.cpp + test/triton/deployment.cpp test/triton/device.cpp + test/triton/input.cpp + test/triton/model.cpp + test/triton/model_instance.cpp + test/triton/requests.cpp + test/triton/responses.cpp test/test.cpp test/utils/const_agnostic.cpp test/utils/narrow.cpp diff --git a/cpp/test/triton/backend.cpp b/cpp/test/triton/backend.cpp new file mode 100644 index 0000000..cacff7d --- /dev/null +++ b/cpp/test/triton/backend.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/cpp/test/triton/config.cpp b/cpp/test/triton/config.cpp new file mode 100644 index 0000000..7a31891 --- /dev/null +++ b/cpp/test/triton/config.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/cpp/test/triton/deployment.cpp b/cpp/test/triton/deployment.cpp new file mode 100644 index 0000000..dbd22b7 --- /dev/null +++ b/cpp/test/triton/deployment.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/cpp/test/triton/input.cpp b/cpp/test/triton/input.cpp new file mode 100644 index 0000000..1cac65d --- /dev/null +++ b/cpp/test/triton/input.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/cpp/test/triton/logging.cpp b/cpp/test/triton/logging.cpp new file mode 100644 index 0000000..e9393b5 --- /dev/null +++ b/cpp/test/triton/logging.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/cpp/test/triton/model.cpp b/cpp/test/triton/model.cpp new file mode 100644 index 0000000..3226e35 --- /dev/null +++ b/cpp/test/triton/model.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/cpp/test/triton/model_instance.cpp b/cpp/test/triton/model_instance.cpp new file mode 100644 index 0000000..3582f75 --- /dev/null +++ b/cpp/test/triton/model_instance.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/cpp/test/triton/requests.cpp b/cpp/test/triton/requests.cpp new file mode 100644 index 0000000..cff2af3 --- /dev/null +++ b/cpp/test/triton/requests.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/cpp/test/triton/responses.cpp b/cpp/test/triton/responses.cpp new file mode 100644 index 0000000..4ca7d52 --- /dev/null +++ b/cpp/test/triton/responses.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include From c0a40232990f0ab2019896d529f406c88e342251 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 30 Aug 2021 13:55:02 -0400 Subject: [PATCH 024/199] Add tests for logging --- cpp/test/CMakeLists.txt | 1 + cpp/test/tensor/dtype.cpp | 2 ++ cpp/test/triton/logging.cpp | 15 +++++++++++++++ 3 files changed, 18 insertions(+) diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 21964ad..800ccf5 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -29,6 +29,7 @@ add_executable(test_rapids_triton test/triton/deployment.cpp test/triton/device.cpp test/triton/input.cpp + test/triton/logging.cpp test/triton/model.cpp test/triton/model_instance.cpp test/triton/requests.cpp diff --git a/cpp/test/tensor/dtype.cpp b/cpp/test/tensor/dtype.cpp index 46c9aaa..52a78f4 100644 --- a/cpp/test/tensor/dtype.cpp +++ b/cpp/test/tensor/dtype.cpp @@ -22,11 +22,13 @@ namespace triton { namespace backend { namespace rapids { + template void check_dtype_conversion() { EXPECT_EQ(D, TritonDtype::type>::value); EXPECT_EQ(D, TritonDtype::type const>::value); } + TEST(RapidsTriton, dtype) { check_dtype_conversion(); check_dtype_conversion(); diff --git a/cpp/test/triton/logging.cpp b/cpp/test/triton/logging.cpp index e9393b5..a265800 100644 --- a/cpp/test/triton/logging.cpp +++ b/cpp/test/triton/logging.cpp @@ -14,4 +14,19 @@ * limitations under the License. */ +#include + #include + +namespace triton { +namespace backend { +namespace rapids { +TEST(RapidsTriton, logging) { + log_debug("Debug test message"); + log_info("Info test message"); + log_warn("Warn test message"); + log_error("Error test message"); +} +} // namespace rapids +} // namespace backend +} // namespace triton From d8177cdf671bd685d414831ec235babbcbb8742f Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 30 Aug 2021 17:42:13 -0400 Subject: [PATCH 025/199] Add tests for Batch --- cpp/include/rapids_triton/batch/batch.hpp | 53 ++++++++++----- cpp/include/rapids_triton/triton/input.hpp | 4 +- cpp/include/rapids_triton/triton/output.hpp | 73 +++++++++++++++++++++ cpp/src/interface/api.cc | 8 +-- cpp/test/CMakeLists.txt | 1 + cpp/test/batch/batch.cpp | 17 +++++ 6 files changed, 132 insertions(+), 24 deletions(-) create mode 100644 cpp/include/rapids_triton/triton/output.hpp create mode 100644 cpp/test/batch/batch.cpp diff --git a/cpp/include/rapids_triton/batch/batch.hpp b/cpp/include/rapids_triton/batch/batch.hpp index 86bdbe3..dc7a3e0 100644 --- a/cpp/include/rapids_triton/batch/batch.hpp +++ b/cpp/include/rapids_triton/batch/batch.hpp @@ -17,34 +17,46 @@ #pragma once #include +#include +#include +#include +#include #include +#include +#include #include #include +#include #include +#include +#include #include #include namespace triton { namespace backend { namespace rapids { + template struct Batch { using size_type = std::size_t; - Batch(TRITONBACKEND_Request** raw_requests, request_size_t count, TRITONBACKEND_MemoryManager& triton_mem_manager, bool use_pinned_input, bool use_pinned_output, size_type max_batch_size, cudaStream_t stream) : - requests_(raw_requests, count), + Batch(ModelState* model_state, ModelInstanceState* instance_state, TRITONBACKEND_Request** raw_requests, request_size_t count, std::vector const& output_shape, size_type max_batch_size, cudaStream_t stream) : + model_state_{model_state}, + instance_state_{instance_state}, + requests_(raw_requests, raw_requests + count), responses_(construct_responses(requests_.begin(), requests_.end())), collector_{ raw_requests, count, &responses_, - &triton_mem_manager, - use_pinned_input, + instance_state->TritonMemoryManager(), + instance_state->EnablePinnedInput(), stream }, responder_{std::make_shared( raw_requests, count, &responses_, - max_batch_size, - use_pinned_output, + instance_state->get_model().get_config_param("max_batch_size"), + instance_state->EnablePinnedOutput(), stream )}, stream_{stream} {} @@ -67,19 +79,19 @@ namespace triton { namespace backend { namespace rapids { {{memory_type, device_id}}, &raw_buffer, &reported_bytes, - &reported_memory_type, + &reported_mem_type, &reported_device_id ); auto buffer = Buffer( reinterpret_cast(raw_buffer), reported_bytes, - reported_memory_type, + reported_mem_type, stream_ ); - if (reported_memory_type != memory_type || reported_device_id != device_id) { - throw(TritonException(Error::Internal, "data collected in wrong location"); + if (reported_mem_type != memory_type || reported_device_id != device_id) { + throw TritonException(Error::Internal, "data collected in wrong location"); } return Tensor(std::move(shape), std::move(buffer)); @@ -87,8 +99,10 @@ namespace triton { namespace backend { namespace rapids { template auto get_output(std::string const& name, MemoryType memory_type, device_id_t device_id) { - auto shape = get_output_shape(requests_.begin(), requests_.end(), name); // TODO(wphicks) - // TODO(wphicks): construct buffer + auto shape = get_output_shape(requests_.begin(), requests_.end(), name); + auto buffer_size = std::reduce( + shape.begin(), shape.end(), std::size_t{1}, std::multiplies<>()); + auto buffer = Buffer(buffer_size, memory_type, device_id, stream_); return OutputTensor(std::move(shape), std::move(buffer), responder_, name); } @@ -105,19 +119,28 @@ namespace triton { namespace backend { namespace rapids { } private: + ModelState* model_state_; + ModelInstanceState* instance_state_; std::vector requests_; std::vector responses_; BackendInputCollector collector_; - std::shared_pointer responder_; + std::shared_ptr responder_; cudaStream_t stream_; auto get_input_shape(std::string const& name) { - auto result = decltype(get_triton_input_shape(*std::begin(requests_))){}; + auto result = std::vector{}; if(!requests_.empty()) { - result = get_triton_input_shape(*std::begin(requests_)) + result = get_triton_input_shape(std::begin(requests_), std::end(requests_), name); } return result; } + + auto get_output_shape(std::string const& name) { + auto result = std::vector{}; + auto& triton_result = model_state_->FindBatchOutput(name)->OutputShape(); + std::transform(std::begin(triton_result), std::end(triton_result), std::back_inserter(result), [](auto& coord) { return narrow(coord); }); + return result; + } }; }}} // namespace triton::backend::rapids diff --git a/cpp/include/rapids_triton/triton/input.hpp b/cpp/include/rapids_triton/triton/input.hpp index fe31af8..c412500 100644 --- a/cpp/include/rapids_triton/triton/input.hpp +++ b/cpp/include/rapids_triton/triton/input.hpp @@ -42,11 +42,11 @@ namespace triton { namespace backend { namespace rapids { auto const* input_shape = static_cast(nullptr); auto input_dims = uint32_t{}; - auto batch_dim = std::reduce( + auto batch_dim = std::accumulate( requests_begin, requests_end, int64_t{}, - [&reported_dtype, &input_shape, &input_dims, &name](auto& request, auto total) { + [&reported_dtype, &input_shape, &input_dims, &name](auto total, auto& request) { auto* input = get_triton_input(request, name); triton_check( TRITONBACKEND_InputProperties( diff --git a/cpp/include/rapids_triton/triton/output.hpp b/cpp/include/rapids_triton/triton/output.hpp new file mode 100644 index 0000000..cb0bf5b --- /dev/null +++ b/cpp/include/rapids_triton/triton/output.hpp @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +namespace triton { namespace backend { namespace rapids { + inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) { + auto result = static_cast(nullptr); + triton_check( + TRITONBACKEND_RequestInput(request, name.c_str(), &result)); + return result; + } + + template + auto get_triton_output_shape(Iter requests_begin, Iter requests_end, std::string const& name) { + auto result = std::vector{}; + + auto reported_dtype = DType{}; + auto const* input_shape = static_cast(nullptr); + auto input_dims = uint32_t{}; + + auto batch_dim = std::reduce( + requests_begin, + requests_end, + int64_t{}, + [&reported_dtype, &input_shape, &input_dims, &name](auto& request, auto total) { + auto* input = get_triton_input(request, name); + triton_check( + TRITONBACKEND_InputProperties( + input, nullptr, &reported_dtype, &input_shape, + &input_dims, nullptr, nullptr)); + + if (reported_dtype != TritonDtype::value) { + throw(TritonException(Error::Internal, "incorrect type for requested input")); + } + + if (input_dims != 0) { + total += *input_shape; + } + return total; + }); + + result.reserve(input_dims); + std::transform( + input_shape, + input_shape + input_dims, + std::back_inserter(result), + [](auto& val) { return narrow(val); } + ); + + if (!result.empty()) { + result[0] = narrow(batch_dim); + } + + return result; + } +}}} diff --git a/cpp/src/interface/api.cc b/cpp/src/interface/api.cc index 9bb369e..76e7cc0 100644 --- a/cpp/src/interface/api.cc +++ b/cpp/src/interface/api.cc @@ -155,13 +155,7 @@ auto* TRITONBACKEND_ModelInstanceExecute(TRITONBACKEND_ModelInstance* instance, auto* instance_state = rapids::get_instance_state(*instance); auto& model = instance_state->get_model(); - auto max_batch_size = model.get_config_param("max_batch_size"); - auto batch = Batch{raw_requests, - request_count, - model_state->TritonMemoryManager(), - model_state->EnablePinnedInput(), - model_state->EnablePinnedOutput(), - max_batch_size, + auto batch = Batch{model_state, instance_state, raw_requests, request_count, model.get_stream()}; model.predict(batch); diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 800ccf5..8e0513b 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -18,6 +18,7 @@ add_executable(test_rapids_triton test/build_control.cpp test/exceptions.cpp + test/batch/batch.cpp test/memory/buffer.cpp test/memory/detail/allocate.cpp test/memory/detail/copy.cpp diff --git a/cpp/test/batch/batch.cpp b/cpp/test/batch/batch.cpp new file mode 100644 index 0000000..e522ecb --- /dev/null +++ b/cpp/test/batch/batch.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include From 23041d8e1f8abf16bc53a7c811668351763f62a4 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 30 Aug 2021 18:16:57 -0400 Subject: [PATCH 026/199] Properly encapsulate state types in Batch --- cpp/include/rapids_triton/batch/batch.hpp | 63 +++++++++++------------ cpp/src/interface/api.cc | 23 ++++++++- 2 files changed, 52 insertions(+), 34 deletions(-) diff --git a/cpp/include/rapids_triton/batch/batch.hpp b/cpp/include/rapids_triton/batch/batch.hpp index dc7a3e0..7c55c31 100644 --- a/cpp/include/rapids_triton/batch/batch.hpp +++ b/cpp/include/rapids_triton/batch/batch.hpp @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -38,28 +39,34 @@ namespace triton { namespace backend { namespace rapids { struct Batch { using size_type = std::size_t; - Batch(ModelState* model_state, ModelInstanceState* instance_state, TRITONBACKEND_Request** raw_requests, request_size_t count, std::vector const& output_shape, size_type max_batch_size, cudaStream_t stream) : - model_state_{model_state}, - instance_state_{instance_state}, - requests_(raw_requests, raw_requests + count), - responses_(construct_responses(requests_.begin(), requests_.end())), - collector_{ - raw_requests, - count, - &responses_, - instance_state->TritonMemoryManager(), - instance_state->EnablePinnedInput(), - stream - }, - responder_{std::make_shared( - raw_requests, - count, - &responses_, - instance_state->get_model().get_config_param("max_batch_size"), - instance_state->EnablePinnedOutput(), - stream - )}, - stream_{stream} {} + Batch(TRITONBACKEND_Request** raw_requests, + request_size_t count, + TRITONBACKEND_MemoryManager& triton_mem_manager, + std::function(std::string const&)> get_output_shape, + bool use_pinned_input, + bool use_pinned_output, + size_type max_batch_size, + cudaStream_t stream) : + requests_(raw_requests, raw_requests + count), + responses_(construct_responses(requests_.begin(), requests_.end())), + get_output_shape_{get_output_shape}, + collector_{ + raw_requests, + count, + &responses_, + triton_mem_manager, + use_pinned_input, + stream + }, + responder_{std::make_shared( + raw_requests, + count, + &responses_, + max_batch_size, + use_pinned_output, + stream + )}, + stream_{stream} {} template @@ -99,7 +106,7 @@ namespace triton { namespace backend { namespace rapids { template auto get_output(std::string const& name, MemoryType memory_type, device_id_t device_id) { - auto shape = get_output_shape(requests_.begin(), requests_.end(), name); + auto shape = get_output_shape_(name); auto buffer_size = std::reduce( shape.begin(), shape.end(), std::size_t{1}, std::multiplies<>()); auto buffer = Buffer(buffer_size, memory_type, device_id, stream_); @@ -119,10 +126,9 @@ namespace triton { namespace backend { namespace rapids { } private: - ModelState* model_state_; - ModelInstanceState* instance_state_; std::vector requests_; std::vector responses_; + std::function(std::string const&)> get_output_shape_; BackendInputCollector collector_; std::shared_ptr responder_; cudaStream_t stream_; @@ -134,13 +140,6 @@ namespace triton { namespace backend { namespace rapids { } return result; } - - auto get_output_shape(std::string const& name) { - auto result = std::vector{}; - auto& triton_result = model_state_->FindBatchOutput(name)->OutputShape(); - std::transform(std::begin(triton_result), std::end(triton_result), std::back_inserter(result), [](auto& coord) { return narrow(coord); }); - return result; - } }; }}} // namespace triton::backend::rapids diff --git a/cpp/src/interface/api.cc b/cpp/src/interface/api.cc index 76e7cc0..8a707c5 100644 --- a/cpp/src/interface/api.cc +++ b/cpp/src/interface/api.cc @@ -155,8 +155,27 @@ auto* TRITONBACKEND_ModelInstanceExecute(TRITONBACKEND_ModelInstance* instance, auto* instance_state = rapids::get_instance_state(*instance); auto& model = instance_state->get_model(); - auto batch = Batch{model_state, instance_state, raw_requests, request_count, - model.get_stream()}; + auto max_batch_size = model.get_config_param("max_batch_size"); + auto batch = Batch{raw_requests, request_count, + model_state->TritonMemoryManager(), + /* Note: It is safe to keep a copy of the model_state + * pointer in this closure because both the batch and + * the original model_state pointer go out of scope at + * the end of this block. */ + [model_state](std::string const& name) { + auto result = std::vector{}; + auto& triton_result = + model_state->FindBatchOutput(name)->OutputShape(); + std::transform( + std::begin(triton_result), std::end(triton_result), + std::back_inserter(result), [](auto& coord) { + return narrow(coord); + }); + return result; + }, + model_state->EnablePinnedInput(), + model_state->EnablePinnedOutput(), + max_batch_size model.get_stream()}; model.predict(batch); batch.finalize(); From 2933d20d8dc3baaa9912adb303232d3bf812d130 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 31 Aug 2021 11:58:37 -0400 Subject: [PATCH 027/199] Begin adding statistics reporting --- cpp/include/rapids_triton/batch/batch.hpp | 54 ++++++++++++- .../rapids_triton/triton/statistics.hpp | 79 +++++++++++++++++++ cpp/src/interface/api.cc | 55 ++++++++----- 3 files changed, 165 insertions(+), 23 deletions(-) create mode 100644 cpp/include/rapids_triton/triton/statistics.hpp diff --git a/cpp/include/rapids_triton/batch/batch.hpp b/cpp/include/rapids_triton/batch/batch.hpp index 7c55c31..fc91818 100644 --- a/cpp/include/rapids_triton/batch/batch.hpp +++ b/cpp/include/rapids_triton/batch/batch.hpp @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -35,6 +36,29 @@ #include namespace triton { namespace backend { namespace rapids { + /** + * @brief A representation of all data about a single batch of inference + * requests + * + * Batch objects are the primary interface point between rapids_triton Models + * and the Triton server itself. By calling the `get_input` and `get_output` + * methods of a batch, Model implementations can retrieve the input Tensors + * necessary for prediction and the output Tensors where results can be + * stored. + * + * Batch objects also handle a variety of other tasks necessary for + * processing a batch in the Triton model. This includes reporting statistics + * on how long it took to process requests and sending responses to the + * client via the Triton server once processing is complete. + * + * It is not recommended that developers of rapids_triton backends try to + * construct Batch objects directly. Instead, you should base your backend + * repository on the rapids_triton_template repo. This template repo includes + * all of the code necessary to construct a Batch and feed it to a Model + * implementation. Model implementers need only retrieve the input/output + * Tensors they require from the Batch and then call `finalize` on the output + * Tensors. The code provided in the template repo takes care of the rest. + */ template struct Batch { using size_type = std::size_t; @@ -43,6 +67,13 @@ namespace triton { namespace backend { namespace rapids { request_size_t count, TRITONBACKEND_MemoryManager& triton_mem_manager, std::function(std::string const&)> get_output_shape, + std::function< + void( + TRITONBACKEND_Request*, + time_point const&, + time_point const&, + time_point const&, + time_point const&)> report_request_statistics, bool use_pinned_input, bool use_pinned_output, size_type max_batch_size, @@ -66,7 +97,9 @@ namespace triton { namespace backend { namespace rapids { use_pinned_output, stream )}, - stream_{stream} {} + stream_{stream}, + start_time_{std::chrono::steady_clock::now()}, + compute_start_time_{std::chrono::steady_clock::now()} {} template @@ -101,6 +134,9 @@ namespace triton { namespace backend { namespace rapids { throw TritonException(Error::Internal, "data collected in wrong location"); } + // Set start time of batch to time latest input tensor was retrieved + compute_start_time_ = std::chrono::steady_clock::now(); + return Tensor(std::move(shape), std::move(buffer)); } @@ -113,15 +149,27 @@ namespace triton { namespace backend { namespace rapids { return OutputTensor(std::move(shape), std::move(buffer), responder_, name); } + auto const& start_time() const { + return compute_start_time_; + } + auto stream() const { return stream_; } void finalize() { - // TODO(wphicks): report statistics + auto compute_end_time = std::chrono::steady_clock::now(); if (responder_->Finalize()) { cuda_check(cudaStreamSynchronize(stream_)); } + std::for_each( + std::begin(requests_), + std::end(requests_), + [&report_statistics_, &start_time_, &compute_start_time_, &compute_end_time]( + auto& request) { + report_statistics_(request, true, start_time_, compute_start_time_, compute_end_time); + } + ); //TODO(wphicks): Send responses } @@ -132,6 +180,8 @@ namespace triton { namespace backend { namespace rapids { BackendInputCollector collector_; std::shared_ptr responder_; cudaStream_t stream_; + std::chrono::time_point start_time_; + std::chrono::time_point compute_start_time_; auto get_input_shape(std::string const& name) { auto result = std::vector{}; diff --git a/cpp/include/rapids_triton/triton/statistics.hpp b/cpp/include/rapids_triton/triton/statistics.hpp new file mode 100644 index 0000000..614111d --- /dev/null +++ b/cpp/include/rapids_triton/triton/statistics.hpp @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +namespace triton { namespace backend { namespace rapids { + using time_point = std::chrono::time_point; + + /** + * @brief Report inference statistics for a single request + * + * @param instance The Triton model instance which is processing this request + * @param request The Triton request object itself + * @param start_time The time at which the backend first received the request + * @param compute_start_time The time at which the backend began actual + * inference on the request + * @param compute_end_time The time at which the backend completed inference + * on the request + * @param end_time The time at which the backend finished all processing on + * the request, including copying out results and returning a response + */ + inline void report_statistics(TRITONBACKEND_ModelInstance& instance, + TRITONBACKEND_Request& request, time_point start_time, time_point + compute_start_time, time_point compute_end_time, time_point end_time) { + triton_check(TRITONBACKEND_ModelInstanceReportStatistics( + &instance, + &request, + true, + start_time.time_since_epoch().count(), + compute_start_time.time_since_epoch().count(), + compute_end_time.time_since_epoch().count(), + end_time.time_since_epoch().count() + )); + } + + /** + * @brief Report inference statistics for a batch of requests of given size + * + * @param instance The Triton model instance which is processing this batch + * @param request_count The number of requests in this batch + * @param start_time The time at which the backend first received the batch + * @param compute_start_time The time at which the backend began actual + * inference on the batch + * @param compute_end_time The time at which the backend completed inference + * on the batch + * @param end_time The time at which the backend finished all processing on + * the batch, including copying out results and returning a response + */ + inline void report_statistics(TRITONBACKEND_ModelInstance& instance, + std::size_t request_count, time_point start_time, time_point + compute_start_time, time_point compute_end_time, time_point end_time) { + triton_check(TRITONBACKEND_ModelInstanceReportBatchStatistics( + &instance, + request_count, + true, + start_time.time_since_epoch().count(), + compute_start_time.time_since_epoch().count(), + compute_end_time.time_since_epoch().count(), + end_time.time_since_epoch().count() + )); + } +}}} diff --git a/cpp/src/interface/api.cc b/cpp/src/interface/api.cc index 8a707c5..b536b84 100644 --- a/cpp/src/interface/api.cc +++ b/cpp/src/interface/api.cc @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -103,7 +104,6 @@ auto* TRITONBACKEND_ModelInstanceInitialize( auto device_id = rapids::get_device_id(*instance); auto deployment_type = rapids::get_deployment_type(*instance); - // TODO (wphicks) // TODO (wphicks): Replace InstanceGroupKindString call rapids::log_info(__FILE__, __LINE__, (std::string("TRITONBACKEND_ModelInstanceInitialize: ") + @@ -148,6 +148,8 @@ auto* TRITONBACKEND_ModelInstanceFinalize( auto* TRITONBACKEND_ModelInstanceExecute(TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** raw_requests, uint32_t const request_count) { + auto start_time = std::chrono::steady_clock::now(); + auto* result = static_cast(nullptr); try { @@ -156,29 +158,40 @@ auto* TRITONBACKEND_ModelInstanceExecute(TRITONBACKEND_ModelInstance* instance, rapids::get_instance_state(*instance); auto& model = instance_state->get_model(); auto max_batch_size = model.get_config_param("max_batch_size"); - auto batch = Batch{raw_requests, request_count, - model_state->TritonMemoryManager(), - /* Note: It is safe to keep a copy of the model_state - * pointer in this closure because both the batch and - * the original model_state pointer go out of scope at - * the end of this block. */ - [model_state](std::string const& name) { - auto result = std::vector{}; - auto& triton_result = - model_state->FindBatchOutput(name)->OutputShape(); - std::transform( - std::begin(triton_result), std::end(triton_result), - std::back_inserter(result), [](auto& coord) { - return narrow(coord); - }); - return result; - }, - model_state->EnablePinnedInput(), - model_state->EnablePinnedOutput(), - max_batch_size model.get_stream()}; + auto batch = Batch{ + raw_requests, request_count, model_state->TritonMemoryManager(), + /* Note: It is safe to keep a copy of the model_state + * pointer in this closure and the instance pointer in the next because + * the batch goes out of scope at the end of this block and Triton + * guarantees that the liftimes of both the instance and model states + * extend beyond this function call. */ + [model_state](std::string const& name) { + auto result = std::vector{}; + auto& triton_result = + model_state->FindBatchOutput(name)->OutputShape(); + std::transform(std::begin(triton_result), std::end(triton_result), + std::back_inserter(result), [](auto& coord) { + return narrow(coord); + }); + return result; + }, + [instance](TRITONBACKEND_Request* request, time_point req_start, + time_point req_comp_start, time_point req_comp_end, + time_point req_end) { + report_statistics(*instance, request, req_start, req_comp_start, + req_comp_end, req_end); + }, + model_state->EnablePinnedInput(), model_state->EnablePinnedOutput(), + max_batch_size model.get_stream()}; model.predict(batch); + auto& compute_start_time = batch.compute_start_time(); + auto compute_end_time = std::chrono::steady_clock::now(); batch.finalize(); + auto end_time = std::chrono::steady_clock::now(); + + report_statistics(*instance, request_count, start_time, compute_start_time, + compute_end_time, end_time); } catch (rapids::TritonException& err) { result = err.error(); } From ec7c42f2f4987a31f08ee53fe774739bb989e43e Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 31 Aug 2021 12:52:21 -0400 Subject: [PATCH 028/199] Refactor API logic into library --- .../rapids_triton/triton/api/execute.hpp | 73 ++++++++ .../rapids_triton/triton/api/initialize.hpp | 41 +++++ .../triton/api/instance_finalize.hpp | 42 +++++ .../triton/api/instance_initialize.hpp | 47 ++++++ .../triton/api/model_finalize.hpp | 40 +++++ .../triton/api/model_initialize.hpp | 45 +++++ .../triton/model_instance_state.hpp | 45 +++++ .../rapids_triton/triton/model_state.hpp | 38 +++++ cpp/src/interface/api.cc | 159 ++---------------- cpp/src/interface/model_instance_state.h | 24 +-- cpp/src/interface/model_state.h | 18 +- 11 files changed, 386 insertions(+), 186 deletions(-) create mode 100644 cpp/include/rapids_triton/triton/api/execute.hpp create mode 100644 cpp/include/rapids_triton/triton/api/initialize.hpp create mode 100644 cpp/include/rapids_triton/triton/api/instance_finalize.hpp create mode 100644 cpp/include/rapids_triton/triton/api/instance_initialize.hpp create mode 100644 cpp/include/rapids_triton/triton/api/model_finalize.hpp create mode 100644 cpp/include/rapids_triton/triton/api/model_initialize.hpp create mode 100644 cpp/include/rapids_triton/triton/model_instance_state.hpp create mode 100644 cpp/include/rapids_triton/triton/model_state.hpp diff --git a/cpp/include/rapids_triton/triton/api/execute.hpp b/cpp/include/rapids_triton/triton/api/execute.hpp new file mode 100644 index 0000000..280bcbd --- /dev/null +++ b/cpp/include/rapids_triton/triton/api/execute.hpp @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { namespace backend { namespace rapids { namespace triton_api { + template + auto* execute(TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** raw_requests, std::size_t request_count) { + auto start_time = std::chrono::steady_clock::now(); + + auto* result = static_cast(nullptr); + + try { + auto* model_state = rapids::get_model_state(*instance); + auto* instance_state = + rapids::get_instance_state(*instance); + auto& model = instance_state->get_model(); + auto max_batch_size = model.get_config_param("max_batch_size"); + auto batch = Batch{ + raw_requests, request_count, model_state->TritonMemoryManager(), + /* Note: It is safe to keep a copy of the model_state + * pointer in this closure and the instance pointer in the next because + * the batch goes out of scope at the end of this block and Triton + * guarantees that the liftimes of both the instance and model states + * extend beyond this function call. */ + [model_state](std::string const& name) { + auto result = std::vector{}; + auto& triton_result = + model_state->FindBatchOutput(name)->OutputShape(); + std::transform(std::begin(triton_result), std::end(triton_result), + std::back_inserter(result), [](auto& coord) { + return narrow(coord); + }); + return result; + }, + [instance](TRITONBACKEND_Request* request, time_point req_start, + time_point req_comp_start, time_point req_comp_end, + time_point req_end) { + report_statistics(*instance, request, req_start, req_comp_start, + req_comp_end, req_end); + }, + model_state->EnablePinnedInput(), model_state->EnablePinnedOutput(), + max_batch_size model.get_stream()}; + + model.predict(batch); + auto& compute_start_time = batch.compute_start_time(); + auto compute_end_time = std::chrono::steady_clock::now(); + batch.finalize(); + auto end_time = std::chrono::steady_clock::now(); + + report_statistics(*instance, request_count, start_time, compute_start_time, + compute_end_time, end_time); + } catch (rapids::TritonException& err) { + result = err.error(); + } + + return result; + } +}}}} diff --git a/cpp/include/rapids_triton/triton/api/initialize.hpp b/cpp/include/rapids_triton/triton/api/initialize.hpp new file mode 100644 index 0000000..5bf9ad0 --- /dev/null +++ b/cpp/include/rapids_triton/triton/api/initialize.hpp @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { namespace backend { namespace rapids { namespace triton_api { + auto* model_initialize(TRITONBACKEND_Model* model) { + auto* result = static_cast(nullptr); + try { + auto name = rapids::get_backend_name(*backend); + + // TODO (wphicks): use sstream + rapids::log_info( + __FILE__, __LINE__, + (std::string("TRITONBACKEND_Initialize: ") + name).c_str()); + + if (!rapids::check_backend_version(*backend)) { + throw rapids::TritonException{ + rapids::Error::Unsupported, + "triton backend API version does not support this backend"}; + } + } catch (rapids::TritonException& err) { + result = err.error(); + } + return result; + } +}}}} diff --git a/cpp/include/rapids_triton/triton/api/instance_finalize.hpp b/cpp/include/rapids_triton/triton/api/instance_finalize.hpp new file mode 100644 index 0000000..ad22ae4 --- /dev/null +++ b/cpp/include/rapids_triton/triton/api/instance_finalize.hpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { namespace backend { namespace rapids { namespace triton_api { + template + auto* instance_finalize(TRITONBACKEND_ModelInstance* instance) { + auto* result = static_cast(nullptr); + try { + auto* instance_state = + rapids::get_instance_state(*instance); + if (instance_state != nullptr) { + instance_state->unload(); + + rapids::log_info( + __FILE__, __LINE__, + "TRITONBACKEND_ModelInstanceFinalize: delete instance state"); + + delete instance_state; + } + } catch (rapids::TritonException& err) { + result = err.error(); + } + + return result; + } +}}}} diff --git a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp new file mode 100644 index 0000000..f7f24e8 --- /dev/null +++ b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { namespace backend { namespace rapids { namespace triton_api { + template + auto* instance_initialize(TRITONBACKEND_ModelInstance* instance) { + auto* result = static_cast(nullptr); + try { + auto name = rapids::get_model_instance_name(*instance); + auto device_id = rapids::get_device_id(*instance); + auto deployment_type = rapids::get_deployment_type(*instance); + + // TODO (wphicks): Use sstream + rapids::log_info(__FILE__, __LINE__, + (std::string("TRITONBACKEND_ModelInstanceInitialize: ") + + name + " (" + TRITONSERVER_InstanceGroupKindString(kind) + + " device " + std::to_string(device_id) + ")") + .c_str()); + + auto* model_state = rapids::get_model_state(*instance); + + auto rapids_model = + std::make_unique(model_state, instance) + + rapids::set_instance_state(*instance, std::move(rapids_model)); + } catch (rapids::TritonException& err) { + result = err.error(); + } + return result; + } +}}}} diff --git a/cpp/include/rapids_triton/triton/api/model_finalize.hpp b/cpp/include/rapids_triton/triton/api/model_finalize.hpp new file mode 100644 index 0000000..b233b62 --- /dev/null +++ b/cpp/include/rapids_triton/triton/api/model_finalize.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { namespace backend { namespace rapids { namespace triton_api { + template + auto* model_finalize(TRITONBACKEND_Model* model) { + auto* result = static_cast(nullptr); + try { + auto model_state = rapids::get_model_state(*model); + if (model_state != nullptr) { + model_state->get_shared_state()->unload(); + } + + rapids::log_info(__FILE__, __LINE__, + "TRITONBACKEND_ModelFinalize: delete model state"); + + delete model_state; + } catch (rapids::TritonException& err) { + result = err.error(); + } + + return result; + } +}}}} diff --git a/cpp/include/rapids_triton/triton/api/model_initialize.hpp b/cpp/include/rapids_triton/triton/api/model_initialize.hpp new file mode 100644 index 0000000..239ead2 --- /dev/null +++ b/cpp/include/rapids_triton/triton/api/model_initialize.hpp @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { namespace backend { namespace rapids { namespace triton_api { + template + auto* model_initialize(TRITONBACKEND_Model* model) { + auto* result = static_cast(nullptr); + try { + auto name = rapids::get_model_name(*model); + + auto version = rapids::get_model_version(*model); + + // TODO (wphicks): Use sstream + rapids::log_info(__FILE__, __LINE__, + (std::string("TRITONBACKEND_ModelInitialize: ") + name + + " (version " + std::to_string(version) + ")") + .c_str()); + + auto rapids_model_state = std::make_unique(*model); + rapids_model_state->load(); + + rapids::set_model_state(*model, std::move(rapids_model_state)); + } catch (rapids::TritonException& err) { + result = err.error(); + } + + return result; + } +}}}} diff --git a/cpp/include/rapids_triton/triton/model_instance_state.hpp b/cpp/include/rapids_triton/triton/model_instance_state.hpp new file mode 100644 index 0000000..53c30bf --- /dev/null +++ b/cpp/include/rapids_triton/triton/model_instance_state.hpp @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include + +namespace triton { namespace backend { namespace rapids { + +template +struct ModelInstanceState : public BackendModelInstance { + ModelInstanceState(TritonModelState& model_state, + TRITONBACKEND_ModelInstance* triton_model_instance) + : BackendModelInstance(&model_state, triton_model_instance), + model_(model_state.get_shared_state(), + rapids::get_device_id(*triton_model_instance), CudaStream(), + Kind(), + JoinPath({RepositoryPath(), std::to_string(Version()), + ArtifactFilename()})) {} + + auto& get_model() const { return model_; } + + void load() { model_->load(); } + void unload() { model_->unload(); } + + private: + RapidsModel model_; +}; + +}}} diff --git a/cpp/include/rapids_triton/triton/model_state.hpp b/cpp/include/rapids_triton/triton/model_state.hpp new file mode 100644 index 0000000..f2b4953 --- /dev/null +++ b/cpp/include/rapids_triton/triton/model_state.hpp @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include + +namespace triton { namespace backend { namespace rapids { +template +struct TritonModelState : public BackendModel { + + TritonModelState(TRITONBACKEND_Model& triton_model) + : state_{std::make_shared( + get_model_config(triton_model))} {} + + void load() { state_->load(); } + void unload() { state_->unload(); } + + auto get_shared_state() { return state_; } + + private: + std::shared_ptr state_; +}; + +}}} diff --git a/cpp/src/interface/api.cc b/cpp/src/interface/api.cc index b536b84..8254426 100644 --- a/cpp/src/interface/api.cc +++ b/cpp/src/interface/api.cc @@ -21,182 +21,47 @@ #include #include -#include -#include +#include namespace triton { namespace backend { namespace NAMESPACE { +using ModelState = TritonModelState; +using ModelInstanceState = TritonModelState; + extern "C" { /** Confirm that backend is compatible with Triton's backend API version */ auto* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { - auto* result = static_cast(nullptr); - try { - auto name = rapids::get_backend_name(*backend); - - // TODO (wphicks) - rapids::log_info( - __FILE__, __LINE__, - (std::string("TRITONBACKEND_Initialize: ") + name).c_str()); - - if (!rapids::check_backend_version(*backend)) { - throw rapids::TritonException{ - rapids::Error::Unsupported, - "triton backend API version does not support this backend"}; - } - } catch (rapids::TritonException& err) { - result = err.error(); - } - return result; + return rapids::triton_api::initialize(backend); } auto* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { - auto* result = static_cast(nullptr); - try { - auto name = rapids::get_model_name(*model); - - auto version = rapids::get_model_version(*model); - - // TODO (wphicks) - rapids::log_info(__FILE__, __LINE__, - (std::string("TRITONBACKEND_ModelInitialize: ") + name + - " (version " + std::to_string(version) + ")") - .c_str()); - - auto rapids_model_state = std::make_unique(*model); - rapids_model_state->load(); - - rapids::set_model_state(*model, std::move(rapids_model_state)); - } catch (rapids::TritonException& err) { - result = err.error(); - } - - return result; + return rapids::triton_api::model_initialize(model); } auto* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { - auto* result = static_cast(nullptr); - try { - auto model_state = rapids::get_model_state(*model); - if (model_state != nullptr) { - model_state->get_shared_state()->unload(); - } - - rapids::log_info(__FILE__, __LINE__, - "TRITONBACKEND_ModelFinalize: delete model state"); - - delete model_state; - } catch (rapids::TritonException& err) { - result = err.error(); - } - - return result; + return rapids::triton_api::model_finalize(model); } auto* TRITONBACKEND_ModelInstanceInitialize( TRITONBACKEND_ModelInstance* instance) { - auto* result = static_cast(nullptr); - try { - auto name = rapids::get_model_instance_name(*instance); - auto device_id = rapids::get_device_id(*instance); - auto deployment_type = rapids::get_deployment_type(*instance); - - // TODO (wphicks): Replace InstanceGroupKindString call - rapids::log_info(__FILE__, __LINE__, - (std::string("TRITONBACKEND_ModelInstanceInitialize: ") + - name + " (" + TRITONSERVER_InstanceGroupKindString(kind) + - " device " + std::to_string(device_id) + ")") - .c_str()); - - auto* model_state = rapids::get_model_state(*instance); - - auto rapids_model = - std::make_unique(model_state, instance) - - rapids::set_instance_state(*instance, std::move(rapids_model)); - } catch (rapids::TritonException& err) { - result = err.error(); - } - return result; + return rapids::triton_api::instance_initialize(instance); } auto* TRITONBACKEND_ModelInstanceFinalize( TRITONBACKEND_ModelInstance* instance) { - auto* result = static_cast(nullptr); - try { - auto* instance_state = - rapids::get_instance_state(*instance); - if (instance_state != nullptr) { - instance_state->unload(); - - rapids::log_info( - __FILE__, __LINE__, - "TRITONBACKEND_ModelInstanceFinalize: delete instance state"); - - delete instance_state; - } - } catch (rapids::TritonException& err) { - result = err.error(); - } - - return result; + return rapids::triton_api::instance_finalize(instance); } auto* TRITONBACKEND_ModelInstanceExecute(TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** raw_requests, uint32_t const request_count) { - auto start_time = std::chrono::steady_clock::now(); - - auto* result = static_cast(nullptr); - - try { - auto* model_state = rapids::get_model_state(*instance); - auto* instance_state = - rapids::get_instance_state(*instance); - auto& model = instance_state->get_model(); - auto max_batch_size = model.get_config_param("max_batch_size"); - auto batch = Batch{ - raw_requests, request_count, model_state->TritonMemoryManager(), - /* Note: It is safe to keep a copy of the model_state - * pointer in this closure and the instance pointer in the next because - * the batch goes out of scope at the end of this block and Triton - * guarantees that the liftimes of both the instance and model states - * extend beyond this function call. */ - [model_state](std::string const& name) { - auto result = std::vector{}; - auto& triton_result = - model_state->FindBatchOutput(name)->OutputShape(); - std::transform(std::begin(triton_result), std::end(triton_result), - std::back_inserter(result), [](auto& coord) { - return narrow(coord); - }); - return result; - }, - [instance](TRITONBACKEND_Request* request, time_point req_start, - time_point req_comp_start, time_point req_comp_end, - time_point req_end) { - report_statistics(*instance, request, req_start, req_comp_start, - req_comp_end, req_end); - }, - model_state->EnablePinnedInput(), model_state->EnablePinnedOutput(), - max_batch_size model.get_stream()}; - - model.predict(batch); - auto& compute_start_time = batch.compute_start_time(); - auto compute_end_time = std::chrono::steady_clock::now(); - batch.finalize(); - auto end_time = std::chrono::steady_clock::now(); - - report_statistics(*instance, request_count, start_time, compute_start_time, - compute_end_time, end_time); - } catch (rapids::TritonException& err) { - result = err.error(); - } - - return result; + return rapids::triton_api::execute( + instance, raw_requests, static_cast(request_count)); } } // extern "C" diff --git a/cpp/src/interface/model_instance_state.h b/cpp/src/interface/model_instance_state.h index 19c69cc..ad5b6db 100644 --- a/cpp/src/interface/model_instance_state.h +++ b/cpp/src/interface/model_instance_state.h @@ -18,33 +18,11 @@ #include #include -#include - -#include -#include namespace triton { namespace backend { namespace NAMESPACE { -struct ModelInstanceState : public BackendModelInstance { - ModelInstanceState(ModelState& model_state, - TRITONBACKEND_ModelInstance* triton_model_instance) - : BackendModelInstance(&model_state, triton_model_instance), - model_(model_state.get_shared_state(), - rapids::get_device_id(*triton_model_instance), CudaStream(), - Kind(), - JoinPath({RepositoryPath(), std::to_string(Version()), - ArtifactFilename()})) {} - - auto& get_model() const { return model_; } - - void load() { model_->load(); } - void unload() { model_->unload(); } - - private: - RapidsModel model_; -}; - +using ModelInstanceState = TritonModelInstanceState; } // namespace NAMESPACE } // namespace backend } // namespace triton diff --git a/cpp/src/interface/model_state.h b/cpp/src/interface/model_state.h index 093bfba..2e3e9cb 100644 --- a/cpp/src/interface/model_state.h +++ b/cpp/src/interface/model_state.h @@ -17,22 +17,8 @@ #pragma once #include -#include +#include namespace triton { namespace backend { namespace NAMESPACE { -struct ModelState : public BackendModel { - - ModelState(TRITONBACKEND_Model& triton_model) - : state_{std::make_shared( - get_model_config(triton_model))} {} - - void load() { state_->load(); } - void unload() { state_->unload(); } - - auto get_shared_state() { return state_; } - - private: - std::shared_ptr state_; -}; - +using ModelState = TritonModelState; }}} // namespace triton::backend::NAMESPACE From b8178dce0c5ec7db0044fca5d1af3cb7da2761d3 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 31 Aug 2021 12:56:02 -0400 Subject: [PATCH 029/199] Correct batch statistics reporting call --- cpp/include/rapids_triton/batch/batch.hpp | 3 ++- cpp/include/rapids_triton/triton/statistics.hpp | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/include/rapids_triton/batch/batch.hpp b/cpp/include/rapids_triton/batch/batch.hpp index fc91818..4a0e4cd 100644 --- a/cpp/include/rapids_triton/batch/batch.hpp +++ b/cpp/include/rapids_triton/batch/batch.hpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -165,7 +166,7 @@ namespace triton { namespace backend { namespace rapids { std::for_each( std::begin(requests_), std::end(requests_), - [&report_statistics_, &start_time_, &compute_start_time_, &compute_end_time]( + [this, &compute_end_time]( auto& request) { report_statistics_(request, true, start_time_, compute_start_time_, compute_end_time); } diff --git a/cpp/include/rapids_triton/triton/statistics.hpp b/cpp/include/rapids_triton/triton/statistics.hpp index 614111d..206dee2 100644 --- a/cpp/include/rapids_triton/triton/statistics.hpp +++ b/cpp/include/rapids_triton/triton/statistics.hpp @@ -69,7 +69,6 @@ namespace triton { namespace backend { namespace rapids { triton_check(TRITONBACKEND_ModelInstanceReportBatchStatistics( &instance, request_count, - true, start_time.time_since_epoch().count(), compute_start_time.time_since_epoch().count(), compute_end_time.time_since_epoch().count(), From af258c9cdcd5608ea786c743dde10a8cf3e56b7e Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 31 Aug 2021 15:54:11 -0400 Subject: [PATCH 030/199] Add Triton API tests --- cpp/include/rapids_triton/batch/batch.hpp | 50 +++++++++++-------- .../rapids_triton/triton/api/execute.hpp | 27 +++++++--- .../rapids_triton/triton/api/initialize.hpp | 19 ++++--- .../triton/api/instance_finalize.hpp | 9 ++-- .../triton/api/instance_initialize.hpp | 24 +++++---- .../triton/api/model_finalize.hpp | 10 ++-- .../triton/api/model_initialize.hpp | 14 ++++-- cpp/include/rapids_triton/triton/requests.hpp | 16 ++++++ .../rapids_triton/triton/responses.hpp | 20 ++++++++ .../rapids_triton/triton/statistics.hpp | 1 + cpp/test/CMakeLists.txt | 11 +++- cpp/test/triton/api/execute.cpp | 17 +++++++ cpp/test/triton/api/initialize.cpp | 17 +++++++ cpp/test/triton/api/instance_finalize.cpp | 17 +++++++ cpp/test/triton/api/instance_initialize.cpp | 17 +++++++ cpp/test/triton/api/model_finalize.cpp | 17 +++++++ cpp/test/triton/api/model_initialize.cpp | 17 +++++++ cpp/test/triton/statistics.cpp | 17 +++++++ 18 files changed, 264 insertions(+), 56 deletions(-) create mode 100644 cpp/test/triton/api/execute.cpp create mode 100644 cpp/test/triton/api/initialize.cpp create mode 100644 cpp/test/triton/api/instance_finalize.cpp create mode 100644 cpp/test/triton/api/instance_initialize.cpp create mode 100644 cpp/test/triton/api/model_finalize.cpp create mode 100644 cpp/test/triton/api/model_initialize.cpp create mode 100644 cpp/test/triton/statistics.cpp diff --git a/cpp/include/rapids_triton/batch/batch.hpp b/cpp/include/rapids_triton/batch/batch.hpp index 4a0e4cd..1d5f97a 100644 --- a/cpp/include/rapids_triton/batch/batch.hpp +++ b/cpp/include/rapids_triton/batch/batch.hpp @@ -53,14 +53,10 @@ namespace triton { namespace backend { namespace rapids { * client via the Triton server once processing is complete. * * It is not recommended that developers of rapids_triton backends try to - * construct Batch objects directly. Instead, you should base your backend - * repository on the rapids_triton_template repo. This template repo includes - * all of the code necessary to construct a Batch and feed it to a Model - * implementation. Model implementers need only retrieve the input/output - * Tensors they require from the Batch and then call `finalize` on the output - * Tensors. The code provided in the template repo takes care of the rest. + * construct Batch objects directly. Instead, you should make use of the + * rapids::triton_api::execute template, which will construct the Batch for + * you. */ - template struct Batch { using size_type = std::size_t; @@ -82,19 +78,20 @@ namespace triton { namespace backend { namespace rapids { requests_(raw_requests, raw_requests + count), responses_(construct_responses(requests_.begin(), requests_.end())), get_output_shape_{get_output_shape}, - collector_{ + collector_( raw_requests, count, &responses_, - triton_mem_manager, + &triton_mem_manager, use_pinned_input, stream - }, + ), responder_{std::make_shared( raw_requests, count, &responses_, max_batch_size, + &triton_mem_manager, use_pinned_output, stream )}, @@ -158,26 +155,39 @@ namespace triton { namespace backend { namespace rapids { return stream_; } - void finalize() { + void finalize(TRITONSERVER_Error* err) { auto compute_end_time = std::chrono::steady_clock::now(); if (responder_->Finalize()) { cuda_check(cudaStreamSynchronize(stream_)); } - std::for_each( - std::begin(requests_), - std::end(requests_), - [this, &compute_end_time]( - auto& request) { - report_statistics_(request, true, start_time_, compute_start_time_, compute_end_time); - } - ); - //TODO(wphicks): Send responses + + send_responses(std::begin(responses_), std::end(responses_), err); + + // Triton resumes ownership of failed requests; only release on success + if (err == nullptr) { + std::for_each( + std::begin(requests_), + std::end(requests_), + [this, &compute_end_time]( + auto& request) { + report_statistics_( + request, + start_time_, + compute_start_time_, + compute_end_time, + std::chrono::steady_clock::now() + ); + } + ); + release_requests(std::begin(requests_), std::end(requests_)); + } } private: std::vector requests_; std::vector responses_; std::function(std::string const&)> get_output_shape_; + std::function report_statistics_; BackendInputCollector collector_; std::shared_ptr responder_; cudaStream_t stream_; diff --git a/cpp/include/rapids_triton/triton/api/execute.hpp b/cpp/include/rapids_triton/triton/api/execute.hpp index 280bcbd..9c94648 100644 --- a/cpp/include/rapids_triton/triton/api/execute.hpp +++ b/cpp/include/rapids_triton/triton/api/execute.hpp @@ -15,6 +15,13 @@ */ #pragma once +#include +#include +#include +#include +#include +#include +#include #include namespace triton { namespace backend { namespace rapids { namespace triton_api { @@ -25,9 +32,9 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { auto* result = static_cast(nullptr); try { - auto* model_state = rapids::get_model_state(*instance); + auto* model_state = get_model_state(*instance); auto* instance_state = - rapids::get_instance_state(*instance); + get_instance_state(*instance); auto& model = instance_state->get_model(); auto max_batch_size = model.get_config_param("max_batch_size"); auto batch = Batch{ @@ -38,7 +45,7 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { * guarantees that the liftimes of both the instance and model states * extend beyond this function call. */ [model_state](std::string const& name) { - auto result = std::vector{}; + auto result = std::vector{}; auto& triton_result = model_state->FindBatchOutput(name)->OutputShape(); std::transform(std::begin(triton_result), std::end(triton_result), @@ -50,21 +57,27 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { [instance](TRITONBACKEND_Request* request, time_point req_start, time_point req_comp_start, time_point req_comp_end, time_point req_end) { - report_statistics(*instance, request, req_start, req_comp_start, + report_statistics(*instance, *request, req_start, req_comp_start, req_comp_end, req_end); }, model_state->EnablePinnedInput(), model_state->EnablePinnedOutput(), max_batch_size model.get_stream()}; - model.predict(batch); + auto predict_err = static_cast(nullptr); + try { + model.predict(batch); + } catch (TritonException& err) { + predict_err = err.error(); + } + auto& compute_start_time = batch.compute_start_time(); auto compute_end_time = std::chrono::steady_clock::now(); - batch.finalize(); + batch.finalize(predict_err); auto end_time = std::chrono::steady_clock::now(); report_statistics(*instance, request_count, start_time, compute_start_time, compute_end_time, end_time); - } catch (rapids::TritonException& err) { + } catch (TritonException& err) { result = err.error(); } diff --git a/cpp/include/rapids_triton/triton/api/initialize.hpp b/cpp/include/rapids_triton/triton/api/initialize.hpp index 5bf9ad0..d9e8360 100644 --- a/cpp/include/rapids_triton/triton/api/initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/initialize.hpp @@ -17,23 +17,28 @@ #pragma once #include +#include +#include +#include +#include + namespace triton { namespace backend { namespace rapids { namespace triton_api { - auto* model_initialize(TRITONBACKEND_Model* model) { + inline auto* initialize(TRITONBACKEND_Backend* backend) { auto* result = static_cast(nullptr); try { - auto name = rapids::get_backend_name(*backend); + auto name = get_backend_name(*backend); // TODO (wphicks): use sstream - rapids::log_info( + log_info( __FILE__, __LINE__, (std::string("TRITONBACKEND_Initialize: ") + name).c_str()); - if (!rapids::check_backend_version(*backend)) { - throw rapids::TritonException{ - rapids::Error::Unsupported, + if (!check_backend_version(*backend)) { + throw TritonException{ + Error::Unsupported, "triton backend API version does not support this backend"}; } - } catch (rapids::TritonException& err) { + } catch (TritonException& err) { result = err.error(); } return result; diff --git a/cpp/include/rapids_triton/triton/api/instance_finalize.hpp b/cpp/include/rapids_triton/triton/api/instance_finalize.hpp index ad22ae4..ed6c7d6 100644 --- a/cpp/include/rapids_triton/triton/api/instance_finalize.hpp +++ b/cpp/include/rapids_triton/triton/api/instance_finalize.hpp @@ -15,6 +15,9 @@ */ #pragma once +#include +#include +#include #include namespace triton { namespace backend { namespace rapids { namespace triton_api { @@ -23,17 +26,17 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { auto* result = static_cast(nullptr); try { auto* instance_state = - rapids::get_instance_state(*instance); + get_instance_state(*instance); if (instance_state != nullptr) { instance_state->unload(); - rapids::log_info( + log_info( __FILE__, __LINE__, "TRITONBACKEND_ModelInstanceFinalize: delete instance state"); delete instance_state; } - } catch (rapids::TritonException& err) { + } catch (TritonException& err) { result = err.error(); } diff --git a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp index f7f24e8..b7b470f 100644 --- a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp @@ -15,31 +15,37 @@ */ #pragma once +#include +#include +#include +#include #include +#include namespace triton { namespace backend { namespace rapids { namespace triton_api { template auto* instance_initialize(TRITONBACKEND_ModelInstance* instance) { auto* result = static_cast(nullptr); try { - auto name = rapids::get_model_instance_name(*instance); - auto device_id = rapids::get_device_id(*instance); - auto deployment_type = rapids::get_deployment_type(*instance); + auto name = get_model_instance_name(*instance); + auto device_id = get_device_id(*instance); + auto deployment_type = get_deployment_type(*instance); // TODO (wphicks): Use sstream - rapids::log_info(__FILE__, __LINE__, + log_info(__FILE__, __LINE__, (std::string("TRITONBACKEND_ModelInstanceInitialize: ") + - name + " (" + TRITONSERVER_InstanceGroupKindString(kind) + + name + " (" + TRITONSERVER_InstanceGroupKindString(deployment_type) + " device " + std::to_string(device_id) + ")") .c_str()); - auto* model_state = rapids::get_model_state(*instance); + auto* triton_model = get_model_from_instance(*instance); + auto* model_state = get_model_state(*triton_model); auto rapids_model = - std::make_unique(model_state, instance) + std::make_unique(model_state, instance); - rapids::set_instance_state(*instance, std::move(rapids_model)); - } catch (rapids::TritonException& err) { + set_instance_state(*instance, std::move(rapids_model)); + } catch (TritonException& err) { result = err.error(); } return result; diff --git a/cpp/include/rapids_triton/triton/api/model_finalize.hpp b/cpp/include/rapids_triton/triton/api/model_finalize.hpp index b233b62..9af0524 100644 --- a/cpp/include/rapids_triton/triton/api/model_finalize.hpp +++ b/cpp/include/rapids_triton/triton/api/model_finalize.hpp @@ -15,23 +15,27 @@ */ #pragma once +#include +#include +#include #include +#include namespace triton { namespace backend { namespace rapids { namespace triton_api { template auto* model_finalize(TRITONBACKEND_Model* model) { auto* result = static_cast(nullptr); try { - auto model_state = rapids::get_model_state(*model); + auto model_state = get_model_state(*model); if (model_state != nullptr) { model_state->get_shared_state()->unload(); } - rapids::log_info(__FILE__, __LINE__, + log_info(__FILE__, __LINE__, "TRITONBACKEND_ModelFinalize: delete model state"); delete model_state; - } catch (rapids::TritonException& err) { + } catch (TritonException& err) { result = err.error(); } diff --git a/cpp/include/rapids_triton/triton/api/model_initialize.hpp b/cpp/include/rapids_triton/triton/api/model_initialize.hpp index 239ead2..913b900 100644 --- a/cpp/include/rapids_triton/triton/api/model_initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/model_initialize.hpp @@ -15,19 +15,23 @@ */ #pragma once +#include +#include +#include #include +#include namespace triton { namespace backend { namespace rapids { namespace triton_api { template auto* model_initialize(TRITONBACKEND_Model* model) { auto* result = static_cast(nullptr); try { - auto name = rapids::get_model_name(*model); + auto name = get_model_name(*model); - auto version = rapids::get_model_version(*model); + auto version = get_model_version(*model); // TODO (wphicks): Use sstream - rapids::log_info(__FILE__, __LINE__, + log_info(__FILE__, __LINE__, (std::string("TRITONBACKEND_ModelInitialize: ") + name + " (version " + std::to_string(version) + ")") .c_str()); @@ -35,8 +39,8 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { auto rapids_model_state = std::make_unique(*model); rapids_model_state->load(); - rapids::set_model_state(*model, std::move(rapids_model_state)); - } catch (rapids::TritonException& err) { + set_model_state(*model, std::move(rapids_model_state)); + } catch (TritonException& err) { result = err.error(); } diff --git a/cpp/include/rapids_triton/triton/requests.hpp b/cpp/include/rapids_triton/triton/requests.hpp index 4839486..927af65 100644 --- a/cpp/include/rapids_triton/triton/requests.hpp +++ b/cpp/include/rapids_triton/triton/requests.hpp @@ -16,9 +16,25 @@ #pragma once #include +#include + +#include +#include +#include namespace triton { namespace backend { namespace rapids { using request_size_t = uint32_t; + + template + void release_requests(Iter begin, Iter end) { + std::for_each(begin, end, [](auto& request) { + try { + triton_check(TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL)); + } catch (TritonException& err) { + log_error(err.what()); + } + }); + } }}} // namespace triton::backend::rapids diff --git a/cpp/include/rapids_triton/triton/responses.hpp b/cpp/include/rapids_triton/triton/responses.hpp index 5693bcb..5de2ac3 100644 --- a/cpp/include/rapids_triton/triton/responses.hpp +++ b/cpp/include/rapids_triton/triton/responses.hpp @@ -19,6 +19,7 @@ #include #include #include +#include #include namespace triton { namespace backend { namespace rapids { @@ -45,6 +46,25 @@ auto construct_responses(Iter requests_begin, Iter requests_end) return responses; } +template +void send_responses(Iter begin, Iter end, TRITONSERVER_Error* err) { + std::for_each(begin, end, [err](auto& response) { + decltype(err) err_copy; + if (err != nullptr) { + err_copy = TRITONSERVER_ErrorNew( + TRITONSERVER_ErrorCode(err), TRITONSERVER_ErrorMessage(err)); + } else { + err_copy = err; + } + + try { + triton_check(TRITONBACKEND_ResponseSend(response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, err_copy)); + } catch (TritonException& err) { + log_error(__FILE__, __LINE__, err.what()); + } + }); +} + }}} // namespace triton::backend::rapids diff --git a/cpp/include/rapids_triton/triton/statistics.hpp b/cpp/include/rapids_triton/triton/statistics.hpp index 206dee2..bb4f685 100644 --- a/cpp/include/rapids_triton/triton/statistics.hpp +++ b/cpp/include/rapids_triton/triton/statistics.hpp @@ -18,6 +18,7 @@ #include #include +#include #include namespace triton { namespace backend { namespace rapids { diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 8e0513b..9b9967c 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -16,15 +16,22 @@ # keep the files in alphabetical order! add_executable(test_rapids_triton + test/batch/batch.cpp test/build_control.cpp test/exceptions.cpp - test/batch/batch.cpp test/memory/buffer.cpp test/memory/detail/allocate.cpp test/memory/detail/copy.cpp test/memory/types.cpp test/tensor/dtype.cpp test/tensor/tensor.cpp + test/test.cpp + # test/triton/api/execute.cpp + test/triton/api/initialize.cpp + test/triton/api/instance_finalize.cpp + test/triton/api/instance_initialize.cpp + test/triton/api/model_finalize.cpp + test/triton/api/model_initialize.cpp test/triton/backend.cpp test/triton/config.cpp test/triton/deployment.cpp @@ -35,7 +42,7 @@ add_executable(test_rapids_triton test/triton/model_instance.cpp test/triton/requests.cpp test/triton/responses.cpp - test/test.cpp + test/triton/statistics.cpp test/utils/const_agnostic.cpp test/utils/narrow.cpp ) diff --git a/cpp/test/triton/api/execute.cpp b/cpp/test/triton/api/execute.cpp new file mode 100644 index 0000000..ba48f8e --- /dev/null +++ b/cpp/test/triton/api/execute.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/cpp/test/triton/api/initialize.cpp b/cpp/test/triton/api/initialize.cpp new file mode 100644 index 0000000..33b95d4 --- /dev/null +++ b/cpp/test/triton/api/initialize.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/cpp/test/triton/api/instance_finalize.cpp b/cpp/test/triton/api/instance_finalize.cpp new file mode 100644 index 0000000..f7218a3 --- /dev/null +++ b/cpp/test/triton/api/instance_finalize.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/cpp/test/triton/api/instance_initialize.cpp b/cpp/test/triton/api/instance_initialize.cpp new file mode 100644 index 0000000..3ed2b91 --- /dev/null +++ b/cpp/test/triton/api/instance_initialize.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/cpp/test/triton/api/model_finalize.cpp b/cpp/test/triton/api/model_finalize.cpp new file mode 100644 index 0000000..367f16c --- /dev/null +++ b/cpp/test/triton/api/model_finalize.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/cpp/test/triton/api/model_initialize.cpp b/cpp/test/triton/api/model_initialize.cpp new file mode 100644 index 0000000..071baa3 --- /dev/null +++ b/cpp/test/triton/api/model_initialize.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/cpp/test/triton/statistics.cpp b/cpp/test/triton/statistics.cpp new file mode 100644 index 0000000..7c01a6d --- /dev/null +++ b/cpp/test/triton/statistics.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include From e70732ac84c7ae78cb50d82394705bc428a4670a Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 31 Aug 2021 15:59:49 -0400 Subject: [PATCH 031/199] Re-enable execute tests --- cpp/include/rapids_triton/batch/batch.hpp | 2 +- cpp/include/rapids_triton/triton/api/execute.hpp | 4 +++- cpp/include/rapids_triton/triton/requests.hpp | 2 +- cpp/test/CMakeLists.txt | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cpp/include/rapids_triton/batch/batch.hpp b/cpp/include/rapids_triton/batch/batch.hpp index 1d5f97a..ca840b7 100644 --- a/cpp/include/rapids_triton/batch/batch.hpp +++ b/cpp/include/rapids_triton/batch/batch.hpp @@ -147,7 +147,7 @@ namespace triton { namespace backend { namespace rapids { return OutputTensor(std::move(shape), std::move(buffer), responder_, name); } - auto const& start_time() const { + auto const& compute_start_time() const { return compute_start_time_; } diff --git a/cpp/include/rapids_triton/triton/api/execute.hpp b/cpp/include/rapids_triton/triton/api/execute.hpp index 9c94648..1500edb 100644 --- a/cpp/include/rapids_triton/triton/api/execute.hpp +++ b/cpp/include/rapids_triton/triton/api/execute.hpp @@ -17,6 +17,7 @@ #pragma once #include #include +#include #include #include #include @@ -61,7 +62,8 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { req_comp_end, req_end); }, model_state->EnablePinnedInput(), model_state->EnablePinnedOutput(), - max_batch_size model.get_stream()}; + max_batch_size, + model.get_stream()}; auto predict_err = static_cast(nullptr); try { diff --git a/cpp/include/rapids_triton/triton/requests.hpp b/cpp/include/rapids_triton/triton/requests.hpp index 927af65..8f9ed05 100644 --- a/cpp/include/rapids_triton/triton/requests.hpp +++ b/cpp/include/rapids_triton/triton/requests.hpp @@ -31,7 +31,7 @@ namespace triton { namespace backend { namespace rapids { try { triton_check(TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL)); } catch (TritonException& err) { - log_error(err.what()); + log_error(__FILE__, __LINE__, err.what()); } }); } diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 9b9967c..1af11bb 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -26,7 +26,7 @@ add_executable(test_rapids_triton test/tensor/dtype.cpp test/tensor/tensor.cpp test/test.cpp - # test/triton/api/execute.cpp + test/triton/api/execute.cpp test/triton/api/initialize.cpp test/triton/api/instance_finalize.cpp test/triton/api/instance_initialize.cpp From 75295c3b6a1f40afd143f6ad2a578c5090a97c5b Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 31 Aug 2021 16:01:20 -0400 Subject: [PATCH 032/199] Correctly handle const in input types --- cpp/include/rapids_triton/model/model.hpp | 3 +-- cpp/include/rapids_triton/tensor/dtype.hpp | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/cpp/include/rapids_triton/model/model.hpp b/cpp/include/rapids_triton/model/model.hpp index 1552fae..99a4377 100644 --- a/cpp/include/rapids_triton/model/model.hpp +++ b/cpp/include/rapids_triton/model/model.hpp @@ -75,10 +75,9 @@ namespace triton { namespace backend { namespace rapids { /** * @brief Get input tensor of a particular named input for an entire batch */ - //TODO(wphicks): Handle const for input type template auto get_input(Batch& batch, std::string const& name, std::optional> const& mem_type, cudaStream_t stream) const { - return batch.get_input(name, mem_type, device_id_, stream); + return batch.get_input(name, mem_type, device_id_, stream); } template auto get_input(Batch& batch, std::string const& name, std::optional const& mem_type) const { diff --git a/cpp/include/rapids_triton/tensor/dtype.hpp b/cpp/include/rapids_triton/tensor/dtype.hpp index 01f8601..04be436 100644 --- a/cpp/include/rapids_triton/tensor/dtype.hpp +++ b/cpp/include/rapids_triton/tensor/dtype.hpp @@ -155,5 +155,3 @@ struct TritonDtype> { }; }}} - -// TODO(whicks): Correctly handle const versions of types From f561f4c41b7e5414b36ece9078ea9efe83ff849a Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 31 Aug 2021 16:12:08 -0400 Subject: [PATCH 033/199] Clean up log message construction --- .../rapids_triton/triton/api/initialize.hpp | 6 +++--- .../triton/api/instance_initialize.hpp | 18 ++++++++++++------ .../triton/api/model_initialize.hpp | 15 ++++++++++----- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/cpp/include/rapids_triton/triton/api/initialize.hpp b/cpp/include/rapids_triton/triton/api/initialize.hpp index d9e8360..8021427 100644 --- a/cpp/include/rapids_triton/triton/api/initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/initialize.hpp @@ -28,10 +28,10 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { try { auto name = get_backend_name(*backend); - // TODO (wphicks): use sstream log_info( - __FILE__, __LINE__, - (std::string("TRITONBACKEND_Initialize: ") + name).c_str()); + __FILE__, __LINE__, + std::string("TRITONBACKEND_Initialize: ") + name + ); if (!check_backend_version(*backend)) { throw TritonException{ diff --git a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp index b7b470f..f0f38b4 100644 --- a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp @@ -15,6 +15,8 @@ */ #pragma once + +#include #include #include #include @@ -31,12 +33,16 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { auto device_id = get_device_id(*instance); auto deployment_type = get_deployment_type(*instance); - // TODO (wphicks): Use sstream - log_info(__FILE__, __LINE__, - (std::string("TRITONBACKEND_ModelInstanceInitialize: ") + - name + " (" + TRITONSERVER_InstanceGroupKindString(deployment_type) + - " device " + std::to_string(device_id) + ")") - .c_str()); + auto log_stream = std::stringstream{}; + log_stream << "TRITONBACKEND_ModelInstanceInitialize: " + << name + << " (" + << TRITONSERVER_InstanceGroupKindString(deployment_type) + << " device " + << device_id + << ")"; + + log_info(__FILE__, __LINE__, log_stream.str()); auto* triton_model = get_model_from_instance(*instance); auto* model_state = get_model_state(*triton_model); diff --git a/cpp/include/rapids_triton/triton/api/model_initialize.hpp b/cpp/include/rapids_triton/triton/api/model_initialize.hpp index 913b900..edbe0ef 100644 --- a/cpp/include/rapids_triton/triton/api/model_initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/model_initialize.hpp @@ -15,6 +15,7 @@ */ #pragma once +#include #include #include #include @@ -30,11 +31,15 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { auto version = get_model_version(*model); - // TODO (wphicks): Use sstream - log_info(__FILE__, __LINE__, - (std::string("TRITONBACKEND_ModelInitialize: ") + name + - " (version " + std::to_string(version) + ")") - .c_str()); + auto log_stream = std::stringstream{}; + + log_stream << "TRITONBACKEND_ModelInitialize: " + << name + << " (version " + << version + << ")"; + + log_info(__FILE__, __LINE__, log_stream.str()); auto rapids_model_state = std::make_unique(*model); rapids_model_state->load(); From 541d682954d0eb0ca40aa9f519c55f8910480469 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 1 Sep 2021 16:01:51 -0400 Subject: [PATCH 034/199] Reorganize and simplify example backend --- cpp/CMakeLists.txt | 8 ++ cpp/include/rapids_triton/batch/batch.hpp | 70 +++++++--- cpp/include/rapids_triton/memory/buffer.hpp | 2 +- cpp/include/rapids_triton/model/model.hpp | 24 ++-- .../rapids_triton/model/shared_state.hpp | 8 +- cpp/include/rapids_triton/tensor/tensor.hpp | 53 +++---- cpp/src/CMakeLists.txt | 52 +++++++ cpp/src/{interface => }/api.cc | 14 +- cpp/src/impl/model.h | 74 ---------- cpp/src/interface/model_instance_state.h | 28 ---- cpp/src/interface/model_state.h | 24 ---- cpp/src/model.h | 131 ++++++++++++++++++ cpp/src/{impl => }/names.h | 0 cpp/src/{impl => }/shared_state.h | 15 +- cpp/test/tensor/tensor.cpp | 60 ++++++++ 15 files changed, 371 insertions(+), 192 deletions(-) create mode 100644 cpp/src/CMakeLists.txt rename cpp/src/{interface => }/api.cc (81%) delete mode 100644 cpp/src/impl/model.h delete mode 100644 cpp/src/interface/model_instance_state.h delete mode 100644 cpp/src/interface/model_state.h create mode 100644 cpp/src/model.h rename cpp/src/{impl => }/names.h (100%) rename cpp/src/{impl => }/shared_state.h (55%) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8ee3748..9f8d397 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -41,6 +41,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # - User Options ------------------------------------------------------------ option(BUILD_TESTS "Build rapids_triton unit-tests" ON) +option(BUILD_EXAMPLE "Build rapids_identity example backend" OFF) option(CUDA_ENABLE_KERNELINFO "Enable kernel resource usage info" OFF) option(CUDA_ENABLE_LINEINFO "Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler)" OFF) option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF) @@ -175,6 +176,13 @@ if(BUILD_TESTS) include(test/CMakeLists.txt) endif() +############################################################################## +# - build example backend ---------------------------------------------------- + +if(BUILD_EXAMPLE) + include(src/CMakeLists.txt) +endif() + ############################################################################## # - doxygen targets ---------------------------------------------------------- diff --git a/cpp/include/rapids_triton/batch/batch.hpp b/cpp/include/rapids_triton/batch/batch.hpp index ca840b7..23e8791 100644 --- a/cpp/include/rapids_triton/batch/batch.hpp +++ b/cpp/include/rapids_triton/batch/batch.hpp @@ -17,6 +17,7 @@ #pragma once #include +#include #include #include #include @@ -27,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -99,33 +101,50 @@ namespace triton { namespace backend { namespace rapids { start_time_{std::chrono::steady_clock::now()}, compute_start_time_{std::chrono::steady_clock::now()} {} + auto get_input_shape(std::string const& name) { + auto result = std::vector{}; + if(!requests_.empty()) { + result = get_triton_input_shape(std::begin(requests_), std::end(requests_), name); + } + return result; + } + template - auto get_input(std::string const& name, MemoryType memory_type, device_id_t device_id) { - auto shape = get_input_shape(requests_.begin(), requests_.end(), name); + auto get_input(std::string const& name, std::optional + memory_type, device_id_t device_id, cudaStream_t stream) { + auto shape = get_input_shape(name); auto size_bytes = sizeof(T) * std::reduce(shape.begin(), shape.end(), std::size_t{1}, std::multiplies<>()); + auto allowed_memory_configs = std::vector>{}; + if (memory_type.has_value()) { + allowed_memory_configs.emplace_back(memory_type.value(), device_id); + } else { + allowed_memory_configs.emplace_back(HostMemory, int64_t{}); + allowed_memory_configs.emplace_back(DeviceMemory, device_id); + } auto const* raw_buffer = static_cast(nullptr); auto reported_bytes = std::size_t{}; - auto reported_mem_type = memory_type; - auto reported_device_id = device_id; + auto reported_mem_type = MemoryType{}; + auto reported_device_id = int64_t{}; - collector_.ProcessTensor( + triton_check(collector_.ProcessTensor( name.c_str(), - nullptr, // Return data without copy if possible + static_cast(nullptr), // Return data without copy if possible size_bytes, - {{memory_type, device_id}}, + allowed_memory_configs, &raw_buffer, &reported_bytes, &reported_mem_type, &reported_device_id - ); + )); auto buffer = Buffer( reinterpret_cast(raw_buffer), reported_bytes, reported_mem_type, - stream_ + reported_device_id, + stream ); if (reported_mem_type != memory_type || reported_device_id != device_id) { @@ -139,12 +158,31 @@ namespace triton { namespace backend { namespace rapids { } template - auto get_output(std::string const& name, MemoryType memory_type, device_id_t device_id) { + auto get_input(std::string const& name, std::optional + memory_type, device_id_t device_id) { + return get_input(name, memory_type, device_id, stream_); + } + + template + auto get_output(std::string const& name, std::optional memory_type, device_id_t device_id, cudaStream_t stream) { auto shape = get_output_shape_(name); auto buffer_size = std::reduce( shape.begin(), shape.end(), std::size_t{1}, std::multiplies<>()); - auto buffer = Buffer(buffer_size, memory_type, device_id, stream_); - return OutputTensor(std::move(shape), std::move(buffer), responder_, name); + auto final_memory_type = MemoryType{}; + if (memory_type.has_value()) { + final_memory_type = memory_type.value(); + } else { + // If consumer doesn't care, use HostMemory to avoid additional copy on + // non-shared-memory responses. + final_memory_type = HostMemory; + } + auto buffer = Buffer(buffer_size, final_memory_type, device_id, stream); + return OutputTensor(std::move(shape), std::move(buffer), name, responder_); + } + + template + auto get_output(std::string const& name, std::optional memory_type, device_id_t device_id) { + return get_output(name, memory_type, device_id, stream_); } auto const& compute_start_time() const { @@ -193,14 +231,6 @@ namespace triton { namespace backend { namespace rapids { cudaStream_t stream_; std::chrono::time_point start_time_; std::chrono::time_point compute_start_time_; - - auto get_input_shape(std::string const& name) { - auto result = std::vector{}; - if(!requests_.empty()) { - result = get_triton_input_shape(std::begin(requests_), std::end(requests_), name); - } - return result; - } }; }}} // namespace triton::backend::rapids diff --git a/cpp/include/rapids_triton/memory/buffer.hpp b/cpp/include/rapids_triton/memory/buffer.hpp index 5de2b69..db0f4db 100644 --- a/cpp/include/rapids_triton/memory/buffer.hpp +++ b/cpp/include/rapids_triton/memory/buffer.hpp @@ -267,6 +267,6 @@ namespace triton { namespace backend { namespace rapids { template void copy(Buffer& dst, Buffer const& src, typename Buffer::size_type src_begin, typename Buffer::size_type src_end) { - copy(dst, src, 0, 0, src.size()); + copy(dst, src, 0, src_begin, src_end); } }}} // namespace triton::backend::rapids diff --git a/cpp/include/rapids_triton/model/model.hpp b/cpp/include/rapids_triton/model/model.hpp index 99a4377..e8d8e15 100644 --- a/cpp/include/rapids_triton/model/model.hpp +++ b/cpp/include/rapids_triton/model/model.hpp @@ -76,32 +76,36 @@ namespace triton { namespace backend { namespace rapids { * @brief Get input tensor of a particular named input for an entire batch */ template - auto get_input(Batch& batch, std::string const& name, std::optional> const& mem_type, cudaStream_t stream) const { + auto get_input(Batch& batch, std::string const& name, std::optional const& mem_type, cudaStream_t stream) const { return batch.get_input(name, mem_type, device_id_, stream); } template auto get_input(Batch& batch, std::string const& name, std::optional const& mem_type) const { - return get_input(name, mem_type, device_id_, default_stream_); + return get_input(batch, name, mem_type, default_stream_); } template auto get_input(Batch& batch, std::string const& name) const { - return get_input(name, preferred_mem_type(batch), device_id_, default_stream_); + return get_input(batch, name, preferred_mem_type(batch), default_stream_); } /** * @brief Get output tensor of a particular named output for an entire batch */ template - auto get_output(Batch& batch, std::string const& name, std::optional> const& mem_type, cudaStream_t stream) const { - return batch.get_output(name, mem_type, device_id_, stream); + auto get_output(Batch& batch, std::string const& name, std::optional const& mem_type, device_id_t device_id, cudaStream_t stream) const { + return batch.get_output(name, mem_type, device_id, stream); + } + template + auto get_output(Batch& batch, std::string const& name, std::optional const& mem_type, cudaStream_t stream) const { + return get_output(batch, name, mem_type, device_id_, stream); } template auto get_output(Batch& batch, std::string const& name, std::optional const& mem_type) const { - return get_output(name, mem_type, device_id_, default_stream_); + return get_output(batch, name, mem_type, device_id_, default_stream_); } template auto get_output(Batch& batch, std::string const& name) const { - return get_output(name, preferred_mem_type(batch), device_id_, default_stream_); + return get_output(batch, name, preferred_mem_type(batch), device_id_, default_stream_); } /** @@ -109,15 +113,15 @@ namespace triton { namespace backend { namespace rapids { */ template auto get_config_param(std::string const& name) const { - return shared_state_->get_config_param(name); + return shared_state_->template get_config_param(name); } template auto get_config_param(std::string const& name, T default_value) const { - return shared_state_->get_config_param(name, default_value); + return shared_state_->template get_config_param(name, default_value); } Model(std::shared_ptr shared_state, device_id_t device_id, cudaStream_t default_stream, DeploymentType deployment_type, std::string const& filepath) : - shared_state_{shared_state}, device_id_{device_id}, default_stream_{default_stream}, deployment_type_{deployment_type} filepath_{filepath} {} + shared_state_{shared_state}, device_id_{device_id}, default_stream_{default_stream}, deployment_type_{deployment_type}, filepath_{filepath} {} auto get_device_id() const { return device_id_; } auto get_deployment_type() const { return deployment_type_; } diff --git a/cpp/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp index 6f4845e..ccf14b3 100644 --- a/cpp/include/rapids_triton/model/shared_state.hpp +++ b/cpp/include/rapids_triton/model/shared_state.hpp @@ -23,7 +23,7 @@ #include #include -#include +#include #include #include #include @@ -40,7 +40,7 @@ namespace triton { namespace backend { namespace rapids { virtual void unload() {} explicit SharedModelState( - common::TritonJSON::Value config) : config_{config}, + common::TritonJson::Value&& config) : config_{std::move(config)}, max_batch_size_(get_max_batch_size(config)) {} template @@ -54,7 +54,7 @@ namespace triton { namespace backend { namespace rapids { } private: - common::TritonJSON::Value config_; + common::TritonJson::Value config_; Batch::size_type max_batch_size_; template @@ -65,7 +65,7 @@ namespace triton { namespace backend { namespace rapids { return result; } auto json_value = common::TritonJson::Value{}; - if (config_.Find(name.c_str(), &json_value) { + if (config_.Find(name.c_str(), &json_value)) { auto string_repr = std::string{}; triton_check(json_value.MemberAsString("string_value", &string_repr)); diff --git a/cpp/include/rapids_triton/tensor/tensor.hpp b/cpp/include/rapids_triton/tensor/tensor.hpp index 0ebc7f7..f55a452 100644 --- a/cpp/include/rapids_triton/tensor/tensor.hpp +++ b/cpp/include/rapids_triton/tensor/tensor.hpp @@ -93,32 +93,6 @@ namespace triton { namespace backend { namespace rapids { template BaseTensor::~BaseTensor() {} - template - void copy(BaseTensor> dst, BaseTensor src) { - copy(dst.buffer(), src.buffer()); - } - - /** - * @brief Copy data from src Tensor into buffers indicated by iterators - * - * This method is provided to assist with distributing data from a single - * Tensor into many smaller buffers which have been set up to receive a part - * of the data from the src Tensor - */ - template - void copy(Iter begin, Iter end, BaseTensor src) { - std::accumulate( - begin, - end, - BaseTensor::size_type(0), - [&src] (auto offset, auto& buffer) { - auto end_offset = offset + buffer.size(); - copy(buffer, src.buffer(), offset, end_offset); - return end_offset; - } - ); - } - template struct Tensor final : BaseTensor { Tensor() : BaseTensor{} {} @@ -165,4 +139,31 @@ namespace triton { namespace backend { namespace rapids { std::shared_ptr responder_; std::string name_; }; + + template + void copy(BaseTensor>& dst, BaseTensor& src) { + copy(dst.buffer(), src.buffer()); + } + + /** + * @brief Copy data from src Tensor into buffers indicated by iterators + * + * This method is provided to assist with distributing data from a single + * Tensor into many smaller buffers which have been set up to receive a part + * of the data from the src Tensor + */ + template + void copy(Iter begin, Iter end, BaseTensor& src) { + std::accumulate( + begin, + end, + typename BaseTensor::size_type{}, + [&src] (auto offset, auto& dst) { + auto end_offset = offset + dst.size(); + copy(dst.buffer(), src.buffer(), offset, end_offset); + return end_offset; + } + ); + } + }}} // namespace triton::backend::rapids diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt new file mode 100644 index 0000000..02d15d2 --- /dev/null +++ b/cpp/src/CMakeLists.txt @@ -0,0 +1,52 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +# keep the files in alphabetical order! +add_library( + rapids-identity SHARED + src/api.cc +) + +set_target_properties(rapids-identity +PROPERTIES BUILD_RPATH "\$ORIGIN" + # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON +) + +target_compile_options(rapids-identity + PRIVATE "$<$:${RAPIDS_TRITON_CXX_FLAGS}>" + "$<$:${RAPIDS_TRITON_CUDA_FLAGS}>" +) + +target_include_directories(rapids-identity + PRIVATE "$" + "${CMAKE_CURRENT_SOURCE_DIR}/src" +) + +target_link_libraries(rapids-identity +PRIVATE + rmm::rmm + raft::raft + triton-core-serverstub + triton-backend-utils + "${TRITONSERVER_LIB}" + $ +) diff --git a/cpp/src/interface/api.cc b/cpp/src/api.cc similarity index 81% rename from cpp/src/interface/api.cc rename to cpp/src/api.cc index 8254426..a9f977a 100644 --- a/cpp/src/interface/api.cc +++ b/cpp/src/api.cc @@ -14,21 +14,29 @@ * limitations under the License. */ -#include +#include +#include +#include #include #include #include #include -#include +#include #include +#include +#include +#include +#include +#include +#include namespace triton { namespace backend { namespace NAMESPACE { using ModelState = TritonModelState; -using ModelInstanceState = TritonModelState; +using ModelInstanceState = TritonModelInstance; extern "C" { diff --git a/cpp/src/impl/model.h b/cpp/src/impl/model.h deleted file mode 100644 index 5b6c016..0000000 --- a/cpp/src/impl/model.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include - -#include // rapids::Batch -#include // rapids::MemoryType -#include // rapids::Model - -namespace triton { -namespace backend { -namespace NAMESPACE { - -struct RapidsModel : rapids::Model { - void load() {} - void unload() {} - void predict(rapids::Batch& batch) { - auto input = model.get_input(batch, "input__0"); - auto output = model.get_output(batch, "output__0"); - copy(output, input); - // TODO(wphicks): Read config and make this interesting - } - - /*************************************************************************** - * ADVANCED FEATURES * - * *********************************************************************** * - * None of the following methods are required to be implemented in order to - * create a valid model, but they are presented here for those who require - * the additional functionality they provide. - **************************************************************************/ - - /*************************************************************************** - * preferred_mem_type / preferred_mem_type_in / preferred_mem_type_out * - * *********************************************************************** * - * If implemented, `preferred_mem_type` allows for control over when input - * and output data are provided on the host versus on device. In the case - * that a model prefers to receive its input on-host but return output - * on-device (or vice versa), `preferred_mem_type_in` and - * `preferred_mem_type_out` can be used for even more precise control. - * - * In this example, we simply return `std::nullopt` to indicate that the - * model has no preference on its input/output data locations. Note that the - * Batch being processed is taken as input to this function to facilitate - * implementations that may switch their preferred memory location based on - * e.g. the size of the batch. - * - * Valid MemoryType options to return are rapids::HostMemory and - * rapids::DeviceMemory. - **************************************************************************/ - std::optional preferred_mem_type( - rapids::Batch& batch) const { - return std::nullopt; - } -}; - -} // namespace NAMESPACE -} // namespace backend -} // namespace triton diff --git a/cpp/src/interface/model_instance_state.h b/cpp/src/interface/model_instance_state.h deleted file mode 100644 index ad5b6db..0000000 --- a/cpp/src/interface/model_instance_state.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include - -namespace triton { -namespace backend { -namespace NAMESPACE { -using ModelInstanceState = TritonModelInstanceState; -} // namespace NAMESPACE -} // namespace backend -} // namespace triton diff --git a/cpp/src/interface/model_state.h b/cpp/src/interface/model_state.h deleted file mode 100644 index 2e3e9cb..0000000 --- a/cpp/src/interface/model_state.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include - -namespace triton { namespace backend { namespace NAMESPACE { -using ModelState = TritonModelState; -}}} // namespace triton::backend::NAMESPACE diff --git a/cpp/src/model.h b/cpp/src/model.h new file mode 100644 index 0000000..bdf1916 --- /dev/null +++ b/cpp/src/model.h @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include +#include // rapids::Batch +#include // rapids::MemoryType +#include // rapids::Model +#include // rapids::copy + +namespace triton { +namespace backend { +namespace NAMESPACE { + +/* Any logic necessary to perform inference with a model and manage its data + * should be implemented in a struct named RapidsModel, as shown here */ + +struct RapidsModel : rapids::Model { + /*************************************************************************** + * BASIC FEATURES * + * *********************************************************************** * + * The only method that *must* be implemented for a viable model is the + * `predict` method, but the others presented here are often used for basic + * model implementations. Filling out these methods should take care of most + * use cases. + **************************************************************************/ + + /*************************************************************************** + * predict * + * *********************************************************************** * + * This method performs the actual inference step on input data. Implementing + * a predict function requires four steps: + * 1. Call `get_input` on the provided `Batch` object for each of the input + * tensors named in the config file for this backend. This provides a + * `Tensor` object containing the input data. + * 2. Call `get_output` on the provided `Batch` object for each of the output + * tensors named in the config file for this backend. This provides a + * `Tensor` object to which output values can be written. + * 3. Perform inference based on the input Tensors and store the results in + * the output Tensors. `some_tensor.data()` can be used to retrieve a raw + * pointer to the underlying data. + * 4. Call the `finalize` method on all output tensors. + **************************************************************************/ + void predict(rapids::Batch& batch) { + // 1. Acquire a tensor representing the input named "input__0" + auto input = get_input(batch, "input__0"); + // 2. Acquire a tensor representing the output named "input__0" + auto output = get_output(batch, "output__0"); + + // 3. Perform inference. In this example, we simply copy the data from the + // input to the output tensor. + rapids::copy(output, input); + + // 4. Call finalize on all output tensors. In this case, we have just one + // output, so we call finalize on it. + output.finalize(); + } + + /*************************************************************************** + * load / unload * + * *********************************************************************** * + * These methods can be used to perform one-time loading/unloading of + * resources when a model is created. For example, data representing the + * model may be loaded onto the GPU in the `load` method and unloaded in the + * `unload` method. This data will then remain loaded while the server is + * running. + * + * While these methods take no arguments, it is typical to read any necessary + * input from the model configuration file by using the `get_config_param` + * method. Any parameters defined in the "parameters" section of the config + * can be accessed by name in this way. The maximum batch size can also be + * retrieved using the name "max_batch_size". + * + * These methods need not be explicitly implemented if no loading/unloading + * logic is required, but we show them here for illustrative purposes. + **************************************************************************/ + void load() {} + void unload() {} + + /*************************************************************************** + * ADVANCED FEATURES * + * *********************************************************************** * + * None of the following methods are required to be implemented in order to + * create a valid model, but they are presented here for those who require + * the additional functionality they provide. + **************************************************************************/ + + /*************************************************************************** + * preferred_mem_type / preferred_mem_type_in / preferred_mem_type_out * + * *********************************************************************** * + * If implemented, `preferred_mem_type` allows for control over when input + * and output data are provided on the host versus on device. In the case + * that a model prefers to receive its input on-host but return output + * on-device (or vice versa), `preferred_mem_type_in` and + * `preferred_mem_type_out` can be used for even more precise control. + * + * In this example, we simply return `std::nullopt` to indicate that the + * model has no preference on its input/output data locations. Note that the + * Batch being processed is taken as input to this function to facilitate + * implementations that may switch their preferred memory location based on + * properties of the batch. + * + * Valid MemoryType options to return are rapids::HostMemory and + * rapids::DeviceMemory. + **************************************************************************/ + std::optional preferred_mem_type( + rapids::Batch& batch) const { + return std::nullopt; + } +}; + +} // namespace NAMESPACE +} // namespace backend +} // namespace triton diff --git a/cpp/src/impl/names.h b/cpp/src/names.h similarity index 100% rename from cpp/src/impl/names.h rename to cpp/src/names.h diff --git a/cpp/src/impl/shared_state.h b/cpp/src/shared_state.h similarity index 55% rename from cpp/src/impl/shared_state.h rename to cpp/src/shared_state.h index 0a292a0..2e55237 100644 --- a/cpp/src/impl/shared_state.h +++ b/cpp/src/shared_state.h @@ -16,7 +16,7 @@ #pragma once -#include +#include #include @@ -24,7 +24,18 @@ namespace triton { namespace backend { namespace NAMESPACE { -struct RapidsSharedState : rapids::SharedState { +/* Triton allows multiple instances of a single model to be instantiated at the + * same time (e.g. on different GPUs). All instances of a model share access to + * an object which manages any state that can be shared across all instances. + * Any logic necessary for managing such state should be implemented in a + * struct named RapidsSharedState, as shown here. Models may access this shared + * state object via the `get_shared_state` method, which returns a shared + * pointer to the RapidsSharedState object. + * + * Not all backends require shared state, so leaving this implementation empty + * is entirely valid */ + +struct RapidsSharedState : rapids::SharedModelState { void load() {} void unload() {} }; diff --git a/cpp/test/tensor/tensor.cpp b/cpp/test/tensor/tensor.cpp index 49ea793..7842f6c 100644 --- a/cpp/test/tensor/tensor.cpp +++ b/cpp/test/tensor/tensor.cpp @@ -72,6 +72,66 @@ TEST(RapidsTriton, multi_buffer_tensor) { EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } +TEST(RapidsTriton, tensor_copy) { + auto shape = std::vector{2, 2}; + auto data = std::vector{1, 2, 3, 4}; + + auto data1 = data; + auto tensor1 = + Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); + auto data2 = std::vector(data1.size()); + auto tensor2 = + Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); + + copy(tensor2, tensor1); + + auto data_out = + std::vector(tensor2.data(), tensor2.data() + tensor2.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + + auto small_shape = std::vector{2}; + auto small_data = std::vector(2); + auto tensor3 = Tensor( + small_shape, + Buffer{small_data.data(), small_data.size(), HostMemory}); + + EXPECT_THROW(copy(tensor3, tensor1), TritonException); +} + +TEST(RapidsTriton, tensor_multi_copy) { + auto shape = std::vector{2, 2}; + auto data = std::vector{1, 2, 3, 4}; + + auto data1 = data; + auto tensor1 = + Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); + + auto receiver_shape = std::vector{1}; + auto receivers = std::vector>{}; + + receivers.reserve(data.size()); + std::transform(data.begin(), data.end(), std::back_inserter(receivers), + [&receiver_shape](auto& val) { + return Tensor(receiver_shape, + Buffer{std::size_t{1}, HostMemory}); + }); + + rapids::copy(receivers.begin(), receivers.end(), tensor1); + + auto data_out = std::vector{}; + data_out.reserve(receivers.size()); + std::transform(receivers.begin(), receivers.end(), + std::back_inserter(data_out), + [](auto& tensor) { return *tensor.data(); }); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + + // Throw if trying to copy to too many outputs + receivers.emplace_back(receiver_shape, + Buffer{std::size_t{1}, HostMemory}); + EXPECT_THROW(rapids::copy(receivers.begin(), receivers.end(), tensor1), + TritonException); +} + } // namespace rapids } // namespace backend } // namespace triton From b951775b906a4705c0fba70ee6d0369696dd909b Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 1 Sep 2021 20:41:34 -0400 Subject: [PATCH 035/199] Add complete compilation of backend library --- .../rapids_triton/memory/detail/allocate.hpp | 7 ++++- cpp/include/rapids_triton/model/model.hpp | 15 +++++++--- .../rapids_triton/model/shared_state.hpp | 8 ++--- cpp/include/rapids_triton/tensor/tensor.hpp | 13 ++++++-- .../rapids_triton/triton/api/execute.hpp | 11 +++---- .../triton/api/instance_initialize.hpp | 2 +- .../triton/model_instance_state.hpp | 9 +++--- .../rapids_triton/triton/model_state.hpp | 3 +- cpp/src/api.cc | 21 ++++++------- cpp/src/model.h | 30 +++++++++++++++---- cpp/src/shared_state.h | 3 ++ 11 files changed, 83 insertions(+), 39 deletions(-) diff --git a/cpp/include/rapids_triton/memory/detail/allocate.hpp b/cpp/include/rapids_triton/memory/detail/allocate.hpp index 3f3f8e4..616fc6f 100644 --- a/cpp/include/rapids_triton/memory/detail/allocate.hpp +++ b/cpp/include/rapids_triton/memory/detail/allocate.hpp @@ -16,6 +16,7 @@ #pragma once #include +#include #include @@ -29,7 +30,11 @@ template struct dev_deallocater { void operator()(T* d_ptr) { if constexpr (IS_GPU_BUILD) { - cudaFree(reinterpret_cast(d_ptr)); + // Note: We allow a const_cast here because this deallocator is only used + // in a RAII context. If we are deallocating this memory, we allocated it + // and made it const. Removing the const qualifier allows the + // deallocation to proceed. + cudaFree(reinterpret_cast(const_cast::type*>(d_ptr))); } else { log_error( __FILE__, diff --git a/cpp/include/rapids_triton/model/model.hpp b/cpp/include/rapids_triton/model/model.hpp index e8d8e15..927b7e0 100644 --- a/cpp/include/rapids_triton/model/model.hpp +++ b/cpp/include/rapids_triton/model/model.hpp @@ -30,7 +30,7 @@ namespace triton { namespace backend { namespace rapids { template struct Model { - virtual void predict(Batch& batch) = 0; + virtual void predict(Batch& batch) const = 0; virtual void load() {} virtual void unload() {} @@ -65,10 +65,9 @@ namespace triton { namespace backend { namespace rapids { * override this in order to provide different streams for use with * successive incoming batches. For instance, one might cycle through * several streams in order to distribute batches across them, but care - * should be taken to ensure proper synchronization in this case. It is - * recommended that this method be overridden only when strictly necessary. + * should be taken to ensure proper synchronization in this case. */ - virtual cudaStream_t get_stream() { + virtual cudaStream_t get_stream() const { return default_stream_; } @@ -119,6 +118,14 @@ namespace triton { namespace backend { namespace rapids { auto get_config_param(std::string const& name, T default_value) const { return shared_state_->template get_config_param(name, default_value); } + template + auto get_config_param(char const* name) const { + return get_config_param(std::string(name)); + } + template + auto get_config_param(char const* name, T default_value) const { + return get_config_param(std::string(name), default_value); + } Model(std::shared_ptr shared_state, device_id_t device_id, cudaStream_t default_stream, DeploymentType deployment_type, std::string const& filepath) : shared_state_{shared_state}, device_id_{device_id}, default_stream_{default_stream}, deployment_type_{deployment_type}, filepath_{filepath} {} diff --git a/cpp/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp index ccf14b3..ca7d604 100644 --- a/cpp/include/rapids_triton/model/shared_state.hpp +++ b/cpp/include/rapids_triton/model/shared_state.hpp @@ -40,8 +40,8 @@ namespace triton { namespace backend { namespace rapids { virtual void unload() {} explicit SharedModelState( - common::TritonJson::Value&& config) : config_{std::move(config)}, - max_batch_size_(get_max_batch_size(config)) {} + std::unique_ptr&& config) : config_{std::move(config)}, + max_batch_size_(get_max_batch_size(*config)) {} template auto get_config_param(std::string const& name) { @@ -54,7 +54,7 @@ namespace triton { namespace backend { namespace rapids { } private: - common::TritonJson::Value config_; + std::unique_ptr config_; Batch::size_type max_batch_size_; template @@ -65,7 +65,7 @@ namespace triton { namespace backend { namespace rapids { return result; } auto json_value = common::TritonJson::Value{}; - if (config_.Find(name.c_str(), &json_value)) { + if (config_->Find(name.c_str(), &json_value)) { auto string_repr = std::string{}; triton_check(json_value.MemberAsString("string_value", &string_repr)); diff --git a/cpp/include/rapids_triton/tensor/tensor.hpp b/cpp/include/rapids_triton/tensor/tensor.hpp index f55a452..dd33999 100644 --- a/cpp/include/rapids_triton/tensor/tensor.hpp +++ b/cpp/include/rapids_triton/tensor/tensor.hpp @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include #include @@ -30,6 +31,7 @@ #include #include #include +#include #include namespace triton { namespace backend { namespace rapids { @@ -121,6 +123,11 @@ namespace triton { namespace backend { namespace rapids { * used to finalize them. */ void finalize() { + auto& shape = BaseTensor::shape(); + auto triton_shape = std::vector{}; + triton_shape.reserve(shape.size()); + std::transform(std::begin(shape), std::end(shape), std::back_inserter(triton_shape), [](auto& val) { return narrow(val);}); + // Must call the following because BackendOutputResponder does not expose // its stream, so we cannot be certain that our data is not being // processed on another stream. @@ -128,16 +135,16 @@ namespace triton { namespace backend { namespace rapids { responder_->ProcessTensor( name_.c_str(), TritonDtype::value, - BaseTensor::shape(), - BaseTensor::data(), + triton_shape, + reinterpret_cast(BaseTensor::data()), BaseTensor::mem_type(), BaseTensor::device() ); } private: - std::shared_ptr responder_; std::string name_; + std::shared_ptr responder_; }; template diff --git a/cpp/include/rapids_triton/triton/api/execute.hpp b/cpp/include/rapids_triton/triton/api/execute.hpp index 1500edb..3d0c6c7 100644 --- a/cpp/include/rapids_triton/triton/api/execute.hpp +++ b/cpp/include/rapids_triton/triton/api/execute.hpp @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include #include @@ -33,13 +34,13 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { auto* result = static_cast(nullptr); try { - auto* model_state = get_model_state(*instance); + auto* model_state = get_model_state(*get_model_from_instance(*instance)); auto* instance_state = get_instance_state(*instance); auto& model = instance_state->get_model(); - auto max_batch_size = model.get_config_param("max_batch_size"); - auto batch = Batch{ - raw_requests, request_count, model_state->TritonMemoryManager(), + auto max_batch_size = model.template get_config_param("max_batch_size"); + auto batch = Batch( + raw_requests, request_count, *(model_state->TritonMemoryManager()), /* Note: It is safe to keep a copy of the model_state * pointer in this closure and the instance pointer in the next because * the batch goes out of scope at the end of this block and Triton @@ -63,7 +64,7 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { }, model_state->EnablePinnedInput(), model_state->EnablePinnedOutput(), max_batch_size, - model.get_stream()}; + model.get_stream()); auto predict_err = static_cast(nullptr); try { diff --git a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp index f0f38b4..24d4e7e 100644 --- a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp @@ -48,7 +48,7 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { auto* model_state = get_model_state(*triton_model); auto rapids_model = - std::make_unique(model_state, instance); + std::make_unique(*model_state, instance); set_instance_state(*instance, std::move(rapids_model)); } catch (TritonException& err) { diff --git a/cpp/include/rapids_triton/triton/model_instance_state.hpp b/cpp/include/rapids_triton/triton/model_instance_state.hpp index 53c30bf..aaca523 100644 --- a/cpp/include/rapids_triton/triton/model_instance_state.hpp +++ b/cpp/include/rapids_triton/triton/model_instance_state.hpp @@ -18,7 +18,8 @@ #include #include #include -#include +#include +#include namespace triton { namespace backend { namespace rapids { @@ -30,13 +31,13 @@ struct ModelInstanceState : public BackendModelInstance { model_(model_state.get_shared_state(), rapids::get_device_id(*triton_model_instance), CudaStream(), Kind(), - JoinPath({RepositoryPath(), std::to_string(Version()), + JoinPath({model_state.RepositoryPath(), std::to_string(model_state.Version()), ArtifactFilename()})) {} auto& get_model() const { return model_; } - void load() { model_->load(); } - void unload() { model_->unload(); } + void load() { model_.load(); } + void unload() { model_.unload(); } private: RapidsModel model_; diff --git a/cpp/include/rapids_triton/triton/model_state.hpp b/cpp/include/rapids_triton/triton/model_state.hpp index f2b4953..38ba6bd 100644 --- a/cpp/include/rapids_triton/triton/model_state.hpp +++ b/cpp/include/rapids_triton/triton/model_state.hpp @@ -23,7 +23,8 @@ template struct TritonModelState : public BackendModel { TritonModelState(TRITONBACKEND_Model& triton_model) - : state_{std::make_shared( + : BackendModel(&triton_model), + state_{std::make_shared( get_model_config(triton_model))} {} void load() { state_->load(); } diff --git a/cpp/src/api.cc b/cpp/src/api.cc index a9f977a..7e62aa6 100644 --- a/cpp/src/api.cc +++ b/cpp/src/api.cc @@ -35,39 +35,40 @@ namespace triton { namespace backend { namespace NAMESPACE { -using ModelState = TritonModelState; -using ModelInstanceState = TritonModelInstance; +using ModelState = rapids::TritonModelState; +using ModelInstanceState = + rapids::ModelInstanceState; extern "C" { /** Confirm that backend is compatible with Triton's backend API version */ -auto* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { +TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { return rapids::triton_api::initialize(backend); } -auto* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { +TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { return rapids::triton_api::model_initialize(model); } -auto* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { +TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { return rapids::triton_api::model_finalize(model); } -auto* TRITONBACKEND_ModelInstanceInitialize( +TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize( TRITONBACKEND_ModelInstance* instance) { return rapids::triton_api::instance_initialize(instance); } -auto* TRITONBACKEND_ModelInstanceFinalize( +TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize( TRITONBACKEND_ModelInstance* instance) { return rapids::triton_api::instance_finalize(instance); } -auto* TRITONBACKEND_ModelInstanceExecute(TRITONBACKEND_ModelInstance* instance, - TRITONBACKEND_Request** raw_requests, - uint32_t const request_count) { +TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( + TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** raw_requests, + uint32_t const request_count) { return rapids::triton_api::execute( instance, raw_requests, static_cast(request_count)); } diff --git a/cpp/src/model.h b/cpp/src/model.h index bdf1916..8b5f64c 100644 --- a/cpp/src/model.h +++ b/cpp/src/model.h @@ -16,14 +16,18 @@ #pragma once +#include #include #include +#include #include -#include // rapids::Batch -#include // rapids::MemoryType -#include // rapids::Model -#include // rapids::copy +#include // rapids::Batch +#include // rapids::MemoryType +#include // rapids::Model +#include // rapids::copy +#include // rapids::DeploymentType +#include // rapids::device_id_t namespace triton { namespace backend { @@ -33,6 +37,20 @@ namespace NAMESPACE { * should be implemented in a struct named RapidsModel, as shown here */ struct RapidsModel : rapids::Model { + /*************************************************************************** + * BOILERPLATE * + * *********************************************************************** * + * The following constructor can be copied directly into any model + * implementation. + **************************************************************************/ + RapidsModel(std::shared_ptr shared_state, + rapids::device_id_t device_id, cudaStream_t default_stream, + rapids::DeploymentType deployment_type, + std::string const& filepath) + : rapids::Model(shared_state, device_id, + default_stream, deployment_type, + filepath) {} + /*************************************************************************** * BASIC FEATURES * * *********************************************************************** * @@ -58,7 +76,7 @@ struct RapidsModel : rapids::Model { * pointer to the underlying data. * 4. Call the `finalize` method on all output tensors. **************************************************************************/ - void predict(rapids::Batch& batch) { + void predict(rapids::Batch& batch) const { // 1. Acquire a tensor representing the input named "input__0" auto input = get_input(batch, "input__0"); // 2. Acquire a tensor representing the output named "input__0" @@ -66,7 +84,7 @@ struct RapidsModel : rapids::Model { // 3. Perform inference. In this example, we simply copy the data from the // input to the output tensor. - rapids::copy(output, input); + rapids::copy(output, input); // 4. Call finalize on all output tensors. In this case, we have just one // output, so we call finalize on it. diff --git a/cpp/src/shared_state.h b/cpp/src/shared_state.h index 2e55237..6de0397 100644 --- a/cpp/src/shared_state.h +++ b/cpp/src/shared_state.h @@ -18,6 +18,7 @@ #include +#include #include namespace triton { @@ -36,6 +37,8 @@ namespace NAMESPACE { * is entirely valid */ struct RapidsSharedState : rapids::SharedModelState { + RapidsSharedState(std::unique_ptr&& config) + : rapids::SharedModelState{std::move(config)} {} void load() {} void unload() {} }; From 5927547998d2b6d71b96b16bc1ede7243c1b4727 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 2 Sep 2021 13:02:07 -0400 Subject: [PATCH 036/199] Do not access moved-from object --- Dockerfile | 39 ++++++++++++++++--- .../rapids_triton_dev_cuda11.4.yml | 2 +- cpp/CMakeLists.txt | 8 ++-- .../rapids_triton/model/shared_state.hpp | 2 +- cpp/include/rapids_triton/triton/model.hpp | 2 +- cpp/src/CMakeLists.txt | 15 ++++--- 6 files changed, 50 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3b9f11f..07e937a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,13 @@ -ARG TRITON_VERSION=21.07 +########################################################################################### +# Arguments for controlling build details +########################################################################################### +# Version of Triton to use +ARG TRITON_VERSION=21.08 +# Base container image ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 +# Whether or not to build indicated components +ARG BUILD_TESTS=OFF +ARG BUILD_EXAMPLE=ON FROM ${BASE_IMAGE} as base @@ -41,13 +49,32 @@ ENV TRITON_VERSION=$TRITON_VERSION ARG BUILD_TYPE=Release ENV BUILD_TYPE=$BUILD_TYPE +ARG BUILD_TESTS +ENV BUILD_TESTS=$BUILD_TESTS +ARG BUILD_EXAMPLE +ENV BUILD_EXAMPLE=$BUILD_EXAMPLE RUN mkdir /rapids_triton/build WORKDIR /rapids_triton/build -RUN cmake -GNinja .. - -RUN ninja - -CMD ["/rapids_triton/build/test_rapids_triton"] +RUN cmake \ + -GNinja \ + -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ + -DBUILD_TESTS="${BUILD_TESTS}" \ + -DBUILD_EXAMPLE="${BUILD_EXAMPLES}" \ + .. + +RUN ninja install + +# FROM ${BASE_IMAGE} +# +# # Remove existing backend install +# RUN if [ -d /opt/tritonserver/backends/rapids_identity ]; \ +# then \ +# rm -rf /opt/tritonserver/backends/rapids_identity/*; \ +# fi +# +# COPY --from=build-stage \ +# /opt/tritonserver/backends/rapids_identity \ +# /opt/tritonserver/backends/rapids_identity diff --git a/conda/environments/rapids_triton_dev_cuda11.4.yml b/conda/environments/rapids_triton_dev_cuda11.4.yml index 18d47e6..409c7bc 100644 --- a/conda/environments/rapids_triton_dev_cuda11.4.yml +++ b/conda/environments/rapids_triton_dev_cuda11.4.yml @@ -4,7 +4,7 @@ channels: - nvidia - conda-forge dependencies: - - cmake=3.20.1 + - cmake>=3.21 - cudatoolkit=11.4 - ninja - rapidjson diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 9f8d397..169d73c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -14,7 +14,7 @@ # limitations under the License. #============================================================================= -cmake_minimum_required(VERSION 3.20.1 FATAL_ERROR) +cmake_minimum_required(VERSION 3.21 FATAL_ERROR) file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-21.10/RAPIDS.cmake ${CMAKE_BINARY_DIR}/RAPIDS.cmake) include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) @@ -50,9 +50,9 @@ option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(NVTX "Enable nvtx markers" OFF) option(TRITON_ENABLE_GPU "Enable GPU support in Triton" ON) option(TRITON_ENABLE_STATS "Enable statistics collection in Triton" ON) -set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo") -set(TRITON_CORE_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/core repo") -set(TRITON_BACKEND_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/backend repo") +set(TRITON_COMMON_REPO_TAG "r21.08" CACHE STRING "Tag for triton-inference-server/common repo") +set(TRITON_CORE_REPO_TAG "r21.08" CACHE STRING "Tag for triton-inference-server/core repo") +set(TRITON_BACKEND_REPO_TAG "r21.08" CACHE STRING "Tag for triton-inference-server/backend repo") message(VERBOSE "RAPIDS_TRITON: Build RAPIDS_TRITON unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAPIDS_TRITON: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") diff --git a/cpp/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp index ca7d604..3aa2242 100644 --- a/cpp/include/rapids_triton/model/shared_state.hpp +++ b/cpp/include/rapids_triton/model/shared_state.hpp @@ -41,7 +41,7 @@ namespace triton { namespace backend { namespace rapids { explicit SharedModelState( std::unique_ptr&& config) : config_{std::move(config)}, - max_batch_size_(get_max_batch_size(*config)) {} + max_batch_size_(get_max_batch_size(*config_)) {} template auto get_config_param(std::string const& name) { diff --git a/cpp/include/rapids_triton/triton/model.hpp b/cpp/include/rapids_triton/triton/model.hpp index 25e06a2..3ea61bf 100644 --- a/cpp/include/rapids_triton/triton/model.hpp +++ b/cpp/include/rapids_triton/triton/model.hpp @@ -55,7 +55,7 @@ inline auto get_model_config(TRITONBACKEND_Model& model) { - TRITONSERVER_Message* config_message = nullptr; + auto* config_message = static_cast(nullptr); triton_check(TRITONBACKEND_ModelConfig(&model, 1, &config_message)); auto* buffer = static_cast(nullptr); diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 02d15d2..32e2863 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -16,11 +16,11 @@ # keep the files in alphabetical order! add_library( - rapids-identity SHARED + triton_rapids-identity SHARED src/api.cc ) -set_target_properties(rapids-identity +set_target_properties(triton_rapids-identity PROPERTIES BUILD_RPATH "\$ORIGIN" # set target compile options CXX_STANDARD 17 @@ -31,17 +31,17 @@ PROPERTIES BUILD_RPATH "\$ORIGIN" INTERFACE_POSITION_INDEPENDENT_CODE ON ) -target_compile_options(rapids-identity +target_compile_options(triton_rapids-identity PRIVATE "$<$:${RAPIDS_TRITON_CXX_FLAGS}>" "$<$:${RAPIDS_TRITON_CUDA_FLAGS}>" ) -target_include_directories(rapids-identity +target_include_directories(triton_rapids-identity PRIVATE "$" "${CMAKE_CURRENT_SOURCE_DIR}/src" ) -target_link_libraries(rapids-identity +target_link_libraries(triton_rapids-identity PRIVATE rmm::rmm raft::raft @@ -50,3 +50,8 @@ PRIVATE "${TRITONSERVER_LIB}" $ ) + +install( + TARGETS triton_rapids-identity + LIBRARY DESTINATION /opt/tritonserver/backends/rapids-identity +) From d0ecbc91d91c3f0207f207ab3468071ab40e6c16 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 2 Sep 2021 14:57:00 -0400 Subject: [PATCH 037/199] Improve logging for type mismatches --- cpp/include/rapids_triton/tensor/dtype.hpp | 6 ++++++ cpp/include/rapids_triton/triton/input.hpp | 8 +++++++- cpp/include/rapids_triton/triton/output.hpp | 7 ++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/cpp/include/rapids_triton/tensor/dtype.hpp b/cpp/include/rapids_triton/tensor/dtype.hpp index 04be436..7dcb236 100644 --- a/cpp/include/rapids_triton/tensor/dtype.hpp +++ b/cpp/include/rapids_triton/tensor/dtype.hpp @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include @@ -154,4 +155,9 @@ struct TritonDtype> { static constexpr DType value = DTypeFloat64; }; +inline std::ostream& operator<<(std::ostream& out, DType const& dtype) { + out << TRITONSERVER_DataTypeString(dtype); + return out; +} + }}} diff --git a/cpp/include/rapids_triton/triton/input.hpp b/cpp/include/rapids_triton/triton/input.hpp index c412500..f2d7ddd 100644 --- a/cpp/include/rapids_triton/triton/input.hpp +++ b/cpp/include/rapids_triton/triton/input.hpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -54,7 +55,12 @@ namespace triton { namespace backend { namespace rapids { &input_dims, nullptr, nullptr)); if (reported_dtype != TritonDtype::value) { - throw(TritonException(Error::Internal, "incorrect type for requested input")); + auto log_stream = std::stringstream{}; + log_stream << "incorrect type " + << reported_dtype + << " for input with required type " + << TritonDtype::value; + throw(TritonException(Error::Internal, log_stream.str())); } if (input_dims != 0) { diff --git a/cpp/include/rapids_triton/triton/output.hpp b/cpp/include/rapids_triton/triton/output.hpp index cb0bf5b..5b258d1 100644 --- a/cpp/include/rapids_triton/triton/output.hpp +++ b/cpp/include/rapids_triton/triton/output.hpp @@ -47,7 +47,12 @@ namespace triton { namespace backend { namespace rapids { &input_dims, nullptr, nullptr)); if (reported_dtype != TritonDtype::value) { - throw(TritonException(Error::Internal, "incorrect type for requested input")); + auto log_stream = std::stringstream{}; + log_stream << "incorrect type " + << reported_dtype + << " for output with required type " + << TritonDtype::value; + throw(TritonException(Error::Internal, log_stream.str())); } if (input_dims != 0) { From abd6e9ed1a6908932f5fde5857818bf33e388cbc Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 3 Sep 2021 17:23:50 -0400 Subject: [PATCH 038/199] Redo output shape determination code --- cpp/include/rapids_triton/batch/batch.hpp | 42 ++++++++++--- cpp/include/rapids_triton/model/model.hpp | 4 ++ .../rapids_triton/model/shared_state.hpp | 60 ++++++++++++++++++- .../rapids_triton/triton/api/execute.hpp | 41 +++++++++---- cpp/src/model.h | 2 +- 5 files changed, 126 insertions(+), 23 deletions(-) diff --git a/cpp/include/rapids_triton/batch/batch.hpp b/cpp/include/rapids_triton/batch/batch.hpp index 23e8791..fa7115b 100644 --- a/cpp/include/rapids_triton/batch/batch.hpp +++ b/cpp/include/rapids_triton/batch/batch.hpp @@ -65,7 +65,7 @@ namespace triton { namespace backend { namespace rapids { Batch(TRITONBACKEND_Request** raw_requests, request_size_t count, TRITONBACKEND_MemoryManager& triton_mem_manager, - std::function(std::string const&)> get_output_shape, + std::function(std::string const&, size_type)> get_output_shape, std::function< void( TRITONBACKEND_Request*, @@ -80,6 +80,7 @@ namespace triton { namespace backend { namespace rapids { requests_(raw_requests, raw_requests + count), responses_(construct_responses(requests_.begin(), requests_.end())), get_output_shape_{get_output_shape}, + report_statistics_{report_request_statistics}, collector_( raw_requests, count, @@ -99,12 +100,30 @@ namespace triton { namespace backend { namespace rapids { )}, stream_{stream}, start_time_{std::chrono::steady_clock::now()}, - compute_start_time_{std::chrono::steady_clock::now()} {} + compute_start_time_{std::chrono::steady_clock::now()}, + batch_size_{} {} + template auto get_input_shape(std::string const& name) { auto result = std::vector{}; if(!requests_.empty()) { - result = get_triton_input_shape(std::begin(requests_), std::end(requests_), name); + result = get_triton_input_shape(std::begin(requests_), std::end(requests_), name); + + auto input_batch_dim = size_type{}; + if (result.size() > 0) { + input_batch_dim = result[0]; + } else { + input_batch_dim = size_type{}; + } + + if(batch_size_.has_value()) { + if(batch_size_.value() != input_batch_dim) { + throw TritonException( + Error::Internal, "all input tensors must have same batch dimension"); + } + } else { + batch_size_ = input_batch_dim; + } } return result; } @@ -113,7 +132,7 @@ namespace triton { namespace backend { namespace rapids { template auto get_input(std::string const& name, std::optional memory_type, device_id_t device_id, cudaStream_t stream) { - auto shape = get_input_shape(name); + auto shape = get_input_shape(name); auto size_bytes = sizeof(T) * std::reduce(shape.begin(), shape.end(), std::size_t{1}, std::multiplies<>()); auto allowed_memory_configs = std::vector>{}; if (memory_type.has_value()) { @@ -141,13 +160,13 @@ namespace triton { namespace backend { namespace rapids { auto buffer = Buffer( reinterpret_cast(raw_buffer), - reported_bytes, + reported_bytes / sizeof(T), reported_mem_type, reported_device_id, stream ); - if (reported_mem_type != memory_type || reported_device_id != device_id) { + if (memory_type && (reported_mem_type != memory_type || reported_device_id != device_id)) { throw TritonException(Error::Internal, "data collected in wrong location"); } @@ -165,7 +184,13 @@ namespace triton { namespace backend { namespace rapids { template auto get_output(std::string const& name, std::optional memory_type, device_id_t device_id, cudaStream_t stream) { - auto shape = get_output_shape_(name); + if (!batch_size_.has_value()) { + throw TritonException( + Error::Internal, + "At least one input must be retrieved before any output" + ); + } + auto shape = get_output_shape_(name, batch_size_.value()); auto buffer_size = std::reduce( shape.begin(), shape.end(), std::size_t{1}, std::multiplies<>()); auto final_memory_type = MemoryType{}; @@ -224,13 +249,14 @@ namespace triton { namespace backend { namespace rapids { private: std::vector requests_; std::vector responses_; - std::function(std::string const&)> get_output_shape_; + std::function(std::string const&, size_type)> get_output_shape_; std::function report_statistics_; BackendInputCollector collector_; std::shared_ptr responder_; cudaStream_t stream_; std::chrono::time_point start_time_; std::chrono::time_point compute_start_time_; + std::optional batch_size_; }; }}} // namespace triton::backend::rapids diff --git a/cpp/include/rapids_triton/model/model.hpp b/cpp/include/rapids_triton/model/model.hpp index 927b7e0..653f266 100644 --- a/cpp/include/rapids_triton/model/model.hpp +++ b/cpp/include/rapids_triton/model/model.hpp @@ -134,6 +134,10 @@ namespace triton { namespace backend { namespace rapids { auto get_deployment_type() const { return deployment_type_; } auto const& get_filepath() const { return filepath_; } + auto get_output_shape(std::string const& name) const { + return shared_state_->get_output_shape(name); + } + protected: auto get_shared_state() const { return shared_state_; } diff --git a/cpp/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp index 3aa2242..5b87654 100644 --- a/cpp/include/rapids_triton/model/shared_state.hpp +++ b/cpp/include/rapids_triton/model/shared_state.hpp @@ -15,6 +15,7 @@ */ #pragma once +#include #include #include #include @@ -41,7 +42,45 @@ namespace triton { namespace backend { namespace rapids { explicit SharedModelState( std::unique_ptr&& config) : config_{std::move(config)}, - max_batch_size_(get_max_batch_size(*config_)) {} + max_batch_size_(get_max_batch_size(*config_)), output_shapes_([this]() { + auto result = std::vector>>{}; + auto output_entries = triton::common::TritonJson::Value{}; + triton_check(config_->MemberAsArray("output", &output_entries)); + + result.reserve(output_entries.ArraySize()); + + // Using a raw loop because TritonJSON::Value access has no iterator interface + for (std::size_t i=0; i < output_entries.ArraySize(); ++i) { + auto output_entry = triton::common::TritonJson::Value{}; + triton_check(output_entries.IndexAsObject(i, &output_entry)); + auto name = std::string{}; + triton_check(output_entry.MemberAsString("name", &name)); + + auto shape = std::vector{}; + auto reshape_entry = triton::common::TritonJson::Value{}; + if (output_entry.Find("reshape", &reshape_entry)) { + ParseShape(reshape_entry, "shape", &shape); + } else { + ParseShape(output_entry, "dims", &shape); + } + if (shape[0] != -1) { + shape.insert(shape.begin(), -1); + } + result.insert( + std::upper_bound( + std::begin(output_shapes_), + std::end(output_shapes_), + name, + [](auto& value, auto& entry) { + return value < entry.first; + } + ), + {name, shape} + ); + } + + return result; + }()) {} template auto get_config_param(std::string const& name) { @@ -53,9 +92,28 @@ namespace triton { namespace backend { namespace rapids { return get_config_param(name, std::make_optional(default_value)); } + auto get_output_shape(std::string const& name) const { + auto cached_shape = std::lower_bound( + std::begin(output_shapes_), + std::end(output_shapes_), + name, + [](auto& entry, auto& value) { + return entry.first < value; + } + ); + if (cached_shape == std::end(output_shapes_)) { + auto log_stream = std::stringstream{}; + log_stream << "No output with name " << name << " in configuration."; + throw TritonException(Error::Internal, log_stream.str()); + } else { + return cached_shape->second; + } + } + private: std::unique_ptr config_; Batch::size_type max_batch_size_; + std::vector>> mutable output_shapes_; template auto get_config_param(std::string const& name, std::optional const& default_value) { diff --git a/cpp/include/rapids_triton/triton/api/execute.hpp b/cpp/include/rapids_triton/triton/api/execute.hpp index 3d0c6c7..d77117d 100644 --- a/cpp/include/rapids_triton/triton/api/execute.hpp +++ b/cpp/include/rapids_triton/triton/api/execute.hpp @@ -15,6 +15,7 @@ */ #pragma once +#include #include #include #include @@ -24,6 +25,7 @@ #include #include #include +#include #include namespace triton { namespace backend { namespace rapids { namespace triton_api { @@ -41,19 +43,32 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { auto max_batch_size = model.template get_config_param("max_batch_size"); auto batch = Batch( raw_requests, request_count, *(model_state->TritonMemoryManager()), - /* Note: It is safe to keep a copy of the model_state - * pointer in this closure and the instance pointer in the next because - * the batch goes out of scope at the end of this block and Triton - * guarantees that the liftimes of both the instance and model states - * extend beyond this function call. */ - [model_state](std::string const& name) { - auto result = std::vector{}; - auto& triton_result = - model_state->FindBatchOutput(name)->OutputShape(); - std::transform(std::begin(triton_result), std::end(triton_result), - std::back_inserter(result), [](auto& coord) { - return narrow(coord); - }); + /* Note: It is safe to keep a reference to the model in htis closure + * and a pointer to the instance in the next because the batch goes + * out of scope at the end of this block and Triton guarantees that + * the liftimes of both the instance and model extend beyond this + * function call. */ + [&model](std::string const& name, Batch::size_type batch_dim) { + auto result = std::vector{}; + auto config_shape = model.get_output_shape(name); + if (config_shape.size() > 0 && config_shape[0] < 0) { + config_shape[0] = batch_dim; + } + std::transform( + std::begin(config_shape), + std::end(config_shape), + std::back_inserter(result), + [](auto& coord) { + if (coord < 0) { + throw TritonException( + Error::Internal, + "Backends with variable-shape outputs must request desired output shape" + ); + } else { + return narrow(coord); + } + } + ); return result; }, [instance](TRITONBACKEND_Request* request, time_point req_start, diff --git a/cpp/src/model.h b/cpp/src/model.h index 8b5f64c..0ca87bc 100644 --- a/cpp/src/model.h +++ b/cpp/src/model.h @@ -79,7 +79,7 @@ struct RapidsModel : rapids::Model { void predict(rapids::Batch& batch) const { // 1. Acquire a tensor representing the input named "input__0" auto input = get_input(batch, "input__0"); - // 2. Acquire a tensor representing the output named "input__0" + // 2. Acquire a tensor representing the output named "output__0" auto output = get_output(batch, "output__0"); // 3. Perform inference. In this example, we simply copy the data from the From 75e55d88d4e064c95248539f28b75b6202c09b0d Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 7 Sep 2021 09:39:57 -0400 Subject: [PATCH 039/199] Default to running example server in Dockerfile --- Dockerfile | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 07e937a..03d2340 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,19 +62,23 @@ RUN cmake \ -GNinja \ -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ -DBUILD_TESTS="${BUILD_TESTS}" \ - -DBUILD_EXAMPLE="${BUILD_EXAMPLES}" \ + -DBUILD_EXAMPLE="${BUILD_EXAMPLE}" \ .. RUN ninja install -# FROM ${BASE_IMAGE} -# -# # Remove existing backend install -# RUN if [ -d /opt/tritonserver/backends/rapids_identity ]; \ -# then \ -# rm -rf /opt/tritonserver/backends/rapids_identity/*; \ -# fi -# -# COPY --from=build-stage \ -# /opt/tritonserver/backends/rapids_identity \ -# /opt/tritonserver/backends/rapids_identity +FROM ${BASE_IMAGE} + +RUN mkdir /models + +# Remove existing backend install +RUN if [ -d /opt/tritonserver/backends/rapids-identity ]; \ + then \ + rm -rf /opt/tritonserver/backends/rapids-identity/*; \ + fi + +COPY --from=build-stage \ + /opt/tritonserver/backends/rapids-identity \ + /opt/tritonserver/backends/rapids-identity + +ENTRYPOINT ["tritonserver", "--model-repository=/models"] From 482b1571b767c6c1b42e244783ef8af793eb6481 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 7 Sep 2021 11:36:45 -0400 Subject: [PATCH 040/199] Begin filling out docs --- README.md | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++ docs/usage.md | 32 ++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 docs/usage.md diff --git a/README.md b/README.md index 82c36c3..916e387 100644 --- a/README.md +++ b/README.md @@ -34,3 +34,65 @@ This project is designed to make it easy to integrate any C++-based algorithm into the NVIDIA Triton Inference Server. Originally developed to assist with the integration of RAPIDS algorithms, this library can be used by anyone to quickly get up and running with a custom backend for Triton. + +## Background + +### Triton + +The NVIDIA Triton Inference Server offers a complete open-source solution for +deployment of machine learning models from a wide variety of ML frameworks +(PyTorch, Tensorflow, ONNX, XGBoost, etc.) on both CPU and GPU hardware. It +allows you to maximize inference performance in production (whether that means +maximizing throughput, minimizing latency, or optimizing some other metric) +regardless of how you may have trained your ML model. Through smart batching, +efficient pipeline handling, and tools to simplify deployments almost anywhere, +Triton helps make production inference serving simpler and more cost-effective. + +### Custom Backends + +While Triton natively supports many common ML frameworks, you may wish to take +advantage of Triton's features for something a little more specialized. Triton +provides support for different kinds of models via "backends:" modular +libraries which provide the specialized logic for those models. Triton allows +you to create custom backends in +[Python](https://github.com/triton-inference-server/python_backend), but for +those who wish to use C++ directly, RAPIDS-Triton can help simplify the process +of developing your backend. + +## Simple Example + +In the `cpp/src` directory of this repository, you can see a complete, +annotated example of a backend built with RAPIDS-Triton. The core of any +backend is defining the `predict` function for your model as shown below: + +``` + void predict(rapids::Batch& batch) const { + rapids::Tensor input = get_input(batch, "input__0"); + rapids::Tensor output = get_output(batch, "output__0"); + + rapids::copy(output, input); + + output.finalize(); + } +``` + +In this example, we ask Triton to provide a tensor named `"input__0"` and copy +it to an output tensor named `"output__0"`. Thus, our "inference" function in +this simple example is just a passthrough from one input tensor to one output +tensor. + +To do something more sophisticated in this `predict` function, we might take +advantage of the `data()` method of Tensor objects, which provides a raw +pointer (on host or device) to the underlying data along with `size()`, and +`mem_type()` to determine the number of elements in the Tensor and whether they +are stored on host or device respectively. Note that `finalize()` must be +called on all output tensors before returning from the predict function. + +For a much more detailed look at developing backends with RAPIDS-Triton, +check out our complete [usage guide](https://github.com/rapidsai/rapids-triton/blob/main/docs/usage.md). + +## Contributing + +If you wish to contribute to RAPIDS-Triton, please see our [contributors' +guide](https://github.com/rapidsai/rapids-triton/blob/main/CONTRIBUTING.md) for +tips and full details on how to get started. diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 0000000..3897ca4 --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,32 @@ + + +# Using RAPIDS-Triton + +To begin developing a custom backend with RAPIDS-Triton, we strongly recommend +that you take advantage of the [rapids-triton-template repo](https://github.com/rapidsai/rapids-triton-template), which provides a basic template for your backend code. From ddcd1c008251a67fbadd6cc6446ece1febddc4ef Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 7 Sep 2021 14:08:31 -0400 Subject: [PATCH 041/199] Add basics from identity example --- CMakeLists.txt | 148 +++++++++++++++++ Dockerfile | 87 ++++++++++ cmake/modules/ConfigureCUDA.cmake | 43 +++++ cmake/thirdparty/get_gtest.cmake | 43 +++++ cmake/thirdparty/get_rapids-triton.cmake | 46 ++++++ .../rapids_triton_dev_cuda11.4.yml | 10 ++ src/api.cc | 80 ++++++++++ src/model.h | 149 ++++++++++++++++++ src/names.h | 29 ++++ src/shared_state.h | 48 ++++++ 10 files changed, 683 insertions(+) create mode 100644 CMakeLists.txt create mode 100644 Dockerfile create mode 100644 cmake/modules/ConfigureCUDA.cmake create mode 100644 cmake/thirdparty/get_gtest.cmake create mode 100644 cmake/thirdparty/get_rapids-triton.cmake create mode 100644 conda/environments/rapids_triton_dev_cuda11.4.yml create mode 100644 src/api.cc create mode 100644 src/model.h create mode 100644 src/names.h create mode 100644 src/shared_state.h diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..cf0f0c4 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,148 @@ +#============================================================================= +# Copyright (c) 2020-2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +cmake_minimum_required(VERSION 3.21 FATAL_ERROR) +file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-21.10/RAPIDS.cmake + ${CMAKE_BINARY_DIR}/RAPIDS.cmake) +include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) +include(rapids-cmake) +include(rapids-cpm) +include(rapids-cuda) +include(rapids-export) +include(rapids-find) + +rapids_cuda_init_architectures(RAPIDS_TRITON_LINEAR) + +project(RAPIDS_TRITON_LINEAR VERSION 21.10.00 LANGUAGES CXX CUDA) + +############################################################################## +# - build type --------------------------------------------------------------- + +# Set a default build type if none was specified +rapids_cmake_build_type(Release) + +# this is needed for clang-tidy runs +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +############################################################################## +# - User Options ------------------------------------------------------------ + +option(BUILD_BACKEND_TESTS "Build rapids_triton_linear unit-tests" ON) +option(CUDA_ENABLE_KERNEL_INFO "Enable kernel resource usage info" OFF) +option(CUDA_ENABLE_LINE_INFO "Enable lineinfo in nvcc" OFF) +option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" ON) +option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) +option(NVTX "Enable nvtx markers" OFF) + +message(VERBOSE "RAPIDS_TRITON_LINEAR: Enabling detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") +message(VERBOSE "RAPIDS_TRITON_LINEAR: Enabling kernelinfo in nvcc: ${CUDA_ENABLE_KERNEL_INFO}") +message(VERBOSE "RAPIDS_TRITON_LINEAR: Enabling lineinfo in nvcc: ${CUDA_ENABLE_LINE_INFO}") +message(VERBOSE "RAPIDS_TRITON_LINEAR: Enabling nvtx markers: ${NVTX}") +message(VERBOSE "RAPIDS_TRITON_LINEAR: Build RAPIDS_TRITON_LINEAR unit-tests: ${BUILD_TESTS}") + +# Set RMM logging level +set(RMM_LOGGING_LEVEL "INFO" CACHE STRING "Choose the logging level.") +set_property(CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF") +message(VERBOSE "RAPIDS_TRITON_LINEAR: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") + +############################################################################## +# - Conda environment detection ---------------------------------------------- + +if(DETECT_CONDA_ENV) + rapids_cmake_support_conda_env( conda_env MODIFY_PREFIX_PATH ) + if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED ENV{CONDA_PREFIX}) + message(STATUS "CUML: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}") + set(CMAKE_INSTALL_PREFIX "$ENV{CONDA_PREFIX}") + endif() +endif() + +############################################################################## +# - compiler options --------------------------------------------------------- + +# * find CUDAToolkit package +# * determine GPU architectures +# * enable the CMake CUDA language +# * set other CUDA compilation flags +rapids_find_package(CUDAToolkit REQUIRED + BUILD_EXPORT_SET cuml-exports + INSTALL_EXPORT_SET cuml-exports + ) +include(cmake/modules/ConfigureCUDA.cmake) + +############################################################################## +# - Requirements ------------------------------------------------------------- + +# add third party dependencies using CPM +rapids_cpm_init() + +# TODO(wphicks) +include(cmake/thirdparty/get_rapids-triton.cmake) + +if(BUILD_TESTS) + include(cmake/thirdparty/get_gtest.cmake) +endif() + + +############################################################################## +# - install targets----------------------------------------------------------- + +add_library( + triton_rapids-linear SHARED + src/api.cc +) + +set_target_properties(triton_rapids-linear +PROPERTIES BUILD_RPATH "\$ORIGIN" + # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON +) + +target_compile_options(triton_rapids-linear + PRIVATE "$<$:${RAPIDS_TRITON_LINEAR_CXX_FLAGS}>" + "$<$:${RAPIDS_TRITON_LINEAR_CUDA_FLAGS}>" +) + +target_include_directories(triton_rapids-linear + PRIVATE "$" + "${CMAKE_CURRENT_SOURCE_DIR}/src" +) + +target_link_libraries(triton_rapids-linear +PRIVATE + rapids_triton::rapids_triton + triton-core-serverstub + triton-backend-utils + "${TRITONSERVER_LIB}" + $ +) + +install( + TARGETS triton_rapids-linear + LIBRARY DESTINATION /opt/tritonserver/backends/rapids-linear +) + +############################################################################## +# - build test executable ---------------------------------------------------- + +# TODO (wphicks) +# if(BUILD_TESTS) +# include(test/CMakeLists.txt) +# endif() diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ca863ee --- /dev/null +++ b/Dockerfile @@ -0,0 +1,87 @@ +########################################################################################### +# Arguments for controlling build details +########################################################################################### +# Version of Triton to use +ARG TRITON_VERSION=21.08 +# Base container image +ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 +# Whether or not to build indicated components +ARG BUILD_TESTS=OFF +ARG BUILD_EXAMPLE=ON + +FROM ${BASE_IMAGE} as base + +ENV PATH="/root/miniconda3/bin:${PATH}" + +RUN apt-get update \ + && apt-get install --no-install-recommends -y wget patchelf \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +ENV PYTHONDONTWRITEBYTECODE=true + +RUN wget \ + https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && mkdir /root/.conda \ + && bash Miniconda3-latest-Linux-x86_64.sh -b \ + && rm -f Miniconda3-latest-Linux-x86_64.sh + +COPY ./conda/environments/rapids_triton_dev_cuda11.4.yml /environment.yml + +RUN conda env update -f /environment.yml \ + && rm /environment.yml \ + && conda clean -afy \ + && find /root/miniconda3/ -follow -type f -name '*.pyc' -delete \ + && find /root/miniconda3/ -follow -type f -name '*.js.map' -delete + +ENV PYTHONDONTWRITEBYTECODE=false + +RUN mkdir /rapids_triton + +COPY ./src /rapids_triton/src +COPY ./CMakeLists.txt /rapids_triton +COPY ./cmake /rapids_triton/cmake + +WORKDIR /rapids_triton + +SHELL ["conda", "run", "--no-capture-output", "-n", "rapids_triton_dev", "/bin/bash", "-c"] + +FROM base as build-stage + +ARG TRITON_VERSION +ENV TRITON_VERSION=$TRITON_VERSION + +ARG BUILD_TYPE=Release +ENV BUILD_TYPE=$BUILD_TYPE +ARG BUILD_TESTS +ENV BUILD_TESTS=$BUILD_TESTS +ARG BUILD_EXAMPLE +ENV BUILD_EXAMPLE=$BUILD_EXAMPLE + +RUN mkdir /rapids_triton/build + +WORKDIR /rapids_triton/build + +RUN cmake \ + -GNinja \ + -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ + -DBUILD_TESTS="${BUILD_TESTS}" \ + .. + +RUN ninja install + +FROM ${BASE_IMAGE} + +RUN mkdir /models + +# Remove existing backend install +RUN if [ -d /opt/tritonserver/backends/rapids-linear ]; \ + then \ + rm -rf /opt/tritonserver/backends/rapids-linear/*; \ + fi + +COPY --from=build-stage \ + /opt/tritonserver/backends/rapids-linear \ + /opt/tritonserver/backends/rapids-linear + +ENTRYPOINT ["tritonserver", "--model-repository=/models"] diff --git a/cmake/modules/ConfigureCUDA.cmake b/cmake/modules/ConfigureCUDA.cmake new file mode 100644 index 0000000..f170102 --- /dev/null +++ b/cmake/modules/ConfigureCUDA.cmake @@ -0,0 +1,43 @@ +#============================================================================= +# Copyright (c) 2018-2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +if(DISABLE_DEPRECATION_WARNINGS) + list(APPEND RAPIDS_TRITON_CXX_FLAGS -Wno-deprecated-declarations) + list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Xcompiler=-Wno-deprecated-declarations) +endif() + +if(CMAKE_COMPILER_IS_GNUCXX) + list(APPEND RAPIDS_TRITON_CXX_FLAGS -Wall -Werror -Wno-unknown-pragmas -Wno-error=deprecated-declarations) +endif() + +list(APPEND RAPIDS_TRITON_CUDA_FLAGS --expt-extended-lambda --expt-relaxed-constexpr) + +# set warnings as errors +if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 11.2.0) + list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Werror=all-warnings) +endif() +list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Xcompiler=-Wall,-Werror,-Wno-error=deprecated-declarations) + +# Option to enable line info in CUDA device compilation to allow introspection when profiling / memchecking +if(CUDA_ENABLE_LINEINFO) + list(APPEND RAPIDS_TRITON_CUDA_FLAGS -lineinfo) +endif() + +# Debug options +if(CMAKE_BUILD_TYPE MATCHES Debug) + message(VERBOSE "RAPIDS_TRITON: Building with debugging flags") + list(APPEND RAPIDS_TRITON_CUDA_FLAGS -G -Xcompiler=-rdynamic) +endif() diff --git a/cmake/thirdparty/get_gtest.cmake b/cmake/thirdparty/get_gtest.cmake new file mode 100644 index 0000000..02cf9cc --- /dev/null +++ b/cmake/thirdparty/get_gtest.cmake @@ -0,0 +1,43 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_gtest VERSION) + + if(TARGET GTest::gtest) + return() + endif() + + rapids_cpm_find(GTest ${VERSION} + GLOBAL_TARGETS gtest gtest_main GTest::gtest GTest::gtest_main gmock gmock_main + CPM_ARGS + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG release-${VERSION} + GIT_SHALLOW TRUE + OPTIONS "INSTALL_GTEST OFF" + # googletest >= 1.10.0 provides a cmake config file -- use it if it exists + FIND_PACKAGE_ARGUMENTS "CONFIG" + ) + + if(NOT TARGET GTest::gtest) + add_library(GTest::gtest ALIAS gtest) + add_library(GTest::gtest_main ALIAS gtest_main) + endif() + +endfunction() + +set(RAFT_MIN_VERSION_gtest 1.10.0) + +find_and_configure_gtest(${RAFT_MIN_VERSION_gtest}) diff --git a/cmake/thirdparty/get_rapids-triton.cmake b/cmake/thirdparty/get_rapids-triton.cmake new file mode 100644 index 0000000..64ba57d --- /dev/null +++ b/cmake/thirdparty/get_rapids-triton.cmake @@ -0,0 +1,46 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_rapids_triton) + + set(oneValueArgs VERSION FORK PINNED_TAG) + cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN} ) + + rapids_cpm_find(rapids_triton ${PKG_VERSION} + GLOBAL_TARGETS rapids_triton::rapids_triton + BUILD_EXPORT_SET rapids_triton_linear-exports + INSTALL_EXPORT_SET rapids_triton_linear-exports + CPM_ARGS + GIT_REPOSITORY https://github.com/${PKG_FORK}/rapids-triton.git + GIT_TAG ${PKG_PINNED_TAG} + SOURCE_SUBDIR cpp + OPTIONS + "BUILD_TESTS OFF" + "BUILD_EXAMPLE OFF" + ) + + message(VERBOSE "RAPIDS_TRITON_LINEAR: Using RAPIDS-Triton located in ${rapids_triton_SOURCE_DIR}") + +endfunction() + +# Change pinned tag here to test a commit in CI +# To use a different RAFT locally, set the CMake variable +# CPM_raft_SOURCE=/path/to/local/raft +find_and_configure_rapids_triton(VERSION 21.10 + FORK rapidsai + PINNED_TAG fea-initial + ) diff --git a/conda/environments/rapids_triton_dev_cuda11.4.yml b/conda/environments/rapids_triton_dev_cuda11.4.yml new file mode 100644 index 0000000..409c7bc --- /dev/null +++ b/conda/environments/rapids_triton_dev_cuda11.4.yml @@ -0,0 +1,10 @@ +--- +name: rapids_triton_dev +channels: + - nvidia + - conda-forge +dependencies: + - cmake>=3.21 + - cudatoolkit=11.4 + - ninja + - rapidjson diff --git a/src/api.cc b/src/api.cc new file mode 100644 index 0000000..7e62aa6 --- /dev/null +++ b/src/api.cc @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace NAMESPACE { + +using ModelState = rapids::TritonModelState; +using ModelInstanceState = + rapids::ModelInstanceState; + +extern "C" { + +/** Confirm that backend is compatible with Triton's backend API version + */ +TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { + return rapids::triton_api::initialize(backend); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { + return rapids::triton_api::model_initialize(model); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { + return rapids::triton_api::model_finalize(model); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize( + TRITONBACKEND_ModelInstance* instance) { + return rapids::triton_api::instance_initialize(instance); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize( + TRITONBACKEND_ModelInstance* instance) { + return rapids::triton_api::instance_finalize(instance); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( + TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** raw_requests, + uint32_t const request_count) { + return rapids::triton_api::execute( + instance, raw_requests, static_cast(request_count)); +} + +} // extern "C" + +} // namespace NAMESPACE +} // namespace backend +} // namespace triton diff --git a/src/model.h b/src/model.h new file mode 100644 index 0000000..0ca87bc --- /dev/null +++ b/src/model.h @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include +#include +#include // rapids::Batch +#include // rapids::MemoryType +#include // rapids::Model +#include // rapids::copy +#include // rapids::DeploymentType +#include // rapids::device_id_t + +namespace triton { +namespace backend { +namespace NAMESPACE { + +/* Any logic necessary to perform inference with a model and manage its data + * should be implemented in a struct named RapidsModel, as shown here */ + +struct RapidsModel : rapids::Model { + /*************************************************************************** + * BOILERPLATE * + * *********************************************************************** * + * The following constructor can be copied directly into any model + * implementation. + **************************************************************************/ + RapidsModel(std::shared_ptr shared_state, + rapids::device_id_t device_id, cudaStream_t default_stream, + rapids::DeploymentType deployment_type, + std::string const& filepath) + : rapids::Model(shared_state, device_id, + default_stream, deployment_type, + filepath) {} + + /*************************************************************************** + * BASIC FEATURES * + * *********************************************************************** * + * The only method that *must* be implemented for a viable model is the + * `predict` method, but the others presented here are often used for basic + * model implementations. Filling out these methods should take care of most + * use cases. + **************************************************************************/ + + /*************************************************************************** + * predict * + * *********************************************************************** * + * This method performs the actual inference step on input data. Implementing + * a predict function requires four steps: + * 1. Call `get_input` on the provided `Batch` object for each of the input + * tensors named in the config file for this backend. This provides a + * `Tensor` object containing the input data. + * 2. Call `get_output` on the provided `Batch` object for each of the output + * tensors named in the config file for this backend. This provides a + * `Tensor` object to which output values can be written. + * 3. Perform inference based on the input Tensors and store the results in + * the output Tensors. `some_tensor.data()` can be used to retrieve a raw + * pointer to the underlying data. + * 4. Call the `finalize` method on all output tensors. + **************************************************************************/ + void predict(rapids::Batch& batch) const { + // 1. Acquire a tensor representing the input named "input__0" + auto input = get_input(batch, "input__0"); + // 2. Acquire a tensor representing the output named "output__0" + auto output = get_output(batch, "output__0"); + + // 3. Perform inference. In this example, we simply copy the data from the + // input to the output tensor. + rapids::copy(output, input); + + // 4. Call finalize on all output tensors. In this case, we have just one + // output, so we call finalize on it. + output.finalize(); + } + + /*************************************************************************** + * load / unload * + * *********************************************************************** * + * These methods can be used to perform one-time loading/unloading of + * resources when a model is created. For example, data representing the + * model may be loaded onto the GPU in the `load` method and unloaded in the + * `unload` method. This data will then remain loaded while the server is + * running. + * + * While these methods take no arguments, it is typical to read any necessary + * input from the model configuration file by using the `get_config_param` + * method. Any parameters defined in the "parameters" section of the config + * can be accessed by name in this way. The maximum batch size can also be + * retrieved using the name "max_batch_size". + * + * These methods need not be explicitly implemented if no loading/unloading + * logic is required, but we show them here for illustrative purposes. + **************************************************************************/ + void load() {} + void unload() {} + + /*************************************************************************** + * ADVANCED FEATURES * + * *********************************************************************** * + * None of the following methods are required to be implemented in order to + * create a valid model, but they are presented here for those who require + * the additional functionality they provide. + **************************************************************************/ + + /*************************************************************************** + * preferred_mem_type / preferred_mem_type_in / preferred_mem_type_out * + * *********************************************************************** * + * If implemented, `preferred_mem_type` allows for control over when input + * and output data are provided on the host versus on device. In the case + * that a model prefers to receive its input on-host but return output + * on-device (or vice versa), `preferred_mem_type_in` and + * `preferred_mem_type_out` can be used for even more precise control. + * + * In this example, we simply return `std::nullopt` to indicate that the + * model has no preference on its input/output data locations. Note that the + * Batch being processed is taken as input to this function to facilitate + * implementations that may switch their preferred memory location based on + * properties of the batch. + * + * Valid MemoryType options to return are rapids::HostMemory and + * rapids::DeviceMemory. + **************************************************************************/ + std::optional preferred_mem_type( + rapids::Batch& batch) const { + return std::nullopt; + } +}; + +} // namespace NAMESPACE +} // namespace backend +} // namespace triton diff --git a/src/names.h b/src/names.h new file mode 100644 index 0000000..e98cb0d --- /dev/null +++ b/src/names.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +/* Triton expects certain definitions within its backend libraries to follow + * specific naming conventions. Specifically, for a backend named + * "rapids_identity," most definitions should appear within a namespace called + * triton::backend::rapids_identity. + * + * In order to facilitate this with minimal effort on the part of backend + * developers, we ask that you put the name of your backend here. This macro is + * then used to propagate the correct namespace name wherever it is needed in + * the impl and interface code. */ + +#define NAMESPACE rapids_linear diff --git a/src/shared_state.h b/src/shared_state.h new file mode 100644 index 0000000..6de0397 --- /dev/null +++ b/src/shared_state.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include + +namespace triton { +namespace backend { +namespace NAMESPACE { + +/* Triton allows multiple instances of a single model to be instantiated at the + * same time (e.g. on different GPUs). All instances of a model share access to + * an object which manages any state that can be shared across all instances. + * Any logic necessary for managing such state should be implemented in a + * struct named RapidsSharedState, as shown here. Models may access this shared + * state object via the `get_shared_state` method, which returns a shared + * pointer to the RapidsSharedState object. + * + * Not all backends require shared state, so leaving this implementation empty + * is entirely valid */ + +struct RapidsSharedState : rapids::SharedModelState { + RapidsSharedState(std::unique_ptr&& config) + : rapids::SharedModelState{std::move(config)} {} + void load() {} + void unload() {} +}; + +} // namespace NAMESPACE +} // namespace backend +} // namespace triton From c256fe1ec7eaff6a46af569325a552726afbb797 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 8 Sep 2021 13:01:10 -0400 Subject: [PATCH 042/199] Correct parameter fetching logic --- Dockerfile | 2 ++ README.md | 8 ++++++++ cpp/include/rapids_triton/model/shared_state.hpp | 5 ++++- cpp/include/rapids_triton/triton/model_state.hpp | 1 + 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 03d2340..6be8a53 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,6 +67,8 @@ RUN cmake \ RUN ninja install +ENTRYPOINT ["/rapids_triton/build/test_rapids_triton"] + FROM ${BASE_IMAGE} RUN mkdir /models diff --git a/README.md b/README.md index 916e387..488d361 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,14 @@ you to create custom backends in those who wish to use C++ directly, RAPIDS-Triton can help simplify the process of developing your backend. +The goal of RAPIDS-Triton is not to facilitate every possible use case of the +Triton backend API but to make the most common uses of this API easier by +providing a simpler interface to them. That being said, if there is a feature +of the Triton backend API which RAPIDS-Triton does not expose and which you +wish to use in a custom backend, please [submit a feature +request](https://github.com/rapidsai/rapids-triton/issues), and we will see if +it can be added. + ## Simple Example In the `cpp/src` directory of this repository, you can see a complete, diff --git a/cpp/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp index 5b87654..63cb09e 100644 --- a/cpp/include/rapids_triton/model/shared_state.hpp +++ b/cpp/include/rapids_triton/model/shared_state.hpp @@ -122,8 +122,11 @@ namespace triton { namespace backend { namespace rapids { result = max_batch_size_; return result; } + auto parameters = common::TritonJson::Value{}; auto json_value = common::TritonJson::Value{}; - if (config_->Find(name.c_str(), &json_value)) { + if ( + config_->Find("parameters", ¶meters) && + parameters.Find(name.c_str(), &json_value)) { auto string_repr = std::string{}; triton_check(json_value.MemberAsString("string_value", &string_repr)); diff --git a/cpp/include/rapids_triton/triton/model_state.hpp b/cpp/include/rapids_triton/triton/model_state.hpp index 38ba6bd..44804e5 100644 --- a/cpp/include/rapids_triton/triton/model_state.hpp +++ b/cpp/include/rapids_triton/triton/model_state.hpp @@ -16,6 +16,7 @@ #pragma once #include +#include #include namespace triton { namespace backend { namespace rapids { From 78b077793d908179b3b2d18033126130fb927fc5 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 8 Sep 2021 15:09:34 -0400 Subject: [PATCH 043/199] Add stream interface for logging --- .../rapids_triton/triton/api/initialize.hpp | 5 +- .../triton/api/instance_finalize.hpp | 4 +- .../triton/api/instance_initialize.hpp | 19 +++--- .../triton/api/model_finalize.hpp | 3 +- .../triton/api/model_initialize.hpp | 15 ++--- cpp/include/rapids_triton/triton/backend.hpp | 19 +++--- cpp/include/rapids_triton/triton/logging.hpp | 60 +++++++++++++++++++ cpp/test/triton/logging.cpp | 9 +++ 8 files changed, 95 insertions(+), 39 deletions(-) diff --git a/cpp/include/rapids_triton/triton/api/initialize.hpp b/cpp/include/rapids_triton/triton/api/initialize.hpp index 8021427..3cf7cc3 100644 --- a/cpp/include/rapids_triton/triton/api/initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/initialize.hpp @@ -28,10 +28,7 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { try { auto name = get_backend_name(*backend); - log_info( - __FILE__, __LINE__, - std::string("TRITONBACKEND_Initialize: ") + name - ); + log_info(__FILE__, __LINE__) << "TRITONBACKEND_Initialize: " << name; if (!check_backend_version(*backend)) { throw TritonException{ diff --git a/cpp/include/rapids_triton/triton/api/instance_finalize.hpp b/cpp/include/rapids_triton/triton/api/instance_finalize.hpp index ed6c7d6..d5b0dc7 100644 --- a/cpp/include/rapids_triton/triton/api/instance_finalize.hpp +++ b/cpp/include/rapids_triton/triton/api/instance_finalize.hpp @@ -30,9 +30,7 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { if (instance_state != nullptr) { instance_state->unload(); - log_info( - __FILE__, __LINE__, - "TRITONBACKEND_ModelInstanceFinalize: delete instance state"); + log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInstanceFinalize: delete instance state"; delete instance_state; } diff --git a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp index 24d4e7e..c3d421e 100644 --- a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp @@ -16,7 +16,6 @@ #pragma once -#include #include #include #include @@ -33,16 +32,14 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { auto device_id = get_device_id(*instance); auto deployment_type = get_deployment_type(*instance); - auto log_stream = std::stringstream{}; - log_stream << "TRITONBACKEND_ModelInstanceInitialize: " - << name - << " (" - << TRITONSERVER_InstanceGroupKindString(deployment_type) - << " device " - << device_id - << ")"; - - log_info(__FILE__, __LINE__, log_stream.str()); + log_info(__FILE__, __LINE__) + << "TRITONBACKEND_ModelInstanceInitialize: " + << name + << " (" + << TRITONSERVER_InstanceGroupKindString(deployment_type) + << " device " + << device_id + << ")"; auto* triton_model = get_model_from_instance(*instance); auto* model_state = get_model_state(*triton_model); diff --git a/cpp/include/rapids_triton/triton/api/model_finalize.hpp b/cpp/include/rapids_triton/triton/api/model_finalize.hpp index 9af0524..5846033 100644 --- a/cpp/include/rapids_triton/triton/api/model_finalize.hpp +++ b/cpp/include/rapids_triton/triton/api/model_finalize.hpp @@ -31,8 +31,7 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { model_state->get_shared_state()->unload(); } - log_info(__FILE__, __LINE__, - "TRITONBACKEND_ModelFinalize: delete model state"); + log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelFinalize: delete model state"; delete model_state; } catch (TritonException& err) { diff --git a/cpp/include/rapids_triton/triton/api/model_initialize.hpp b/cpp/include/rapids_triton/triton/api/model_initialize.hpp index edbe0ef..ac18195 100644 --- a/cpp/include/rapids_triton/triton/api/model_initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/model_initialize.hpp @@ -15,7 +15,6 @@ */ #pragma once -#include #include #include #include @@ -31,15 +30,11 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { auto version = get_model_version(*model); - auto log_stream = std::stringstream{}; - - log_stream << "TRITONBACKEND_ModelInitialize: " - << name - << " (version " - << version - << ")"; - - log_info(__FILE__, __LINE__, log_stream.str()); + log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInitialize: " + << name + << " (version " + << version + << ")"; auto rapids_model_state = std::make_unique(*model); rapids_model_state->load(); diff --git a/cpp/include/rapids_triton/triton/backend.hpp b/cpp/include/rapids_triton/triton/backend.hpp index 0adcb13..b5629e5 100644 --- a/cpp/include/rapids_triton/triton/backend.hpp +++ b/cpp/include/rapids_triton/triton/backend.hpp @@ -54,18 +54,19 @@ namespace triton { namespace backend { namespace rapids { auto version = backend_version{}; triton_check(TRITONBACKEND_ApiVersion(&version.major, &version.minor)); - log_info( - (std::string("Triton TRITONBACKEND API version: ") + - std::to_string(version.major) + "." + std::to_string(version.minor)) - .c_str()); + log_info(__FILE__, __LINE__) << "Triton TRITONBACKEND API version: " + << version.major + << "." + << version.minor; auto name = get_backend_name(backend); - log_info( - (std::string("'") + name + "' TRITONBACKEND API version: " + - std::to_string(TRITONBACKEND_API_VERSION_MAJOR) + "." + - std::to_string(TRITONBACKEND_API_VERSION_MINOR)) - .c_str()); + log_info(__FILE__, __LINE__) << "'" + << name + << "' TRITONBACKEND API version: " + << TRITONBACKEND_API_VERSION_MAJOR + << "." + << TRITONBACKEND_API_VERSION_MINOR; return ( (version.major == TRITONBACKEND_API_VERSION_MAJOR) && diff --git a/cpp/include/rapids_triton/triton/logging.hpp b/cpp/include/rapids_triton/triton/logging.hpp index dbc7c26..1ecfcd5 100644 --- a/cpp/include/rapids_triton/triton/logging.hpp +++ b/cpp/include/rapids_triton/triton/logging.hpp @@ -18,6 +18,8 @@ #include #include +#include +#include #include namespace triton { namespace backend { namespace rapids { @@ -31,6 +33,40 @@ namespace triton { namespace backend { namespace rapids { } } + struct log_stream : public std::ostream { + log_stream(TRITONSERVER_LogLevel level, char const* filename, int line) : std::ostream{}, buffer_{level, filename, line} { rdbuf(&buffer_); } + log_stream(TRITONSERVER_LogLevel level) : std::ostream{}, buffer_{level, __FILE__, __LINE__} { rdbuf(&buffer_); } + + ~log_stream() { + try { + flush(); + } catch (std::ios_base::failure const& ignored_err) { + // Ignore error if flush fails + } + } + + private: + struct log_buffer : public std::stringbuf { + log_buffer(TRITONSERVER_LogLevel level, char const* filename, int line) : level_{level}, filename_{filename}, line_{line} {} + + virtual int sync() { + auto msg = str(); + if(!msg.empty()) { + log(level_, filename_, line_, msg.c_str()); + str(""); + } + return 0; + } + + private: + TRITONSERVER_LogLevel level_; + char const* filename_; + int line_; + }; + + log_buffer buffer_; + }; + /** Log message at INFO level */ inline void log_info(const char* filename, const int line, const char* message) { @@ -45,6 +81,12 @@ namespace triton { namespace backend { namespace rapids { inline void log_info(std::string const& message) { log_info(__FILE__, __LINE__, message.c_str()); } + inline auto log_info(const char* filename, const int line) { + return log_stream(TRITONSERVER_LOG_INFO, filename, line); + } + inline auto log_info() { + return log_stream(TRITONSERVER_LOG_INFO); + } /** Log message at WARN level */ @@ -60,6 +102,12 @@ namespace triton { namespace backend { namespace rapids { inline void log_warn(std::string const& message) { log_warn(__FILE__, __LINE__, message.c_str()); } + inline auto log_warn(const char* filename, const int line) { + return log_stream(TRITONSERVER_LOG_WARN, filename, line); + } + inline auto log_warn() { + return log_stream(TRITONSERVER_LOG_WARN); + } /** Log message at ERROR level */ @@ -75,6 +123,12 @@ namespace triton { namespace backend { namespace rapids { inline void log_error(std::string const& message) { log_error(__FILE__, __LINE__, message.c_str()); } + inline auto log_error(const char* filename, const int line) { + return log_stream(TRITONSERVER_LOG_ERROR, filename, line); + } + inline auto log_error() { + return log_stream(TRITONSERVER_LOG_ERROR); + } /** Log message at VERBOSE level */ @@ -90,5 +144,11 @@ namespace triton { namespace backend { namespace rapids { inline void log_debug(std::string const& message) { log_debug(__FILE__, __LINE__, message.c_str()); } + inline auto log_debug(const char* filename, const int line) { + return log_stream(TRITONSERVER_LOG_VERBOSE, filename, line); + } + inline auto log_debug() { + return log_stream(TRITONSERVER_LOG_VERBOSE); + } }}} // namespace triton::backend::rapids diff --git a/cpp/test/triton/logging.cpp b/cpp/test/triton/logging.cpp index a265800..e4dc37e 100644 --- a/cpp/test/triton/logging.cpp +++ b/cpp/test/triton/logging.cpp @@ -16,6 +16,7 @@ #include +#include #include namespace triton { @@ -27,6 +28,14 @@ TEST(RapidsTriton, logging) { log_warn("Warn test message"); log_error("Error test message"); } + +TEST(RapidsTriton, stream_logging) { + log_debug() << "Streamed debug test message"; + log_info() << "Streamed info test message"; + log_warn() << "Streamed warn test message"; + log_error() << "Streamed error test message"; +} + } // namespace rapids } // namespace backend } // namespace triton From ed47a541a92e88e4f6001012105ec00485aa3161 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 8 Sep 2021 15:28:42 -0400 Subject: [PATCH 044/199] Clean up const declarations in logging --- cpp/include/rapids_triton/triton/logging.hpp | 36 ++++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/cpp/include/rapids_triton/triton/logging.hpp b/cpp/include/rapids_triton/triton/logging.hpp index 1ecfcd5..dbefd68 100644 --- a/cpp/include/rapids_triton/triton/logging.hpp +++ b/cpp/include/rapids_triton/triton/logging.hpp @@ -27,8 +27,8 @@ namespace triton { namespace backend { namespace rapids { namespace { /** Log message at indicated level */ inline void log( - TRITONSERVER_LogLevel level, const char* filename, const int line, - const char* message) { + TRITONSERVER_LogLevel level, char const* filename, int line, + char const* message) { triton_check(TRITONSERVER_LogMessage(level, filename, line, message)); } } @@ -69,19 +69,19 @@ namespace triton { namespace backend { namespace rapids { /** Log message at INFO level */ - inline void log_info(const char* filename, const int line, const char* message) { + inline void log_info(char const* filename, int line, char const* message) { log(TRITONSERVER_LOG_INFO, filename, line, message); } - inline void log_info(const char* filename, const int line, std::string const& message) { + inline void log_info(char const* filename, int line, std::string const& message) { log_info(filename, line, message.c_str()); } - inline void log_info(const char* message) { + inline void log_info(char const* message) { log_info(__FILE__, __LINE__, message); } inline void log_info(std::string const& message) { log_info(__FILE__, __LINE__, message.c_str()); } - inline auto log_info(const char* filename, const int line) { + inline auto log_info(char const* filename, int line) { return log_stream(TRITONSERVER_LOG_INFO, filename, line); } inline auto log_info() { @@ -90,19 +90,19 @@ namespace triton { namespace backend { namespace rapids { /** Log message at WARN level */ - inline void log_warn(const char* filename, const int line, const char* message) { + inline void log_warn(char const* filename, int line, char const* message) { log(TRITONSERVER_LOG_WARN, filename, line, message); } - inline void log_warn(const char* filename, const int line, std::string const& message) { + inline void log_warn(char const* filename, int line, std::string const& message) { log_warn(filename, line, message.c_str()); } - inline void log_warn(const char* message) { + inline void log_warn(char const* message) { log_warn(__FILE__, __LINE__, message); } inline void log_warn(std::string const& message) { log_warn(__FILE__, __LINE__, message.c_str()); } - inline auto log_warn(const char* filename, const int line) { + inline auto log_warn(char const* filename, int line) { return log_stream(TRITONSERVER_LOG_WARN, filename, line); } inline auto log_warn() { @@ -111,19 +111,19 @@ namespace triton { namespace backend { namespace rapids { /** Log message at ERROR level */ - inline void log_error(const char* filename, const int line, const char* message) { + inline void log_error(char const* filename, int line, char const* message) { log(TRITONSERVER_LOG_ERROR, filename, line, message); } - inline void log_error(const char* filename, const int line, std::string const& message) { + inline void log_error(char const* filename, int line, std::string const& message) { log_error(filename, line, message.c_str()); } - inline void log_error(const char* message) { + inline void log_error(char const* message) { log_error(__FILE__, __LINE__, message); } inline void log_error(std::string const& message) { log_error(__FILE__, __LINE__, message.c_str()); } - inline auto log_error(const char* filename, const int line) { + inline auto log_error(char const* filename, int line) { return log_stream(TRITONSERVER_LOG_ERROR, filename, line); } inline auto log_error() { @@ -132,19 +132,19 @@ namespace triton { namespace backend { namespace rapids { /** Log message at VERBOSE level */ - inline void log_debug(const char* filename, const int line, const char* message) { + inline void log_debug(char const* filename, int line, char const* message) { log(TRITONSERVER_LOG_VERBOSE, filename, line, message); } - inline void log_debug(const char* filename, const int line, std::string const& message) { + inline void log_debug(char const* filename, int line, std::string const& message) { log_debug(filename, line, message.c_str()); } - inline void log_debug(const char* message) { + inline void log_debug(char const* message) { log_debug(__FILE__, __LINE__, message); } inline void log_debug(std::string const& message) { log_debug(__FILE__, __LINE__, message.c_str()); } - inline auto log_debug(const char* filename, const int line) { + inline auto log_debug(char const* filename, int line) { return log_stream(TRITONSERVER_LOG_VERBOSE, filename, line); } inline auto log_debug() { From ff2032a54f5c87fca84b0b2431d53ef6ad1be148 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 8 Sep 2021 15:41:14 -0400 Subject: [PATCH 045/199] Provide shared state example docs --- CMakeLists.txt | 45 +++++---- Dockerfile | 11 ++- README.md | 233 ++++++++++++++++++++++++++++++++++++++++++++- src/names.h | 4 +- src/shared_state.h | 8 +- 5 files changed, 274 insertions(+), 27 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cf0f0c4..5af8428 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,15 @@ #============================================================================= cmake_minimum_required(VERSION 3.21 FATAL_ERROR) + +############################################################################## +# - Target names ------------------------------------------------------------- +set(BACKEND_NAME "rapids_linear") +set(BACKEND_TARGET "triton_${BACKEND_NAME}") + + +############################################################################## +# - Prepare rapids-cmake ----------------------------------------------------- file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-21.10/RAPIDS.cmake ${CMAKE_BINARY_DIR}/RAPIDS.cmake) include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) @@ -24,9 +33,9 @@ include(rapids-cuda) include(rapids-export) include(rapids-find) -rapids_cuda_init_architectures(RAPIDS_TRITON_LINEAR) +rapids_cuda_init_architectures(RAPIDS_TRITON_BACKEND) -project(RAPIDS_TRITON_LINEAR VERSION 21.10.00 LANGUAGES CXX CUDA) +project(RAPIDS_TRITON_BACKEND VERSION 21.10.00 LANGUAGES CXX CUDA) ############################################################################## # - build type --------------------------------------------------------------- @@ -47,16 +56,16 @@ option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(NVTX "Enable nvtx markers" OFF) -message(VERBOSE "RAPIDS_TRITON_LINEAR: Enabling detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") -message(VERBOSE "RAPIDS_TRITON_LINEAR: Enabling kernelinfo in nvcc: ${CUDA_ENABLE_KERNEL_INFO}") -message(VERBOSE "RAPIDS_TRITON_LINEAR: Enabling lineinfo in nvcc: ${CUDA_ENABLE_LINE_INFO}") -message(VERBOSE "RAPIDS_TRITON_LINEAR: Enabling nvtx markers: ${NVTX}") -message(VERBOSE "RAPIDS_TRITON_LINEAR: Build RAPIDS_TRITON_LINEAR unit-tests: ${BUILD_TESTS}") +message(VERBOSE "RAPIDS_TRITON_BACKEND: Enabling detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") +message(VERBOSE "RAPIDS_TRITON_BACKEND: Enabling kernelinfo in nvcc: ${CUDA_ENABLE_KERNEL_INFO}") +message(VERBOSE "RAPIDS_TRITON_BACKEND: Enabling lineinfo in nvcc: ${CUDA_ENABLE_LINE_INFO}") +message(VERBOSE "RAPIDS_TRITON_BACKEND: Enabling nvtx markers: ${NVTX}") +message(VERBOSE "RAPIDS_TRITON_BACKEND: Build RAPIDS_TRITON_BACKEND unit-tests: ${BUILD_TESTS}") # Set RMM logging level set(RMM_LOGGING_LEVEL "INFO" CACHE STRING "Choose the logging level.") set_property(CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF") -message(VERBOSE "RAPIDS_TRITON_LINEAR: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") +message(VERBOSE "RAPIDS_TRITON_BACKEND: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") ############################################################################## # - Conda environment detection ---------------------------------------------- @@ -100,11 +109,11 @@ endif() # - install targets----------------------------------------------------------- add_library( - triton_rapids-linear SHARED + ${BACKEND_TARGET} SHARED src/api.cc ) -set_target_properties(triton_rapids-linear +set_target_properties(${BACKEND_TARGET} PROPERTIES BUILD_RPATH "\$ORIGIN" # set target compile options CXX_STANDARD 17 @@ -115,17 +124,17 @@ PROPERTIES BUILD_RPATH "\$ORIGIN" INTERFACE_POSITION_INDEPENDENT_CODE ON ) -target_compile_options(triton_rapids-linear - PRIVATE "$<$:${RAPIDS_TRITON_LINEAR_CXX_FLAGS}>" - "$<$:${RAPIDS_TRITON_LINEAR_CUDA_FLAGS}>" +target_compile_options(${BACKEND_TARGET} + PRIVATE "$<$:${RAPIDS_TRITON_BACKEND_CXX_FLAGS}>" + "$<$:${RAPIDS_TRITON_BACKEND_CUDA_FLAGS}>" ) -target_include_directories(triton_rapids-linear - PRIVATE "$" +target_include_directories(${BACKEND_TARGET} + PRIVATE "$" "${CMAKE_CURRENT_SOURCE_DIR}/src" ) -target_link_libraries(triton_rapids-linear +target_link_libraries(${BACKEND_TARGET} PRIVATE rapids_triton::rapids_triton triton-core-serverstub @@ -135,8 +144,8 @@ PRIVATE ) install( - TARGETS triton_rapids-linear - LIBRARY DESTINATION /opt/tritonserver/backends/rapids-linear + TARGETS ${BACKEND_TARGET} + LIBRARY DESTINATION /opt/tritonserver/backends/${BACKEND_NAME} ) ############################################################################## diff --git a/Dockerfile b/Dockerfile index ca863ee..5e16147 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,16 +72,19 @@ RUN ninja install FROM ${BASE_IMAGE} +ARG BACKEND_NAME=rapids_linear +ENV BACKEND_NAME=$BACKEND_NAME + RUN mkdir /models # Remove existing backend install -RUN if [ -d /opt/tritonserver/backends/rapids-linear ]; \ +RUN if [ -d /opt/tritonserver/backends/${BACKEND_NAME} ]; \ then \ - rm -rf /opt/tritonserver/backends/rapids-linear/*; \ + rm -rf /opt/tritonserver/backends/${BACKEND_NAME}/*; \ fi COPY --from=build-stage \ - /opt/tritonserver/backends/rapids-linear \ - /opt/tritonserver/backends/rapids-linear + /opt/tritonserver/backends/$BACKEND_NAME \ + /opt/tritonserver/backends/$BACKEND_NAME ENTRYPOINT ["tritonserver", "--model-repository=/models"] diff --git a/README.md b/README.md index 82e5bc3..e69a2fe 100644 --- a/README.md +++ b/README.md @@ -31,4 +31,235 @@ # The RAPIDS-Triton Linear Example This repository offers an annotated example of how to create a custom Triton -backend using the RAPIDS-Triton library. +backend using the RAPIDS-Triton library. In the following, we will demonstrate +step-by-step how to create a backend with RAPIDS-Triton that, when given two +vectors (**u** and **v**) as input will return a vector **r** according to the +following equation: + +**r** = \alpha * **u** + **v** + **c** + +where \alpha is a scalar constant read from a configuration file and **c** is a +constant vector read from a "model" file. Along the way, we will illustrate a +variety of useful operations in RAPIDS-Triton, including retrieving data from a +configuration file and loading model resources. + +## 1. Getting Started + +It is strongly recommended that you start from the [RAPIDS-Triton template +repo](https://github.com/rapidsai/rapids-triton-template) whenever you begin +developing a custom backend. This repo provides the boilerplate that would +otherwise be necessary to get started with your custom backend. Go ahead and +[create a new +repository](https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/creating-a-repository-on-github/creating-a-repository-from-a-template) based on this template. + +## 2. Pick a Name + +Your custom backend will need a name that uniquely identifies it to the Triton +server. For this example, we will use the name `rapids_linear`. We will need to +provide this name in two places: + + +### 2.1 Update `names.h` +In the `src/names.h` file, adjust the definition of NAMESPACE to read + +```cpp +#define NAMESPACE rapids_linear +``` + +Triton conventions require that backend definition be placed in a namespace of +the form `triton::backend::NAME_OF_BACKEND`, so we define the namespace name +here and use it where required in the `rapids-triton-template` code. + +### 2.2 Update `CMakeLists.txt` +Near the top of `CMakeLists.txt` in the section labeled "Target names", there +is an option to provide a `BACKEND_NAME`. In this case, we will set this as +follows: + +```cmake +set(BACKEND_NAME "rapids_linear") +``` + +## 2. Create an Example Configuration + +[Configuration +files](https://github.com/triton-inference-server/server/blob/main/docs/model_configuration.md) +provide settings for model deployments which determine how +the model will behave throughout its entire deployment (as opposed to values +which vary on a request-by-request basis). It is often helpful to think through +what your configuration file will look like before actually writing any code +for your custom backend. In the present example, we need to specify a few +different things: + +1. The name of the backend which will be used to serve this model (the name + chosen in step 2) +2. The names of the input vectors +3. The value of \alpha +4. Where to load **c** from + +With this in mind, an example configuration file for the `rapids_linear` +backend might look something like this: + +```protobuf +name: "linear_example" +backend: "rapids_linear" +max_batch_size: 32768 +input [ + { + name: "u" + data_type: TYPE_FP32 + dims: [ 4 ] + }, + { + name: "v" + data_type: TYPE_FP32 + dims: [ 4 ] + } +] +output [ + { + name: "r" + data_type: TYPE_FP32 + dims: [ 4 ] + } +] +instance_group [{ kind: KIND_GPU }] +parameters [ + { + key: "alpha" + value: { string_value: "2.0" } + } +] +dynamic_batching { + max_queue_delay_microseconds: 100 +} +``` + +Let's review the pieces of this configuration file in a bit more detail, since +they will introduce several important concepts for development of our backend. + +### `name` +This provides a name for an individual model. Remember that a backend is used +to serve a particular *kind* of model, but Triton can serve many different +models of this kind. The name given here specifies the specific model being +served in order to allow clients to submit requests to that model. + +### `backend` +This is the name of the custom backend we are developing, in this case +`rapids_linear`. + +### `max_batch_size` +All RAPIDS-Triton backends require that models specify some maximum batch size, +although this value can be arbitrarily large. + +### `input` +This section specifies the input tensors used for this backend. Each input +tensor has a name (which we will use later in the actual custom backend code), +a +[datatype](https://github.com/triton-inference-server/server/blob/main/docs/model_configuration.md#datatypes) +(we use 32-bit floats in this example), and a shape. In this case, the **u** +and **v** vectors must be of the same shape as one another and as **c**, so we +will arbitrarily choose to make them of dimension 4 for our example +configuration. + +### `output` +This section follows the same conventions as `input`. Again, for this +particular example, we must match the dimensions of the inputs and **c**. + +### `instance_group` +This section determines the hardware on which this model can be deployed (GPU +or CPU). For this example backend, we will demonstrate how to ensure that +models can perform inference on both GPUs and CPUs, buy we will start with GPU +inference. + +### `parameters` +This section allows us to specify any other settings that may be required for +our model. Note that we specify all settings as `string_value` entries, but +RAPIDS-Triton will allow us to easily parse those values into bools, floats, +ints, or any other basic type. + +### `dynamic_batching` +This section specifies how Triton's dynamic batcher will be used to combine +requests into batches for this model. By setting `max_queue_delay_microseconds` +to 100, we are allowing Triton to gather requests in a window of up to 100 +microseconds before passing them to the backend. Triton's [model +analyzer](https://github.com/triton-inference-server/model_analyzer) can help +determine the optimal value for this window, but we will arbitrarily set it to +100 for this example. + +Note that batching can happen both client-side and server-side. If a client +were to submit a request with 5x4 input tensors, for instance, the Triton +server will correctly interpret this as 5 different **u** and **v** vectors. It +can then batch this request with the next, which may carry 7x4 inputs or 1x4 +inputs or any other valid mini-batch shape. + +## 4. Define RapidsSharedState + +Triton allows multiple copies or "instances" of a model to be loaded at the +same time. One obvious use case for this feature is to load the same model on +multiple GPUs, but it can also be used to "oversubscribe" hardware when this is +beneficial for meeting e.g. throughput or latency requirements. When multiple +instances of a model are loaded, they all have access to an object which +manages state that is relevant to all instances. + +The most basic state that is shared among instances of a model is the model +configuration itself (as defined in step 3), but we can also use it to share +data that is specifically required by our backend. In our particular case, it +would be useful to cache the value of \alpha so that we do not have to retrieve +it from the configuration each time (which may involve additional parsing). + +All RAPIDS-Triton backends store their shared state in a class called +`RapidsSharedState` in their own namespace, which inherits from +`rapids::SharedModelState`. A basic implementation of this class for the +current backend might look something like the following: + +```cpp +struct RapidsSharedState : rapids::SharedModelState { + RapidsSharedState(std::unique_ptr&& config) + : rapids::SharedModelState{std::move(config)} {} + void load() {} + void unload() {} + + float alpha = 1.0f; +}; +``` + +You may safely ignore the constructor in this example. It is boilerplate code +that should be included for all `RapidsSharedState` definitions and will not +change between backends. + +Note that we have added two public member variables to this class definition +which will be used to store \alpha and **c**. One could equally well have made +these private members with getter functions or added arbitrarily complex logic +to this class definition, but we will leave them as is for simplicity. + +### Accessing configuration parameters + +Next, we need to actually load values into these newly-defined members. We can +do this by filling out the logic for our `load` method. For example, in order +to load \alpha, we could implement something like: +```cpp +void load() { alpha = get_config_param("alpha"); } +``` + +Here, the `get_config_param` function allows us to directly access the `alpha` +parameter we defined in our example configuration file. By invoking the `float` +instantiation of this template, we ensure that we will retrieve the value as a +float. + +### Unloading resources + +In this particular case, our shared state does not include any resources that +need to be unloaded, but the `unload` method is available to do precisely that. +We will instead use it here simply to illustrate the use of RAPIDS-Triton +logging functions. Logging functions are defined in +`rapids_triton/triton/logging.hpp` and may be invoked as follows: +```cpp +void unload() { log_info(__FILE__, __LINE__) << "Unloading shared state..."; } +``` +The arguments to `log_info` and associated functions may be omitted, but if +included may provide some use in debugging. + +### Accessing model serialization files +Most backends will also need to deserialize their models from some file on-disk. +In our case, we are using the **c** vector as a stand-in for some more +interesting model representation. diff --git a/src/names.h b/src/names.h index e98cb0d..4331bcc 100644 --- a/src/names.h +++ b/src/names.h @@ -18,8 +18,8 @@ /* Triton expects certain definitions within its backend libraries to follow * specific naming conventions. Specifically, for a backend named - * "rapids_identity," most definitions should appear within a namespace called - * triton::backend::rapids_identity. + * "rapids_linear," most definitions should appear within a namespace called + * triton::backend::rapids_linear. * * In order to facilitate this with minimal effort on the part of backend * developers, we ask that you put the name of your backend here. This macro is diff --git a/src/shared_state.h b/src/shared_state.h index 6de0397..5d6c7c6 100644 --- a/src/shared_state.h +++ b/src/shared_state.h @@ -20,6 +20,8 @@ #include #include +#include +#include namespace triton { namespace backend { @@ -39,8 +41,10 @@ namespace NAMESPACE { struct RapidsSharedState : rapids::SharedModelState { RapidsSharedState(std::unique_ptr&& config) : rapids::SharedModelState{std::move(config)} {} - void load() {} - void unload() {} + void load() { alpha = get_config_param("alpha"); } + void unload() { log_info(__FILE__, __LINE__) << "Unloading shared state..."; } + + float alpha = 1.0f; }; } // namespace NAMESPACE From c91cd943c037e97a1395c1995f1cfdbde54cef22 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 8 Sep 2021 16:40:12 -0400 Subject: [PATCH 046/199] Add missing load in instance initialization --- cpp/include/rapids_triton/triton/api/instance_initialize.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp index c3d421e..46a3962 100644 --- a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp @@ -46,6 +46,7 @@ namespace triton { namespace backend { namespace rapids { namespace triton_api { auto rapids_model = std::make_unique(*model_state, instance); + rapids_model->load(); set_instance_state(*instance, std::move(rapids_model)); } catch (TritonException& err) { From 17e9024dbe58916a77261ce4d50c53b5664dda0b Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 8 Sep 2021 19:40:05 -0400 Subject: [PATCH 047/199] Add move assignment operator to Buffer --- cpp/include/rapids_triton/memory/buffer.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cpp/include/rapids_triton/memory/buffer.hpp b/cpp/include/rapids_triton/memory/buffer.hpp index db0f4db..da1cfb3 100644 --- a/cpp/include/rapids_triton/memory/buffer.hpp +++ b/cpp/include/rapids_triton/memory/buffer.hpp @@ -104,6 +104,16 @@ namespace triton { namespace backend { namespace rapids { Buffer(Buffer&& other) noexcept : device_{other.device_}, data_{std::move(other.data_)}, size_{other.size_}, stream_{other.stream_} {} + auto& operator=(Buffer&& other) noexcept { + if (this != &other) { + device_ = other.device_; + data_ = std::move(other.data_); + size_ = other.size_; + stream_ = other.stream_; + } + return *this; + } + ~Buffer() = default; /** From 78dcb56160778e153fb4bfc63a428b40e01881fa Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 8 Sep 2021 19:50:47 -0400 Subject: [PATCH 048/199] Add Buffer move assignment test --- cpp/test/memory/buffer.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cpp/test/memory/buffer.cpp b/cpp/test/memory/buffer.cpp index 164cbf6..b320178 100644 --- a/cpp/test/memory/buffer.cpp +++ b/cpp/test/memory/buffer.cpp @@ -149,6 +149,16 @@ TEST(RapidsTriton, move_buffer) { EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } +TEST(RapidsTriton, move_assignment_buffer) { + auto data = std::vector{1, 2, 3}; + + auto buffer = Buffer{data.data(), data.size() - 1, DeviceMemory}; + buffer = Buffer{data.size(), HostMemory}; + + ASSERT_EQ(buffer.mem_type(), HostMemory); + ASSERT_EQ(buffer.size(), data.size()); +} + } // namespace rapids } // namespace backend } // namespace triton From 95f20684dbee51e8e7bc1376589600c6f6bb98d3 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 8 Sep 2021 20:56:59 -0400 Subject: [PATCH 049/199] Fill out model loading implementation --- README.md | 119 ++++++++++++++++++++++++++++++--- src/model.h | 160 ++++++++++++++++++--------------------------- src/shared_state.h | 4 +- 3 files changed, 177 insertions(+), 106 deletions(-) diff --git a/README.md b/README.md index e69a2fe..5cdabee 100644 --- a/README.md +++ b/README.md @@ -225,16 +225,17 @@ struct RapidsSharedState : rapids::SharedModelState { You may safely ignore the constructor in this example. It is boilerplate code that should be included for all `RapidsSharedState` definitions and will not -change between backends. +change between backends. Take a look at `src/shared_state.h` to see this +implementation in context. -Note that we have added two public member variables to this class definition -which will be used to store \alpha and **c**. One could equally well have made +Note that we have added a public member variables to this class definition +which will be used to store \alpha. One could equally well have made these private members with getter functions or added arbitrarily complex logic to this class definition, but we will leave them as is for simplicity. ### Accessing configuration parameters -Next, we need to actually load values into these newly-defined members. We can +Next, we need to actually load a value into this newly-defined member. We can do this by filling out the logic for our `load` method. For example, in order to load \alpha, we could implement something like: ```cpp @@ -254,12 +255,112 @@ We will instead use it here simply to illustrate the use of RAPIDS-Triton logging functions. Logging functions are defined in `rapids_triton/triton/logging.hpp` and may be invoked as follows: ```cpp -void unload() { log_info(__FILE__, __LINE__) << "Unloading shared state..."; } +void unload() { rapids::log_info(__FILE__, __LINE__) << "Unloading shared state..."; } ``` -The arguments to `log_info` and associated functions may be omitted, but if +The arguments to `log_info` and related functions may be omitted, but if included may provide some use in debugging. -### Accessing model serialization files -Most backends will also need to deserialize their models from some file on-disk. +## 5. Define RapidsModel + +The real heart of any RAPIDS-Triton backend is its RapidsModel definition. This +class implements the actual logic to deserialize a model and use it to perform +inference. + +### 5.1 Deserialize the model +Most backends will need to deserialize their models from some file on-disk. In our case, we are using the **c** vector as a stand-in for some more -interesting model representation. +interesting model structure. For simplicity, we will assume that **c** is +just stored as space-separated floats in a text file. + +A natural question is why we did not perform this deserialization in +`RapidsSharedState::load` since **c** is the same for all instances of a model. +In general, instances may be initialized on different GPUs, or some may be on +GPUs while others are on CPUs, so we defer model deserialization until we know +where the model should be deserialized *to*. + +Given that **c** could be stored on either host or device, the question of how +to represent it as a member variable becomes a little more fraught. We could +represent it as a raw pointer, but this requires a great deal of manual +tracking to determine whether to use `std::malloc` or `cudaMalloc` for the +initial allocation in the model `load()` method and `std::free` or `cudaFree` +in `unload()`. + +To help simplify how situations like this are handled, RAPIDS-Triton introduces +a lightweight `Buffer` class, which can provide unified RAII access to memory +on host or device **or** a non-owning view on host/device memory that was +allocated elsewhere. We'll introduce a `Buffer` member to RapidsModel to store **c**: + +```cpp +rapids::Buffer c{}; +``` + +Now, we are ready to actually load **c** from its file representation. +`rapids::Model`, the abstract parent class of `RapidsModel` implementations, +defines several methods to help with model deserialization, including: +- `get_filepath`, which returns the path to the model file if it is specified + in the config or the model directory if it is not +- `get_deployment_type`, which returns whether this model should be deployed on + CPU or GPU +- `get_device_id`, which returns the id of the device on which the model should + be deployed (always 0 for CPU deployments) + +Using these functions, we can define our `load()` function as follows. First, +we determine the full filepath to the model file, defaulting to a name of +`c.txt` if it is not specified in the config file: +```cpp +if (std::filesystem::is_directory(path)) { + path /= "c.txt"; +} +``` + +Next, we actually read the values of **c** into a temporary vector from its +text file representation: +```cpp + auto model_vec = std::vector{}; + auto model_file = std::ifstream(path.string()); + auto input_line = std::string{}; + std::getline(model_file, input_line); + auto input_stream = std::stringstream{input_line}; + auto value = 0.0f; + while (input_stream >> value) { + model_vec.push_back(value); + } +``` + +We then query the model to figure out exactly what sort of Buffer will be +needed to store **c**: +```cpp +auto memory_type = rapids::MemoryType{}; +if (get_deployment_type() == rapids::GPUDeployment) { + memory_type = rapids::DeviceMemory; +} else { + memory_type = rapids::HostMemory; +} +c = rapids::Buffer(model_vec.size(), memory_type, get_device_id()); +``` + +Finally, we use the helper function `rapids::copy` to copy the values of **c** +from a Buffer-based view of `model_vec` to the `c` Buffer itself: +```cpp +rapids::copy(c, rapids::Buffer(model_vec.data(), model_vec.size(), + rapids::HostMemory)); +``` + +By taking advantage of Buffer's RAII semantics, we eliminate the need to +explicitly implement an `unload` function, but we could do so if necessary. + +### 5.2 Write the predict function +Now all that remains is to use our loaded "model" for inference. The `predict` +method of a RapidsModel implementation takes in a `Batch` object as argument, +which can be used to retrieve the input and output tensors that we will operate +on during inference. We can retrieve these tensors using the names we +originally specified in the config file: + +```cpp +auto u = get_input(batch, "u"); +auto v = get_input(batch, "v"); + +auto r = get_output(batch, "r"); +``` + +The returned tensors will be diff --git a/src/model.h b/src/model.h index 0ca87bc..60f4a41 100644 --- a/src/model.h +++ b/src/model.h @@ -20,29 +20,26 @@ #include #include +#include +#include +#include #include #include #include // rapids::Batch +#include // rapids::Buffer, rapids::copy #include // rapids::MemoryType #include // rapids::Model #include // rapids::copy #include // rapids::DeploymentType #include // rapids::device_id_t +#include // rapids::log_info +#include namespace triton { namespace backend { namespace NAMESPACE { -/* Any logic necessary to perform inference with a model and manage its data - * should be implemented in a struct named RapidsModel, as shown here */ - struct RapidsModel : rapids::Model { - /*************************************************************************** - * BOILERPLATE * - * *********************************************************************** * - * The following constructor can be copied directly into any model - * implementation. - **************************************************************************/ RapidsModel(std::shared_ptr shared_state, rapids::device_id_t device_id, cudaStream_t default_stream, rapids::DeploymentType deployment_type, @@ -51,97 +48,68 @@ struct RapidsModel : rapids::Model { default_stream, deployment_type, filepath) {} - /*************************************************************************** - * BASIC FEATURES * - * *********************************************************************** * - * The only method that *must* be implemented for a viable model is the - * `predict` method, but the others presented here are often used for basic - * model implementations. Filling out these methods should take care of most - * use cases. - **************************************************************************/ - - /*************************************************************************** - * predict * - * *********************************************************************** * - * This method performs the actual inference step on input data. Implementing - * a predict function requires four steps: - * 1. Call `get_input` on the provided `Batch` object for each of the input - * tensors named in the config file for this backend. This provides a - * `Tensor` object containing the input data. - * 2. Call `get_output` on the provided `Batch` object for each of the output - * tensors named in the config file for this backend. This provides a - * `Tensor` object to which output values can be written. - * 3. Perform inference based on the input Tensors and store the results in - * the output Tensors. `some_tensor.data()` can be used to retrieve a raw - * pointer to the underlying data. - * 4. Call the `finalize` method on all output tensors. - **************************************************************************/ void predict(rapids::Batch& batch) const { - // 1. Acquire a tensor representing the input named "input__0" - auto input = get_input(batch, "input__0"); - // 2. Acquire a tensor representing the output named "output__0" - auto output = get_output(batch, "output__0"); - - // 3. Perform inference. In this example, we simply copy the data from the - // input to the output tensor. - rapids::copy(output, input); - - // 4. Call finalize on all output tensors. In this case, we have just one - // output, so we call finalize on it. - output.finalize(); + auto u = get_input(batch, "u"); + auto v = get_input(batch, "v"); + + auto r = get_output(batch, "r"); + + if (u.mem_type() == rapids::HostMemory) { + auto alpha = get_shared_state()->alpha; + for (std::size_t i{}; i < u.size(); ++i) { + auto u_val = *(u.data() + i); + auto v_val = *(v.data() + i); + auto& r_val = *(r.data() + i); + auto c_val = *(c.data() + (i % c.size())); + + r_val = alpha * u_val + v_val + c_val; + } + } + + r.finalize(); } - /*************************************************************************** - * load / unload * - * *********************************************************************** * - * These methods can be used to perform one-time loading/unloading of - * resources when a model is created. For example, data representing the - * model may be loaded onto the GPU in the `load` method and unloaded in the - * `unload` method. This data will then remain loaded while the server is - * running. - * - * While these methods take no arguments, it is typical to read any necessary - * input from the model configuration file by using the `get_config_param` - * method. Any parameters defined in the "parameters" section of the config - * can be accessed by name in this way. The maximum batch size can also be - * retrieved using the name "max_batch_size". - * - * These methods need not be explicitly implemented if no loading/unloading - * logic is required, but we show them here for illustrative purposes. - **************************************************************************/ - void load() {} - void unload() {} - - /*************************************************************************** - * ADVANCED FEATURES * - * *********************************************************************** * - * None of the following methods are required to be implemented in order to - * create a valid model, but they are presented here for those who require - * the additional functionality they provide. - **************************************************************************/ - - /*************************************************************************** - * preferred_mem_type / preferred_mem_type_in / preferred_mem_type_out * - * *********************************************************************** * - * If implemented, `preferred_mem_type` allows for control over when input - * and output data are provided on the host versus on device. In the case - * that a model prefers to receive its input on-host but return output - * on-device (or vice versa), `preferred_mem_type_in` and - * `preferred_mem_type_out` can be used for even more precise control. - * - * In this example, we simply return `std::nullopt` to indicate that the - * model has no preference on its input/output data locations. Note that the - * Batch being processed is taken as input to this function to facilitate - * implementations that may switch their preferred memory location based on - * properties of the batch. - * - * Valid MemoryType options to return are rapids::HostMemory and - * rapids::DeviceMemory. - **************************************************************************/ - std::optional preferred_mem_type( - rapids::Batch& batch) const { - return std::nullopt; + void load() { + auto path = std::filesystem::path(get_filepath()); + /* If the config file does not specify a filepath for the model, + * get_filepath returns the directory where the serialized model should be + * found. It is generally good practice to provide logic to allow the use + * of a default filename so that model configurations do not always have to + * specify a path to their model */ + if (std::filesystem::is_directory(path)) { + path /= "c.txt"; + } + + // Read space-separated text file into a vector of floats + auto model_vec = std::vector{}; + auto model_file = std::ifstream(path.string()); + auto input_line = std::string{}; + std::getline(model_file, input_line); + auto input_stream = std::stringstream{input_line}; + auto value = 0.0f; + while (input_stream >> value) { + model_vec.push_back(value); + } + + // Construct buffer to hold c based on details of this model deployment + auto memory_type = rapids::MemoryType{}; + if (get_deployment_type() == rapids::GPUDeployment) { + memory_type = rapids::DeviceMemory; + } else { + memory_type = rapids::HostMemory; + } + c = rapids::Buffer(model_vec.size(), memory_type, get_device_id()); + + /* Use a Buffer view on model_vec to safely copy data to its final + * location. Making use of rapids::copy here provides additional safety + * checks to avoid buffer overruns. Note that the destination buffer comes + * first in rapids::copy calls, so we are copying *into* c */ + rapids::copy(c, rapids::Buffer(model_vec.data(), model_vec.size(), + rapids::HostMemory)); } + + private: + rapids::Buffer c{}; }; } // namespace NAMESPACE diff --git a/src/shared_state.h b/src/shared_state.h index 5d6c7c6..df9da52 100644 --- a/src/shared_state.h +++ b/src/shared_state.h @@ -42,7 +42,9 @@ struct RapidsSharedState : rapids::SharedModelState { RapidsSharedState(std::unique_ptr&& config) : rapids::SharedModelState{std::move(config)} {} void load() { alpha = get_config_param("alpha"); } - void unload() { log_info(__FILE__, __LINE__) << "Unloading shared state..."; } + void unload() { + rapids::log_info(__FILE__, __LINE__) << "Unloading shared state..."; + } float alpha = 1.0f; }; From 2d6d53ce545634d810203a29b12e3ab9b302429f Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 9 Sep 2021 21:33:27 -0400 Subject: [PATCH 050/199] Use default move operators for Buffer --- cpp/include/rapids_triton/memory/buffer.hpp | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/cpp/include/rapids_triton/memory/buffer.hpp b/cpp/include/rapids_triton/memory/buffer.hpp index da1cfb3..58ac3a1 100644 --- a/cpp/include/rapids_triton/memory/buffer.hpp +++ b/cpp/include/rapids_triton/memory/buffer.hpp @@ -101,18 +101,9 @@ namespace triton { namespace backend { namespace rapids { return result; }()}, size_{other.size_}, stream_{other.stream_} {} - Buffer(Buffer&& other) noexcept : device_{other.device_}, data_{std::move(other.data_)}, size_{other.size_}, - stream_{other.stream_} {} - - auto& operator=(Buffer&& other) noexcept { - if (this != &other) { - device_ = other.device_; - data_ = std::move(other.data_); - size_ = other.size_; - stream_ = other.stream_; - } - return *this; - } + Buffer(Buffer&& other) = default; + + Buffer& operator=(Buffer&& other) = default; ~Buffer() = default; From 7f672343c3cc418538fcfdc879aae075e9557f50 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 9 Sep 2021 23:24:22 -0400 Subject: [PATCH 051/199] Add full prediction logic --- CMakeLists.txt | 1 + README.md | 66 ++++++++++++++++++++++++++++++++++++++++++++++-- src/gpu_infer.cu | 45 +++++++++++++++++++++++++++++++++ src/gpu_infer.h | 33 ++++++++++++++++++++++++ src/model.h | 16 ++++++------ 5 files changed, 151 insertions(+), 10 deletions(-) create mode 100644 src/gpu_infer.cu create mode 100644 src/gpu_infer.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 5af8428..354990a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,6 +111,7 @@ endif() add_library( ${BACKEND_TARGET} SHARED src/api.cc + src/gpu_infer.cu ) set_target_properties(${BACKEND_TARGET} diff --git a/README.md b/README.md index 5cdabee..1a65685 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,10 @@ constant vector read from a "model" file. Along the way, we will illustrate a variety of useful operations in RAPIDS-Triton, including retrieving data from a configuration file and loading model resources. +All of the following steps are written as if you were starting from scratch in +creating this backend, but you can also just browse the files in this repo to +see how the final version of the backend might look. + ## 1. Getting Started It is strongly recommended that you start from the [RAPIDS-Triton template @@ -359,8 +363,66 @@ originally specified in the config file: ```cpp auto u = get_input(batch, "u"); auto v = get_input(batch, "v"); - auto r = get_output(batch, "r"); ``` -The returned tensors will be +By default, the location of the returned tensors (host or device) is determined +by whether the model is deployed on host or device and whether or not the +backend was compiled with GPU-support enabled. You may choose to override the +`preferred_mem_type` method of your RapidsModel implementation in order to +specify a different general rule, or you can optionally pass a `MemoryType` to +`get_input` and `get_output` for even finer-grained control. Here, we will +simply accept the default behavior and use the `mem_type` method of the +returned tensor to determine how inference will proceed. + +For tensors on the host, our inference logic might look something like: + +```cpp +if (u.mem_type() == rapids::HostMemory) { + auto alpha = get_shared_state()->alpha; + for (std::size_t i{}; i < u.size(); ++i) { + r.data()[i] = + alpha * u.data()[i] + v.data()[i] + c.data()[i % c.size()]; + } +} +``` + +We'll define the logic for GPU inference in a separate `.cu` file like so: +```cuda +__global__ void cu_gpu_infer(float* r, float const* u, float const* v, + float* c, float alpha, std::size_t features, + std::size_t length) { + auto id = blockIdx.x * blockDim.x + threadIdx.x; + if (id < length) { + r[id] = alpha * u[id] + v[id] + c[id % features]; + } +} +void gpu_infer(float* r, float const* u, float const* v, float* c, float alpha, + std::size_t features, std::size_t length) { + auto constexpr block_size = 1024; + auto grid_size = static_cast(std::max(1.0f, std::ceil(length / + static_cast(block_size))));; + cu_gpu_infer<<>>(r, u, v, c, alpha, features, length); +} +``` + +and then call it within our RapidsModel `predict` method via: + +```cpp +gpu_infer(r.data(), u.data(), v.data(), c.data(), alpha, c.size(), + u.size()); +``` + +After the actual inference has been performed, the one remaining task is to +call the `finalize` method of all output tensors. In this example, we have +exactly one, so the final line of our `predict method is just: + +```cpp +r.finalize(); +``` + +## 6. Build the backend +TODO (wphicks) + +## 7. Test inference +TODO (wphicks) diff --git a/src/gpu_infer.cu b/src/gpu_infer.cu new file mode 100644 index 0000000..dba8383 --- /dev/null +++ b/src/gpu_infer.cu @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include + +namespace triton { namespace backend { namespace NAMESPACE { + +namespace { +__global__ void cu_gpu_infer(float* r, float const* u, float const* v, + float* c, float alpha, std::size_t features, + std::size_t length) { + auto id = blockIdx.x * blockDim.x + threadIdx.x; + if (id < length) { + r[id] = alpha * u[id] + v[id] + c[id % features]; + } +} +} + +void gpu_infer(float* r, float const* u, float const* v, float* c, float alpha, + std::size_t features, std::size_t length) { + auto constexpr block_size = 1024; + auto grid_size = static_cast(std::max(1.0f, std::ceil(length / + static_cast(block_size))));; + cu_gpu_infer<<>>(r, u, v, c, alpha, features, length); +} + +}}} diff --git a/src/gpu_infer.h b/src/gpu_infer.h new file mode 100644 index 0000000..9eed9ef --- /dev/null +++ b/src/gpu_infer.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace NAMESPACE { + +void gpu_infer(float* r, float const* u, float const* v, float* c, float alpha, + std::size_t features, std::size_t length); +} +} // namespace backend +} // namespace triton diff --git a/src/model.h b/src/model.h index 60f4a41..d1107e8 100644 --- a/src/model.h +++ b/src/model.h @@ -17,6 +17,7 @@ #pragma once #include +#include #include #include @@ -24,7 +25,6 @@ #include #include #include -#include #include // rapids::Batch #include // rapids::Buffer, rapids::copy #include // rapids::MemoryType @@ -54,16 +54,15 @@ struct RapidsModel : rapids::Model { auto r = get_output(batch, "r"); + auto alpha = get_shared_state()->alpha; if (u.mem_type() == rapids::HostMemory) { - auto alpha = get_shared_state()->alpha; for (std::size_t i{}; i < u.size(); ++i) { - auto u_val = *(u.data() + i); - auto v_val = *(v.data() + i); - auto& r_val = *(r.data() + i); - auto c_val = *(c.data() + (i % c.size())); - - r_val = alpha * u_val + v_val + c_val; + r.data()[i] = + alpha * u.data()[i] + v.data()[i] + c.data()[i % c.size()]; } + } else { + gpu_infer(r.data(), u.data(), v.data(), c.data(), alpha, c.size(), + u.size()); } r.finalize(); @@ -98,6 +97,7 @@ struct RapidsModel : rapids::Model { } else { memory_type = rapids::HostMemory; } + c = rapids::Buffer(model_vec.size(), memory_type, get_device_id()); /* Use a Buffer view on model_vec to safely copy data to its final From 6d42da2bf46d1b283c265942b02954e039c19a15 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 10 Sep 2021 11:13:29 -0400 Subject: [PATCH 052/199] Add Python testing package --- conda/environments/rapids_triton_test.yml | 11 ++ python/pyproject.toml | 3 + python/rapids_triton/__init__.py | 16 +++ python/rapids_triton/client.py | 141 ++++++++++++++++++++++ python/rapids_triton/logging.py | 18 +++ python/rapids_triton/testing.py | 123 +++++++++++++++++++ python/rapids_triton/triton/__init__.py | 13 ++ python/rapids_triton/triton/client.py | 48 ++++++++ python/rapids_triton/triton/dtype.py | 34 ++++++ python/rapids_triton/triton/io.py | 139 +++++++++++++++++++++ python/rapids_triton/triton/message.py | 29 +++++ python/rapids_triton/triton/response.py | 33 +++++ python/setup.py | 28 +++++ 13 files changed, 636 insertions(+) create mode 100644 conda/environments/rapids_triton_test.yml create mode 100644 python/pyproject.toml create mode 100644 python/rapids_triton/__init__.py create mode 100644 python/rapids_triton/client.py create mode 100644 python/rapids_triton/logging.py create mode 100644 python/rapids_triton/testing.py create mode 100644 python/rapids_triton/triton/__init__.py create mode 100644 python/rapids_triton/triton/client.py create mode 100644 python/rapids_triton/triton/dtype.py create mode 100644 python/rapids_triton/triton/io.py create mode 100644 python/rapids_triton/triton/message.py create mode 100644 python/rapids_triton/triton/response.py create mode 100644 python/setup.py diff --git a/conda/environments/rapids_triton_test.yml b/conda/environments/rapids_triton_test.yml new file mode 100644 index 0000000..c58d9d3 --- /dev/null +++ b/conda/environments/rapids_triton_test.yml @@ -0,0 +1,11 @@ +--- +name: rapids_triton_dev +channels: + - conda-forge +dependencies: + - flake8 + - pip + - python + - numpy + - pip: + - nvidia-pyindex diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000..9787c3b --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/python/rapids_triton/__init__.py b/python/rapids_triton/__init__.py new file mode 100644 index 0000000..e71e0bd --- /dev/null +++ b/python/rapids_triton/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from rapids_triton.logging import logger +from rapids_triton.client import Client diff --git a/python/rapids_triton/client.py b/python/rapids_triton/client.py new file mode 100644 index 0000000..7a70a00 --- /dev/null +++ b/python/rapids_triton/client.py @@ -0,0 +1,141 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +from rapids_triton.triton.client import get_triton_client +from rapids_triton.triton.io import create_triton_input, create_triton_output + +from rapids_triton.triton.dtype import dtype_to_triton_name +from rapids_triton.triton.response import get_response_data +from tritonclient import utils as triton_utils + + +class Client(object): + def __init__( + self, + protocol='grpc', + host='localhost', + port=None, + concurrency=4): + self.triton_client = get_triton_client( + protocol=protocol, + host=host, + port=port, + concurrency=concurrency + ) + self._protocol = protocol + + @property + def protocol(self): + return self._protocol + + def create_input(self, data, name, dtype, shared_mem=None): + return create_triton_input( + self.triton_client, + data, + name, + dtype, + protocol=self.protocol, + shared_mem=shared_mem + ) + + def create_output(self, size, name, shared_mem=None): + return create_triton_output( + self.triton_client, + size, + name, + protocol=self.protocol, + shared_mem=shared_mem + ) + + def wait_for_server(self, timeout): + server_wait_start = time.time() + while True: + try: + if self.triton_client.is_server_ready(): + break + except triton_utils.InferenceServerException: + pass + if time.time() - server_wait_start > timeout: + raise RuntimeError("Server startup timeout expired") + time.sleep(1) + + def clear_shared_memory(self): + self.triton_client.unregister_cuda_shared_memory() + self.triton_client.unregister_system_shared_memory() + + def get_model_config(self, model_name): + return self.triton_client.get_model_config(model_name).config + + def predict( + self, + model_name, + input_data, + output_sizes, + model_version='1', + shared_mem=None, + attempts=1): + model_version = str(model_version) + + try: + inputs = [ + self.create_input( + arr, + name, + dtype_to_triton_name(arr.dtype), + shared_mem=shared_mem + ) + for name, arr in input_data.items() + ] + + outputs = { + name: self.create_output(size, name, shared_mem=shared_mem) + for name, size in output_sizes.items() + } + + response = self.triton_client.infer( + model_name, + model_version=model_version, + inputs=[input_.input for input_ in inputs], + outputs=[output_.output for output_ in outputs.values()] + ) + + except triton_utils.InferenceServerException: + if attempts > 1: + return self.predict( + model_name, + input_data, + output_sizes, + model_version=model_version, + shared_mem=shared_mem, + attempts=attempts - 1 + ) + raise + result = { + name: get_response_data(response, handle, name) + for name, (_, handle, _) in outputs.items() + } + + for input_ in inputs: + if input_.name is not None: + self.triton_client.unregister_cuda_shared_memory( + name=input_.name + ) + for output_ in outputs.values(): + if output_.name is not None: + self.triton_client.unregister_cuda_shared_memory( + name=output_.name + ) + return result diff --git a/python/rapids_triton/logging.py b/python/rapids_triton/logging.py new file mode 100644 index 0000000..9b8e2c4 --- /dev/null +++ b/python/rapids_triton/logging.py @@ -0,0 +1,18 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +logger = logging.getLogger('rapids_triton') +logger.setLevel(logging.INFO) diff --git a/python/rapids_triton/testing.py b/python/rapids_triton/testing.py new file mode 100644 index 0000000..882f2ed --- /dev/null +++ b/python/rapids_triton/testing.py @@ -0,0 +1,123 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import numpy as np + +from rapids_triton.logging import logger +from rapids_triton.triton.client import STANDARD_PORTS +from rapids_triton.client import Client + + +def arrays_close( + a, + b, + atol=None, + rtol=None, + total_atol=None, + total_rtol=None, + assert_close=False): + """ + Compare numpy arrays for approximate equality + + :param numpy.array a: The array to compare against a reference value + :param numpy.array b: The reference array to compare against + :param float atol: The maximum absolute difference allowed between an + element in a and an element in b before they are considered non-close. + If both atol and rtol are set to None, atol is assumed to be 0. If atol + is set to None and rtol is not None, no absolute threshold is used in + comparisons. + :param float rtol: The maximum relative difference allowed between an + element in a and an element in b before they are considered non-close. + If rtol is set to None, no relative threshold is used in comparisons. + :param int total_atol: The maximum number of elements allowed to be + non-close before the arrays are considered non-close. + :param float total_rtol: The maximum proportion of elements allowed to be + non-close before the arrays are considered non-close. + """ + + if np.any(a.shape != b.shape): + if assert_close: + raise AssertionError( + "Arrays have different shapes:\n{} vs. {}".format( + a.shape, b.shape + ) + ) + return False + + if a.size == 0 and b.size == 0: + return True + + if atol is None and rtol is None: + atol = 0 + if total_atol is None and total_rtol is None: + total_atol = 0 + + diff_mask = np.ones(a.shape, dtype='bool') + + diff = np.abs(a-b) + + if atol is not None: + diff_mask = np.logical_and(diff_mask, diff > atol) + + if rtol is not None: + diff_mask = np.logical_and(diff_mask, diff > rtol * np.abs(b)) + + is_close = True + + mismatch_count = np.sum(diff_mask) + + if total_atol is not None and mismatch_count > total_atol: + is_close = False + + mismatch_proportion = mismatch_count / a.size + if total_rtol is not None and mismatch_proportion > total_rtol: + is_close = False + + if assert_close and not is_close: + total_tol_desc = [] + if total_atol is not None: + total_tol_desc.append(str(int(total_atol))) + if total_rtol is not None: + total_tol_desc.append( + "{:.2f} %".format(total_rtol * 100) + ) + total_tol_desc = " or ".join(total_tol_desc) + + msg = """Arrays have more than {} mismatched elements. + +Mismatch in {} ({:.2f} %) elements + a: {} + b: {} + + Mismatched indices: {}""".format( + total_tol_desc, mismatch_count, mismatch_proportion * 100, a, b, + np.transpose(np.nonzero(diff_mask))) + raise AssertionError(msg) + return is_close + + +def get_random_seed(): + """Provide random seed to allow for easer reproduction of testing failures + + Note: Code taken directly from cuML testing infrastructure""" + current_random_seed = os.getenv('PYTEST_RANDOM_SEED') + if current_random_seed is not None and current_random_seed.isdigit(): + random_seed = int(current_random_seed) + else: + random_seed = np.random.randint(0, 1e6) + os.environ['PYTEST_RANDOM_SEED'] = str(random_seed) + logger.info("Random seed value:", random_seed) + return random_seed diff --git a/python/rapids_triton/triton/__init__.py b/python/rapids_triton/triton/__init__.py new file mode 100644 index 0000000..3291348 --- /dev/null +++ b/python/rapids_triton/triton/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/rapids_triton/triton/client.py b/python/rapids_triton/triton/client.py new file mode 100644 index 0000000..eaa86e3 --- /dev/null +++ b/python/rapids_triton/triton/client.py @@ -0,0 +1,48 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tritonclient.http as triton_http +import tritonclient.grpc as triton_grpc + +STANDARD_PORTS = { + 'http': 8000, + 'grpc': 8001 +} + + +def get_triton_client( + protocol="grpc", + host='localhost', + port=None, + concurrency=4): + """Get Triton client instance of desired type """ + + if port is None: + port = STANDARD_PORTS[protocol] + + if protocol == 'grpc': + client = triton_grpc.InferenceServerClient( + url=f'{host}:{port}', + verbose=False + ) + elif protocol == 'http': + client = triton_http.InferenceServerClient( + url=f'{host}:{port}', + verbose=False, + concurrency=concurrency + ) + else: + raise RuntimeError('Bad protocol: "{}"'.format(protocol)) + + return client diff --git a/python/rapids_triton/triton/dtype.py b/python/rapids_triton/triton/dtype.py new file mode 100644 index 0000000..e3077a9 --- /dev/null +++ b/python/rapids_triton/triton/dtype.py @@ -0,0 +1,34 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np + +DTYPE_NAMES = { + np.dtype('bool').str: 'BOOL', + np.dtype('uint8').str: 'UINT8', + np.dtype('uint16').str: 'UINT16', + np.dtype('uint32').str: 'UINT32', + np.dtype('uint64').str: 'UINT64', + np.dtype('int8').str: 'INT8', + np.dtype('int16').str: 'INT16', + np.dtype('int32').str: 'INT32', + np.dtype('int64').str: 'INT64', + np.dtype('float16').str: 'FP16', + np.dtype('float32').str: 'FP32', + np.dtype('float64').str: 'FP64' +} + +def dtype_to_triton_name(dtype): + dtype = np.dtype(dtype).str + return DTYPE_NAMES.get(dtype, 'BYTES') diff --git a/python/rapids_triton/triton/io.py b/python/rapids_triton/triton/io.py new file mode 100644 index 0000000..c3bdedc --- /dev/null +++ b/python/rapids_triton/triton/io.py @@ -0,0 +1,139 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import namedtuple +from uuid import uuid4 + +import tritonclient.http as triton_http +import tritonclient.grpc as triton_grpc +import tritonclient.utils.cuda_shared_memory as shm +from tritonclient import utils as triton_utils + + +TritonInput = namedtuple('TritonInput', ('name', 'input')) +TritonOutput = namedtuple('TritonOutput', ('name', 'handle', 'output')) + +def set_unshared_input_data(triton_input, data, protocol='grpc'): + if protocol == 'grpc': + triton_input.set_data_from_numpy(data) + else: + triton_input.set_data_from_numpy(data, binary_data=True) + + return None + + +def set_shared_input_data(triton_client, triton_input, data, protocol='grpc'): + input_size = data.size * data.itemsize + + input_name = 'input_{}'.format(uuid4().hex) + + input_handle = shm.create_shared_memory_region( + input_name, input_size, 0 + ) + + shm.set_shared_memory_region(input_handle, [data]) + + triton_client.register_cuda_shared_memory( + input_name, shm.get_raw_handle(input_handle), 0, input_size + ) + + triton_input.set_shared_memory(input_name, input_size) + + return input_name + + +def set_input_data( + triton_client, + triton_input, + data, + protocol='grpc', + shared_mem=None): + if shared_mem is None: + return set_unshared_input_data( + triton_input, data, protocol=protocol + ) + if shared_mem == 'cuda': + return set_shared_input_data( + triton_client, triton_input, data, protocol=protocol + ) + raise RuntimeError("Unsupported shared memory type") + + +def create_triton_input( + triton_client, data, name, dtype, protocol='grpc', shared_mem=None): + if protocol == 'grpc': + triton_input = triton_grpc.InferInput(name, data.shape, dtype) + else: + triton_input = triton_http.InferInput(name, data.shape, dtype) + + input_name = set_input_data( + triton_client, + triton_input, + data, + protocol=protocol, + shared_mem=shared_mem + ) + + return TritonInput(name=input_name, input=triton_input) + + +def create_output_handle(triton_client, triton_output, size, shared_mem=None): + if shared_mem is None: + return (None, None) + + output_name = 'output_{}'.format(uuid4().hex) + output_handle = shm.create_shared_memory_region( + output_name, size, 0 + ) + + triton_client.register_cuda_shared_memory( + output_name, shm.get_raw_handle(output_handle), 0, size + ) + + triton_output.set_shared_memory(output_name, size) + + return output_name, output_handle + + +def create_triton_output( + triton_client, size, name, protocol='grpc', shared_mem=None): + """Set up output memory in Triton + + Parameters + ---------- + triton_client : Triton client object + The client used to set output parameters + size : int + The size of the output in bytes + name : str + The model-defined name for this output + protocol : 'grpc' or 'http' + The protocol used for communication with the server + """ + if protocol == 'grpc': + triton_output = triton_grpc.InferRequestedOutput(name) + else: + triton_output = triton_grpc.InferRequestedOutput( + name, binary_data=True + ) + + output_name, output_handle = create_output_handle( + triton_client, triton_output, size, shared_mem=shared_mem + ) + + return TritonOutput( + name=output_name, + handle=output_handle, + output=triton_output + ) diff --git a/python/rapids_triton/triton/message.py b/python/rapids_triton/triton/message.py new file mode 100644 index 0000000..ae9df1e --- /dev/null +++ b/python/rapids_triton/triton/message.py @@ -0,0 +1,29 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class TritonMessage: + """Adapter to read output from both GRPC and HTTP responses""" + def __init__(self, message): + self.message = message + + def __getattr__(self, attr): + try: + return getattr(self.message, attr) + except AttributeError: + try: + return self.message[attr] + except Exception: # Re-raise AttributeError + pass + raise diff --git a/python/rapids_triton/triton/response.py b/python/rapids_triton/triton/response.py new file mode 100644 index 0000000..af8c511 --- /dev/null +++ b/python/rapids_triton/triton/response.py @@ -0,0 +1,33 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tritonclient.utils.cuda_shared_memory as shm +from tritonclient import utils as triton_utils + +from rapids_triton.triton.message import TritonMessage + + +def get_response_data(response, output_handle, output_name): + """Convert Triton response to NumPy array""" + if output_handle is None: + return response.as_numpy(output_name) + else: + network_result = TritonMessage( + response.get_output(output_name) + ) + return shm.get_contents_as_numpy( + output_handle, + triton_utils.triton_to_np_dtype(network_result.datatype), + network_result.shape + ) diff --git a/python/setup.py b/python/setup.py new file mode 100644 index 0000000..f32ee17 --- /dev/null +++ b/python/setup.py @@ -0,0 +1,28 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import setup + +setup( + name='rapids_triton', + description="Tools for clients to RAPIDS-Triton backends", + version='21.10.00', # TODO(wphicks): versioneer + author='NVIDIA Corporation', + license='Apache', + packages=['rapids_triton'], + install_requires=[ + 'numpy', + 'tritonclient[all]' + ] +) From 19fa354cbb96a846f3f8bf0c614d9deab7eb5ff1 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 10 Sep 2021 11:27:00 -0400 Subject: [PATCH 053/199] Remove circular import --- python/rapids_triton/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/rapids_triton/__init__.py b/python/rapids_triton/__init__.py index e71e0bd..3291348 100644 --- a/python/rapids_triton/__init__.py +++ b/python/rapids_triton/__init__.py @@ -11,6 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from rapids_triton.logging import logger -from rapids_triton.client import Client From 15b36edcdcb9ee39806178212efbcab27d792a1a Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 10 Sep 2021 12:10:48 -0400 Subject: [PATCH 054/199] Use find_packages correctly --- python/rapids_triton/__init__.py | 3 +++ python/setup.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/python/rapids_triton/__init__.py b/python/rapids_triton/__init__.py index 3291348..5c99549 100644 --- a/python/rapids_triton/__init__.py +++ b/python/rapids_triton/__init__.py @@ -11,3 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from rapids_triton.client import Client +from rapids_triton.logging import logger diff --git a/python/setup.py b/python/setup.py index f32ee17..9b35976 100644 --- a/python/setup.py +++ b/python/setup.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import setup +from setuptools import setup, find_packages setup( name='rapids_triton', @@ -20,7 +20,7 @@ version='21.10.00', # TODO(wphicks): versioneer author='NVIDIA Corporation', license='Apache', - packages=['rapids_triton'], + packages=find_packages(), install_requires=[ 'numpy', 'tritonclient[all]' From 720c64f43ee755e3bc8faacaaa265f300f8b891e Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 10 Sep 2021 12:45:39 -0400 Subject: [PATCH 055/199] Add example models --- Dockerfile | 2 +- README.md | 101 +++++++++++++++++- conda/environments/rapids_triton_test.yml | 12 +++ .../model_repository/linear_example/1/c.txt | 1 + .../linear_example/config.pbtxt | 32 ++++++ .../linear_example_cpu/1/c.txt | 1 + .../linear_example_cpu/config.pbtxt | 32 ++++++ 7 files changed, 175 insertions(+), 6 deletions(-) create mode 100644 conda/environments/rapids_triton_test.yml create mode 100644 qa/L0_e2e/model_repository/linear_example/1/c.txt create mode 100644 qa/L0_e2e/model_repository/linear_example/config.pbtxt create mode 100644 qa/L0_e2e/model_repository/linear_example_cpu/1/c.txt create mode 100644 qa/L0_e2e/model_repository/linear_example_cpu/config.pbtxt diff --git a/Dockerfile b/Dockerfile index 5e16147..9f0c1f9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,7 +72,7 @@ RUN ninja install FROM ${BASE_IMAGE} -ARG BACKEND_NAME=rapids_linear +ARG BACKEND_NAME ENV BACKEND_NAME=$BACKEND_NAME RUN mkdir /models diff --git a/README.md b/README.md index 1a65685..4386836 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,13 @@ constant vector read from a "model" file. Along the way, we will illustrate a variety of useful operations in RAPIDS-Triton, including retrieving data from a configuration file and loading model resources. +This example is intended to provide a fair amount of depth about backend +development with RAPIDS-Triton. For a simpler example, check out the +pass-through backend in the main [RAPIDS-Triton +repo](https://github.com/rapidsai/rapids-triton/tree/fea-initial#simple-example). +For even more detail on RAPIDS-Triton features introduced here, check out the +API documentation. + All of the following steps are written as if you were starting from scratch in creating this backend, but you can also just browse the files in this repo to see how the final version of the backend might look. @@ -83,7 +90,7 @@ follows: set(BACKEND_NAME "rapids_linear") ``` -## 2. Create an Example Configuration +## 3. Create an Example Configuration [Configuration files](https://github.com/triton-inference-server/server/blob/main/docs/model_configuration.md) @@ -415,14 +422,98 @@ gpu_infer(r.data(), u.data(), v.data(), c.data(), alpha, c.size(), After the actual inference has been performed, the one remaining task is to call the `finalize` method of all output tensors. In this example, we have -exactly one, so the final line of our `predict method is just: +exactly one, so the final line of our `predict` method is just: ```cpp r.finalize(); ``` +To see all of this in context, check out the `src/gpu_infer.h` and +`src/gpu_infer.cu` files where GPU inference has been implemented as well as +`src/model.h` where it is used. When introducing a new source file, don't +forget to add it to CMakeLists.txt so that it will be included in the build. + ## 6. Build the backend -TODO (wphicks) +Having defined all the necessary logic for serving our model, we can now +actually build the server container with the new backend included. To do so, +run the following command from the base of the repository: + +```bash +docker build --build-arg BACKEND_NAME=rapids_linear -t rapids_linear . +``` + +## 7. Test + +All that remains is to test that the backend performs correctly. In this repo, +we have provided two model directories in `qa/L0_e2e/model_repository`. These +models are identical except that one is deployed on CPU and the other on GPU. +The `config.pbtxt` files are laid out exactly as discussed in step 3. + +To start the server with these models, run the following: + +```bash +docker run \ + --gpus=all \ + --rm \ + -p 8000:8000 \ + -p 8001:8001 \ + -p 8002:8002 \ + -v $PWD/qa/L0_e2e/model_repository:/models + rapids_linear +``` -## 7. Test inference -TODO (wphicks) +You can now submit inference requests via any Triton client. For convenience, +we will use the client provided by the `rapids_triton` package in the main +RAPIDS-Triton repo. This package is primarily designed to assist with writing +end-to-end tests for RAPIDS-Triton backends. We can install it into a conda +environment as follows: + +```bash +conda env create -f conda/environments/rapids_triton_test.yml +conda activate rapids_triton_test +python -m pip install git+https://github.com/rapidsai/rapids-triton.git@fea-initial#subdirectory=python +``` + +To use it for a basic test, we might execute something like the following + +```python +from rapids_triton import Client + +u = np.array([[2, 2, 3, 3]], dtype='float32') +v = np.array([[-1, 1, 1, -1]], dtype='float32') +# The value of the c vector specified in c.txt +c = np.array([[1, 2, 3, 4]], dtype='float32') +alpha = 2 + +ground_truth = alpha * u + v + np.repeat(c, u.shape[0], axis=0) + +print({'r': ground_truth}) + +print(client.predict( + # Specify name of model to use for prediction + 'linear_example', + # Provide input arrays + {'u': u, 'v': v}, + # Provide size in bytes of expected output(s) + {'r': u.shape[0] * u.shape[1] * np.dtype('float32').itemsize}, + # Optionally submit request with Triton's shared memory mode + shared_mem='cuda' +)) +print(client.predict( + 'linear_example_cpu', + {'u': u, 'v': v}, + {'r': u.shape[0] * u.shape[1] * np.dtype('float32').itemsize} +)) +``` +which should give us the following output: +``` +{'r': array([[ 4., 7., 10., 9.]], dtype=float32)} +{'r': array([[ 4., 7., 10., 9.]], dtype=float32)} +{'r': array([[ 4., 7., 10., 9.]], dtype=float32)} +``` + +While this suggests that the backend is operating correctly, we probably want +to offer a more robust test, one that might form the basis for end-to-end +testing in CI. For this, we can make use of `pytest` to write something like +```python +``` diff --git a/conda/environments/rapids_triton_test.yml b/conda/environments/rapids_triton_test.yml new file mode 100644 index 0000000..8a4c4c0 --- /dev/null +++ b/conda/environments/rapids_triton_test.yml @@ -0,0 +1,12 @@ +--- +name: rapids_triton_test +channels: + - conda-forge +dependencies: + - flake8 + - pip + - python + - pytest + - numpy + - pip: + - nvidia-pyindex diff --git a/qa/L0_e2e/model_repository/linear_example/1/c.txt b/qa/L0_e2e/model_repository/linear_example/1/c.txt new file mode 100644 index 0000000..a91c0af --- /dev/null +++ b/qa/L0_e2e/model_repository/linear_example/1/c.txt @@ -0,0 +1 @@ +1.0 2.0 3.0 4.0 diff --git a/qa/L0_e2e/model_repository/linear_example/config.pbtxt b/qa/L0_e2e/model_repository/linear_example/config.pbtxt new file mode 100644 index 0000000..8dbfe20 --- /dev/null +++ b/qa/L0_e2e/model_repository/linear_example/config.pbtxt @@ -0,0 +1,32 @@ +name: "linear_example" +backend: "rapids_linear" +max_batch_size: 32768 +input [ + { + name: "u" + data_type: TYPE_FP32 + dims: [ 4 ] + }, + { + name: "v" + data_type: TYPE_FP32 + dims: [ 4 ] + } +] +output [ + { + name: "r" + data_type: TYPE_FP32 + dims: [ 4 ] + } +] +instance_group [{ kind: KIND_GPU }] +parameters [ + { + key: "alpha" + value: { string_value: "2.0" } + } +] +dynamic_batching { + max_queue_delay_microseconds: 100 +} diff --git a/qa/L0_e2e/model_repository/linear_example_cpu/1/c.txt b/qa/L0_e2e/model_repository/linear_example_cpu/1/c.txt new file mode 100644 index 0000000..a91c0af --- /dev/null +++ b/qa/L0_e2e/model_repository/linear_example_cpu/1/c.txt @@ -0,0 +1 @@ +1.0 2.0 3.0 4.0 diff --git a/qa/L0_e2e/model_repository/linear_example_cpu/config.pbtxt b/qa/L0_e2e/model_repository/linear_example_cpu/config.pbtxt new file mode 100644 index 0000000..5bf4d75 --- /dev/null +++ b/qa/L0_e2e/model_repository/linear_example_cpu/config.pbtxt @@ -0,0 +1,32 @@ +name: "linear_example_cpu" +backend: "rapids_linear" +max_batch_size: 32768 +input [ + { + name: "u" + data_type: TYPE_FP32 + dims: [ 4 ] + }, + { + name: "v" + data_type: TYPE_FP32 + dims: [ 4 ] + } +] +output [ + { + name: "r" + data_type: TYPE_FP32 + dims: [ 4 ] + } +] +instance_group [{ kind: KIND_CPU }] +parameters [ + { + key: "alpha" + value: { string_value: "2.0" } + } +] +dynamic_batching { + max_queue_delay_microseconds: 100 +} From 74adfe51b78867028d31fca449c52cfa6d3e3839 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 10 Sep 2021 12:58:34 -0400 Subject: [PATCH 056/199] Add conclusion --- README.md | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4386836..93a42c9 100644 --- a/README.md +++ b/README.md @@ -513,7 +513,38 @@ which should give us the following output: ``` While this suggests that the backend is operating correctly, we probably want -to offer a more robust test, one that might form the basis for end-to-end -testing in CI. For this, we can make use of `pytest` to write something like -```python -``` +to set up a more robust test with larger input data for use in CI and +development testing. See `qa/L0_e2e/test_model.py` for an example of how such a +test might be created. + +## Conclusion +This walkthrough has provided an in-depth look at how to create a Triton +backend using RAPIDS-Triton, from initial description of the backend behavior +to end-to-end testing of models deployed using this backend. Following similar +steps, you should be able to integrate almost any algorithm for deployment with +Triton. + +While we have tried to cover a wide variety of possible use cases with this +example, there is much more to explore in the RAPIDS-Triton API documentation. +If there is something you would like to do with RAPIDS-Triton which does not +seem to be covered by the available API or if something is not working as +expected, please submit a feature request or bug report to the [RAPIDS-Triton +issue tracker](https://github.com/rapidsai/rapids-triton/issues). If you think +this example could be improved or expanded in some way, please [submit a pull +request](https://github.com/rapidsai/rapids-triton-linear-example/pulls) or +[issue](https://github.com/rapidsai/rapids-triton-linear-example/issues) to +this repo. + +For additional information about using and deploying Triton after creating a +backend like this, check out the [main Triton +repo](https://github.com/triton-inference-server/server/). There you will find +information about many more tools to help you get the most out of Triton, +including: +- `perf_analyzer`: A tool to help measure throughput and latency for models + deployed with Triton +- `model_analyzer`: A tool to help determine what parameters will optimize + throughput, latency, or any other metric for your deployed models +- [Helm + charts](https://ngc.nvidia.com/catalog/helm-charts/nvidia:tritoninferenceserver/) + and other information to help you easily deploy Triton in any + cloud service or orchestration environment From deaa77fc41526c9811f070b1e1702be86ebc4268 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 10 Sep 2021 14:54:33 -0400 Subject: [PATCH 057/199] Correct logging message in get_random_seed --- python/rapids_triton/testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/rapids_triton/testing.py b/python/rapids_triton/testing.py index 882f2ed..ece5e6a 100644 --- a/python/rapids_triton/testing.py +++ b/python/rapids_triton/testing.py @@ -119,5 +119,5 @@ def get_random_seed(): else: random_seed = np.random.randint(0, 1e6) os.environ['PYTEST_RANDOM_SEED'] = str(random_seed) - logger.info("Random seed value:", random_seed) + logger.info("Random seed value: %d", random_seed) return random_seed From 7f5533d19b8845b794033695d01cbbb009d46c22 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 10 Sep 2021 14:59:07 -0400 Subject: [PATCH 058/199] Add links to Triton docs --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 93a42c9..7a9796e 100644 --- a/README.md +++ b/README.md @@ -540,11 +540,12 @@ backend like this, check out the [main Triton repo](https://github.com/triton-inference-server/server/). There you will find information about many more tools to help you get the most out of Triton, including: -- `perf_analyzer`: A tool to help measure throughput and latency for models - deployed with Triton -- `model_analyzer`: A tool to help determine what parameters will optimize - throughput, latency, or any other metric for your deployed models +- [`perf_analyzer`](https://github.com/triton-inference-server/server/blob/main/docs/perf_analyzer.md): + A tool to help measure throughput and latency for models deployed with Triton +- [`model_analyzer`](https://github.com/triton-inference-server/server/blob/main/docs/model_analyzer.md): + A tool to help determine what parameters will optimize throughput, latency, + or any other metric for your deployed models - [Helm charts](https://ngc.nvidia.com/catalog/helm-charts/nvidia:tritoninferenceserver/) - and other information to help you easily deploy Triton in any - cloud service or orchestration environment + and other information to help you easily deploy Triton in any cloud service + or orchestration environment From c29835f26dc5b86b56800c098977b5789f6de9e2 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 10 Sep 2021 14:59:23 -0400 Subject: [PATCH 059/199] Add E2E tests --- qa/L0_e2e/test_model.py | 68 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 qa/L0_e2e/test_model.py diff --git a/qa/L0_e2e/test_model.py b/qa/L0_e2e/test_model.py new file mode 100644 index 0000000..2ce0ccf --- /dev/null +++ b/qa/L0_e2e/test_model.py @@ -0,0 +1,68 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import pytest + +from rapids_triton import Client +from rapids_triton.testing import get_random_seed, arrays_close + +TOTAL_SAMPLES = 8192 +FEATURE_COUNT = 4 +ALPHA = 2 +C = np.array([[1, 2, 3, 4]], dtype='float32') + +@pytest.fixture +def model_inputs(): + np.random.seed(get_random_seed()) + return { + input_name: + np.random.rand(TOTAL_SAMPLES, FEATURE_COUNT).astype('float32') + for input_name in ('u', 'v') + } + +@pytest.fixture +def model_output_sizes(): + return {'r': TOTAL_SAMPLES * FEATURE_COUNT * np.dtype('float32').itemsize} + +def get_ground_truth(inputs): + u = inputs['u'] + v = inputs['v'] + return {'r': ALPHA * u + v + np.repeat(C, u.shape[0], axis=0)} + + +@pytest.mark.parametrize( + "model_name", ['linear_example', 'linear_example_cpu'] +) +def test_model(model_name, model_inputs, model_output_sizes): + client = Client() + result = client.predict(model_name, model_inputs, model_output_sizes) + shm_result = client.predict( + model_name, model_inputs, model_output_sizes, shared_mem='cuda' + ) + ground_truth = get_ground_truth(model_inputs) + + for output_name in sorted(ground_truth.keys()): + arrays_close( + result[output_name], + ground_truth[output_name], + atol=1e-5, + assert_close=True + ) + arrays_close( + shm_result[output_name], + ground_truth[output_name], + atol=1e-5, + assert_close=True + ) From 2d97788e3746ef175b4b5997a53f8e156f955323 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 10 Sep 2021 15:10:40 -0400 Subject: [PATCH 060/199] Remove copied comments --- src/names.h | 11 ----------- src/shared_state.h | 11 ----------- 2 files changed, 22 deletions(-) diff --git a/src/names.h b/src/names.h index 4331bcc..b134abc 100644 --- a/src/names.h +++ b/src/names.h @@ -15,15 +15,4 @@ */ #pragma once - -/* Triton expects certain definitions within its backend libraries to follow - * specific naming conventions. Specifically, for a backend named - * "rapids_linear," most definitions should appear within a namespace called - * triton::backend::rapids_linear. - * - * In order to facilitate this with minimal effort on the part of backend - * developers, we ask that you put the name of your backend here. This macro is - * then used to propagate the correct namespace name wherever it is needed in - * the impl and interface code. */ - #define NAMESPACE rapids_linear diff --git a/src/shared_state.h b/src/shared_state.h index df9da52..790751a 100644 --- a/src/shared_state.h +++ b/src/shared_state.h @@ -27,17 +27,6 @@ namespace triton { namespace backend { namespace NAMESPACE { -/* Triton allows multiple instances of a single model to be instantiated at the - * same time (e.g. on different GPUs). All instances of a model share access to - * an object which manages any state that can be shared across all instances. - * Any logic necessary for managing such state should be implemented in a - * struct named RapidsSharedState, as shown here. Models may access this shared - * state object via the `get_shared_state` method, which returns a shared - * pointer to the RapidsSharedState object. - * - * Not all backends require shared state, so leaving this implementation empty - * is entirely valid */ - struct RapidsSharedState : rapids::SharedModelState { RapidsSharedState(std::unique_ptr&& config) : rapids::SharedModelState{std::move(config)} {} From 0a6e9395d80c0f619c72e69ba8306bfcca11ddc0 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 10 Sep 2021 16:42:08 -0400 Subject: [PATCH 061/199] Add contributing and usage docs --- CONTRIBUTING.md | 71 +++++++++++++++++++++++++++++++++++++++++++++++++ docs/usage.md | 9 ++++++- 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..15a544a --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,71 @@ + +# Contributing to RAPIDS-Triton + +You can help improve RAPIDS-Triton in any of the following ways: +- Submitting a bug report, feature request or documentation issue +- Proposing and implementing a new feature +- Implementing a feature or bug-fix for an outstanding issue + +## Bug reports +When submitting a bug report, please include a *minimum* *reproducible* +example. Ideally, this should be a snippet of code that other developers can +copy, paste, and immediately run to try to reproduce the error. Please: +- Do include import statements and any other code necessary to immediately run + your example +- Avoid examples that require other developers to download models or data + unless you cannot reproduce the problem with synthetically-generated data + +## Code Contributions +To contribute code to this project, please follow these steps: +1. Find an issue to work on or submit an issue documenting the problem you + would like to work on. +2. Comment on the issue saying that you plan to work on it. +3. Review the implementation details section below for information to help you + make your changes in a way that is consistent with the rest of the codebase. +4. Code! +5. Create your pull request. +6. Wait for other developers to review your code and update your PR as needed. +7. Once a PR is approved, it will be merged into the main branch. + +### Coding Conventions +* RAPIDS-Triton follows [Almost Always Auto + (AAA)](https://herbsutter.com/2013/08/12/gotw-94-solution-aaa-style-almost-always-auto/) + style. Please maintain this style in any contributions. +* Avoid raw loops where possible + +### Signing Your Work +* We require that all contributors "sign-off" on their commits. This certifies that the contribution is your original work, or you have rights to submit it under the same license, or a compatible license. + * Any contribution which contains commits that are not Signed-Off will not be accepted. +* To sign off on a commit you simply use the `--signoff` (or `-s`) option when committing your changes: + ```bash + $ git commit -s -m "Add cool feature." + ``` + This will append the following to your commit message: + ``` + Signed-off-by: Your Name + ``` +* Full text of the DCO: + ``` + Developer Certificate of Origin + Version 1.1 + + Copyright (C) 2004, 2006 The Linux Foundation and its contributors. + 1 Letterman Drive + Suite D4700 + San Francisco, CA, 94129 + + Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. + ``` + ``` + Developer's Certificate of Origin 1.1 + + By making a contribution to this project, I certify that: + + (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or + + (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or + + (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. + + (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. + ``` diff --git a/docs/usage.md b/docs/usage.md index 3897ca4..4ce4852 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -29,4 +29,11 @@ # Using RAPIDS-Triton To begin developing a custom backend with RAPIDS-Triton, we strongly recommend -that you take advantage of the [rapids-triton-template repo](https://github.com/rapidsai/rapids-triton-template), which provides a basic template for your backend code. +that you take advantage of the [rapids-triton-template +repo](https://github.com/rapidsai/rapids-triton-template), which provides a +basic template for your backend code. For a detailed example of how to make use +of this template, check out the [Linear Example +repo](https://github.com/rapidsai/rapids-triton-linear-example), which provides +a complete walkthrough of how to build a backend using RAPIDS-Triton + + From 1241f0b0ab85624a78e52b0a2b85e63cb3e8d2e8 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 10 Sep 2021 16:52:22 -0400 Subject: [PATCH 062/199] Fix style --- conda/environments/rapids_triton_test.yml | 3 +- cpp/include/rapids_triton.hpp | 12 +- cpp/include/rapids_triton/batch/batch.hpp | 395 ++++++++------- cpp/include/rapids_triton/build_control.hpp | 8 +- cpp/include/rapids_triton/exceptions.hpp | 58 +-- cpp/include/rapids_triton/memory/buffer.hpp | 451 +++++++++--------- .../rapids_triton/memory/detail/allocate.hpp | 24 +- .../rapids_triton/memory/detail/copy.hpp | 42 +- cpp/include/rapids_triton/memory/types.hpp | 14 +- cpp/include/rapids_triton/model/model.hpp | 270 ++++++----- .../rapids_triton/model/shared_state.hpp | 198 ++++---- cpp/include/rapids_triton/tensor/dtype.hpp | 41 +- cpp/include/rapids_triton/tensor/tensor.hpp | 291 +++++------ .../rapids_triton/triton/api/execute.hpp | 146 +++--- .../rapids_triton/triton/api/initialize.hpp | 38 +- .../triton/api/instance_finalize.hpp | 42 +- .../triton/api/instance_initialize.hpp | 59 +-- .../triton/api/model_finalize.hpp | 43 +- .../triton/api/model_initialize.hpp | 50 +- cpp/include/rapids_triton/triton/backend.hpp | 70 ++- cpp/include/rapids_triton/triton/config.hpp | 21 +- .../rapids_triton/triton/deployment.hpp | 20 +- cpp/include/rapids_triton/triton/device.hpp | 12 +- cpp/include/rapids_triton/triton/input.hpp | 98 ++-- cpp/include/rapids_triton/triton/logging.hpp | 237 ++++----- cpp/include/rapids_triton/triton/model.hpp | 51 +- .../rapids_triton/triton/model_instance.hpp | 141 +++--- .../triton/model_instance_state.hpp | 28 +- .../rapids_triton/triton/model_state.hpp | 20 +- cpp/include/rapids_triton/triton/output.hpp | 94 ++-- cpp/include/rapids_triton/triton/requests.hpp | 34 +- .../rapids_triton/triton/responses.hpp | 37 +- .../rapids_triton/triton/statistics.hpp | 120 ++--- .../rapids_triton/utils/const_agnostic.hpp | 11 +- cpp/include/rapids_triton/utils/narrow.hpp | 14 +- cpp/src/model.h | 17 +- cpp/src/shared_state.h | 4 +- cpp/test/build_control.cpp | 3 +- cpp/test/exceptions.cpp | 22 +- cpp/test/memory/buffer.cpp | 70 +-- cpp/test/memory/detail/allocate.cpp | 22 +- cpp/test/memory/detail/copy.cpp | 45 +- cpp/test/tensor/dtype.cpp | 6 +- cpp/test/tensor/tensor.cpp | 94 ++-- cpp/test/test.cpp | 9 +- cpp/test/triton/logging.cpp | 6 +- cpp/test/utils/const_agnostic.cpp | 6 +- cpp/test/utils/narrow.cpp | 6 +- 48 files changed, 1830 insertions(+), 1673 deletions(-) diff --git a/conda/environments/rapids_triton_test.yml b/conda/environments/rapids_triton_test.yml index c58d9d3..ded958f 100644 --- a/conda/environments/rapids_triton_test.yml +++ b/conda/environments/rapids_triton_test.yml @@ -1,8 +1,9 @@ --- -name: rapids_triton_dev +name: rapids_triton_test channels: - conda-forge dependencies: + - clang-tools=11.0.0 - flake8 - pip - python diff --git a/cpp/include/rapids_triton.hpp b/cpp/include/rapids_triton.hpp index de73942..16cf368 100644 --- a/cpp/include/rapids_triton.hpp +++ b/cpp/include/rapids_triton.hpp @@ -17,13 +17,15 @@ #pragma once #include -namespace triton { namespace backend { namespace rapids { +namespace triton { +namespace backend { +namespace rapids { /* Function for testing rapids_triton include * * @return message indicating rapids_triton has been included succesfully*/ -inline auto test_install() { - return std::string("rapids_triton set up successfully"); -} +inline auto test_install() { return std::string("rapids_triton set up successfully"); } -}}} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/batch/batch.hpp b/cpp/include/rapids_triton/batch/batch.hpp index fa7115b..2811e44 100644 --- a/cpp/include/rapids_triton/batch/batch.hpp +++ b/cpp/include/rapids_triton/batch/batch.hpp @@ -18,14 +18,14 @@ #include #include +#include +#include #include #include #include #include #include #include -#include -#include #include #include #include @@ -35,228 +35,223 @@ #include #include #include -#include -#include +#include +#include -namespace triton { namespace backend { namespace rapids { - /** - * @brief A representation of all data about a single batch of inference - * requests - * - * Batch objects are the primary interface point between rapids_triton Models - * and the Triton server itself. By calling the `get_input` and `get_output` - * methods of a batch, Model implementations can retrieve the input Tensors - * necessary for prediction and the output Tensors where results can be - * stored. - * - * Batch objects also handle a variety of other tasks necessary for - * processing a batch in the Triton model. This includes reporting statistics - * on how long it took to process requests and sending responses to the - * client via the Triton server once processing is complete. - * - * It is not recommended that developers of rapids_triton backends try to - * construct Batch objects directly. Instead, you should make use of the - * rapids::triton_api::execute template, which will construct the Batch for - * you. - */ - struct Batch { - using size_type = std::size_t; +namespace triton { +namespace backend { +namespace rapids { +/** + * @brief A representation of all data about a single batch of inference + * requests + * + * Batch objects are the primary interface point between rapids_triton Models + * and the Triton server itself. By calling the `get_input` and `get_output` + * methods of a batch, Model implementations can retrieve the input Tensors + * necessary for prediction and the output Tensors where results can be + * stored. + * + * Batch objects also handle a variety of other tasks necessary for + * processing a batch in the Triton model. This includes reporting statistics + * on how long it took to process requests and sending responses to the + * client via the Triton server once processing is complete. + * + * It is not recommended that developers of rapids_triton backends try to + * construct Batch objects directly. Instead, you should make use of the + * rapids::triton_api::execute template, which will construct the Batch for + * you. + */ +struct Batch { + using size_type = std::size_t; - Batch(TRITONBACKEND_Request** raw_requests, - request_size_t count, - TRITONBACKEND_MemoryManager& triton_mem_manager, - std::function(std::string const&, size_type)> get_output_shape, - std::function< - void( - TRITONBACKEND_Request*, - time_point const&, - time_point const&, - time_point const&, - time_point const&)> report_request_statistics, - bool use_pinned_input, - bool use_pinned_output, - size_type max_batch_size, - cudaStream_t stream) : - requests_(raw_requests, raw_requests + count), - responses_(construct_responses(requests_.begin(), requests_.end())), - get_output_shape_{get_output_shape}, - report_statistics_{report_request_statistics}, - collector_( - raw_requests, - count, - &responses_, - &triton_mem_manager, - use_pinned_input, - stream - ), - responder_{std::make_shared( - raw_requests, - count, - &responses_, - max_batch_size, - &triton_mem_manager, - use_pinned_output, - stream - )}, - stream_{stream}, - start_time_{std::chrono::steady_clock::now()}, - compute_start_time_{std::chrono::steady_clock::now()}, - batch_size_{} {} + Batch(TRITONBACKEND_Request** raw_requests, + request_size_t count, + TRITONBACKEND_MemoryManager& triton_mem_manager, + std::function(std::string const&, size_type)> get_output_shape, + std::function report_request_statistics, + bool use_pinned_input, + bool use_pinned_output, + size_type max_batch_size, + cudaStream_t stream) + : requests_(raw_requests, raw_requests + count), + responses_(construct_responses(requests_.begin(), requests_.end())), + get_output_shape_{get_output_shape}, + report_statistics_{report_request_statistics}, + collector_(raw_requests, count, &responses_, &triton_mem_manager, use_pinned_input, stream), + responder_{std::make_shared(raw_requests, + count, + &responses_, + max_batch_size, + &triton_mem_manager, + use_pinned_output, + stream)}, + stream_{stream}, + start_time_{std::chrono::steady_clock::now()}, + compute_start_time_{std::chrono::steady_clock::now()}, + batch_size_{} + { + } - template - auto get_input_shape(std::string const& name) { - auto result = std::vector{}; - if(!requests_.empty()) { - result = get_triton_input_shape(std::begin(requests_), std::end(requests_), name); + template + auto get_input_shape(std::string const& name) + { + auto result = std::vector{}; + if (!requests_.empty()) { + result = get_triton_input_shape(std::begin(requests_), std::end(requests_), name); - auto input_batch_dim = size_type{}; - if (result.size() > 0) { - input_batch_dim = result[0]; - } else { - input_batch_dim = size_type{}; - } + auto input_batch_dim = size_type{}; + if (result.size() > 0) { + input_batch_dim = result[0]; + } else { + input_batch_dim = size_type{}; + } - if(batch_size_.has_value()) { - if(batch_size_.value() != input_batch_dim) { - throw TritonException( - Error::Internal, "all input tensors must have same batch dimension"); - } - } else { - batch_size_ = input_batch_dim; + if (batch_size_.has_value()) { + if (batch_size_.value() != input_batch_dim) { + throw TritonException(Error::Internal, + "all input tensors must have same batch dimension"); } + } else { + batch_size_ = input_batch_dim; } - return result; } + return result; + } + template + auto get_input(std::string const& name, + std::optional memory_type, + device_id_t device_id, + cudaStream_t stream) + { + auto shape = get_input_shape(name); + auto size_bytes = + sizeof(T) * std::reduce(shape.begin(), shape.end(), std::size_t{1}, std::multiplies<>()); + auto allowed_memory_configs = std::vector>{}; + if (memory_type.has_value()) { + allowed_memory_configs.emplace_back(memory_type.value(), device_id); + } else { + allowed_memory_configs.emplace_back(HostMemory, int64_t{}); + allowed_memory_configs.emplace_back(DeviceMemory, device_id); + } - template - auto get_input(std::string const& name, std::optional - memory_type, device_id_t device_id, cudaStream_t stream) { - auto shape = get_input_shape(name); - auto size_bytes = sizeof(T) * std::reduce(shape.begin(), shape.end(), std::size_t{1}, std::multiplies<>()); - auto allowed_memory_configs = std::vector>{}; - if (memory_type.has_value()) { - allowed_memory_configs.emplace_back(memory_type.value(), device_id); - } else { - allowed_memory_configs.emplace_back(HostMemory, int64_t{}); - allowed_memory_configs.emplace_back(DeviceMemory, device_id); - } + auto const* raw_buffer = static_cast(nullptr); + auto reported_bytes = std::size_t{}; + auto reported_mem_type = MemoryType{}; + auto reported_device_id = int64_t{}; - auto const* raw_buffer = static_cast(nullptr); - auto reported_bytes = std::size_t{}; - auto reported_mem_type = MemoryType{}; - auto reported_device_id = int64_t{}; + triton_check( + collector_.ProcessTensor(name.c_str(), + static_cast(nullptr), // Return data without copy if possible + size_bytes, + allowed_memory_configs, + &raw_buffer, + &reported_bytes, + &reported_mem_type, + &reported_device_id)); - triton_check(collector_.ProcessTensor( - name.c_str(), - static_cast(nullptr), // Return data without copy if possible - size_bytes, - allowed_memory_configs, - &raw_buffer, - &reported_bytes, - &reported_mem_type, - &reported_device_id - )); + auto buffer = Buffer(reinterpret_cast(raw_buffer), + reported_bytes / sizeof(T), + reported_mem_type, + reported_device_id, + stream); - auto buffer = Buffer( - reinterpret_cast(raw_buffer), - reported_bytes / sizeof(T), - reported_mem_type, - reported_device_id, - stream - ); + if (memory_type && (reported_mem_type != memory_type || reported_device_id != device_id)) { + throw TritonException(Error::Internal, "data collected in wrong location"); + } - if (memory_type && (reported_mem_type != memory_type || reported_device_id != device_id)) { - throw TritonException(Error::Internal, "data collected in wrong location"); - } + // Set start time of batch to time latest input tensor was retrieved + compute_start_time_ = std::chrono::steady_clock::now(); - // Set start time of batch to time latest input tensor was retrieved - compute_start_time_ = std::chrono::steady_clock::now(); + return Tensor(std::move(shape), std::move(buffer)); + } - return Tensor(std::move(shape), std::move(buffer)); - } + template + auto get_input(std::string const& name, + std::optional memory_type, + device_id_t device_id) + { + return get_input(name, memory_type, device_id, stream_); + } - template - auto get_input(std::string const& name, std::optional - memory_type, device_id_t device_id) { - return get_input(name, memory_type, device_id, stream_); + template + auto get_output(std::string const& name, + std::optional memory_type, + device_id_t device_id, + cudaStream_t stream) + { + if (!batch_size_.has_value()) { + throw TritonException(Error::Internal, + "At least one input must be retrieved before any output"); } - - template - auto get_output(std::string const& name, std::optional memory_type, device_id_t device_id, cudaStream_t stream) { - if (!batch_size_.has_value()) { - throw TritonException( - Error::Internal, - "At least one input must be retrieved before any output" - ); - } - auto shape = get_output_shape_(name, batch_size_.value()); - auto buffer_size = std::reduce( - shape.begin(), shape.end(), std::size_t{1}, std::multiplies<>()); - auto final_memory_type = MemoryType{}; - if (memory_type.has_value()) { - final_memory_type = memory_type.value(); - } else { - // If consumer doesn't care, use HostMemory to avoid additional copy on - // non-shared-memory responses. - final_memory_type = HostMemory; - } - auto buffer = Buffer(buffer_size, final_memory_type, device_id, stream); - return OutputTensor(std::move(shape), std::move(buffer), name, responder_); + auto shape = get_output_shape_(name, batch_size_.value()); + auto buffer_size = std::reduce(shape.begin(), shape.end(), std::size_t{1}, std::multiplies<>()); + auto final_memory_type = MemoryType{}; + if (memory_type.has_value()) { + final_memory_type = memory_type.value(); + } else { + // If consumer doesn't care, use HostMemory to avoid additional copy on + // non-shared-memory responses. + final_memory_type = HostMemory; } + auto buffer = Buffer(buffer_size, final_memory_type, device_id, stream); + return OutputTensor(std::move(shape), std::move(buffer), name, responder_); + } - template - auto get_output(std::string const& name, std::optional memory_type, device_id_t device_id) { - return get_output(name, memory_type, device_id, stream_); - } + template + auto get_output(std::string const& name, + std::optional memory_type, + device_id_t device_id) + { + return get_output(name, memory_type, device_id, stream_); + } - auto const& compute_start_time() const { - return compute_start_time_; - } + auto const& compute_start_time() const { return compute_start_time_; } - auto stream() const { - return stream_; - } + auto stream() const { return stream_; } - void finalize(TRITONSERVER_Error* err) { - auto compute_end_time = std::chrono::steady_clock::now(); - if (responder_->Finalize()) { - cuda_check(cudaStreamSynchronize(stream_)); - } + void finalize(TRITONSERVER_Error* err) + { + auto compute_end_time = std::chrono::steady_clock::now(); + if (responder_->Finalize()) { cuda_check(cudaStreamSynchronize(stream_)); } - send_responses(std::begin(responses_), std::end(responses_), err); + send_responses(std::begin(responses_), std::end(responses_), err); - // Triton resumes ownership of failed requests; only release on success - if (err == nullptr) { - std::for_each( - std::begin(requests_), - std::end(requests_), - [this, &compute_end_time]( - auto& request) { - report_statistics_( - request, - start_time_, - compute_start_time_, - compute_end_time, - std::chrono::steady_clock::now() - ); - } - ); - release_requests(std::begin(requests_), std::end(requests_)); - } + // Triton resumes ownership of failed requests; only release on success + if (err == nullptr) { + std::for_each( + std::begin(requests_), std::end(requests_), [this, &compute_end_time](auto& request) { + report_statistics_(request, + start_time_, + compute_start_time_, + compute_end_time, + std::chrono::steady_clock::now()); + }); + release_requests(std::begin(requests_), std::end(requests_)); } + } - private: - std::vector requests_; - std::vector responses_; - std::function(std::string const&, size_type)> get_output_shape_; - std::function report_statistics_; - BackendInputCollector collector_; - std::shared_ptr responder_; - cudaStream_t stream_; - std::chrono::time_point start_time_; - std::chrono::time_point compute_start_time_; - std::optional batch_size_; - }; -}}} // namespace triton::backend::rapids - + private: + std::vector requests_; + std::vector responses_; + std::function(std::string const&, size_type)> get_output_shape_; + std::function + report_statistics_; + BackendInputCollector collector_; + std::shared_ptr responder_; + cudaStream_t stream_; + std::chrono::time_point start_time_; + std::chrono::time_point compute_start_time_; + std::optional batch_size_; +}; +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/build_control.hpp b/cpp/include/rapids_triton/build_control.hpp index e05dc12..c2a9fec 100644 --- a/cpp/include/rapids_triton/build_control.hpp +++ b/cpp/include/rapids_triton/build_control.hpp @@ -17,7 +17,9 @@ #pragma once #include -namespace triton { namespace backend { namespace rapids { +namespace triton { +namespace backend { +namespace rapids { #ifdef TRITON_ENABLE_GPU auto constexpr IS_GPU_BUILD = true; @@ -25,4 +27,6 @@ auto constexpr IS_GPU_BUILD = true; auto constexpr IS_GPU_BUILD = false; #endif -}}} // namespace triton::backend::rapids +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/exceptions.hpp b/cpp/include/rapids_triton/exceptions.hpp index 76040f0..dfdab76 100644 --- a/cpp/include/rapids_triton/exceptions.hpp +++ b/cpp/include/rapids_triton/exceptions.hpp @@ -15,24 +15,26 @@ */ #pragma once -#include #include +#include #include #include -namespace triton { namespace backend { namespace rapids { +namespace triton { +namespace backend { +namespace rapids { using ErrorCode = TRITONSERVER_Error_Code; namespace Error { - auto constexpr Unknown = ErrorCode::TRITONSERVER_ERROR_UNKNOWN; - auto constexpr Internal = ErrorCode::TRITONSERVER_ERROR_INTERNAL; - auto constexpr NotFound = ErrorCode::TRITONSERVER_ERROR_NOT_FOUND; - auto constexpr InvalidArg = ErrorCode::TRITONSERVER_ERROR_INVALID_ARG; - auto constexpr Unavailable = ErrorCode::TRITONSERVER_ERROR_UNAVAILABLE; - auto constexpr Unsupported = ErrorCode::TRITONSERVER_ERROR_UNSUPPORTED; - auto constexpr AlreadyExists = ErrorCode::TRITONSERVER_ERROR_ALREADY_EXISTS; -} +auto constexpr Unknown = ErrorCode::TRITONSERVER_ERROR_UNKNOWN; +auto constexpr Internal = ErrorCode::TRITONSERVER_ERROR_INTERNAL; +auto constexpr NotFound = ErrorCode::TRITONSERVER_ERROR_NOT_FOUND; +auto constexpr InvalidArg = ErrorCode::TRITONSERVER_ERROR_INVALID_ARG; +auto constexpr Unavailable = ErrorCode::TRITONSERVER_ERROR_UNAVAILABLE; +auto constexpr Unsupported = ErrorCode::TRITONSERVER_ERROR_UNSUPPORTED; +auto constexpr AlreadyExists = ErrorCode::TRITONSERVER_ERROR_ALREADY_EXISTS; +} // namespace Error /** * @brief Exception thrown if processing cannot continue for a request @@ -45,30 +47,19 @@ namespace Error { * requests, including requests to other models. */ struct TritonException : std::exception { - public: - TritonException() - : error_(TRITONSERVER_ErrorNew(Error::Unknown, - "encountered unknown error")) - { - } + TritonException() : error_(TRITONSERVER_ErrorNew(Error::Unknown, "encountered unknown error")) {} - TritonException(ErrorCode code, std::string const & msg) - : error_(TRITONSERVER_ErrorNew(code, msg.c_str())) + TritonException(ErrorCode code, std::string const& msg) + : error_(TRITONSERVER_ErrorNew(code, msg.c_str())) { } - TritonException(ErrorCode code, char const* msg) - : error_{TRITONSERVER_ErrorNew(code, msg)} - { - } + TritonException(ErrorCode code, char const* msg) : error_{TRITONSERVER_ErrorNew(code, msg)} {} TritonException(TRITONSERVER_Error* prev_error) : error_(prev_error) {} - virtual char const* what() const noexcept - { - return TRITONSERVER_ErrorMessage(error_); - } + virtual char const* what() const noexcept { return TRITONSERVER_ErrorMessage(error_); } auto* error() const { return error_; } @@ -76,18 +67,19 @@ struct TritonException : std::exception { TRITONSERVER_Error* error_; }; -inline void triton_check(TRITONSERVER_Error* err) { - if (err != nullptr) { - throw TritonException(err); - } +inline void triton_check(TRITONSERVER_Error* err) +{ + if (err != nullptr) { throw TritonException(err); } } -inline void cuda_check(cudaError_t const& err) { +inline void cuda_check(cudaError_t const& err) +{ if (err != cudaSuccess) { cudaGetLastError(); throw TritonException(Error::Internal, cudaGetErrorString(err)); } } -}}} // namespace triton::backend::rapids - +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/memory/buffer.hpp b/cpp/include/rapids_triton/memory/buffer.hpp index 58ac3a1..c334495 100644 --- a/cpp/include/rapids_triton/memory/buffer.hpp +++ b/cpp/include/rapids_triton/memory/buffer.hpp @@ -23,251 +23,270 @@ #include #include -#include #include #include +#include #include -namespace triton { namespace backend { namespace rapids { - template - struct Buffer { - using size_type = std::size_t; - using value_type = T; - - using h_ptr = T*; - using d_ptr = T*; - using owned_h_ptr = std::unique_ptr; - using owned_d_ptr = std::unique_ptr>; - using data_ptr = std::variant; - - Buffer() noexcept : device_{}, data_{std::in_place_index<0>, nullptr}, size_{}, stream_{} {} - - /** - * @brief Construct buffer of given size in given memory location (either - * on host or on device) - * A buffer constructed in this way is owning and will release allocated - * resources on deletion - */ - Buffer(size_type size, MemoryType memory_type=DeviceMemory, device_id_t device=0, cudaStream_t - stream=0) : - device_{device}, data_{allocate(size, device, memory_type, stream)}, size_{size}, stream_{stream} {} - - /** - * @brief Construct buffer from given source in given memory location (either - * on host or on device) - * A buffer constructed in this way is non-owning; the caller is - * responsible for freeing any resources associated with the input pointer - */ - Buffer(T* input_data, size_type size, MemoryType memory_type=DeviceMemory, - device_id_t device=0, cudaStream_t stream=0) : - device_{device}, data_{ - [&memory_type, &input_data](){ - auto result = data_ptr{}; - if(memory_type == HostMemory) { - result = data_ptr{std::in_place_index<0>, input_data}; - } else { - result = data_ptr{std::in_place_index<1>, input_data}; - } - return result; - }() - }, size_{size}, stream_{stream} {} - - /** - * @brief Construct one buffer from another in the given memory location - * (either on host or on device) - * A buffer constructed in this way is owning and will copy the data from - * the original location - */ - Buffer(Buffer const& other, MemoryType memory_type, device_id_t device=0) : device_{device}, data_([&other, &memory_type, &device](){ - auto result = allocate(other.size_, device, memory_type, other.stream_); - copy(result, other.data_, other.size_, other.stream_); - return result; - }()), size_{other.size_}, stream_{other.stream_} {} - - /** - * @brief Create owning copy of existing buffer - * The memory type of this new buffer will be the same as the original - */ - Buffer(Buffer const& other) : Buffer(other, other.mem_type(), other.device()) {} - - Buffer(Buffer&& other, MemoryType memory_type) : device_{other.device()}, data_{[&other, memory_type](){ - data_ptr result; - if(memory_type == other.mem_type()) { - result = std::move(other.data_); - } else { - result = allocate(other.size_, memory_type, other.device(), other.stream()); - copy(result, other.data_, other.size_, other.stream_); - } - return result; - }()}, size_{other.size_}, stream_{other.stream_} {} +namespace triton { +namespace backend { +namespace rapids { +template +struct Buffer { + using size_type = std::size_t; + using value_type = T; - Buffer(Buffer&& other) = default; + using h_ptr = T*; + using d_ptr = T*; + using owned_h_ptr = std::unique_ptr; + using owned_d_ptr = std::unique_ptr>; + using data_ptr = std::variant; - Buffer& operator=(Buffer&& other) = default; + Buffer() noexcept : device_{}, data_{std::in_place_index<0>, nullptr}, size_{}, stream_{} {} - ~Buffer() = default; + /** + * @brief Construct buffer of given size in given memory location (either + * on host or on device) + * A buffer constructed in this way is owning and will release allocated + * resources on deletion + */ + Buffer(size_type size, + MemoryType memory_type = DeviceMemory, + device_id_t device = 0, + cudaStream_t stream = 0) + : device_{device}, + data_{allocate(size, device, memory_type, stream)}, + size_{size}, + stream_{stream} + { + } - /** - * @brief Return where memory for this buffer is located (host or device) - */ - auto mem_type() const noexcept { - return data_.index() % 2 == 0 ? HostMemory : DeviceMemory; - } + /** + * @brief Construct buffer from given source in given memory location (either + * on host or on device) + * A buffer constructed in this way is non-owning; the caller is + * responsible for freeing any resources associated with the input pointer + */ + Buffer(T* input_data, + size_type size, + MemoryType memory_type = DeviceMemory, + device_id_t device = 0, + cudaStream_t stream = 0) + : device_{device}, + data_{[&memory_type, &input_data]() { + auto result = data_ptr{}; + if (memory_type == HostMemory) { + result = data_ptr{std::in_place_index<0>, input_data}; + } else { + result = data_ptr{std::in_place_index<1>, input_data}; + } + return result; + }()}, + size_{size}, + stream_{stream} + { + } - /** - * @brief Return number of elements in buffer - */ - auto size() const noexcept { - return size_; - } + /** + * @brief Construct one buffer from another in the given memory location + * (either on host or on device) + * A buffer constructed in this way is owning and will copy the data from + * the original location + */ + Buffer(Buffer const& other, MemoryType memory_type, device_id_t device = 0) + : device_{device}, + data_([&other, &memory_type, &device]() { + auto result = allocate(other.size_, device, memory_type, other.stream_); + copy(result, other.data_, other.size_, other.stream_); + return result; + }()), + size_{other.size_}, + stream_{other.stream_} + { + } - /** - * @brief Return pointer to data stored in buffer - */ - auto* data() const noexcept { - return get_raw_ptr(data_); - } + /** + * @brief Create owning copy of existing buffer + * The memory type of this new buffer will be the same as the original + */ + Buffer(Buffer const& other) : Buffer(other, other.mem_type(), other.device()) {} + + Buffer(Buffer&& other, MemoryType memory_type) + : device_{other.device()}, + data_{[&other, memory_type]() { + data_ptr result; + if (memory_type == other.mem_type()) { + result = std::move(other.data_); + } else { + result = allocate(other.size_, memory_type, other.device(), other.stream()); + copy(result, other.data_, other.size_, other.stream_); + } + return result; + }()}, + size_{other.size_}, + stream_{other.stream_} + { + } - auto device() const noexcept { - return device_; - } + Buffer(Buffer&& other) = default; - /** - * @brief Return CUDA stream associated with this buffer - */ - auto stream() const noexcept { - return stream_; - } + Buffer& operator=(Buffer&& other) = default; - void stream_synchronize() const { - if constexpr (IS_GPU_BUILD) { - cuda_check(cudaStreamSynchronize(stream_)); - } - } - - /** - * @brief Set CUDA stream for this buffer to new value - * - * @warning This method calls cudaStreamSynchronize on the old stream - * before updating. Be aware of performance implications and try to avoid - * interactions between buffers on different streams where possible. - */ - void set_stream(cudaStream_t new_stream) { - stream_synchronize(); - stream_ = new_stream; - } + ~Buffer() = default; - private: - device_id_t device_; - data_ptr data_; - size_type size_; - cudaStream_t stream_; - - // Helper function for accessing raw pointer to underlying data of data_ptr - static auto* get_raw_ptr(data_ptr const& ptr) noexcept { - /* Switch statement is an optimization relative to std::visit to avoid - * vtable overhead for a small number of alternatives */ - auto* result = static_cast(nullptr); - switch (ptr.index()) { - case 0: - result = std::get<0>(ptr); - break; - case 1: - result = std::get<1>(ptr); - break; - case 2: - result = std::get<2>(ptr).get(); - break; - case 3: - result = std::get<3>(ptr).get(); - break; - } - return result; - } + /** + * @brief Return where memory for this buffer is located (host or device) + */ + auto mem_type() const noexcept { return data_.index() % 2 == 0 ? HostMemory : DeviceMemory; } - // Helper function for allocating memory in constructors - static auto allocate(size_type size, device_id_t device=0, MemoryType memory_type=DeviceMemory, cudaStream_t stream=0) { - auto result = data_ptr{}; - if (memory_type == DeviceMemory) { - if constexpr (IS_GPU_BUILD) { - cuda_check(cudaSetDevice(device)); - result = data_ptr{owned_d_ptr{detail::dev_allocate(size, stream)}}; - } else { - throw TritonException( - Error::Internal, - "DeviceMemory requested in CPU-only build of FIL backend" - ); - } - } else { - result = std::make_unique(size); - } - return result; - } + /** + * @brief Return number of elements in buffer + */ + auto size() const noexcept { return size_; } - // Helper function for copying memory in constructors, where there are - // stronger guarantees on conditions that would otherwise need to be - // checked - static void copy( - data_ptr const& dst, data_ptr const& src, size_type len, cudaStream_t stream) { - auto raw_dst = get_raw_ptr(dst); - auto raw_src = get_raw_ptr(src); + /** + * @brief Return pointer to data stored in buffer + */ + auto* data() const noexcept { return get_raw_ptr(data_); } - auto dst_mem_type = dst.index() % 2 == 0 ? HostMemory : DeviceMemory; - auto src_mem_type = src.index() % 2 == 0 ? HostMemory : DeviceMemory; + auto device() const noexcept { return device_; } - detail::copy(raw_dst, raw_src, len, stream, dst_mem_type, src_mem_type); - } + /** + * @brief Return CUDA stream associated with this buffer + */ + auto stream() const noexcept { return stream_; } - }; + void stream_synchronize() const + { + if constexpr (IS_GPU_BUILD) { cuda_check(cudaStreamSynchronize(stream_)); } + } /** - * @brief Copy data from one Buffer to another - * - * @param dst The destination buffer - * @param src The source buffer - * @param dst_begin The offset from the beginning of the destination buffer - * at which to begin copying to. - * @param src_begin The offset from the beginning of the source buffer - * at which to begin copying from. - * @param src_end The offset from the beginning of the source buffer - * before which to end copying from. + * @brief Set CUDA stream for this buffer to new value * - * @warning This method is NOT thread-safe. If the stream of the src buffer - * changes while a copy is in progress, dst may receive incorrect data from - * src. Avoid interactions between buffers on different streams, *especially* - * when those buffers may be modified on different host threads as well. + * @warning This method calls cudaStreamSynchronize on the old stream + * before updating. Be aware of performance implications and try to avoid + * interactions between buffers on different streams where possible. */ - template - void copy(Buffer& dst, Buffer const& src, typename Buffer::size_type dst_begin, - typename Buffer::size_type src_begin, typename Buffer::size_type src_end) { - if(dst.stream() != src.stream()) { - dst.set_stream(src.stream()); + void set_stream(cudaStream_t new_stream) + { + stream_synchronize(); + stream_ = new_stream; + } + + private: + device_id_t device_; + data_ptr data_; + size_type size_; + cudaStream_t stream_; + + // Helper function for accessing raw pointer to underlying data of data_ptr + static auto* get_raw_ptr(data_ptr const& ptr) noexcept + { + /* Switch statement is an optimization relative to std::visit to avoid + * vtable overhead for a small number of alternatives */ + auto* result = static_cast(nullptr); + switch (ptr.index()) { + case 0: result = std::get<0>(ptr); break; + case 1: result = std::get<1>(ptr); break; + case 2: result = std::get<2>(ptr).get(); break; + case 3: result = std::get<3>(ptr).get(); break; } - auto len = src_end - src_begin; - if (len < 0 || src_end > src.size() || len > dst.size() - dst_begin) { - throw TritonException(Error::Internal, "bad copy between buffers"); + return result; + } + + // Helper function for allocating memory in constructors + static auto allocate(size_type size, + device_id_t device = 0, + MemoryType memory_type = DeviceMemory, + cudaStream_t stream = 0) + { + auto result = data_ptr{}; + if (memory_type == DeviceMemory) { + if constexpr (IS_GPU_BUILD) { + cuda_check(cudaSetDevice(device)); + result = data_ptr{owned_d_ptr{detail::dev_allocate(size, stream)}}; + } else { + throw TritonException(Error::Internal, + "DeviceMemory requested in CPU-only build of FIL backend"); + } + } else { + result = std::make_unique(size); } + return result; + } - auto raw_dst = dst.data() + dst_begin; - auto raw_src = src.data() + src_begin; + // Helper function for copying memory in constructors, where there are + // stronger guarantees on conditions that would otherwise need to be + // checked + static void copy(data_ptr const& dst, data_ptr const& src, size_type len, cudaStream_t stream) + { + auto raw_dst = get_raw_ptr(dst); + auto raw_src = get_raw_ptr(src); - detail::copy(raw_dst, raw_src, len, dst.stream(), dst.mem_type(), - src.mem_type()); - } + auto dst_mem_type = dst.index() % 2 == 0 ? HostMemory : DeviceMemory; + auto src_mem_type = src.index() % 2 == 0 ? HostMemory : DeviceMemory; - template - void copy(Buffer& dst, Buffer const& src) { - copy(dst, src, 0, 0, src.size()); + detail::copy(raw_dst, raw_src, len, stream, dst_mem_type, src_mem_type); } +}; - template - void copy(Buffer& dst, Buffer const& src, typename Buffer::size_type dst_begin) { - copy(dst, src, dst_begin, 0, src.size()); +/** + * @brief Copy data from one Buffer to another + * + * @param dst The destination buffer + * @param src The source buffer + * @param dst_begin The offset from the beginning of the destination buffer + * at which to begin copying to. + * @param src_begin The offset from the beginning of the source buffer + * at which to begin copying from. + * @param src_end The offset from the beginning of the source buffer + * before which to end copying from. + * + * @warning This method is NOT thread-safe. If the stream of the src buffer + * changes while a copy is in progress, dst may receive incorrect data from + * src. Avoid interactions between buffers on different streams, *especially* + * when those buffers may be modified on different host threads as well. + */ +template +void copy(Buffer& dst, + Buffer const& src, + typename Buffer::size_type dst_begin, + typename Buffer::size_type src_begin, + typename Buffer::size_type src_end) +{ + if (dst.stream() != src.stream()) { dst.set_stream(src.stream()); } + auto len = src_end - src_begin; + if (len < 0 || src_end > src.size() || len > dst.size() - dst_begin) { + throw TritonException(Error::Internal, "bad copy between buffers"); } - template - void copy(Buffer& dst, Buffer const& src, typename Buffer::size_type src_begin, - typename Buffer::size_type src_end) { - copy(dst, src, 0, src_begin, src_end); - } -}}} // namespace triton::backend::rapids + auto raw_dst = dst.data() + dst_begin; + auto raw_src = src.data() + src_begin; + + detail::copy(raw_dst, raw_src, len, dst.stream(), dst.mem_type(), src.mem_type()); +} + +template +void copy(Buffer& dst, Buffer const& src) +{ + copy(dst, src, 0, 0, src.size()); +} + +template +void copy(Buffer& dst, Buffer const& src, typename Buffer::size_type dst_begin) +{ + copy(dst, src, dst_begin, 0, src.size()); +} + +template +void copy(Buffer& dst, + Buffer const& src, + typename Buffer::size_type src_begin, + typename Buffer::size_type src_end) +{ + copy(dst, src, 0, src_begin, src_end); +} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/memory/detail/allocate.hpp b/cpp/include/rapids_triton/memory/detail/allocate.hpp index 616fc6f..460f92f 100644 --- a/cpp/include/rapids_triton/memory/detail/allocate.hpp +++ b/cpp/include/rapids_triton/memory/detail/allocate.hpp @@ -25,10 +25,14 @@ #include #include -namespace triton { namespace backend { namespace rapids { namespace detail { +namespace triton { +namespace backend { +namespace rapids { +namespace detail { template struct dev_deallocater { - void operator()(T* d_ptr) { + void operator()(T* d_ptr) + { if constexpr (IS_GPU_BUILD) { // Note: We allow a const_cast here because this deallocator is only used // in a RAII context. If we are deallocating this memory, we allocated it @@ -37,10 +41,7 @@ struct dev_deallocater { cudaFree(reinterpret_cast(const_cast::type*>(d_ptr))); } else { log_error( - __FILE__, - __LINE__, - "ERROR: device deallocation cannot be performed in non-GPU build!" - ); + __FILE__, __LINE__, "ERROR: device deallocation cannot be performed in non-GPU build!"); } } }; @@ -49,16 +50,17 @@ struct dev_deallocater { * @brief Allocate given number of elements on GPU and return device pointer */ template -[[nodiscard]] T* -dev_allocate(std::size_t count, cudaStream_t stream) +[[nodiscard]] T* dev_allocate(std::size_t count, cudaStream_t stream) { if constexpr (!IS_GPU_BUILD) { throw TritonException(Error::Internal, "device allocation attempted in non-GPU build"); } auto* ptr_d = - static_cast(rmm::mr::get_current_device_resource()->allocate( - sizeof(T) * count, stream)); + static_cast(rmm::mr::get_current_device_resource()->allocate(sizeof(T) * count, stream)); return ptr_d; } -}}}} // namespace triton::backend::rapids::detail +} // namespace detail +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/memory/detail/copy.hpp b/cpp/include/rapids_triton/memory/detail/copy.hpp index 6774656..de0efe2 100644 --- a/cpp/include/rapids_triton/memory/detail/copy.hpp +++ b/cpp/include/rapids_triton/memory/detail/copy.hpp @@ -20,19 +20,20 @@ #include -#include #include +#include - -namespace triton { namespace backend { namespace rapids { namespace detail { +namespace triton { +namespace backend { +namespace rapids { +namespace detail { /** * @brief Copy given number of elements from one place to another, with either * source or destination on device */ template -void -dev_copy(T* dst, T const* src, std::size_t len, cudaStream_t stream) +void dev_copy(T* dst, T const* src, std::size_t len, cudaStream_t stream) { if constexpr (IS_GPU_BUILD) { try { @@ -41,10 +42,8 @@ dev_copy(T* dst, T const* src, std::size_t len, cudaStream_t stream) throw TritonException(Error::Internal, err.what()); } } else { - throw TritonException( - Error::Internal, - "copy to or from device memory cannot be used in CPU-only builds" - ); + throw TritonException(Error::Internal, + "copy to or from device memory cannot be used in CPU-only builds"); } } @@ -53,28 +52,31 @@ dev_copy(T* dst, T const* src, std::size_t len, cudaStream_t stream) * source or destination on device */ template -void -host_copy(T* dst, T const* src, std::size_t len) +void host_copy(T* dst, T const* src, std::size_t len) { std::memcpy(dst, src, len * sizeof(T)); } -template -void -copy(T* dst, T const* src, std::size_t len, cudaStream_t stream, MemoryType - dst_type, MemoryType src_type) { +template +void copy(T* dst, + T const* src, + std::size_t len, + cudaStream_t stream, + MemoryType dst_type, + MemoryType src_type) +{ if (dst_type == DeviceMemory || src_type == DeviceMemory) { if constexpr (IS_GPU_BUILD) { dev_copy(dst, src, len, stream); } else { - throw TritonException( - Error::Internal, - "DeviceMemory copy cannot be used in CPU-only builds" - ); + throw TritonException(Error::Internal, "DeviceMemory copy cannot be used in CPU-only builds"); } } else { host_copy(dst, src, len); } } -}}}} // namespace triton::backend::rapids::detail +} // namespace detail +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/memory/types.hpp b/cpp/include/rapids_triton/memory/types.hpp index 8540775..884f315 100644 --- a/cpp/include/rapids_triton/memory/types.hpp +++ b/cpp/include/rapids_triton/memory/types.hpp @@ -17,8 +17,12 @@ #pragma once #include -namespace triton { namespace backend { namespace rapids { - using MemoryType = TRITONSERVER_MemoryType; - auto constexpr DeviceMemory = TRITONSERVER_MEMORY_GPU; - auto constexpr HostMemory = TRITONSERVER_MEMORY_CPU; -}}} // namespace triton::backend::rapids +namespace triton { +namespace backend { +namespace rapids { +using MemoryType = TRITONSERVER_MemoryType; +auto constexpr DeviceMemory = TRITONSERVER_MEMORY_GPU; +auto constexpr HostMemory = TRITONSERVER_MEMORY_CPU; +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/model/model.hpp b/cpp/include/rapids_triton/model/model.hpp index 653f266..249ea89 100644 --- a/cpp/include/rapids_triton/model/model.hpp +++ b/cpp/include/rapids_triton/model/model.hpp @@ -15,137 +15,177 @@ */ #pragma once -#include #include -#include -#include +#include #include #include #include #include #include #include +#include +#include -namespace triton { namespace backend { namespace rapids { - template - struct Model { - - virtual void predict(Batch& batch) const = 0; +namespace triton { +namespace backend { +namespace rapids { +template +struct Model { + virtual void predict(Batch& batch) const = 0; - virtual void load() {} - virtual void unload() {} + virtual void load() {} + virtual void unload() {} - /** - * @brief Return the preferred memory type in which to store data for this - * batch or std::nullopt to accept whatever Triton returns - * - * The base implementation of this method will require data on-host if the - * model itself is deployed on the host OR if this backend has not been - * compiled with GPU support. Otherwise, models deployed on device will - * receive memory on device. Overriding this method will allow derived - * model classes to select a preferred memory location based on properties - * of the batch or to simply return std::nullopt if device memory or host - * memory will do equally well. - */ - virtual std::optional preferred_mem_type(Batch& batch) const { - return (IS_GPU_BUILD && deployment_type_ == GPUDeployment) ? DeviceMemory : HostMemory; - } - virtual std::optional preferred_mem_type_in(Batch& batch) const { - return preferred_mem_type(batch); - } - virtual std::optional preferred_mem_type_out(Batch& batch) const { - return preferred_mem_type(batch); - } + /** + * @brief Return the preferred memory type in which to store data for this + * batch or std::nullopt to accept whatever Triton returns + * + * The base implementation of this method will require data on-host if the + * model itself is deployed on the host OR if this backend has not been + * compiled with GPU support. Otherwise, models deployed on device will + * receive memory on device. Overriding this method will allow derived + * model classes to select a preferred memory location based on properties + * of the batch or to simply return std::nullopt if device memory or host + * memory will do equally well. + */ + virtual std::optional preferred_mem_type(Batch& batch) const + { + return (IS_GPU_BUILD && deployment_type_ == GPUDeployment) ? DeviceMemory : HostMemory; + } + virtual std::optional preferred_mem_type_in(Batch& batch) const + { + return preferred_mem_type(batch); + } + virtual std::optional preferred_mem_type_out(Batch& batch) const + { + return preferred_mem_type(batch); + } - /** - * @brief Retrieve a stream used to set up batches for this model - * - * The base implementation of this method simply returns the default stream - * provided by Triton for use with this model. Child classes may choose to - * override this in order to provide different streams for use with - * successive incoming batches. For instance, one might cycle through - * several streams in order to distribute batches across them, but care - * should be taken to ensure proper synchronization in this case. - */ - virtual cudaStream_t get_stream() const { - return default_stream_; - } + /** + * @brief Retrieve a stream used to set up batches for this model + * + * The base implementation of this method simply returns the default stream + * provided by Triton for use with this model. Child classes may choose to + * override this in order to provide different streams for use with + * successive incoming batches. For instance, one might cycle through + * several streams in order to distribute batches across them, but care + * should be taken to ensure proper synchronization in this case. + */ + virtual cudaStream_t get_stream() const { return default_stream_; } - /** - * @brief Get input tensor of a particular named input for an entire batch - */ - template - auto get_input(Batch& batch, std::string const& name, std::optional const& mem_type, cudaStream_t stream) const { - return batch.get_input(name, mem_type, device_id_, stream); - } - template - auto get_input(Batch& batch, std::string const& name, std::optional const& mem_type) const { - return get_input(batch, name, mem_type, default_stream_); - } - template - auto get_input(Batch& batch, std::string const& name) const { - return get_input(batch, name, preferred_mem_type(batch), default_stream_); - } + /** + * @brief Get input tensor of a particular named input for an entire batch + */ + template + auto get_input(Batch& batch, + std::string const& name, + std::optional const& mem_type, + cudaStream_t stream) const + { + return batch.get_input(name, mem_type, device_id_, stream); + } + template + auto get_input(Batch& batch, + std::string const& name, + std::optional const& mem_type) const + { + return get_input(batch, name, mem_type, default_stream_); + } + template + auto get_input(Batch& batch, std::string const& name) const + { + return get_input(batch, name, preferred_mem_type(batch), default_stream_); + } - /** - * @brief Get output tensor of a particular named output for an entire batch - */ - template - auto get_output(Batch& batch, std::string const& name, std::optional const& mem_type, device_id_t device_id, cudaStream_t stream) const { - return batch.get_output(name, mem_type, device_id, stream); - } - template - auto get_output(Batch& batch, std::string const& name, std::optional const& mem_type, cudaStream_t stream) const { - return get_output(batch, name, mem_type, device_id_, stream); - } - template - auto get_output(Batch& batch, std::string const& name, std::optional const& mem_type) const { - return get_output(batch, name, mem_type, device_id_, default_stream_); - } - template - auto get_output(Batch& batch, std::string const& name) const { - return get_output(batch, name, preferred_mem_type(batch), device_id_, default_stream_); - } + /** + * @brief Get output tensor of a particular named output for an entire batch + */ + template + auto get_output(Batch& batch, + std::string const& name, + std::optional const& mem_type, + device_id_t device_id, + cudaStream_t stream) const + { + return batch.get_output(name, mem_type, device_id, stream); + } + template + auto get_output(Batch& batch, + std::string const& name, + std::optional const& mem_type, + cudaStream_t stream) const + { + return get_output(batch, name, mem_type, device_id_, stream); + } + template + auto get_output(Batch& batch, + std::string const& name, + std::optional const& mem_type) const + { + return get_output(batch, name, mem_type, device_id_, default_stream_); + } + template + auto get_output(Batch& batch, std::string const& name) const + { + return get_output(batch, name, preferred_mem_type(batch), device_id_, default_stream_); + } - /** - * @brief Retrieve value of configuration parameter - */ - template - auto get_config_param(std::string const& name) const { - return shared_state_->template get_config_param(name); - } - template - auto get_config_param(std::string const& name, T default_value) const { - return shared_state_->template get_config_param(name, default_value); - } - template - auto get_config_param(char const* name) const { - return get_config_param(std::string(name)); - } - template - auto get_config_param(char const* name, T default_value) const { - return get_config_param(std::string(name), default_value); - } + /** + * @brief Retrieve value of configuration parameter + */ + template + auto get_config_param(std::string const& name) const + { + return shared_state_->template get_config_param(name); + } + template + auto get_config_param(std::string const& name, T default_value) const + { + return shared_state_->template get_config_param(name, default_value); + } + template + auto get_config_param(char const* name) const + { + return get_config_param(std::string(name)); + } + template + auto get_config_param(char const* name, T default_value) const + { + return get_config_param(std::string(name), default_value); + } - Model(std::shared_ptr shared_state, device_id_t device_id, cudaStream_t default_stream, DeploymentType deployment_type, std::string const& filepath) : - shared_state_{shared_state}, device_id_{device_id}, default_stream_{default_stream}, deployment_type_{deployment_type}, filepath_{filepath} {} + Model(std::shared_ptr shared_state, + device_id_t device_id, + cudaStream_t default_stream, + DeploymentType deployment_type, + std::string const& filepath) + : shared_state_{shared_state}, + device_id_{device_id}, + default_stream_{default_stream}, + deployment_type_{deployment_type}, + filepath_{filepath} + { + } - auto get_device_id() const { return device_id_; } - auto get_deployment_type() const { return deployment_type_; } - auto const& get_filepath() const { return filepath_; } + auto get_device_id() const { return device_id_; } + auto get_deployment_type() const { return deployment_type_; } + auto const& get_filepath() const { return filepath_; } - auto get_output_shape(std::string const& name) const { - return shared_state_->get_output_shape(name); - } + auto get_output_shape(std::string const& name) const + { + return shared_state_->get_output_shape(name); + } - protected: - auto get_shared_state() const { return shared_state_; } + protected: + auto get_shared_state() const { return shared_state_; } - private: - std::shared_ptr shared_state_; - device_id_t device_id_; - cudaStream_t default_stream_; - DeploymentType deployment_type_; - std::string filepath_; - }; -}}} // namespace triton::backend::rapids + private: + std::shared_ptr shared_state_; + device_id_t device_id_; + cudaStream_t default_stream_; + DeploymentType deployment_type_; + std::string filepath_; +}; +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp index 63cb09e..71c2f10 100644 --- a/cpp/include/rapids_triton/model/shared_state.hpp +++ b/cpp/include/rapids_triton/model/shared_state.hpp @@ -15,9 +15,9 @@ */ #pragma once +#include #include #include -#include #include #include #include @@ -31,136 +31,128 @@ #include #include -namespace triton { namespace backend { namespace rapids { - /** - * @brief Stores shared state for multiple instances of the same model - */ - struct SharedModelState { - - virtual void load() {} - virtual void unload() {} - - explicit SharedModelState( - std::unique_ptr&& config) : config_{std::move(config)}, - max_batch_size_(get_max_batch_size(*config_)), output_shapes_([this]() { - auto result = std::vector>>{}; +namespace triton { +namespace backend { +namespace rapids { +/** + * @brief Stores shared state for multiple instances of the same model + */ +struct SharedModelState { + virtual void load() {} + virtual void unload() {} + + explicit SharedModelState(std::unique_ptr&& config) + : config_{std::move(config)}, + max_batch_size_(get_max_batch_size(*config_)), + output_shapes_([this]() { + auto result = std::vector>>{}; auto output_entries = triton::common::TritonJson::Value{}; triton_check(config_->MemberAsArray("output", &output_entries)); result.reserve(output_entries.ArraySize()); // Using a raw loop because TritonJSON::Value access has no iterator interface - for (std::size_t i=0; i < output_entries.ArraySize(); ++i) { + for (std::size_t i = 0; i < output_entries.ArraySize(); ++i) { auto output_entry = triton::common::TritonJson::Value{}; triton_check(output_entries.IndexAsObject(i, &output_entry)); auto name = std::string{}; triton_check(output_entry.MemberAsString("name", &name)); - auto shape = std::vector{}; + auto shape = std::vector{}; auto reshape_entry = triton::common::TritonJson::Value{}; if (output_entry.Find("reshape", &reshape_entry)) { ParseShape(reshape_entry, "shape", &shape); } else { ParseShape(output_entry, "dims", &shape); } - if (shape[0] != -1) { - shape.insert(shape.begin(), -1); - } + if (shape[0] != -1) { shape.insert(shape.begin(), -1); } result.insert( - std::upper_bound( - std::begin(output_shapes_), - std::end(output_shapes_), - name, - [](auto& value, auto& entry) { - return value < entry.first; - } - ), - {name, shape} - ); + std::upper_bound(std::begin(output_shapes_), + std::end(output_shapes_), + name, + [](auto& value, auto& entry) { return value < entry.first; }), + {name, shape}); } return result; - }()) {} - - template - auto get_config_param(std::string const& name) { - return get_config_param(name, std::optional{}); + }()) + { + } + + template + auto get_config_param(std::string const& name) + { + return get_config_param(name, std::optional{}); + } + + template + auto get_config_param(std::string const& name, T default_value) + { + return get_config_param(name, std::make_optional(default_value)); + } + + auto get_output_shape(std::string const& name) const + { + auto cached_shape = std::lower_bound( + std::begin(output_shapes_), std::end(output_shapes_), name, [](auto& entry, auto& value) { + return entry.first < value; + }); + if (cached_shape == std::end(output_shapes_)) { + auto log_stream = std::stringstream{}; + log_stream << "No output with name " << name << " in configuration."; + throw TritonException(Error::Internal, log_stream.str()); + } else { + return cached_shape->second; } - - template - auto get_config_param(std::string const& name, T default_value) { - return get_config_param(name, std::make_optional(default_value)); + } + + private: + std::unique_ptr config_; + Batch::size_type max_batch_size_; + std::vector>> mutable output_shapes_; + + template + auto get_config_param(std::string const& name, std::optional const& default_value) + { + auto result = T{}; + if (name == std::string("max_batch_size")) { + result = max_batch_size_; + return result; } + auto parameters = common::TritonJson::Value{}; + auto json_value = common::TritonJson::Value{}; + if (config_->Find("parameters", ¶meters) && parameters.Find(name.c_str(), &json_value)) { + auto string_repr = std::string{}; + triton_check(json_value.MemberAsString("string_value", &string_repr)); - auto get_output_shape(std::string const& name) const { - auto cached_shape = std::lower_bound( - std::begin(output_shapes_), - std::end(output_shapes_), - name, - [](auto& entry, auto& value) { - return entry.first < value; - } - ); - if (cached_shape == std::end(output_shapes_)) { - auto log_stream = std::stringstream{}; - log_stream << "No output with name " << name << " in configuration."; - throw TritonException(Error::Internal, log_stream.str()); + auto input_stream = std::istringstream{string_repr}; + + if (std::is_same::value) { + input_stream >> std::boolalpha >> result; } else { - return cached_shape->second; + input_stream >> result; } - } - private: - std::unique_ptr config_; - Batch::size_type max_batch_size_; - std::vector>> mutable output_shapes_; - - template - auto get_config_param(std::string const& name, std::optional const& default_value) { - auto result = T{}; - if (name == std::string("max_batch_size")) { - result = max_batch_size_; - return result; - } - auto parameters = common::TritonJson::Value{}; - auto json_value = common::TritonJson::Value{}; - if ( - config_->Find("parameters", ¶meters) && - parameters.Find(name.c_str(), &json_value)) { - auto string_repr = std::string{}; - triton_check(json_value.MemberAsString("string_value", &string_repr)); - - auto input_stream = std::istringstream{string_repr}; - - if (std::is_same::value) { - input_stream >> std::boolalpha >> result; - } else { - input_stream >> result; - } - - if (input_stream.fail()) { - if (default_value) { - result = *default_value; - } else { - throw TritonException( - Error::InvalidArg, - std::string("Bad input for parameter ") + name - ); - } - } + if (input_stream.fail()) { + if (default_value) { + result = *default_value; } else { - if (default_value) { - result = *default_value; - } else { - throw TritonException( - Error::InvalidArg, - std::string("Required parameter ") + name + std::string(" not found in config") - ); - } + throw TritonException(Error::InvalidArg, std::string("Bad input for parameter ") + name); } - - return result; } - }; -}}} // namespace triton::backend::rapids + } else { + if (default_value) { + result = *default_value; + } else { + throw TritonException( + Error::InvalidArg, + std::string("Required parameter ") + name + std::string(" not found in config")); + } + } + return result; + } +}; +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/tensor/dtype.hpp b/cpp/include/rapids_triton/tensor/dtype.hpp index 7dcb236..325ef4d 100644 --- a/cpp/include/rapids_triton/tensor/dtype.hpp +++ b/cpp/include/rapids_triton/tensor/dtype.hpp @@ -15,25 +15,27 @@ */ #pragma once +#include #include #include #include -#include -namespace triton { namespace backend { namespace rapids { - -using DType = TRITONSERVER_DataType; -auto constexpr DTypeBool = TRITONSERVER_TYPE_BOOL; -auto constexpr DTypeUint8 = TRITONSERVER_TYPE_UINT8; -auto constexpr DTypeChar = DTypeUint8; -auto constexpr DTypeByte = DTypeUint8; -auto constexpr DTypeUint16 = TRITONSERVER_TYPE_UINT16; -auto constexpr DTypeUint32 = TRITONSERVER_TYPE_UINT32; -auto constexpr DTypeUint64 = TRITONSERVER_TYPE_UINT64; -auto constexpr DTypeInt8 = TRITONSERVER_TYPE_INT8; -auto constexpr DTypeInt16 = TRITONSERVER_TYPE_INT16; -auto constexpr DTypeInt32 = TRITONSERVER_TYPE_INT32; -auto constexpr DTypeInt64 = TRITONSERVER_TYPE_INT64; +namespace triton { +namespace backend { +namespace rapids { + +using DType = TRITONSERVER_DataType; +auto constexpr DTypeBool = TRITONSERVER_TYPE_BOOL; +auto constexpr DTypeUint8 = TRITONSERVER_TYPE_UINT8; +auto constexpr DTypeChar = DTypeUint8; +auto constexpr DTypeByte = DTypeUint8; +auto constexpr DTypeUint16 = TRITONSERVER_TYPE_UINT16; +auto constexpr DTypeUint32 = TRITONSERVER_TYPE_UINT32; +auto constexpr DTypeUint64 = TRITONSERVER_TYPE_UINT64; +auto constexpr DTypeInt8 = TRITONSERVER_TYPE_INT8; +auto constexpr DTypeInt16 = TRITONSERVER_TYPE_INT16; +auto constexpr DTypeInt32 = TRITONSERVER_TYPE_INT32; +auto constexpr DTypeInt64 = TRITONSERVER_TYPE_INT64; auto constexpr DTypeFloat32 = TRITONSERVER_TYPE_FP32; auto constexpr DTypeFloat64 = TRITONSERVER_TYPE_FP64; @@ -41,7 +43,7 @@ template struct TritonType { }; -template +template struct TritonDtype { }; @@ -155,9 +157,12 @@ struct TritonDtype> { static constexpr DType value = DTypeFloat64; }; -inline std::ostream& operator<<(std::ostream& out, DType const& dtype) { +inline std::ostream& operator<<(std::ostream& out, DType const& dtype) +{ out << TRITONSERVER_DataTypeString(dtype); return out; } -}}} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/tensor/tensor.hpp b/cpp/include/rapids_triton/tensor/tensor.hpp index dd33999..c7088ca 100644 --- a/cpp/include/rapids_triton/tensor/tensor.hpp +++ b/cpp/include/rapids_triton/tensor/tensor.hpp @@ -26,151 +26,174 @@ #include +#include #include #include #include #include #include #include -#include -namespace triton { namespace backend { namespace rapids { - template - struct BaseTensor { - using size_type = typename Buffer::size_type; - - BaseTensor() : shape_{}, buffer_{} {} - BaseTensor(std::vector const& shape, Buffer&& buffer) : shape_(shape), buffer_{std::move(buffer)} {} - - virtual ~BaseTensor() = 0; - - /** - * @brief Construct a BaseTensor from a collection of buffers - * - * Given a collection of buffers, collate them all into one buffer stored in - * a new BaseTensor - */ - template - BaseTensor(std::vector const& shape, Iter begin, Iter end, MemoryType mem_type, device_id_t device, cudaStream_t stream) : - shape_(shape), - buffer_([&begin, &end, &mem_type, &device, &stream] () { - auto total_size = std::transform_reduce( - begin, end, size_type{}, std::plus<>{}, [](auto&& buffer) { return buffer.size(); } - ); - - auto result = Buffer(total_size, mem_type, device, stream); - - std::accumulate(begin, end, size_type{}, [&result](auto offset, auto& buffer) { - copy(result, buffer, offset); - return offset + buffer.size(); - }); - return result; - }()) {} - - auto const& shape() const { return shape_; } - auto size() const { return buffer_.size(); } - auto data() const { return buffer_.data(); } - auto& buffer() { return buffer_; } - - auto constexpr dtype() { return TritonDtype::value; } - auto mem_type() const { return buffer_.mem_type(); } - auto stream() const { return buffer_.stream(); } - auto device() const { return buffer_.device(); } - - void stream_synchronize() const { - if (mem_type() == DeviceMemory) { - buffer_.stream_synchronize(); - } - } - - void set_stream(cudaStream_t new_stream) { - buffer_.set_stream(new_stream); - } - - private: - std::vector shape_; - Buffer buffer_; - }; - - template - BaseTensor::~BaseTensor() {} - - template - struct Tensor final : BaseTensor { - Tensor() : BaseTensor{} {} - Tensor(std::vector::size_type> const& shape, Buffer&& buffer) : BaseTensor(shape, std::move(buffer)) {} - - template - Tensor(std::vector::size_type> const& shape, Iter begin, Iter end, MemoryType mem_type, device_id_t device, cudaStream_t stream) : BaseTensor(shape, begin, end, mem_type, device, stream) {} - - }; - - template - struct OutputTensor final : BaseTensor { - OutputTensor(std::vector::size_type>&& shape, Buffer&& buffer, - std::string const& name, std::shared_ptr responder) : - BaseTensor(std::move(shape), std::move(buffer)), name_{name}, responder_{responder} - {} - /** - * @brief Prepare final output data from this tensor for responding to - * request - * - * This method *must* be called by rapids_triton backends on all of their - * output tensors before returning from their `predict` methods. Because we - * cannot known a priori what names backends might have for their tensors - * and what types will be stored in those tensors, the rapids_triton - * library cannot store references to those tensors that might otherwise be - * used to finalize them. - */ - void finalize() { - auto& shape = BaseTensor::shape(); - auto triton_shape = std::vector{}; - triton_shape.reserve(shape.size()); - std::transform(std::begin(shape), std::end(shape), std::back_inserter(triton_shape), [](auto& val) { return narrow(val);}); - - // Must call the following because BackendOutputResponder does not expose - // its stream, so we cannot be certain that our data is not being - // processed on another stream. - BaseTensor::stream_synchronize(); - responder_->ProcessTensor( - name_.c_str(), - TritonDtype::value, - triton_shape, - reinterpret_cast(BaseTensor::data()), - BaseTensor::mem_type(), - BaseTensor::device() - ); - } - - private: - std::string name_; - std::shared_ptr responder_; - }; - - template - void copy(BaseTensor>& dst, BaseTensor& src) { - copy(dst.buffer(), src.buffer()); +namespace triton { +namespace backend { +namespace rapids { +template +struct BaseTensor { + using size_type = typename Buffer::size_type; + + BaseTensor() : shape_{}, buffer_{} {} + BaseTensor(std::vector const& shape, Buffer&& buffer) + : shape_(shape), buffer_{std::move(buffer)} + { } + virtual ~BaseTensor() = 0; + /** - * @brief Copy data from src Tensor into buffers indicated by iterators + * @brief Construct a BaseTensor from a collection of buffers * - * This method is provided to assist with distributing data from a single - * Tensor into many smaller buffers which have been set up to receive a part - * of the data from the src Tensor + * Given a collection of buffers, collate them all into one buffer stored in + * a new BaseTensor */ - template - void copy(Iter begin, Iter end, BaseTensor& src) { - std::accumulate( - begin, - end, - typename BaseTensor::size_type{}, - [&src] (auto offset, auto& dst) { - auto end_offset = offset + dst.size(); - copy(dst.buffer(), src.buffer(), offset, end_offset); - return end_offset; - } - ); + template + BaseTensor(std::vector const& shape, + Iter begin, + Iter end, + MemoryType mem_type, + device_id_t device, + cudaStream_t stream) + : shape_(shape), buffer_([&begin, &end, &mem_type, &device, &stream]() { + auto total_size = std::transform_reduce( + begin, end, size_type{}, std::plus<>{}, [](auto&& buffer) { return buffer.size(); }); + + auto result = Buffer(total_size, mem_type, device, stream); + + std::accumulate(begin, end, size_type{}, [&result](auto offset, auto& buffer) { + copy(result, buffer, offset); + return offset + buffer.size(); + }); + return result; + }()) + { + } + + auto const& shape() const { return shape_; } + auto size() const { return buffer_.size(); } + auto data() const { return buffer_.data(); } + auto& buffer() { return buffer_; } + + auto constexpr dtype() { return TritonDtype::value; } + auto mem_type() const { return buffer_.mem_type(); } + auto stream() const { return buffer_.stream(); } + auto device() const { return buffer_.device(); } + + void stream_synchronize() const + { + if (mem_type() == DeviceMemory) { buffer_.stream_synchronize(); } + } + + void set_stream(cudaStream_t new_stream) { buffer_.set_stream(new_stream); } + + private: + std::vector shape_; + Buffer buffer_; +}; + +template +BaseTensor::~BaseTensor() +{ +} + +template +struct Tensor final : BaseTensor { + Tensor() : BaseTensor{} {} + Tensor(std::vector::size_type> const& shape, Buffer&& buffer) + : BaseTensor(shape, std::move(buffer)) + { } -}}} // namespace triton::backend::rapids + template + Tensor(std::vector::size_type> const& shape, + Iter begin, + Iter end, + MemoryType mem_type, + device_id_t device, + cudaStream_t stream) + : BaseTensor(shape, begin, end, mem_type, device, stream) + { + } +}; + +template +struct OutputTensor final : BaseTensor { + OutputTensor(std::vector::size_type>&& shape, + Buffer&& buffer, + std::string const& name, + std::shared_ptr responder) + : BaseTensor(std::move(shape), std::move(buffer)), name_{name}, responder_{responder} + { + } + /** + * @brief Prepare final output data from this tensor for responding to + * request + * + * This method *must* be called by rapids_triton backends on all of their + * output tensors before returning from their `predict` methods. Because we + * cannot known a priori what names backends might have for their tensors + * and what types will be stored in those tensors, the rapids_triton + * library cannot store references to those tensors that might otherwise be + * used to finalize them. + */ + void finalize() + { + auto& shape = BaseTensor::shape(); + auto triton_shape = std::vector{}; + triton_shape.reserve(shape.size()); + std::transform( + std::begin(shape), std::end(shape), std::back_inserter(triton_shape), [](auto& val) { + return narrow(val); + }); + + // Must call the following because BackendOutputResponder does not expose + // its stream, so we cannot be certain that our data is not being + // processed on another stream. + BaseTensor::stream_synchronize(); + responder_->ProcessTensor(name_.c_str(), + TritonDtype::value, + triton_shape, + reinterpret_cast(BaseTensor::data()), + BaseTensor::mem_type(), + BaseTensor::device()); + } + + private: + std::string name_; + std::shared_ptr responder_; +}; + +template +void copy(BaseTensor>& dst, BaseTensor& src) +{ + copy(dst.buffer(), src.buffer()); +} + +/** + * @brief Copy data from src Tensor into buffers indicated by iterators + * + * This method is provided to assist with distributing data from a single + * Tensor into many smaller buffers which have been set up to receive a part + * of the data from the src Tensor + */ +template +void copy(Iter begin, Iter end, BaseTensor& src) +{ + std::accumulate(begin, end, typename BaseTensor::size_type{}, [&src](auto offset, auto& dst) { + auto end_offset = offset + dst.size(); + copy(dst.buffer(), src.buffer(), offset, end_offset); + return end_offset; + }); +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/api/execute.hpp b/cpp/include/rapids_triton/triton/api/execute.hpp index d77117d..38cdf8c 100644 --- a/cpp/include/rapids_triton/triton/api/execute.hpp +++ b/cpp/include/rapids_triton/triton/api/execute.hpp @@ -15,90 +15,98 @@ */ #pragma once +#include #include #include #include #include -#include -#include #include +#include #include #include #include #include -#include +#include -namespace triton { namespace backend { namespace rapids { namespace triton_api { - template - auto* execute(TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** raw_requests, std::size_t request_count) { - auto start_time = std::chrono::steady_clock::now(); +namespace triton { +namespace backend { +namespace rapids { +namespace triton_api { +template +auto* execute(TRITONBACKEND_ModelInstance* instance, + TRITONBACKEND_Request** raw_requests, + std::size_t request_count) +{ + auto start_time = std::chrono::steady_clock::now(); - auto* result = static_cast(nullptr); + auto* result = static_cast(nullptr); - try { - auto* model_state = get_model_state(*get_model_from_instance(*instance)); - auto* instance_state = - get_instance_state(*instance); - auto& model = instance_state->get_model(); - auto max_batch_size = model.template get_config_param("max_batch_size"); - auto batch = Batch( - raw_requests, request_count, *(model_state->TritonMemoryManager()), - /* Note: It is safe to keep a reference to the model in htis closure - * and a pointer to the instance in the next because the batch goes - * out of scope at the end of this block and Triton guarantees that - * the liftimes of both the instance and model extend beyond this - * function call. */ - [&model](std::string const& name, Batch::size_type batch_dim) { - auto result = std::vector{}; - auto config_shape = model.get_output_shape(name); - if (config_shape.size() > 0 && config_shape[0] < 0) { - config_shape[0] = batch_dim; + try { + auto* model_state = get_model_state(*get_model_from_instance(*instance)); + auto* instance_state = get_instance_state(*instance); + auto& model = instance_state->get_model(); + auto max_batch_size = model.template get_config_param("max_batch_size"); + auto batch = Batch( + raw_requests, + request_count, + *(model_state->TritonMemoryManager()), + /* Note: It is safe to keep a reference to the model in htis closure + * and a pointer to the instance in the next because the batch goes + * out of scope at the end of this block and Triton guarantees that + * the liftimes of both the instance and model extend beyond this + * function call. */ + [&model](std::string const& name, Batch::size_type batch_dim) { + auto result = std::vector{}; + auto config_shape = model.get_output_shape(name); + if (config_shape.size() > 0 && config_shape[0] < 0) { config_shape[0] = batch_dim; } + std::transform( + std::begin(config_shape), + std::end(config_shape), + std::back_inserter(result), + [](auto& coord) { + if (coord < 0) { + throw TritonException( + Error::Internal, + "Backends with variable-shape outputs must request desired output shape"); + } else { + return narrow(coord); } - std::transform( - std::begin(config_shape), - std::end(config_shape), - std::back_inserter(result), - [](auto& coord) { - if (coord < 0) { - throw TritonException( - Error::Internal, - "Backends with variable-shape outputs must request desired output shape" - ); - } else { - return narrow(coord); - } - } - ); - return result; - }, - [instance](TRITONBACKEND_Request* request, time_point req_start, - time_point req_comp_start, time_point req_comp_end, - time_point req_end) { - report_statistics(*instance, *request, req_start, req_comp_start, - req_comp_end, req_end); - }, - model_state->EnablePinnedInput(), model_state->EnablePinnedOutput(), - max_batch_size, - model.get_stream()); - - auto predict_err = static_cast(nullptr); - try { - model.predict(batch); - } catch (TritonException& err) { - predict_err = err.error(); - } - - auto& compute_start_time = batch.compute_start_time(); - auto compute_end_time = std::chrono::steady_clock::now(); - batch.finalize(predict_err); - auto end_time = std::chrono::steady_clock::now(); + }); + return result; + }, + [instance](TRITONBACKEND_Request* request, + time_point req_start, + time_point req_comp_start, + time_point req_comp_end, + time_point req_end) { + report_statistics(*instance, *request, req_start, req_comp_start, req_comp_end, req_end); + }, + model_state->EnablePinnedInput(), + model_state->EnablePinnedOutput(), + max_batch_size, + model.get_stream()); - report_statistics(*instance, request_count, start_time, compute_start_time, - compute_end_time, end_time); + auto predict_err = static_cast(nullptr); + try { + model.predict(batch); } catch (TritonException& err) { - result = err.error(); + predict_err = err.error(); } - return result; + auto& compute_start_time = batch.compute_start_time(); + auto compute_end_time = std::chrono::steady_clock::now(); + batch.finalize(predict_err); + auto end_time = std::chrono::steady_clock::now(); + + report_statistics( + *instance, request_count, start_time, compute_start_time, compute_end_time, end_time); + } catch (TritonException& err) { + result = err.error(); } -}}}} + + return result; +} +} // namespace triton_api +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/api/initialize.hpp b/cpp/include/rapids_triton/triton/api/initialize.hpp index 3cf7cc3..cccc054 100644 --- a/cpp/include/rapids_triton/triton/api/initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/initialize.hpp @@ -17,27 +17,33 @@ #pragma once #include -#include #include #include #include +#include -namespace triton { namespace backend { namespace rapids { namespace triton_api { - inline auto* initialize(TRITONBACKEND_Backend* backend) { - auto* result = static_cast(nullptr); - try { - auto name = get_backend_name(*backend); +namespace triton { +namespace backend { +namespace rapids { +namespace triton_api { +inline auto* initialize(TRITONBACKEND_Backend* backend) +{ + auto* result = static_cast(nullptr); + try { + auto name = get_backend_name(*backend); - log_info(__FILE__, __LINE__) << "TRITONBACKEND_Initialize: " << name; + log_info(__FILE__, __LINE__) << "TRITONBACKEND_Initialize: " << name; - if (!check_backend_version(*backend)) { - throw TritonException{ - Error::Unsupported, - "triton backend API version does not support this backend"}; - } - } catch (TritonException& err) { - result = err.error(); + if (!check_backend_version(*backend)) { + throw TritonException{Error::Unsupported, + "triton backend API version does not support this backend"}; } - return result; + } catch (TritonException& err) { + result = err.error(); } -}}}} + return result; +} +} // namespace triton_api +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/api/instance_finalize.hpp b/cpp/include/rapids_triton/triton/api/instance_finalize.hpp index d5b0dc7..2d1797c 100644 --- a/cpp/include/rapids_triton/triton/api/instance_finalize.hpp +++ b/cpp/include/rapids_triton/triton/api/instance_finalize.hpp @@ -15,29 +15,35 @@ */ #pragma once +#include #include #include #include -#include -namespace triton { namespace backend { namespace rapids { namespace triton_api { - template - auto* instance_finalize(TRITONBACKEND_ModelInstance* instance) { - auto* result = static_cast(nullptr); - try { - auto* instance_state = - get_instance_state(*instance); - if (instance_state != nullptr) { - instance_state->unload(); +namespace triton { +namespace backend { +namespace rapids { +namespace triton_api { +template +auto* instance_finalize(TRITONBACKEND_ModelInstance* instance) +{ + auto* result = static_cast(nullptr); + try { + auto* instance_state = get_instance_state(*instance); + if (instance_state != nullptr) { + instance_state->unload(); - log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInstanceFinalize: delete instance state"; + log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInstanceFinalize: delete instance state"; - delete instance_state; - } - } catch (TritonException& err) { - result = err.error(); + delete instance_state; } - - return result; + } catch (TritonException& err) { + result = err.error(); } -}}}} + + return result; +} +} // namespace triton_api +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp index 46a3962..0efab9d 100644 --- a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp @@ -16,42 +16,43 @@ #pragma once +#include +#include #include #include #include #include -#include -#include -namespace triton { namespace backend { namespace rapids { namespace triton_api { - template - auto* instance_initialize(TRITONBACKEND_ModelInstance* instance) { - auto* result = static_cast(nullptr); - try { - auto name = get_model_instance_name(*instance); - auto device_id = get_device_id(*instance); - auto deployment_type = get_deployment_type(*instance); +namespace triton { +namespace backend { +namespace rapids { +namespace triton_api { +template +auto* instance_initialize(TRITONBACKEND_ModelInstance* instance) +{ + auto* result = static_cast(nullptr); + try { + auto name = get_model_instance_name(*instance); + auto device_id = get_device_id(*instance); + auto deployment_type = get_deployment_type(*instance); - log_info(__FILE__, __LINE__) - << "TRITONBACKEND_ModelInstanceInitialize: " - << name - << " (" - << TRITONSERVER_InstanceGroupKindString(deployment_type) - << " device " - << device_id - << ")"; + log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInstanceInitialize: " << name << " (" + << TRITONSERVER_InstanceGroupKindString(deployment_type) + << " device " << device_id << ")"; - auto* triton_model = get_model_from_instance(*instance); - auto* model_state = get_model_state(*triton_model); + auto* triton_model = get_model_from_instance(*instance); + auto* model_state = get_model_state(*triton_model); - auto rapids_model = - std::make_unique(*model_state, instance); - rapids_model->load(); + auto rapids_model = std::make_unique(*model_state, instance); + rapids_model->load(); - set_instance_state(*instance, std::move(rapids_model)); - } catch (TritonException& err) { - result = err.error(); - } - return result; + set_instance_state(*instance, std::move(rapids_model)); + } catch (TritonException& err) { + result = err.error(); } -}}}} + return result; +} +} // namespace triton_api +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/api/model_finalize.hpp b/cpp/include/rapids_triton/triton/api/model_finalize.hpp index 5846033..f4d6d6a 100644 --- a/cpp/include/rapids_triton/triton/api/model_finalize.hpp +++ b/cpp/include/rapids_triton/triton/api/model_finalize.hpp @@ -15,29 +15,34 @@ */ #pragma once +#include +#include #include #include #include -#include -#include - -namespace triton { namespace backend { namespace rapids { namespace triton_api { - template - auto* model_finalize(TRITONBACKEND_Model* model) { - auto* result = static_cast(nullptr); - try { - auto model_state = get_model_state(*model); - if (model_state != nullptr) { - model_state->get_shared_state()->unload(); - } - log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelFinalize: delete model state"; +namespace triton { +namespace backend { +namespace rapids { +namespace triton_api { +template +auto* model_finalize(TRITONBACKEND_Model* model) +{ + auto* result = static_cast(nullptr); + try { + auto model_state = get_model_state(*model); + if (model_state != nullptr) { model_state->get_shared_state()->unload(); } - delete model_state; - } catch (TritonException& err) { - result = err.error(); - } + log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelFinalize: delete model state"; - return result; + delete model_state; + } catch (TritonException& err) { + result = err.error(); } -}}}} + + return result; +} +} // namespace triton_api +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/api/model_initialize.hpp b/cpp/include/rapids_triton/triton/api/model_initialize.hpp index ac18195..aa48013 100644 --- a/cpp/include/rapids_triton/triton/api/model_initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/model_initialize.hpp @@ -15,35 +15,39 @@ */ #pragma once +#include +#include #include #include #include -#include -#include - -namespace triton { namespace backend { namespace rapids { namespace triton_api { - template - auto* model_initialize(TRITONBACKEND_Model* model) { - auto* result = static_cast(nullptr); - try { - auto name = get_model_name(*model); - auto version = get_model_version(*model); +namespace triton { +namespace backend { +namespace rapids { +namespace triton_api { +template +auto* model_initialize(TRITONBACKEND_Model* model) +{ + auto* result = static_cast(nullptr); + try { + auto name = get_model_name(*model); - log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInitialize: " - << name - << " (version " - << version - << ")"; + auto version = get_model_version(*model); - auto rapids_model_state = std::make_unique(*model); - rapids_model_state->load(); + log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInitialize: " << name << " (version " + << version << ")"; - set_model_state(*model, std::move(rapids_model_state)); - } catch (TritonException& err) { - result = err.error(); - } + auto rapids_model_state = std::make_unique(*model); + rapids_model_state->load(); - return result; + set_model_state(*model, std::move(rapids_model_state)); + } catch (TritonException& err) { + result = err.error(); } -}}}} + + return result; +} +} // namespace triton_api +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/backend.hpp b/cpp/include/rapids_triton/triton/backend.hpp index b5629e5..414b212 100644 --- a/cpp/include/rapids_triton/triton/backend.hpp +++ b/cpp/include/rapids_triton/triton/backend.hpp @@ -26,50 +26,46 @@ #pragma once +#include #include -#include #include #include -#include +#include -namespace triton { namespace backend { namespace rapids { - inline auto - get_backend_name(TRITONBACKEND_Backend& backend) - { - const char* cname; - triton_check(TRITONBACKEND_BackendName(&backend, &cname)); - return std::string(cname); - } +namespace triton { +namespace backend { +namespace rapids { +inline auto get_backend_name(TRITONBACKEND_Backend& backend) +{ + const char* cname; + triton_check(TRITONBACKEND_BackendName(&backend, &cname)); + return std::string(cname); +} - namespace { - struct backend_version { - std::uint32_t major; - std::uint32_t minor; - }; - } +namespace { +struct backend_version { + std::uint32_t major; + std::uint32_t minor; +}; +} // namespace - inline auto - check_backend_version(TRITONBACKEND_Backend& backend) - { - auto version = backend_version{}; - triton_check(TRITONBACKEND_ApiVersion(&version.major, &version.minor)); +inline auto check_backend_version(TRITONBACKEND_Backend& backend) +{ + auto version = backend_version{}; + triton_check(TRITONBACKEND_ApiVersion(&version.major, &version.minor)); - log_info(__FILE__, __LINE__) << "Triton TRITONBACKEND API version: " - << version.major - << "." - << version.minor; + log_info(__FILE__, __LINE__) << "Triton TRITONBACKEND API version: " << version.major << "." + << version.minor; - auto name = get_backend_name(backend); + auto name = get_backend_name(backend); - log_info(__FILE__, __LINE__) << "'" - << name - << "' TRITONBACKEND API version: " - << TRITONBACKEND_API_VERSION_MAJOR - << "." - << TRITONBACKEND_API_VERSION_MINOR; + log_info(__FILE__, __LINE__) << "'" << name + << "' TRITONBACKEND API version: " << TRITONBACKEND_API_VERSION_MAJOR + << "." << TRITONBACKEND_API_VERSION_MINOR; - return ( - (version.major == TRITONBACKEND_API_VERSION_MAJOR) && - (version.minor >= TRITONBACKEND_API_VERSION_MINOR)); - } -}}} // namespace triton::backend::rapids + return ((version.major == TRITONBACKEND_API_VERSION_MAJOR) && + (version.minor >= TRITONBACKEND_API_VERSION_MINOR)); +} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/config.hpp b/cpp/include/rapids_triton/triton/config.hpp index 393599b..4a8300b 100644 --- a/cpp/include/rapids_triton/triton/config.hpp +++ b/cpp/include/rapids_triton/triton/config.hpp @@ -19,14 +19,19 @@ #include #include +#include #include #include -#include -namespace triton { namespace backend { namespace rapids { - inline auto get_max_batch_size(common::TritonJson::Value& config) { - auto reported = int64_t{}; - triton_check(config.MemberAsInt("max_batch_size", &reported)); - return narrow(reported); - } -}}} // namespace triton::backend::rapids +namespace triton { +namespace backend { +namespace rapids { +inline auto get_max_batch_size(common::TritonJson::Value& config) +{ + auto reported = int64_t{}; + triton_check(config.MemberAsInt("max_batch_size", &reported)); + return narrow(reported); +} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/deployment.hpp b/cpp/include/rapids_triton/triton/deployment.hpp index 48aa4d5..e663c47 100644 --- a/cpp/include/rapids_triton/triton/deployment.hpp +++ b/cpp/include/rapids_triton/triton/deployment.hpp @@ -17,11 +17,15 @@ #pragma once #include -namespace triton { namespace backend { namespace rapids { - using DeploymentType = TRITONSERVER_InstanceGroupKind; - auto constexpr GPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_GPU; - auto constexpr CPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_CPU; - // Note (wphicks): We currently are not including "Auto" or "Model" because I - // am not sure exactly how those would be used in context. If there is a - // demand, they can be added. -}}} +namespace triton { +namespace backend { +namespace rapids { +using DeploymentType = TRITONSERVER_InstanceGroupKind; +auto constexpr GPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_GPU; +auto constexpr CPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_CPU; +// Note (wphicks): We currently are not including "Auto" or "Model" because I +// am not sure exactly how those would be used in context. If there is a +// demand, they can be added. +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/device.hpp b/cpp/include/rapids_triton/triton/device.hpp index 257a6ef..73b28c4 100644 --- a/cpp/include/rapids_triton/triton/device.hpp +++ b/cpp/include/rapids_triton/triton/device.hpp @@ -17,8 +17,10 @@ #pragma once #include -namespace triton { namespace backend { namespace rapids { - using device_id_t = std::int32_t; -}}} // namespace triton::backend::rapids - - +namespace triton { +namespace backend { +namespace rapids { +using device_id_t = std::int32_t; +} +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/input.hpp b/cpp/include/rapids_triton/triton/input.hpp index f2d7ddd..668875a 100644 --- a/cpp/include/rapids_triton/triton/input.hpp +++ b/cpp/include/rapids_triton/triton/input.hpp @@ -17,70 +17,64 @@ #pragma once #include +#include #include #include -#include -#include -#include #include #include #include -#include +#include +#include +#include -namespace triton { namespace backend { namespace rapids { - inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) { - auto result = static_cast(nullptr); - triton_check( - TRITONBACKEND_RequestInput(request, name.c_str(), &result)); - return result; - } +namespace triton { +namespace backend { +namespace rapids { +inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) +{ + auto result = static_cast(nullptr); + triton_check(TRITONBACKEND_RequestInput(request, name.c_str(), &result)); + return result; +} - template - auto get_triton_input_shape(Iter requests_begin, Iter requests_end, std::string const& name) { - auto result = std::vector{}; +template +auto get_triton_input_shape(Iter requests_begin, Iter requests_end, std::string const& name) +{ + auto result = std::vector{}; - auto reported_dtype = DType{}; - auto const* input_shape = static_cast(nullptr); - auto input_dims = uint32_t{}; + auto reported_dtype = DType{}; + auto const* input_shape = static_cast(nullptr); + auto input_dims = uint32_t{}; - auto batch_dim = std::accumulate( - requests_begin, - requests_end, - int64_t{}, - [&reported_dtype, &input_shape, &input_dims, &name](auto total, auto& request) { - auto* input = get_triton_input(request, name); - triton_check( - TRITONBACKEND_InputProperties( - input, nullptr, &reported_dtype, &input_shape, - &input_dims, nullptr, nullptr)); + auto batch_dim = std::accumulate( + requests_begin, + requests_end, + int64_t{}, + [&reported_dtype, &input_shape, &input_dims, &name](auto total, auto& request) { + auto* input = get_triton_input(request, name); + triton_check(TRITONBACKEND_InputProperties( + input, nullptr, &reported_dtype, &input_shape, &input_dims, nullptr, nullptr)); - if (reported_dtype != TritonDtype::value) { - auto log_stream = std::stringstream{}; - log_stream << "incorrect type " - << reported_dtype - << " for input with required type " - << TritonDtype::value; - throw(TritonException(Error::Internal, log_stream.str())); - } + if (reported_dtype != TritonDtype::value) { + auto log_stream = std::stringstream{}; + log_stream << "incorrect type " << reported_dtype << " for input with required type " + << TritonDtype::value; + throw(TritonException(Error::Internal, log_stream.str())); + } - if (input_dims != 0) { - total += *input_shape; - } - return total; + if (input_dims != 0) { total += *input_shape; } + return total; }); - result.reserve(input_dims); - std::transform( - input_shape, - input_shape + input_dims, - std::back_inserter(result), - [](auto& val) { return narrow(val); } - ); + result.reserve(input_dims); + std::transform(input_shape, input_shape + input_dims, std::back_inserter(result), [](auto& val) { + return narrow(val); + }); - if (!result.empty()) { - result[0] = narrow(batch_dim); - } + if (!result.empty()) { result[0] = narrow(batch_dim); } - return result; - } -}}} + return result; +} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/logging.hpp b/cpp/include/rapids_triton/triton/logging.hpp index dbefd68..96455f1 100644 --- a/cpp/include/rapids_triton/triton/logging.hpp +++ b/cpp/include/rapids_triton/triton/logging.hpp @@ -15,29 +15,38 @@ */ #pragma once -#include #include +#include #include #include #include -namespace triton { namespace backend { namespace rapids { +namespace triton { +namespace backend { +namespace rapids { - namespace { - /** Log message at indicated level */ - inline void log( - TRITONSERVER_LogLevel level, char const* filename, int line, - char const* message) { - triton_check(TRITONSERVER_LogMessage(level, filename, line, message)); - } - } +namespace { +/** Log message at indicated level */ +inline void log(TRITONSERVER_LogLevel level, char const* filename, int line, char const* message) +{ + triton_check(TRITONSERVER_LogMessage(level, filename, line, message)); +} +} // namespace - struct log_stream : public std::ostream { - log_stream(TRITONSERVER_LogLevel level, char const* filename, int line) : std::ostream{}, buffer_{level, filename, line} { rdbuf(&buffer_); } - log_stream(TRITONSERVER_LogLevel level) : std::ostream{}, buffer_{level, __FILE__, __LINE__} { rdbuf(&buffer_); } +struct log_stream : public std::ostream { + log_stream(TRITONSERVER_LogLevel level, char const* filename, int line) + : std::ostream{}, buffer_{level, filename, line} + { + rdbuf(&buffer_); + } + log_stream(TRITONSERVER_LogLevel level) : std::ostream{}, buffer_{level, __FILE__, __LINE__} + { + rdbuf(&buffer_); + } - ~log_stream() { + ~log_stream() + { try { flush(); } catch (std::ios_base::failure const& ignored_err) { @@ -45,110 +54,106 @@ namespace triton { namespace backend { namespace rapids { } } - private: - struct log_buffer : public std::stringbuf { - log_buffer(TRITONSERVER_LogLevel level, char const* filename, int line) : level_{level}, filename_{filename}, line_{line} {} + private: + struct log_buffer : public std::stringbuf { + log_buffer(TRITONSERVER_LogLevel level, char const* filename, int line) + : level_{level}, filename_{filename}, line_{line} + { + } - virtual int sync() { - auto msg = str(); - if(!msg.empty()) { - log(level_, filename_, line_, msg.c_str()); - str(""); - } - return 0; + virtual int sync() + { + auto msg = str(); + if (!msg.empty()) { + log(level_, filename_, line_, msg.c_str()); + str(""); } + return 0; + } - private: - TRITONSERVER_LogLevel level_; - char const* filename_; - int line_; - }; - - log_buffer buffer_; + private: + TRITONSERVER_LogLevel level_; + char const* filename_; + int line_; }; - - /** Log message at INFO level */ - inline void log_info(char const* filename, int line, char const* message) { - log(TRITONSERVER_LOG_INFO, filename, line, message); - } - inline void log_info(char const* filename, int line, std::string const& message) { - log_info(filename, line, message.c_str()); - } - inline void log_info(char const* message) { - log_info(__FILE__, __LINE__, message); - } - inline void log_info(std::string const& message) { - log_info(__FILE__, __LINE__, message.c_str()); - } - inline auto log_info(char const* filename, int line) { - return log_stream(TRITONSERVER_LOG_INFO, filename, line); - } - inline auto log_info() { - return log_stream(TRITONSERVER_LOG_INFO); - } - - - /** Log message at WARN level */ - inline void log_warn(char const* filename, int line, char const* message) { - log(TRITONSERVER_LOG_WARN, filename, line, message); - } - inline void log_warn(char const* filename, int line, std::string const& message) { - log_warn(filename, line, message.c_str()); - } - inline void log_warn(char const* message) { - log_warn(__FILE__, __LINE__, message); - } - inline void log_warn(std::string const& message) { - log_warn(__FILE__, __LINE__, message.c_str()); - } - inline auto log_warn(char const* filename, int line) { - return log_stream(TRITONSERVER_LOG_WARN, filename, line); - } - inline auto log_warn() { - return log_stream(TRITONSERVER_LOG_WARN); - } - - - /** Log message at ERROR level */ - inline void log_error(char const* filename, int line, char const* message) { - log(TRITONSERVER_LOG_ERROR, filename, line, message); - } - inline void log_error(char const* filename, int line, std::string const& message) { - log_error(filename, line, message.c_str()); - } - inline void log_error(char const* message) { - log_error(__FILE__, __LINE__, message); - } - inline void log_error(std::string const& message) { - log_error(__FILE__, __LINE__, message.c_str()); - } - inline auto log_error(char const* filename, int line) { - return log_stream(TRITONSERVER_LOG_ERROR, filename, line); - } - inline auto log_error() { - return log_stream(TRITONSERVER_LOG_ERROR); - } - - - /** Log message at VERBOSE level */ - inline void log_debug(char const* filename, int line, char const* message) { - log(TRITONSERVER_LOG_VERBOSE, filename, line, message); - } - inline void log_debug(char const* filename, int line, std::string const& message) { - log_debug(filename, line, message.c_str()); - } - inline void log_debug(char const* message) { - log_debug(__FILE__, __LINE__, message); - } - inline void log_debug(std::string const& message) { - log_debug(__FILE__, __LINE__, message.c_str()); - } - inline auto log_debug(char const* filename, int line) { - return log_stream(TRITONSERVER_LOG_VERBOSE, filename, line); - } - inline auto log_debug() { - return log_stream(TRITONSERVER_LOG_VERBOSE); - } - -}}} // namespace triton::backend::rapids + log_buffer buffer_; +}; + +/** Log message at INFO level */ +inline void log_info(char const* filename, int line, char const* message) +{ + log(TRITONSERVER_LOG_INFO, filename, line, message); +} +inline void log_info(char const* filename, int line, std::string const& message) +{ + log_info(filename, line, message.c_str()); +} +inline void log_info(char const* message) { log_info(__FILE__, __LINE__, message); } +inline void log_info(std::string const& message) { log_info(__FILE__, __LINE__, message.c_str()); } +inline auto log_info(char const* filename, int line) +{ + return log_stream(TRITONSERVER_LOG_INFO, filename, line); +} +inline auto log_info() { return log_stream(TRITONSERVER_LOG_INFO); } + +/** Log message at WARN level */ +inline void log_warn(char const* filename, int line, char const* message) +{ + log(TRITONSERVER_LOG_WARN, filename, line, message); +} +inline void log_warn(char const* filename, int line, std::string const& message) +{ + log_warn(filename, line, message.c_str()); +} +inline void log_warn(char const* message) { log_warn(__FILE__, __LINE__, message); } +inline void log_warn(std::string const& message) { log_warn(__FILE__, __LINE__, message.c_str()); } +inline auto log_warn(char const* filename, int line) +{ + return log_stream(TRITONSERVER_LOG_WARN, filename, line); +} +inline auto log_warn() { return log_stream(TRITONSERVER_LOG_WARN); } + +/** Log message at ERROR level */ +inline void log_error(char const* filename, int line, char const* message) +{ + log(TRITONSERVER_LOG_ERROR, filename, line, message); +} +inline void log_error(char const* filename, int line, std::string const& message) +{ + log_error(filename, line, message.c_str()); +} +inline void log_error(char const* message) { log_error(__FILE__, __LINE__, message); } +inline void log_error(std::string const& message) +{ + log_error(__FILE__, __LINE__, message.c_str()); +} +inline auto log_error(char const* filename, int line) +{ + return log_stream(TRITONSERVER_LOG_ERROR, filename, line); +} +inline auto log_error() { return log_stream(TRITONSERVER_LOG_ERROR); } + +/** Log message at VERBOSE level */ +inline void log_debug(char const* filename, int line, char const* message) +{ + log(TRITONSERVER_LOG_VERBOSE, filename, line, message); +} +inline void log_debug(char const* filename, int line, std::string const& message) +{ + log_debug(filename, line, message.c_str()); +} +inline void log_debug(char const* message) { log_debug(__FILE__, __LINE__, message); } +inline void log_debug(std::string const& message) +{ + log_debug(__FILE__, __LINE__, message.c_str()); +} +inline auto log_debug(char const* filename, int line) +{ + return log_stream(TRITONSERVER_LOG_VERBOSE, filename, line); +} +inline auto log_debug() { return log_stream(TRITONSERVER_LOG_VERBOSE); } + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/model.hpp b/cpp/include/rapids_triton/triton/model.hpp index 3ea61bf..384d428 100644 --- a/cpp/include/rapids_triton/triton/model.hpp +++ b/cpp/include/rapids_triton/triton/model.hpp @@ -25,53 +25,46 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once +#include +#include #include #include #include -#include #include -#include -#include +#include -namespace triton { namespace backend { namespace rapids { +namespace triton { +namespace backend { +namespace rapids { -inline auto -get_model_version(TRITONBACKEND_Model& model) +inline auto get_model_version(TRITONBACKEND_Model& model) { auto version = std::uint64_t{}; triton_check(TRITONBACKEND_ModelVersion(&model, &version)); return version; } -inline auto -get_model_name(TRITONBACKEND_Model& model) +inline auto get_model_name(TRITONBACKEND_Model& model) { auto* cname = static_cast(nullptr); triton_check(TRITONBACKEND_ModelName(&model, &cname)); return std::string(cname); } -inline -auto -get_model_config(TRITONBACKEND_Model& model) +inline auto get_model_config(TRITONBACKEND_Model& model) { auto* config_message = static_cast(nullptr); triton_check(TRITONBACKEND_ModelConfig(&model, 1, &config_message)); - auto* buffer = static_cast(nullptr); + auto* buffer = static_cast(nullptr); auto byte_size = std::size_t{}; - triton_check( - TRITONSERVER_MessageSerializeToJson(config_message, &buffer, &byte_size)); + triton_check(TRITONSERVER_MessageSerializeToJson(config_message, &buffer, &byte_size)); auto model_config = std::make_unique(); - auto* err = model_config->Parse(buffer, byte_size); - auto* result = TRITONSERVER_MessageDelete(config_message); - if (err != nullptr) { - throw(TritonException(err)); - } - if (result != nullptr) { - throw(TritonException(result)); - } + auto* err = model_config->Parse(buffer, byte_size); + auto* result = TRITONSERVER_MessageDelete(config_message); + if (err != nullptr) { throw(TritonException(err)); } + if (result != nullptr) { throw(TritonException(result)); } return model_config; } @@ -85,18 +78,14 @@ get_model_config(TRITONBACKEND_Model& model) * SharedModelState and provide additional interface compatibility. */ template -void -set_model_state( - TRITONBACKEND_Model& model, std::unique_ptr&& model_state) +void set_model_state(TRITONBACKEND_Model& model, std::unique_ptr&& model_state) { - triton_check(TRITONBACKEND_ModelSetState( - &model, reinterpret_cast(model_state.release()))); + triton_check(TRITONBACKEND_ModelSetState(&model, reinterpret_cast(model_state.release()))); } /** Given a model, return its associated ModelState object */ template -auto* -get_model_state(TRITONBACKEND_Model& model) +auto* get_model_state(TRITONBACKEND_Model& model) { auto vstate = static_cast(nullptr); triton_check(TRITONBACKEND_ModelState(&model, &vstate)); @@ -106,4 +95,6 @@ get_model_state(TRITONBACKEND_Model& model) return model_state; } -}}} // namespace triton::backend::rapids +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/model_instance.hpp b/cpp/include/rapids_triton/triton/model_instance.hpp index 01e7727..66c3ac2 100644 --- a/cpp/include/rapids_triton/triton/model_instance.hpp +++ b/cpp/include/rapids_triton/triton/model_instance.hpp @@ -25,86 +25,83 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #pragma once +#include #include -#include #include #include #include -#include +#include -namespace triton { namespace backend { namespace rapids { - /** Get the name of a Triton model instance from the instance itself */ - inline auto - get_model_instance_name(TRITONBACKEND_ModelInstance& instance) - { - auto cname = static_cast(nullptr); - triton_check(TRITONBACKEND_ModelInstanceName(&instance, &cname)); - return std::string(cname); - } +namespace triton { +namespace backend { +namespace rapids { +/** Get the name of a Triton model instance from the instance itself */ +inline auto get_model_instance_name(TRITONBACKEND_ModelInstance& instance) +{ + auto cname = static_cast(nullptr); + triton_check(TRITONBACKEND_ModelInstanceName(&instance, &cname)); + return std::string(cname); +} - /** Get the device on which a Triton model instance is loaded - * - * If this instance is loaded on the host, 0 will be returned. Otherwise the - * GPU device id will be returned.*/ - inline auto - get_device_id(TRITONBACKEND_ModelInstance& instance) - { - auto device_id = device_id_t{}; - triton_check(TRITONBACKEND_ModelInstanceDeviceId(&instance, &device_id)); - return device_id; - } +/** Get the device on which a Triton model instance is loaded + * + * If this instance is loaded on the host, 0 will be returned. Otherwise the + * GPU device id will be returned.*/ +inline auto get_device_id(TRITONBACKEND_ModelInstance& instance) +{ + auto device_id = device_id_t{}; + triton_check(TRITONBACKEND_ModelInstanceDeviceId(&instance, &device_id)); + return device_id; +} - /** Determine how a Triton model instance is deployed - * - * Returns enum value indicating whether the instance is deployed on device - * or on the host - */ - inline auto - get_deployment_type(TRITONBACKEND_ModelInstance& instance) - { - auto kind = GPUDeployment; - triton_check(TRITONBACKEND_ModelInstanceKind(&instance, &kind)); - return kind; - } +/** Determine how a Triton model instance is deployed + * + * Returns enum value indicating whether the instance is deployed on device + * or on the host + */ +inline auto get_deployment_type(TRITONBACKEND_ModelInstance& instance) +{ + auto kind = GPUDeployment; + triton_check(TRITONBACKEND_ModelInstanceKind(&instance, &kind)); + return kind; +} - /** Return the Triton model from one of its instances - */ - inline auto* - get_model_from_instance(TRITONBACKEND_ModelInstance& instance) - { - auto* model = static_cast(nullptr); - triton_check(TRITONBACKEND_ModelInstanceModel(&instance, &model)); - return model; - } +/** Return the Triton model from one of its instances + */ +inline auto* get_model_from_instance(TRITONBACKEND_ModelInstance& instance) +{ + auto* model = static_cast(nullptr); + triton_check(TRITONBACKEND_ModelInstanceModel(&instance, &model)); + return model; +} - /** - * @brief Set Triton model instance state to given object - * - * This function accepts a unique_ptr to an object derived from a Triton - * BackendModelInstance object and sets it as the stored state for a model in the - * Triton server. Note that this object is not the same as a RAPIDS-Triton - * "Model" object. The object that Triton expects must wrap this Model and - * provide additional interface compatibility. - */ - template - void - set_instance_state( - TRITONBACKEND_ModelInstance& instance, - std::unique_ptr&& model_instance_state) - { - triton_check(TRITONBACKEND_ModelInstanceSetState( - &instance, reinterpret_cast(model_instance_state.release()))); - } +/** + * @brief Set Triton model instance state to given object + * + * This function accepts a unique_ptr to an object derived from a Triton + * BackendModelInstance object and sets it as the stored state for a model in the + * Triton server. Note that this object is not the same as a RAPIDS-Triton + * "Model" object. The object that Triton expects must wrap this Model and + * provide additional interface compatibility. + */ +template +void set_instance_state(TRITONBACKEND_ModelInstance& instance, + std::unique_ptr&& model_instance_state) +{ + triton_check(TRITONBACKEND_ModelInstanceSetState( + &instance, reinterpret_cast(model_instance_state.release()))); +} - /** Get model instance state from instance */ - template - auto* - get_instance_state(TRITONBACKEND_ModelInstance& instance) - { - auto instance_state = static_cast(nullptr); - triton_check(TRITONBACKEND_ModelInstanceState( - &instance, reinterpret_cast(&instance_state))); - return instance_state; - } +/** Get model instance state from instance */ +template +auto* get_instance_state(TRITONBACKEND_ModelInstance& instance) +{ + auto instance_state = static_cast(nullptr); + triton_check( + TRITONBACKEND_ModelInstanceState(&instance, reinterpret_cast(&instance_state))); + return instance_state; +} -}}} // namespace triton::backend::rapids +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/model_instance_state.hpp b/cpp/include/rapids_triton/triton/model_instance_state.hpp index aaca523..ba05009 100644 --- a/cpp/include/rapids_triton/triton/model_instance_state.hpp +++ b/cpp/include/rapids_triton/triton/model_instance_state.hpp @@ -15,24 +15,30 @@ */ #pragma once +#include #include #include #include #include -#include -namespace triton { namespace backend { namespace rapids { +namespace triton { +namespace backend { +namespace rapids { -template +template struct ModelInstanceState : public BackendModelInstance { ModelInstanceState(TritonModelState& model_state, TRITONBACKEND_ModelInstance* triton_model_instance) - : BackendModelInstance(&model_state, triton_model_instance), - model_(model_state.get_shared_state(), - rapids::get_device_id(*triton_model_instance), CudaStream(), - Kind(), - JoinPath({model_state.RepositoryPath(), std::to_string(model_state.Version()), - ArtifactFilename()})) {} + : BackendModelInstance(&model_state, triton_model_instance), + model_(model_state.get_shared_state(), + rapids::get_device_id(*triton_model_instance), + CudaStream(), + Kind(), + JoinPath({model_state.RepositoryPath(), + std::to_string(model_state.Version()), + ArtifactFilename()})) + { + } auto& get_model() const { return model_; } @@ -43,4 +49,6 @@ struct ModelInstanceState : public BackendModelInstance { RapidsModel model_; }; -}}} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/model_state.hpp b/cpp/include/rapids_triton/triton/model_state.hpp index 44804e5..582b679 100644 --- a/cpp/include/rapids_triton/triton/model_state.hpp +++ b/cpp/include/rapids_triton/triton/model_state.hpp @@ -15,18 +15,20 @@ */ #pragma once +#include #include #include -#include -namespace triton { namespace backend { namespace rapids { -template +namespace triton { +namespace backend { +namespace rapids { +template struct TritonModelState : public BackendModel { - TritonModelState(TRITONBACKEND_Model& triton_model) - : BackendModel(&triton_model), - state_{std::make_shared( - get_model_config(triton_model))} {} + : BackendModel(&triton_model), + state_{std::make_shared(get_model_config(triton_model))} + { + } void load() { state_->load(); } void unload() { state_->unload(); } @@ -37,4 +39,6 @@ struct TritonModelState : public BackendModel { std::shared_ptr state_; }; -}}} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/output.hpp b/cpp/include/rapids_triton/triton/output.hpp index 5b258d1..b4910c8 100644 --- a/cpp/include/rapids_triton/triton/output.hpp +++ b/cpp/include/rapids_triton/triton/output.hpp @@ -16,63 +16,57 @@ #pragma once -#include #include +#include -namespace triton { namespace backend { namespace rapids { - inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) { - auto result = static_cast(nullptr); - triton_check( - TRITONBACKEND_RequestInput(request, name.c_str(), &result)); - return result; - } +namespace triton { +namespace backend { +namespace rapids { +inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) +{ + auto result = static_cast(nullptr); + triton_check(TRITONBACKEND_RequestInput(request, name.c_str(), &result)); + return result; +} - template - auto get_triton_output_shape(Iter requests_begin, Iter requests_end, std::string const& name) { - auto result = std::vector{}; +template +auto get_triton_output_shape(Iter requests_begin, Iter requests_end, std::string const& name) +{ + auto result = std::vector{}; - auto reported_dtype = DType{}; - auto const* input_shape = static_cast(nullptr); - auto input_dims = uint32_t{}; + auto reported_dtype = DType{}; + auto const* input_shape = static_cast(nullptr); + auto input_dims = uint32_t{}; - auto batch_dim = std::reduce( - requests_begin, - requests_end, - int64_t{}, - [&reported_dtype, &input_shape, &input_dims, &name](auto& request, auto total) { - auto* input = get_triton_input(request, name); - triton_check( - TRITONBACKEND_InputProperties( - input, nullptr, &reported_dtype, &input_shape, - &input_dims, nullptr, nullptr)); + auto batch_dim = + std::reduce(requests_begin, + requests_end, + int64_t{}, + [&reported_dtype, &input_shape, &input_dims, &name](auto& request, auto total) { + auto* input = get_triton_input(request, name); + triton_check(TRITONBACKEND_InputProperties( + input, nullptr, &reported_dtype, &input_shape, &input_dims, nullptr, nullptr)); - if (reported_dtype != TritonDtype::value) { - auto log_stream = std::stringstream{}; - log_stream << "incorrect type " - << reported_dtype - << " for output with required type " - << TritonDtype::value; - throw(TritonException(Error::Internal, log_stream.str())); - } + if (reported_dtype != TritonDtype::value) { + auto log_stream = std::stringstream{}; + log_stream << "incorrect type " << reported_dtype + << " for output with required type " << TritonDtype::value; + throw(TritonException(Error::Internal, log_stream.str())); + } - if (input_dims != 0) { - total += *input_shape; - } - return total; - }); + if (input_dims != 0) { total += *input_shape; } + return total; + }); - result.reserve(input_dims); - std::transform( - input_shape, - input_shape + input_dims, - std::back_inserter(result), - [](auto& val) { return narrow(val); } - ); + result.reserve(input_dims); + std::transform(input_shape, input_shape + input_dims, std::back_inserter(result), [](auto& val) { + return narrow(val); + }); - if (!result.empty()) { - result[0] = narrow(batch_dim); - } + if (!result.empty()) { result[0] = narrow(batch_dim); } - return result; - } -}}} + return result; +} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/requests.hpp b/cpp/include/rapids_triton/triton/requests.hpp index 8f9ed05..c24c8bb 100644 --- a/cpp/include/rapids_triton/triton/requests.hpp +++ b/cpp/include/rapids_triton/triton/requests.hpp @@ -22,20 +22,22 @@ #include #include -namespace triton { namespace backend { namespace rapids { - using request_size_t = uint32_t; - - template - void release_requests(Iter begin, Iter end) { - std::for_each(begin, end, [](auto& request) { - try { - triton_check(TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL)); - } catch (TritonException& err) { - log_error(__FILE__, __LINE__, err.what()); - } - }); - } -}}} // namespace triton::backend::rapids - - +namespace triton { +namespace backend { +namespace rapids { +using request_size_t = uint32_t; +template +void release_requests(Iter begin, Iter end) +{ + std::for_each(begin, end, [](auto& request) { + try { + triton_check(TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL)); + } catch (TritonException& err) { + log_error(__FILE__, __LINE__, err.what()); + } + }); +} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/responses.hpp b/cpp/include/rapids_triton/triton/responses.hpp index 5de2ac3..b4309f0 100644 --- a/cpp/include/rapids_triton/triton/responses.hpp +++ b/cpp/include/rapids_triton/triton/responses.hpp @@ -15,14 +15,16 @@ */ #pragma once +#include #include #include -#include #include #include -#include +#include -namespace triton { namespace backend { namespace rapids { +namespace triton { +namespace backend { +namespace rapids { template auto construct_responses(Iter requests_begin, Iter requests_end) @@ -32,39 +34,38 @@ auto construct_responses(Iter requests_begin, Iter requests_end) auto requests_size = std::distance(requests_begin, requests_end); if (!(requests_size > 0)) { throw TritonException(Error::Internal, - "Invalid iterators for requests when constructing responses"); + "Invalid iterators for requests when constructing responses"); } responses.reserve(requests_size); - std::transform( - requests_begin, requests_end, std::back_inserter(responses), - [](auto* request) { - auto* response = static_cast(nullptr); - triton_check(TRITONBACKEND_ResponseNew(&response, request)); - return response; - }); + std::transform(requests_begin, requests_end, std::back_inserter(responses), [](auto* request) { + auto* response = static_cast(nullptr); + triton_check(TRITONBACKEND_ResponseNew(&response, request)); + return response; + }); return responses; } template -void send_responses(Iter begin, Iter end, TRITONSERVER_Error* err) { +void send_responses(Iter begin, Iter end, TRITONSERVER_Error* err) +{ std::for_each(begin, end, [err](auto& response) { decltype(err) err_copy; if (err != nullptr) { - err_copy = TRITONSERVER_ErrorNew( - TRITONSERVER_ErrorCode(err), TRITONSERVER_ErrorMessage(err)); + err_copy = TRITONSERVER_ErrorNew(TRITONSERVER_ErrorCode(err), TRITONSERVER_ErrorMessage(err)); } else { err_copy = err; } try { - triton_check(TRITONBACKEND_ResponseSend(response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, err_copy)); + triton_check( + TRITONBACKEND_ResponseSend(response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, err_copy)); } catch (TritonException& err) { log_error(__FILE__, __LINE__, err.what()); } }); } -}}} // namespace triton::backend::rapids - - +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/triton/statistics.hpp b/cpp/include/rapids_triton/triton/statistics.hpp index bb4f685..5f4f130 100644 --- a/cpp/include/rapids_triton/triton/statistics.hpp +++ b/cpp/include/rapids_triton/triton/statistics.hpp @@ -16,64 +16,74 @@ #pragma once +#include #include #include #include -#include -namespace triton { namespace backend { namespace rapids { - using time_point = std::chrono::time_point; +namespace triton { +namespace backend { +namespace rapids { +using time_point = std::chrono::time_point; - /** - * @brief Report inference statistics for a single request - * - * @param instance The Triton model instance which is processing this request - * @param request The Triton request object itself - * @param start_time The time at which the backend first received the request - * @param compute_start_time The time at which the backend began actual - * inference on the request - * @param compute_end_time The time at which the backend completed inference - * on the request - * @param end_time The time at which the backend finished all processing on - * the request, including copying out results and returning a response - */ - inline void report_statistics(TRITONBACKEND_ModelInstance& instance, - TRITONBACKEND_Request& request, time_point start_time, time_point - compute_start_time, time_point compute_end_time, time_point end_time) { - triton_check(TRITONBACKEND_ModelInstanceReportStatistics( - &instance, - &request, - true, - start_time.time_since_epoch().count(), - compute_start_time.time_since_epoch().count(), - compute_end_time.time_since_epoch().count(), - end_time.time_since_epoch().count() - )); - } +/** + * @brief Report inference statistics for a single request + * + * @param instance The Triton model instance which is processing this request + * @param request The Triton request object itself + * @param start_time The time at which the backend first received the request + * @param compute_start_time The time at which the backend began actual + * inference on the request + * @param compute_end_time The time at which the backend completed inference + * on the request + * @param end_time The time at which the backend finished all processing on + * the request, including copying out results and returning a response + */ +inline void report_statistics(TRITONBACKEND_ModelInstance& instance, + TRITONBACKEND_Request& request, + time_point start_time, + time_point compute_start_time, + time_point compute_end_time, + time_point end_time) +{ + triton_check( + TRITONBACKEND_ModelInstanceReportStatistics(&instance, + &request, + true, + start_time.time_since_epoch().count(), + compute_start_time.time_since_epoch().count(), + compute_end_time.time_since_epoch().count(), + end_time.time_since_epoch().count())); +} - /** - * @brief Report inference statistics for a batch of requests of given size - * - * @param instance The Triton model instance which is processing this batch - * @param request_count The number of requests in this batch - * @param start_time The time at which the backend first received the batch - * @param compute_start_time The time at which the backend began actual - * inference on the batch - * @param compute_end_time The time at which the backend completed inference - * on the batch - * @param end_time The time at which the backend finished all processing on - * the batch, including copying out results and returning a response - */ - inline void report_statistics(TRITONBACKEND_ModelInstance& instance, - std::size_t request_count, time_point start_time, time_point - compute_start_time, time_point compute_end_time, time_point end_time) { - triton_check(TRITONBACKEND_ModelInstanceReportBatchStatistics( - &instance, - request_count, - start_time.time_since_epoch().count(), - compute_start_time.time_since_epoch().count(), - compute_end_time.time_since_epoch().count(), - end_time.time_since_epoch().count() - )); - } -}}} +/** + * @brief Report inference statistics for a batch of requests of given size + * + * @param instance The Triton model instance which is processing this batch + * @param request_count The number of requests in this batch + * @param start_time The time at which the backend first received the batch + * @param compute_start_time The time at which the backend began actual + * inference on the batch + * @param compute_end_time The time at which the backend completed inference + * on the batch + * @param end_time The time at which the backend finished all processing on + * the batch, including copying out results and returning a response + */ +inline void report_statistics(TRITONBACKEND_ModelInstance& instance, + std::size_t request_count, + time_point start_time, + time_point compute_start_time, + time_point compute_end_time, + time_point end_time) +{ + triton_check( + TRITONBACKEND_ModelInstanceReportBatchStatistics(&instance, + request_count, + start_time.time_since_epoch().count(), + compute_start_time.time_since_epoch().count(), + compute_end_time.time_since_epoch().count(), + end_time.time_since_epoch().count())); +} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/utils/const_agnostic.hpp b/cpp/include/rapids_triton/utils/const_agnostic.hpp index da3acad..f9e1308 100644 --- a/cpp/include/rapids_triton/utils/const_agnostic.hpp +++ b/cpp/include/rapids_triton/utils/const_agnostic.hpp @@ -17,9 +17,14 @@ #pragma once #include -namespace triton { namespace backend { namespace rapids { +namespace triton { +namespace backend { +namespace rapids { template -using const_agnostic_same_t = std::enable_if_t, std::remove_const_t>::value>; +using const_agnostic_same_t = + std::enable_if_t, std::remove_const_t>::value>; -}}} // namespace triton::backend::rapids +} +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/utils/narrow.hpp b/cpp/include/rapids_triton/utils/narrow.hpp index 0d74cff..37465c0 100644 --- a/cpp/include/rapids_triton/utils/narrow.hpp +++ b/cpp/include/rapids_triton/utils/narrow.hpp @@ -18,22 +18,24 @@ #include #include -namespace triton { namespace backend { namespace rapids { +namespace triton { +namespace backend { +namespace rapids { template -auto -narrow(F from) +auto narrow(F from) { auto to = static_cast(from); if (static_cast(to) != from || (std::is_signed::value && !std::is_signed::value && from < F{}) || (std::is_signed::value && !std::is_signed::value && to < T{}) || - (std::is_signed::value == std::is_signed::value && - ((to < T{}) != (from < F{})))) { + (std::is_signed::value == std::is_signed::value && ((to < T{}) != (from < F{})))) { throw TritonException(Error::Internal, "invalid narrowing"); } return to; } -}}} // namespace triton::backend::fil +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/src/model.h b/cpp/src/model.h index 0ca87bc..1232221 100644 --- a/cpp/src/model.h +++ b/cpp/src/model.h @@ -44,12 +44,14 @@ struct RapidsModel : rapids::Model { * implementation. **************************************************************************/ RapidsModel(std::shared_ptr shared_state, - rapids::device_id_t device_id, cudaStream_t default_stream, + rapids::device_id_t device_id, + cudaStream_t default_stream, rapids::DeploymentType deployment_type, std::string const& filepath) - : rapids::Model(shared_state, device_id, - default_stream, deployment_type, - filepath) {} + : rapids::Model( + shared_state, device_id, default_stream, deployment_type, filepath) + { + } /*************************************************************************** * BASIC FEATURES * @@ -76,7 +78,8 @@ struct RapidsModel : rapids::Model { * pointer to the underlying data. * 4. Call the `finalize` method on all output tensors. **************************************************************************/ - void predict(rapids::Batch& batch) const { + void predict(rapids::Batch& batch) const + { // 1. Acquire a tensor representing the input named "input__0" auto input = get_input(batch, "input__0"); // 2. Acquire a tensor representing the output named "output__0" @@ -138,8 +141,8 @@ struct RapidsModel : rapids::Model { * Valid MemoryType options to return are rapids::HostMemory and * rapids::DeviceMemory. **************************************************************************/ - std::optional preferred_mem_type( - rapids::Batch& batch) const { + std::optional preferred_mem_type(rapids::Batch& batch) const + { return std::nullopt; } }; diff --git a/cpp/src/shared_state.h b/cpp/src/shared_state.h index 6de0397..b6f8981 100644 --- a/cpp/src/shared_state.h +++ b/cpp/src/shared_state.h @@ -38,7 +38,9 @@ namespace NAMESPACE { struct RapidsSharedState : rapids::SharedModelState { RapidsSharedState(std::unique_ptr&& config) - : rapids::SharedModelState{std::move(config)} {} + : rapids::SharedModelState{std::move(config)} + { + } void load() {} void unload() {} }; diff --git a/cpp/test/build_control.cpp b/cpp/test/build_control.cpp index 4696fad..18392b0 100644 --- a/cpp/test/build_control.cpp +++ b/cpp/test/build_control.cpp @@ -22,7 +22,8 @@ namespace triton { namespace backend { namespace rapids { -TEST(RapidsTriton, build_control) { +TEST(RapidsTriton, build_control) +{ #ifdef TRITON_ENABLE_GPU ASSERT_EQ(IS_GPU_BUILD, true) << "IS_GPU_BUILD constant has wrong value\n"; #else diff --git a/cpp/test/exceptions.cpp b/cpp/test/exceptions.cpp index 09ad631..bc0fc59 100644 --- a/cpp/test/exceptions.cpp +++ b/cpp/test/exceptions.cpp @@ -24,16 +24,17 @@ namespace triton { namespace backend { namespace rapids { -TEST(RapidsTriton, default_except) { +TEST(RapidsTriton, default_except) +{ try { throw TritonException(); } catch (TritonException const& err) { - EXPECT_EQ(std::string(err.what()), - std::string("encountered unknown error")); + EXPECT_EQ(std::string(err.what()), std::string("encountered unknown error")); } } -TEST(RapidsTriton, msg_except) { +TEST(RapidsTriton, msg_except) +{ auto msg = std::string("TEST ERROR MESSAGE"); try { throw TritonException(Error::Internal, msg); @@ -56,17 +57,16 @@ TEST(RapidsTriton, msg_except) { } } -TEST(RapidsTriton, triton_check) { +TEST(RapidsTriton, triton_check) +{ auto msg = std::string("TEST ERROR MESSAGE"); - EXPECT_THROW( - triton_check(TRITONSERVER_ErrorNew(Error::Internal, msg.c_str())), - TritonException); + EXPECT_THROW(triton_check(TRITONSERVER_ErrorNew(Error::Internal, msg.c_str())), TritonException); triton_check(nullptr); } -TEST(RapidsTriton, cuda_check) { - EXPECT_THROW(cuda_check(cudaError::cudaErrorMissingConfiguration), - TritonException); +TEST(RapidsTriton, cuda_check) +{ + EXPECT_THROW(cuda_check(cudaError::cudaErrorMissingConfiguration), TritonException); cuda_check(cudaError::cudaSuccess); } diff --git a/cpp/test/memory/buffer.cpp b/cpp/test/memory/buffer.cpp index b320178..1867f52 100644 --- a/cpp/test/memory/buffer.cpp +++ b/cpp/test/memory/buffer.cpp @@ -28,7 +28,8 @@ namespace triton { namespace backend { namespace rapids { -TEST(RapidsTriton, default_buffer) { +TEST(RapidsTriton, default_buffer) +{ auto buffer = Buffer(); EXPECT_EQ(buffer.mem_type(), HostMemory); EXPECT_EQ(buffer.size(), 0); @@ -44,7 +45,8 @@ TEST(RapidsTriton, default_buffer) { } } -TEST(RapidsTriton, device_buffer) { +TEST(RapidsTriton, device_buffer) +{ auto data = std::vector{1, 2, 3}; if (IS_GPU_BUILD) { auto buffer = Buffer(data.size(), DeviceMemory, 0, 0); @@ -55,10 +57,12 @@ TEST(RapidsTriton, device_buffer) { auto data_out = std::vector(data.size()); cudaMemcpy(static_cast(buffer.data()), - static_cast(data.data()), sizeof(int) * data.size(), + static_cast(data.data()), + sizeof(int) * data.size(), cudaMemcpyHostToDevice); cudaMemcpy(static_cast(data_out.data()), - static_cast(buffer.data()), sizeof(int) * data.size(), + static_cast(buffer.data()), + sizeof(int) * data.size(), cudaMemcpyDeviceToHost); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); @@ -67,13 +71,16 @@ TEST(RapidsTriton, device_buffer) { } } -TEST(RapidsTriton, non_owning_device_buffer) { +TEST(RapidsTriton, non_owning_device_buffer) +{ auto data = std::vector{1, 2, 3}; if constexpr (IS_GPU_BUILD) { auto* ptr_d = static_cast(nullptr); cudaMalloc(reinterpret_cast(&ptr_d), sizeof(int) * data.size()); - cudaMemcpy(static_cast(ptr_d), static_cast(data.data()), - sizeof(int) * data.size(), cudaMemcpyHostToDevice); + cudaMemcpy(static_cast(ptr_d), + static_cast(data.data()), + sizeof(int) * data.size(), + cudaMemcpyHostToDevice); auto buffer = Buffer(ptr_d, data.size(), DeviceMemory); ASSERT_EQ(buffer.mem_type(), DeviceMemory); @@ -82,78 +89,79 @@ TEST(RapidsTriton, non_owning_device_buffer) { auto data_out = std::vector(data.size()); cudaMemcpy(static_cast(data_out.data()), - static_cast(buffer.data()), sizeof(int) * data.size(), + static_cast(buffer.data()), + sizeof(int) * data.size(), cudaMemcpyDeviceToHost); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); cudaFree(reinterpret_cast(ptr_d)); } else { - ASSERT_THROW(Buffer(data.data(), data.size(), DeviceMemory), - TritonException); + ASSERT_THROW(Buffer(data.data(), data.size(), DeviceMemory), TritonException); } } -TEST(RapidsTriton, host_buffer) { - auto data = std::vector{1, 2, 3}; +TEST(RapidsTriton, host_buffer) +{ + auto data = std::vector{1, 2, 3}; auto buffer = Buffer(data.size(), HostMemory, 0, 0); ASSERT_EQ(buffer.mem_type(), HostMemory); ASSERT_EQ(buffer.size(), data.size()); ASSERT_NE(buffer.data(), nullptr); - std::memcpy(static_cast(buffer.data()), - static_cast(data.data()), data.size() * sizeof(int)); + std::memcpy( + static_cast(buffer.data()), static_cast(data.data()), data.size() * sizeof(int)); - auto data_out = - std::vector(buffer.data(), buffer.data() + buffer.size()); + auto data_out = std::vector(buffer.data(), buffer.data() + buffer.size()); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, non_owning_host_buffer) { - auto data = std::vector{1, 2, 3}; +TEST(RapidsTriton, non_owning_host_buffer) +{ + auto data = std::vector{1, 2, 3}; auto buffer = Buffer(data.data(), data.size(), HostMemory); ASSERT_EQ(buffer.mem_type(), HostMemory); ASSERT_EQ(buffer.size(), data.size()); ASSERT_EQ(buffer.data(), data.data()); - auto data_out = - std::vector(buffer.data(), buffer.data() + buffer.size()); + auto data_out = std::vector(buffer.data(), buffer.data() + buffer.size()); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, copy_buffer) { - auto data = std::vector{1, 2, 3}; +TEST(RapidsTriton, copy_buffer) +{ + auto data = std::vector{1, 2, 3}; auto orig_buffer = Buffer(data.data(), data.size(), HostMemory); - auto buffer = Buffer(orig_buffer); + auto buffer = Buffer(orig_buffer); ASSERT_EQ(buffer.mem_type(), HostMemory); ASSERT_EQ(buffer.size(), data.size()); ASSERT_NE(buffer.data(), orig_buffer.data()); - auto data_out = - std::vector(buffer.data(), buffer.data() + buffer.size()); + auto data_out = std::vector(buffer.data(), buffer.data() + buffer.size()); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, move_buffer) { - auto data = std::vector{1, 2, 3}; +TEST(RapidsTriton, move_buffer) +{ + auto data = std::vector{1, 2, 3}; auto buffer = Buffer(Buffer(data.data(), data.size(), HostMemory)); ASSERT_EQ(buffer.mem_type(), HostMemory); ASSERT_EQ(buffer.size(), data.size()); ASSERT_EQ(buffer.data(), data.data()); - auto data_out = - std::vector(buffer.data(), buffer.data() + buffer.size()); + auto data_out = std::vector(buffer.data(), buffer.data() + buffer.size()); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, move_assignment_buffer) { +TEST(RapidsTriton, move_assignment_buffer) +{ auto data = std::vector{1, 2, 3}; auto buffer = Buffer{data.data(), data.size() - 1, DeviceMemory}; - buffer = Buffer{data.size(), HostMemory}; + buffer = Buffer{data.size(), HostMemory}; ASSERT_EQ(buffer.mem_type(), HostMemory); ASSERT_EQ(buffer.size(), data.size()); diff --git a/cpp/test/memory/detail/allocate.cpp b/cpp/test/memory/detail/allocate.cpp index dcfce83..ff315a5 100644 --- a/cpp/test/memory/detail/allocate.cpp +++ b/cpp/test/memory/detail/allocate.cpp @@ -26,21 +26,25 @@ namespace triton { namespace backend { namespace rapids { -TEST(RapidsTriton, dev_allocate) { +TEST(RapidsTriton, dev_allocate) +{ auto data = std::vector{1, 2, 3}; if constexpr (IS_GPU_BUILD) { - auto ptr_d = detail::dev_allocate(data.size(), 0); + auto ptr_d = detail::dev_allocate(data.size(), 0); auto data_out = std::vector(data.size()); - cudaMemcpy(static_cast(ptr_d), static_cast(data.data()), - sizeof(int) * data.size(), cudaMemcpyHostToDevice); - cudaMemcpy(static_cast(data_out.data()), static_cast(ptr_d), - sizeof(int) * data.size(), cudaMemcpyDeviceToHost); + cudaMemcpy(static_cast(ptr_d), + static_cast(data.data()), + sizeof(int) * data.size(), + cudaMemcpyHostToDevice); + cudaMemcpy(static_cast(data_out.data()), + static_cast(ptr_d), + sizeof(int) * data.size(), + cudaMemcpyDeviceToHost); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); detail::dev_deallocater{}(ptr_d); } else { - EXPECT_THROW( - [&data]() { return detail::dev_allocate(data.size(), 0); }(), - TritonException); + EXPECT_THROW([&data]() { return detail::dev_allocate(data.size(), 0); }(), + TritonException); } } diff --git a/cpp/test/memory/detail/copy.cpp b/cpp/test/memory/detail/copy.cpp index beb2fe2..ec29e22 100644 --- a/cpp/test/memory/detail/copy.cpp +++ b/cpp/test/memory/detail/copy.cpp @@ -27,36 +27,39 @@ namespace triton { namespace backend { namespace rapids { -TEST(RapidsTriton, dev_copy) { - auto data = std::vector{1, 2, 3}; +TEST(RapidsTriton, dev_copy) +{ + auto data = std::vector{1, 2, 3}; auto data_out = std::vector(data.size()); if constexpr (IS_GPU_BUILD) { auto* ptr_d = static_cast(nullptr); cudaMalloc(reinterpret_cast(&ptr_d), sizeof(int) * data.size()); detail::dev_copy(ptr_d, data.data(), data.size(), 0); - cudaMemcpy(static_cast(data_out.data()), static_cast(ptr_d), - sizeof(int) * data.size(), cudaMemcpyDeviceToHost); + cudaMemcpy(static_cast(data_out.data()), + static_cast(ptr_d), + sizeof(int) * data.size(), + cudaMemcpyDeviceToHost); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); cudaFree(reinterpret_cast(ptr_d)); } else { - ASSERT_THROW(detail::dev_copy(data_out.data(), data.data(), data.size(), 0), - TritonException); + ASSERT_THROW(detail::dev_copy(data_out.data(), data.data(), data.size(), 0), TritonException); } } -TEST(RapidsTriton, host_copy) { - auto data = std::vector{1, 2, 3}; +TEST(RapidsTriton, host_copy) +{ + auto data = std::vector{1, 2, 3}; auto data_out = std::vector(data.size()); detail::host_copy(data_out.data(), data.data(), data.size()); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, copy) { - auto data = std::vector{1, 2, 3}; +TEST(RapidsTriton, copy) +{ + auto data = std::vector{1, 2, 3}; auto data_out = std::vector(data.size()); - detail::copy(data_out.data(), data.data(), data.size(), 0, HostMemory, - HostMemory); + detail::copy(data_out.data(), data.data(), data.size(), 0, HostMemory, HostMemory); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); data_out = std::vector(data.size()); @@ -65,17 +68,19 @@ TEST(RapidsTriton, copy) { cudaMalloc(reinterpret_cast(&ptr_d), sizeof(int) * data.size()); detail::copy(ptr_d, data.data(), data.size(), 0, DeviceMemory, HostMemory); - cudaMemcpy(static_cast(data_out.data()), static_cast(ptr_d), - sizeof(int) * data.size(), cudaMemcpyDeviceToHost); + cudaMemcpy(static_cast(data_out.data()), + static_cast(ptr_d), + sizeof(int) * data.size(), + cudaMemcpyDeviceToHost); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); cudaFree(reinterpret_cast(ptr_d)); } else { - EXPECT_THROW(detail::copy(data_out.data(), data.data(), data.size(), 0, - HostMemory, DeviceMemory), - TritonException); - EXPECT_THROW(detail::copy(data_out.data(), data.data(), data.size(), 0, - DeviceMemory, HostMemory), - TritonException); + EXPECT_THROW( + detail::copy(data_out.data(), data.data(), data.size(), 0, HostMemory, DeviceMemory), + TritonException); + EXPECT_THROW( + detail::copy(data_out.data(), data.data(), data.size(), 0, DeviceMemory, HostMemory), + TritonException); } } diff --git a/cpp/test/tensor/dtype.cpp b/cpp/test/tensor/dtype.cpp index 52a78f4..ef1a9ca 100644 --- a/cpp/test/tensor/dtype.cpp +++ b/cpp/test/tensor/dtype.cpp @@ -24,12 +24,14 @@ namespace backend { namespace rapids { template -void check_dtype_conversion() { +void check_dtype_conversion() +{ EXPECT_EQ(D, TritonDtype::type>::value); EXPECT_EQ(D, TritonDtype::type const>::value); } -TEST(RapidsTriton, dtype) { +TEST(RapidsTriton, dtype) +{ check_dtype_conversion(); check_dtype_conversion(); check_dtype_conversion(); diff --git a/cpp/test/tensor/tensor.cpp b/cpp/test/tensor/tensor.cpp index 7842f6c..d6b53f4 100644 --- a/cpp/test/tensor/tensor.cpp +++ b/cpp/test/tensor/tensor.cpp @@ -26,17 +26,18 @@ namespace triton { namespace backend { namespace rapids { -TEST(RapidsTriton, default_tensor) { +TEST(RapidsTriton, default_tensor) +{ auto tensor = Tensor(); EXPECT_EQ(tensor.buffer().size(), 0); EXPECT_EQ(tensor.shape().size(), 0); } -TEST(RapidsTriton, move_buffer_tensor) { - auto shape = std::vector{2, 2}; - auto data = std::vector{1, 2, 3, 4}; - auto tensor = - Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); +TEST(RapidsTriton, move_buffer_tensor) +{ + auto shape = std::vector{2, 2}; + auto data = std::vector{1, 2, 3, 4}; + auto tensor = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); EXPECT_EQ(data.data(), tensor.data()); EXPECT_EQ(data.size(), tensor.size()); @@ -47,89 +48,84 @@ TEST(RapidsTriton, move_buffer_tensor) { EXPECT_EQ(tensor.stream(), cudaStream_t{}); EXPECT_EQ(tensor.device(), 0); - auto data_out = - std::vector(tensor.data(), tensor.data() + tensor.size()); + auto data_out = std::vector(tensor.data(), tensor.data() + tensor.size()); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, multi_buffer_tensor) { +TEST(RapidsTriton, multi_buffer_tensor) +{ auto shape = std::vector{2, 2}; - auto data = std::vector{1, 2, 3, 4}; + auto data = std::vector{1, 2, 3, 4}; auto all_buffers = std::vector>{}; all_buffers.reserve(data.size()); - std::transform(data.begin(), data.end(), std::back_inserter(all_buffers), - [](auto& elem) { - return Buffer{&elem, 1, DeviceMemory}; - }); - auto tensor = Tensor(shape, all_buffers.begin(), all_buffers.end(), - DeviceMemory, 0, cudaStream_t{}); + std::transform(data.begin(), data.end(), std::back_inserter(all_buffers), [](auto& elem) { + return Buffer{&elem, 1, DeviceMemory}; + }); + auto tensor = + Tensor(shape, all_buffers.begin(), all_buffers.end(), DeviceMemory, 0, cudaStream_t{}); auto data_out = std::vector(data.size()); cudaMemcpy(static_cast(data_out.data()), - static_cast(tensor.data()), sizeof(int) * tensor.size(), + static_cast(tensor.data()), + sizeof(int) * tensor.size(), cudaMemcpyDeviceToHost); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, tensor_copy) { +TEST(RapidsTriton, tensor_copy) +{ auto shape = std::vector{2, 2}; - auto data = std::vector{1, 2, 3, 4}; + auto data = std::vector{1, 2, 3, 4}; - auto data1 = data; - auto tensor1 = - Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); - auto data2 = std::vector(data1.size()); - auto tensor2 = - Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); + auto data1 = data; + auto tensor1 = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); + auto data2 = std::vector(data1.size()); + auto tensor2 = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); copy(tensor2, tensor1); - auto data_out = - std::vector(tensor2.data(), tensor2.data() + tensor2.size()); + auto data_out = std::vector(tensor2.data(), tensor2.data() + tensor2.size()); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); auto small_shape = std::vector{2}; - auto small_data = std::vector(2); - auto tensor3 = Tensor( - small_shape, - Buffer{small_data.data(), small_data.size(), HostMemory}); + auto small_data = std::vector(2); + auto tensor3 = + Tensor(small_shape, Buffer{small_data.data(), small_data.size(), HostMemory}); EXPECT_THROW(copy(tensor3, tensor1), TritonException); } -TEST(RapidsTriton, tensor_multi_copy) { +TEST(RapidsTriton, tensor_multi_copy) +{ auto shape = std::vector{2, 2}; - auto data = std::vector{1, 2, 3, 4}; + auto data = std::vector{1, 2, 3, 4}; - auto data1 = data; - auto tensor1 = - Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); + auto data1 = data; + auto tensor1 = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); auto receiver_shape = std::vector{1}; - auto receivers = std::vector>{}; + auto receivers = std::vector>{}; receivers.reserve(data.size()); - std::transform(data.begin(), data.end(), std::back_inserter(receivers), - [&receiver_shape](auto& val) { - return Tensor(receiver_shape, - Buffer{std::size_t{1}, HostMemory}); - }); + std::transform( + data.begin(), data.end(), std::back_inserter(receivers), [&receiver_shape](auto& val) { + return Tensor(receiver_shape, Buffer{std::size_t{1}, HostMemory}); + }); rapids::copy(receivers.begin(), receivers.end(), tensor1); auto data_out = std::vector{}; data_out.reserve(receivers.size()); - std::transform(receivers.begin(), receivers.end(), - std::back_inserter(data_out), - [](auto& tensor) { return *tensor.data(); }); + std::transform( + receivers.begin(), receivers.end(), std::back_inserter(data_out), [](auto& tensor) { + return *tensor.data(); + }); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); // Throw if trying to copy to too many outputs - receivers.emplace_back(receiver_shape, - Buffer{std::size_t{1}, HostMemory}); - EXPECT_THROW(rapids::copy(receivers.begin(), receivers.end(), tensor1), - TritonException); + receivers.emplace_back(receiver_shape, Buffer{std::size_t{1}, HostMemory}); + EXPECT_THROW(rapids::copy(receivers.begin(), receivers.end(), tensor1), TritonException); } } // namespace rapids diff --git a/cpp/test/test.cpp b/cpp/test/test.cpp index 26e60e3..4a7d7f4 100644 --- a/cpp/test/test.cpp +++ b/cpp/test/test.cpp @@ -18,8 +18,11 @@ #include -namespace triton { namespace backend { namespace rapids { +namespace triton { +namespace backend { +namespace rapids { TEST(RapidsTriton, installed) { std::cout << test_install() << "\n"; } -}}} - +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/test/triton/logging.cpp b/cpp/test/triton/logging.cpp index e4dc37e..f1f60cd 100644 --- a/cpp/test/triton/logging.cpp +++ b/cpp/test/triton/logging.cpp @@ -22,14 +22,16 @@ namespace triton { namespace backend { namespace rapids { -TEST(RapidsTriton, logging) { +TEST(RapidsTriton, logging) +{ log_debug("Debug test message"); log_info("Info test message"); log_warn("Warn test message"); log_error("Error test message"); } -TEST(RapidsTriton, stream_logging) { +TEST(RapidsTriton, stream_logging) +{ log_debug() << "Streamed debug test message"; log_info() << "Streamed info test message"; log_warn() << "Streamed warn test message"; diff --git a/cpp/test/utils/const_agnostic.cpp b/cpp/test/utils/const_agnostic.cpp index 592ecc6..2a74c69 100644 --- a/cpp/test/utils/const_agnostic.cpp +++ b/cpp/test/utils/const_agnostic.cpp @@ -22,9 +22,9 @@ namespace triton { namespace backend { namespace rapids { -TEST(RapidsTriton, const_agnostic) { - static_assert( - std::is_same, void>::value); +TEST(RapidsTriton, const_agnostic) +{ + static_assert(std::is_same, void>::value); static_assert(std::is_same, void>::value); } diff --git a/cpp/test/utils/narrow.cpp b/cpp/test/utils/narrow.cpp index e0543d4..9263e6e 100644 --- a/cpp/test/utils/narrow.cpp +++ b/cpp/test/utils/narrow.cpp @@ -23,11 +23,11 @@ namespace triton { namespace backend { namespace rapids { -TEST(RapidsTriton, narrow) { +TEST(RapidsTriton, narrow) +{ EXPECT_THROW(narrow(-1), TritonException); narrow(int{5}); - EXPECT_THROW(narrow(std::numeric_limits::max()), - TritonException); + EXPECT_THROW(narrow(std::numeric_limits::max()), TritonException); narrow(std::size_t{5}); } From cf75925ab862eb93c94d6674df010016245716d9 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 20 Sep 2021 10:50:26 -0400 Subject: [PATCH 063/199] Use remove_const_t where useful Co-authored-by: Divye Gala --- cpp/include/rapids_triton/memory/detail/allocate.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/memory/detail/allocate.hpp b/cpp/include/rapids_triton/memory/detail/allocate.hpp index 460f92f..1db6f64 100644 --- a/cpp/include/rapids_triton/memory/detail/allocate.hpp +++ b/cpp/include/rapids_triton/memory/detail/allocate.hpp @@ -38,7 +38,7 @@ struct dev_deallocater { // in a RAII context. If we are deallocating this memory, we allocated it // and made it const. Removing the const qualifier allows the // deallocation to proceed. - cudaFree(reinterpret_cast(const_cast::type*>(d_ptr))); + cudaFree(reinterpret_cast(const_cast*>(d_ptr))); } else { log_error( __FILE__, __LINE__, "ERROR: device deallocation cannot be performed in non-GPU build!"); From dc6f4fe71b9418d8c1a7124b91e093c2fb1b17e5 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 20 Sep 2021 10:51:10 -0400 Subject: [PATCH 064/199] Use is_same_v where useful Co-authored-by: Divye Gala --- cpp/include/rapids_triton/model/shared_state.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp index 71c2f10..524ddbe 100644 --- a/cpp/include/rapids_triton/model/shared_state.hpp +++ b/cpp/include/rapids_triton/model/shared_state.hpp @@ -127,7 +127,7 @@ struct SharedModelState { auto input_stream = std::istringstream{string_repr}; - if (std::is_same::value) { + if constexpr (std::is_same_v) { input_stream >> std::boolalpha >> result; } else { input_stream >> result; From a83f1365fdae3d82b1d0410d96818c7f790febfd Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 20 Sep 2021 10:53:41 -0400 Subject: [PATCH 065/199] Use is_same_v where useful Co-authored-by: Divye Gala --- cpp/include/rapids_triton/utils/const_agnostic.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/utils/const_agnostic.hpp b/cpp/include/rapids_triton/utils/const_agnostic.hpp index f9e1308..cd55aeb 100644 --- a/cpp/include/rapids_triton/utils/const_agnostic.hpp +++ b/cpp/include/rapids_triton/utils/const_agnostic.hpp @@ -23,7 +23,7 @@ namespace rapids { template using const_agnostic_same_t = - std::enable_if_t, std::remove_const_t>::value>; + std::enable_if_t, std::remove_const_t>>; } } // namespace backend From 4fc3028f4be7aacafdc1fecb1d59aac201a97891 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 20 Sep 2021 11:57:37 -0400 Subject: [PATCH 066/199] Use braced initialization more consistently --- cpp/include/rapids_triton/batch/batch.hpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cpp/include/rapids_triton/batch/batch.hpp b/cpp/include/rapids_triton/batch/batch.hpp index 2811e44..d96ef72 100644 --- a/cpp/include/rapids_triton/batch/batch.hpp +++ b/cpp/include/rapids_triton/batch/batch.hpp @@ -77,8 +77,8 @@ struct Batch { bool use_pinned_output, size_type max_batch_size, cudaStream_t stream) - : requests_(raw_requests, raw_requests + count), - responses_(construct_responses(requests_.begin(), requests_.end())), + : requests_{raw_requests, raw_requests + count}, + responses_{construct_responses(requests_.begin(), requests_.end())}, get_output_shape_{get_output_shape}, report_statistics_{report_request_statistics}, collector_(raw_requests, count, &responses_, &triton_mem_manager, use_pinned_input, stream), @@ -106,8 +106,6 @@ struct Batch { auto input_batch_dim = size_type{}; if (result.size() > 0) { input_batch_dim = result[0]; - } else { - input_batch_dim = size_type{}; } if (batch_size_.has_value()) { From 17b8499e195ec87cb57b938321b71b009c9eb234 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 20 Sep 2021 11:57:59 -0400 Subject: [PATCH 067/199] Add C++ linting infrastructure --- cpp/.clang-format | 164 ++++++++++++++++++++++++++++++++ cpp/scripts/run-clang-format.py | 148 ++++++++++++++++++++++++++++ 2 files changed, 312 insertions(+) create mode 100644 cpp/.clang-format create mode 100755 cpp/scripts/run-clang-format.py diff --git a/cpp/.clang-format b/cpp/.clang-format new file mode 100644 index 0000000..0c05436 --- /dev/null +++ b/cpp/.clang-format @@ -0,0 +1,164 @@ +--- +# Refer to the following link for the explanation of each params: +# http://releases.llvm.org/8.0.0/tools/clang/docs/ClangFormatStyleOptions.html +Language: Cpp +# BasedOnStyle: Google +AccessModifierOffset: -1 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: true +AlignConsecutiveBitFields: true +AlignConsecutiveDeclarations: false +AlignConsecutiveMacros: true +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: true +AllowShortCaseLabelsOnASingleLine: true +AllowShortEnumsOnASingleLine: true +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: true +AllowShortLambdasOnASingleLine: true +AllowShortLoopsOnASingleLine: false +# This is deprecated +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + # disabling the below splits, else, they'll just add to the vertical length of source files! + SplitEmptyFunction: false + SplitEmptyRecord: false + SplitEmptyNamespace: false +BreakAfterJavaFieldAnnotations: false +BreakBeforeBinaryOperators: None +BreakBeforeBraces: WebKit +BreakBeforeInheritanceComma: false +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakInheritanceList: BeforeColon +BreakStringLiterals: true +ColumnLimit: 100 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +# Kept the below 2 to be the same as `IndentWidth` to keep everything uniform +ConstructorInitializerIndentWidth: 2 +ContinuationIndentWidth: 2 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^' + Priority: 2 + - Regex: '^<.*\.h>' + Priority: 1 + - Regex: '^<.*' + Priority: 2 + - Regex: '.*' + Priority: 3 +IncludeIsMainRegex: '([-_](test|unittest))?$' +IndentCaseLabels: true +IndentPPDirectives: None +IndentWidth: 2 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Never +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +RawStringFormats: + - Language: Cpp + Delimiters: + - cc + - CC + - cpp + - Cpp + - CPP + - 'c++' + - 'C++' + CanonicalDelimiter: '' + - Language: TextProto + Delimiters: + - pb + - PB + - proto + - PROTO + EnclosingFunctions: + - EqualsProto + - EquivToProto + - PARSE_PARTIAL_TEXT_PROTO + - PARSE_TEST_PROTO + - PARSE_TEXT_PROTO + - ParseTextOrDie + - ParseTextProtoOrDie + CanonicalDelimiter: '' + BasedOnStyle: google +# Enabling comment reflow causes doxygen comments to be messed up in their formats! +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceBeforeSquareBrackets: false +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: c++17 +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +# Be consistent with indent-width, even for people who use tab for indentation! +TabWidth: 2 +UseTab: Never diff --git a/cpp/scripts/run-clang-format.py b/cpp/scripts/run-clang-format.py new file mode 100755 index 0000000..614697b --- /dev/null +++ b/cpp/scripts/run-clang-format.py @@ -0,0 +1,148 @@ +# Copyright (c) 2019-2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file was copied from the rapidsai/cuml repo + +from __future__ import print_function +import sys +import re +import os +import subprocess +import argparse +import tempfile +import shutil + + +EXPECTED_VERSION = "11.0.0" +VERSION_REGEX = re.compile(r"clang-format version ([0-9.]+)") +DEFAULT_DIRS = ["cpp/src", + "cpp/include", + "cpp/test"] + + +def parse_args(): + argparser = argparse.ArgumentParser("Runs clang-format on a project") + argparser.add_argument("-dstdir", type=str, default=None, + help="Directory to store the temporary outputs of" + " clang-format. If nothing is passed for this, then" + " a temporary dir will be created using `mkdtemp`") + argparser.add_argument("-exe", type=str, default="clang-format", + help="Path to clang-format exe") + argparser.add_argument("-inplace", default=False, action="store_true", + help="Replace the source files itself.") + argparser.add_argument("-regex", type=str, + default=r"[.](cu|cuh|h|hpp|cpp)$", + help="Regex string to filter in sources") + argparser.add_argument("-ignore", type=str, default=r"cannylab/bh[.]cu$", + help="Regex used to ignore files from matched list") + argparser.add_argument("-v", dest="verbose", action="store_true", + help="Print verbose messages") + argparser.add_argument("dirs", type=str, nargs="*", + help="List of dirs where to find sources") + args = argparser.parse_args() + args.regex_compiled = re.compile(args.regex) + args.ignore_compiled = re.compile(args.ignore) + if args.dstdir is None: + args.dstdir = tempfile.mkdtemp() + ret = subprocess.check_output("%s --version" % args.exe, shell=True) + ret = ret.decode("utf-8") + version = VERSION_REGEX.match(ret) + if version is None: + raise Exception("Failed to figure out clang-format version!") + version = version.group(1) + if version != EXPECTED_VERSION: + raise Exception("clang-format exe must be v%s found '%s'" % \ + (EXPECTED_VERSION, version)) + if len(args.dirs) == 0: + args.dirs = DEFAULT_DIRS + return args + + +def list_all_src_files(file_regex, ignore_regex, srcdirs, dstdir, inplace): + allFiles = [] + for srcdir in srcdirs: + for root, dirs, files in os.walk(srcdir): + for f in files: + if re.search(file_regex, f): + src = os.path.join(root, f) + if re.search(ignore_regex, src): + continue + if inplace: + _dir = root + else: + _dir = os.path.join(dstdir, root) + dst = os.path.join(_dir, f) + allFiles.append((src, dst)) + return allFiles + + +def run_clang_format(src, dst, exe, verbose): + dstdir = os.path.dirname(dst) + if not os.path.exists(dstdir): + os.makedirs(dstdir) + # run the clang format command itself + if src == dst: + cmd = "%s -i %s" % (exe, src) + else: + cmd = "%s %s > %s" % (exe, src, dst) + try: + subprocess.check_call(cmd, shell=True) + except subprocess.CalledProcessError: + print("Failed to run clang-format! Maybe your env is not proper?") + raise + # run the diff to check if there are any formatting issues + cmd = "diff -q %s %s >/dev/null" % (src, dst) + try: + subprocess.check_call(cmd, shell=True) + if verbose: + print("%s passed" % os.path.basename(src)) + except subprocess.CalledProcessError: + print("%s failed! 'diff %s %s' will show formatting violations!" % \ + (os.path.basename(src), src, dst)) + return False + return True + + +def main(): + args = parse_args() + # Attempt to making sure that we run this script from root of repo always + if not os.path.exists(".git"): + print("Error!! This needs to always be run from the root of repo") + sys.exit(-1) + all_files = list_all_src_files(args.regex_compiled, args.ignore_compiled, + args.dirs, args.dstdir, args.inplace) + + # Check whether clang-format exists + if shutil.which("clang-format") is None: + print("clang-format not found. Exiting...") + return + + # actual format checker + status = True + for src, dst in all_files: + if not run_clang_format(src, dst, args.exe, args.verbose): + status = False + if not status: + print("clang-format failed! You have 2 options:") + print(" 1. Look at formatting differences above and fix them manually") + print(" 2. Or run the below command to bulk-fix all these at once") + print("Bulk-fix command: ") + print(" python cpp/scripts/run-clang-format.py %s -inplace" % \ + " ".join(sys.argv[1:])) + sys.exit(-1) + return + + +if __name__ == "__main__": + main() From 31d60fcfdb8ac5156c80d343be0f2048dc3d6952 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 20 Sep 2021 14:47:10 -0400 Subject: [PATCH 068/199] Use braced initialization where possible --- cpp/include/rapids_triton/model/shared_state.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp index 524ddbe..f8570e0 100644 --- a/cpp/include/rapids_triton/model/shared_state.hpp +++ b/cpp/include/rapids_triton/model/shared_state.hpp @@ -43,7 +43,7 @@ struct SharedModelState { explicit SharedModelState(std::unique_ptr&& config) : config_{std::move(config)}, - max_batch_size_(get_max_batch_size(*config_)), + max_batch_size_{get_max_batch_size(*config_)}, output_shapes_([this]() { auto result = std::vector>>{}; auto output_entries = triton::common::TritonJson::Value{}; From b7787c1df5e9d999b7bff4811f8698be977297ba Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 20 Sep 2021 15:00:45 -0400 Subject: [PATCH 069/199] Correct copyright year in cpp/CMakeLists.txt Co-authored-by: Robert Maynard --- cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 169d73c..cab677c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,5 +1,5 @@ #============================================================================= -# Copyright (c) 2020-2021, NVIDIA CORPORATION. +# Copyright (c) 2021, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 78f219c2601e425171c5630c945f73beff7ae54c Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 20 Sep 2021 15:04:36 -0400 Subject: [PATCH 070/199] Make use of rapids_cmake_install_lib_dir --- cpp/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index cab677c..49447c1 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -125,8 +125,9 @@ INTERFACE target_compile_features(rapids_triton INTERFACE cxx_std_17 $) +rapids_cmake_install_lib_dir(lib_dir) install(TARGETS rapids_triton - DESTINATION lib + DESTINATION ${lib_dir} EXPORT rapids_triton-exports ) From b74694327911c9c1bf031758be29cfc785a1c6a7 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 20 Sep 2021 15:07:03 -0400 Subject: [PATCH 071/199] Add comment on thrust inclusion --- cpp/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 49447c1..d97a95e 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -97,6 +97,8 @@ include(cmake/modules/ConfigureCUDA.cmake) # add third party dependencies using CPM rapids_cpm_init() +# NOTE(wphicks): Thrust is not being properly included by an upstream +# dependency, so we include it here include(cmake/thirdparty/get_thrust.cmake) include(cmake/thirdparty/get_rmm.cmake) include(cmake/thirdparty/get_raft.cmake) From 467360e1469e68467f43cb8e9a92a4116c77d202 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 20 Sep 2021 15:11:21 -0400 Subject: [PATCH 072/199] Make use of rapids-cmake for gtest and rmm --- cpp/cmake/thirdparty/get_gtest.cmake | 29 ++++---------------------- cpp/cmake/thirdparty/get_rmm.cmake | 31 +++------------------------- 2 files changed, 7 insertions(+), 53 deletions(-) diff --git a/cpp/cmake/thirdparty/get_gtest.cmake b/cpp/cmake/thirdparty/get_gtest.cmake index 02cf9cc..a29af07 100644 --- a/cpp/cmake/thirdparty/get_gtest.cmake +++ b/cpp/cmake/thirdparty/get_gtest.cmake @@ -14,30 +14,9 @@ # limitations under the License. #============================================================================= -function(find_and_configure_gtest VERSION) - - if(TARGET GTest::gtest) - return() - endif() - - rapids_cpm_find(GTest ${VERSION} - GLOBAL_TARGETS gtest gtest_main GTest::gtest GTest::gtest_main gmock gmock_main - CPM_ARGS - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG release-${VERSION} - GIT_SHALLOW TRUE - OPTIONS "INSTALL_GTEST OFF" - # googletest >= 1.10.0 provides a cmake config file -- use it if it exists - FIND_PACKAGE_ARGUMENTS "CONFIG" - ) - - if(NOT TARGET GTest::gtest) - add_library(GTest::gtest ALIAS gtest) - add_library(GTest::gtest_main ALIAS gtest_main) - endif() - +function(find_and_configure_gtest) + include(${rapids-cmake-dir}/cpm/gtest.cmake) + rapids_cpm_gtest() endfunction() -set(RAFT_MIN_VERSION_gtest 1.10.0) - -find_and_configure_gtest(${RAFT_MIN_VERSION_gtest}) +find_and_configure_gtest() diff --git a/cpp/cmake/thirdparty/get_rmm.cmake b/cpp/cmake/thirdparty/get_rmm.cmake index b4a6782..b73d07a 100644 --- a/cpp/cmake/thirdparty/get_rmm.cmake +++ b/cpp/cmake/thirdparty/get_rmm.cmake @@ -15,33 +15,8 @@ #============================================================================= function(find_and_configure_rmm VERSION) - - if(TARGET rmm::rmm) - return() - endif() - - if(${VERSION} MATCHES [=[([0-9]+)\.([0-9]+)\.([0-9]+)]=]) - set(MAJOR_AND_MINOR "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}") - else() - set(MAJOR_AND_MINOR "${VERSION}") - endif() - - rapids_cpm_find(rmm ${VERSION} - GLOBAL_TARGETS rmm::rmm - BUILD_EXPORT_SET rapids_triton-exports - INSTALL_EXPORT_SET rapids_triton-exports - CPM_ARGS - GIT_REPOSITORY https://github.com/rapidsai/rmm.git - GIT_TAG 23bbe745af1d988224b5498f7b8e3fe3720532d4 - GIT_SHALLOW FALSE - OPTIONS "BUILD_TESTS OFF" - "BUILD_BENCHMARKS OFF" - "CUDA_STATIC_RUNTIME ${CUDA_STATIC_RUNTIME}" - "DISABLE_DEPRECATION_WARNING ${DISABLE_DEPRECATION_WARNING}" - ) - + include(${rapids-cmake-dir}/cpm/rmm.cmake) + rapids_cpm_rmm() endfunction() -set(RAPIDS_TRITON_MIN_VERSION_rmm "${RAPIDS_TRITON_VERSION_MAJOR}.${RAPIDS_TRITON_VERSION_MINOR}.00") - -find_and_configure_rmm(${RAPIDS_TRITON_MIN_VERSION_rmm}) +find_and_configure_rmm() From bb73c2aa239fd1d26367546e8177ec71f78b03af Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 20 Sep 2021 15:23:53 -0400 Subject: [PATCH 073/199] Eliminate unnecessary Thrust inclusion --- cpp/CMakeLists.txt | 3 --- cpp/cmake/thirdparty/get_rmm.cmake | 2 +- cpp/cmake/thirdparty/get_thrust.cmake | 30 --------------------------- 3 files changed, 1 insertion(+), 34 deletions(-) delete mode 100644 cpp/cmake/thirdparty/get_thrust.cmake diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d97a95e..4f7722a 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -97,9 +97,6 @@ include(cmake/modules/ConfigureCUDA.cmake) # add third party dependencies using CPM rapids_cpm_init() -# NOTE(wphicks): Thrust is not being properly included by an upstream -# dependency, so we include it here -include(cmake/thirdparty/get_thrust.cmake) include(cmake/thirdparty/get_rmm.cmake) include(cmake/thirdparty/get_raft.cmake) include(cmake/thirdparty/get_rapidjson.cmake) diff --git a/cpp/cmake/thirdparty/get_rmm.cmake b/cpp/cmake/thirdparty/get_rmm.cmake index b73d07a..4a3ca4a 100644 --- a/cpp/cmake/thirdparty/get_rmm.cmake +++ b/cpp/cmake/thirdparty/get_rmm.cmake @@ -14,7 +14,7 @@ # limitations under the License. #============================================================================= -function(find_and_configure_rmm VERSION) +function(find_and_configure_rmm) include(${rapids-cmake-dir}/cpm/rmm.cmake) rapids_cpm_rmm() endfunction() diff --git a/cpp/cmake/thirdparty/get_thrust.cmake b/cpp/cmake/thirdparty/get_thrust.cmake deleted file mode 100644 index 3764192..0000000 --- a/cpp/cmake/thirdparty/get_thrust.cmake +++ /dev/null @@ -1,30 +0,0 @@ -# ============================================================================= -# Copyright (c) 2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Use CPM to find or clone thrust -function(find_and_configure_thrust VERSION) - - rapids_cpm_find( - Thrust ${VERSION} - BUILD_EXPORT_SET rapids_triton-exports - INSTALL_EXPORT_SET rapids_triton-exports - CPM_ARGS - GIT_REPOSITORY https://github.com/NVIDIA/thrust.git - GIT_TAG ${VERSION} - GIT_SHALLOW TRUE - OPTIONS "THRUST_INSTALL OFF") - -endfunction() - -find_and_configure_thrust(1.12.0) From d1c29ec010a742e4108d6e71224e473e4f76f63e Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 20 Sep 2021 16:26:09 -0400 Subject: [PATCH 074/199] Link to Triton from README --- README.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 488d361..ebbbf4a 100644 --- a/README.md +++ b/README.md @@ -39,14 +39,16 @@ quickly get up and running with a custom backend for Triton. ### Triton -The NVIDIA Triton Inference Server offers a complete open-source solution for -deployment of machine learning models from a wide variety of ML frameworks -(PyTorch, Tensorflow, ONNX, XGBoost, etc.) on both CPU and GPU hardware. It -allows you to maximize inference performance in production (whether that means -maximizing throughput, minimizing latency, or optimizing some other metric) -regardless of how you may have trained your ML model. Through smart batching, -efficient pipeline handling, and tools to simplify deployments almost anywhere, -Triton helps make production inference serving simpler and more cost-effective. +The [NVIDIA Triton Inference +Server](https://developer.nvidia.com/nvidia-triton-inference-server) offers a +complete open-source solution for deployment of machine learning models from a +wide variety of ML frameworks (PyTorch, Tensorflow, ONNX, XGBoost, etc.) on +both CPU and GPU hardware. It allows you to maximize inference performance in +production (whether that means maximizing throughput, minimizing latency, or +optimizing some other metric) regardless of how you may have trained your ML +model. Through smart batching, efficient pipeline handling, and tools to +simplify deployments almost anywhere, Triton helps make production inference +serving simpler and more cost-effective. ### Custom Backends From 39d5e5dfa49e81d7ef342d5439600721b429a02a Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 20 Sep 2021 16:31:01 -0400 Subject: [PATCH 075/199] Add logging of Triton-specific build options --- cpp/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4f7722a..217223a 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -61,6 +61,11 @@ message(VERBOSE "RAPIDS_TRITON: Enable kernel resource usage info: ${CUDA_ENABLE message(VERBOSE "RAPIDS_TRITON: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") message(VERBOSE "RAPIDS_TRITON: Enable nvtx markers: ${NVTX}") message(VERBOSE "RAPIDS_TRITON: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") +message(VERBOSE "RAPIDS_TRITON: Enable GPU support: ${TRITON_ENABLE_GPU}") +message(VERBOSE "RAPIDS_TRITON: Enable statistics collection in Triton: ${TRITON_ENABLE_STATS}") +message(VERBOSE "RAPIDS_TRITON: Triton common repo tag: ${TRITON_COMMON_REPO_TAG}") +message(VERBOSE "RAPIDS_TRITON: Triton core repo tag: ${TRITON_CORE_REPO_TAG}") +message(VERBOSE "RAPIDS_TRITON: Triton backend repo tag: ${TRITON_BACKEND_REPO_TAG}") # Set RMM logging level set(RMM_LOGGING_LEVEL "INFO" CACHE STRING "Choose the logging level.") From b7e0abc0b47ab7057568369460f16e9d17396260 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 20 Sep 2021 16:33:22 -0400 Subject: [PATCH 076/199] Correct copyright year range Co-authored-by: Dante Gama Dessavre --- cpp/cmake/modules/ConfigureCUDA.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/modules/ConfigureCUDA.cmake b/cpp/cmake/modules/ConfigureCUDA.cmake index f170102..20826ab 100644 --- a/cpp/cmake/modules/ConfigureCUDA.cmake +++ b/cpp/cmake/modules/ConfigureCUDA.cmake @@ -1,5 +1,5 @@ #============================================================================= -# Copyright (c) 2018-2021, NVIDIA CORPORATION. +# Copyright (c) 2021, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 4b3d85706e67ccbe61ae7fe8cb02777f514caf4a Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 20 Sep 2021 16:46:58 -0400 Subject: [PATCH 077/199] Add CUDA 11.2 development environment --- conda/environments/rapids_triton_dev_cuda11.2.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 conda/environments/rapids_triton_dev_cuda11.2.yml diff --git a/conda/environments/rapids_triton_dev_cuda11.2.yml b/conda/environments/rapids_triton_dev_cuda11.2.yml new file mode 100644 index 0000000..0755643 --- /dev/null +++ b/conda/environments/rapids_triton_dev_cuda11.2.yml @@ -0,0 +1,10 @@ +--- +name: rapids_triton_dev +channels: + - nvidia + - conda-forge +dependencies: + - cmake>=3.21 + - cudatoolkit=11.2 + - ninja + - rapidjson From 08597653eaad9f98ca00cf6516fa53d7fe58d6ba Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 20 Sep 2021 16:48:53 -0400 Subject: [PATCH 078/199] Force double linebreak with spaces in DCO --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 15a544a..0880e4d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,8 +55,8 @@ To contribute code to this project, please follow these steps: San Francisco, CA, 94129 Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. - ``` - ``` + + Developer's Certificate of Origin 1.1 By making a contribution to this project, I certify that: From 1b83010e156670aa001469ecf0192e9d5f4ea0cc Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 20 Sep 2021 17:36:55 -0400 Subject: [PATCH 079/199] Correct mismatched copyright headers --- CONTRIBUTING.md | 15 +++++++ README.md | 38 ++++++------------ cpp/include/rapids_triton/triton/backend.hpp | 40 +++++++------------ cpp/include/rapids_triton/triton/model.hpp | 40 +++++++------------ .../rapids_triton/triton/model_instance.hpp | 40 +++++++------------ docs/usage.md | 38 ++++++------------ 6 files changed, 86 insertions(+), 125 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0880e4d..f73039e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,18 @@ + # Contributing to RAPIDS-Triton diff --git a/README.md b/README.md index ebbbf4a..efcbfa0 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,17 @@ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) diff --git a/cpp/include/rapids_triton/triton/backend.hpp b/cpp/include/rapids_triton/triton/backend.hpp index 414b212..f0f9a5a 100644 --- a/cpp/include/rapids_triton/triton/backend.hpp +++ b/cpp/include/rapids_triton/triton/backend.hpp @@ -1,28 +1,18 @@ -// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #pragma once diff --git a/cpp/include/rapids_triton/triton/model.hpp b/cpp/include/rapids_triton/triton/model.hpp index 384d428..94d0711 100644 --- a/cpp/include/rapids_triton/triton/model.hpp +++ b/cpp/include/rapids_triton/triton/model.hpp @@ -1,28 +1,18 @@ -// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #pragma once #include diff --git a/cpp/include/rapids_triton/triton/model_instance.hpp b/cpp/include/rapids_triton/triton/model_instance.hpp index 66c3ac2..97b0dd9 100644 --- a/cpp/include/rapids_triton/triton/model_instance.hpp +++ b/cpp/include/rapids_triton/triton/model_instance.hpp @@ -1,28 +1,18 @@ -// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #pragma once #include diff --git a/docs/usage.md b/docs/usage.md index 4ce4852..3c39701 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1,29 +1,17 @@ # Using RAPIDS-Triton From 0b6d1d3f35415d2b97b9af703766758632fb579b Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 21 Sep 2021 11:01:54 -0400 Subject: [PATCH 080/199] Add missing client definition in README code --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7a9796e..b9a7d6a 100644 --- a/README.md +++ b/README.md @@ -479,6 +479,8 @@ To use it for a basic test, we might execute something like the following ```python from rapids_triton import Client +client = Client() + u = np.array([[2, 2, 3, 3]], dtype='float32') v = np.array([[-1, 1, 1, -1]], dtype='float32') # The value of the c vector specified in c.txt From e0aa081836a6803921feaa87a9172111c425ddc1 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 22 Sep 2021 10:08:38 -0400 Subject: [PATCH 081/199] Pass optional by reference Co-authored-by: Micka <9810050+lowener@users.noreply.github.com> --- cpp/include/rapids_triton/batch/batch.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/batch/batch.hpp b/cpp/include/rapids_triton/batch/batch.hpp index d96ef72..5b04065 100644 --- a/cpp/include/rapids_triton/batch/batch.hpp +++ b/cpp/include/rapids_triton/batch/batch.hpp @@ -122,7 +122,7 @@ struct Batch { template auto get_input(std::string const& name, - std::optional memory_type, + std::optional const& memory_type, device_id_t device_id, cudaStream_t stream) { From 8a2e3c3ba9139b3406d681ee9d07f4b76c95ac4b Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 22 Sep 2021 10:11:21 -0400 Subject: [PATCH 082/199] Pass optionals by reference Co-authored-by: Micka <9810050+lowener@users.noreply.github.com> --- cpp/include/rapids_triton/batch/batch.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/include/rapids_triton/batch/batch.hpp b/cpp/include/rapids_triton/batch/batch.hpp index 5b04065..e231234 100644 --- a/cpp/include/rapids_triton/batch/batch.hpp +++ b/cpp/include/rapids_triton/batch/batch.hpp @@ -170,7 +170,7 @@ struct Batch { template auto get_input(std::string const& name, - std::optional memory_type, + std::optional const& memory_type, device_id_t device_id) { return get_input(name, memory_type, device_id, stream_); @@ -178,7 +178,7 @@ struct Batch { template auto get_output(std::string const& name, - std::optional memory_type, + std::optional const& memory_type, device_id_t device_id, cudaStream_t stream) { @@ -202,7 +202,7 @@ struct Batch { template auto get_output(std::string const& name, - std::optional memory_type, + std::optional const& memory_type, device_id_t device_id) { return get_output(name, memory_type, device_id, stream_); From 207d154e0b05fc318e66e5d7a7672302f8beab3a Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 22 Sep 2021 10:12:33 -0400 Subject: [PATCH 083/199] Correct typo in Tensor documentation Co-authored-by: Micka <9810050+lowener@users.noreply.github.com> --- cpp/include/rapids_triton/tensor/tensor.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/tensor/tensor.hpp b/cpp/include/rapids_triton/tensor/tensor.hpp index c7088ca..17abc74 100644 --- a/cpp/include/rapids_triton/tensor/tensor.hpp +++ b/cpp/include/rapids_triton/tensor/tensor.hpp @@ -139,7 +139,7 @@ struct OutputTensor final : BaseTensor { * * This method *must* be called by rapids_triton backends on all of their * output tensors before returning from their `predict` methods. Because we - * cannot known a priori what names backends might have for their tensors + * cannot know a priori what names backends might have for their tensors * and what types will be stored in those tensors, the rapids_triton * library cannot store references to those tensors that might otherwise be * used to finalize them. From 7ddc0afcabc898a040619a4b0b65a5f7b28d0896 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 22 Sep 2021 10:22:54 -0400 Subject: [PATCH 084/199] Explicitly label auto pointers Co-authored-by: Micka <9810050+lowener@users.noreply.github.com> --- cpp/include/rapids_triton/triton/model_instance.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/include/rapids_triton/triton/model_instance.hpp b/cpp/include/rapids_triton/triton/model_instance.hpp index 97b0dd9..0ba03bd 100644 --- a/cpp/include/rapids_triton/triton/model_instance.hpp +++ b/cpp/include/rapids_triton/triton/model_instance.hpp @@ -28,7 +28,7 @@ namespace rapids { /** Get the name of a Triton model instance from the instance itself */ inline auto get_model_instance_name(TRITONBACKEND_ModelInstance& instance) { - auto cname = static_cast(nullptr); + auto* cname = static_cast(nullptr); triton_check(TRITONBACKEND_ModelInstanceName(&instance, &cname)); return std::string(cname); } @@ -86,7 +86,7 @@ void set_instance_state(TRITONBACKEND_ModelInstance& instance, template auto* get_instance_state(TRITONBACKEND_ModelInstance& instance) { - auto instance_state = static_cast(nullptr); + auto* instance_state = static_cast(nullptr); triton_check( TRITONBACKEND_ModelInstanceState(&instance, reinterpret_cast(&instance_state))); return instance_state; From 9786d1aa8282a6692e7bab9c663bc7f73b72a6d2 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 22 Sep 2021 10:23:39 -0400 Subject: [PATCH 085/199] Explicitly label auto pointer Co-authored-by: Micka <9810050+lowener@users.noreply.github.com> --- cpp/include/rapids_triton/triton/output.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/triton/output.hpp b/cpp/include/rapids_triton/triton/output.hpp index b4910c8..a48d156 100644 --- a/cpp/include/rapids_triton/triton/output.hpp +++ b/cpp/include/rapids_triton/triton/output.hpp @@ -24,7 +24,7 @@ namespace backend { namespace rapids { inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) { - auto result = static_cast(nullptr); + auto* result = static_cast(nullptr); triton_check(TRITONBACKEND_RequestInput(request, name.c_str(), &result)); return result; } From 07ea40b776ee095af07ab801e845385b2037ce8f Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 22 Sep 2021 10:26:55 -0400 Subject: [PATCH 086/199] Explicitly label auto pointer --- cpp/include/rapids_triton/triton/model.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/triton/model.hpp b/cpp/include/rapids_triton/triton/model.hpp index 94d0711..1591763 100644 --- a/cpp/include/rapids_triton/triton/model.hpp +++ b/cpp/include/rapids_triton/triton/model.hpp @@ -77,7 +77,7 @@ void set_model_state(TRITONBACKEND_Model& model, std::unique_ptr template auto* get_model_state(TRITONBACKEND_Model& model) { - auto vstate = static_cast(nullptr); + auto* vstate = static_cast(nullptr); triton_check(TRITONBACKEND_ModelState(&model, &vstate)); auto* model_state = reinterpret_cast(vstate); From e2d3026064795113206c1d4afa1fee30108c2fed Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 22 Sep 2021 10:55:37 -0400 Subject: [PATCH 087/199] Allow modifications on state in get_stream --- cpp/include/rapids_triton/model/model.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/model/model.hpp b/cpp/include/rapids_triton/model/model.hpp index 249ea89..7c8a510 100644 --- a/cpp/include/rapids_triton/model/model.hpp +++ b/cpp/include/rapids_triton/model/model.hpp @@ -71,7 +71,7 @@ struct Model { * several streams in order to distribute batches across them, but care * should be taken to ensure proper synchronization in this case. */ - virtual cudaStream_t get_stream() const { return default_stream_; } + virtual cudaStream_t get_stream() { return default_stream_; } /** * @brief Get input tensor of a particular named input for an entire batch From 05b50b2d806b4703dc349685c2cfa85353b181f1 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 23 Sep 2021 14:00:03 -0400 Subject: [PATCH 088/199] Construct closures for Batch separately --- cpp/include/rapids_triton/batch/batch.hpp | 12 ++- .../rapids_triton/triton/api/execute.hpp | 81 ++++++++++--------- 2 files changed, 47 insertions(+), 46 deletions(-) diff --git a/cpp/include/rapids_triton/batch/batch.hpp b/cpp/include/rapids_triton/batch/batch.hpp index e231234..fed959d 100644 --- a/cpp/include/rapids_triton/batch/batch.hpp +++ b/cpp/include/rapids_triton/batch/batch.hpp @@ -67,20 +67,20 @@ struct Batch { Batch(TRITONBACKEND_Request** raw_requests, request_size_t count, TRITONBACKEND_MemoryManager& triton_mem_manager, - std::function(std::string const&, size_type)> get_output_shape, + std::function(std::string const&, size_type)>&& get_output_shape, std::function report_request_statistics, + time_point const&)>&& report_request_statistics, bool use_pinned_input, bool use_pinned_output, size_type max_batch_size, cudaStream_t stream) : requests_{raw_requests, raw_requests + count}, responses_{construct_responses(requests_.begin(), requests_.end())}, - get_output_shape_{get_output_shape}, - report_statistics_{report_request_statistics}, + get_output_shape_{std::move(get_output_shape)}, + report_statistics_{std::move(report_request_statistics)}, collector_(raw_requests, count, &responses_, &triton_mem_manager, use_pinned_input, stream), responder_{std::make_shared(raw_requests, count, @@ -104,9 +104,7 @@ struct Batch { result = get_triton_input_shape(std::begin(requests_), std::end(requests_), name); auto input_batch_dim = size_type{}; - if (result.size() > 0) { - input_batch_dim = result[0]; - } + if (result.size() > 0) { input_batch_dim = result[0]; } if (batch_size_.has_value()) { if (batch_size_.value() != input_batch_dim) { diff --git a/cpp/include/rapids_triton/triton/api/execute.hpp b/cpp/include/rapids_triton/triton/api/execute.hpp index 38cdf8c..bce6bc6 100644 --- a/cpp/include/rapids_triton/triton/api/execute.hpp +++ b/cpp/include/rapids_triton/triton/api/execute.hpp @@ -46,45 +46,48 @@ auto* execute(TRITONBACKEND_ModelInstance* instance, auto* instance_state = get_instance_state(*instance); auto& model = instance_state->get_model(); auto max_batch_size = model.template get_config_param("max_batch_size"); - auto batch = Batch( - raw_requests, - request_count, - *(model_state->TritonMemoryManager()), - /* Note: It is safe to keep a reference to the model in htis closure - * and a pointer to the instance in the next because the batch goes - * out of scope at the end of this block and Triton guarantees that - * the liftimes of both the instance and model extend beyond this - * function call. */ - [&model](std::string const& name, Batch::size_type batch_dim) { - auto result = std::vector{}; - auto config_shape = model.get_output_shape(name); - if (config_shape.size() > 0 && config_shape[0] < 0) { config_shape[0] = batch_dim; } - std::transform( - std::begin(config_shape), - std::end(config_shape), - std::back_inserter(result), - [](auto& coord) { - if (coord < 0) { - throw TritonException( - Error::Internal, - "Backends with variable-shape outputs must request desired output shape"); - } else { - return narrow(coord); - } - }); - return result; - }, - [instance](TRITONBACKEND_Request* request, - time_point req_start, - time_point req_comp_start, - time_point req_comp_end, - time_point req_end) { - report_statistics(*instance, *request, req_start, req_comp_start, req_comp_end, req_end); - }, - model_state->EnablePinnedInput(), - model_state->EnablePinnedOutput(), - max_batch_size, - model.get_stream()); + + /* Note: It is safe to keep a reference to the model in this closure + * and a pointer to the instance in the next because the batch goes + * out of scope at the end of this block and Triton guarantees that + * the liftimes of both the instance and model extend beyond this + * function call. */ + auto output_shape_fetcher = [&model](std::string const& name, Batch::size_type batch_dim) { + auto result = std::vector{}; + auto config_shape = model.get_output_shape(name); + if (config_shape.size() > 0 && config_shape[0] < 0) { config_shape[0] = batch_dim; } + std::transform( + std::begin(config_shape), + std::end(config_shape), + std::back_inserter(result), + [](auto& coord) { + if (coord < 0) { + throw TritonException( + Error::Internal, + "Backends with variable-shape outputs must request desired output shape"); + } else { + return narrow(coord); + } + }); + return result; + }; + auto statistics_reporter = [instance](TRITONBACKEND_Request* request, + time_point req_start, + time_point req_comp_start, + time_point req_comp_end, + time_point req_end) { + report_statistics(*instance, *request, req_start, req_comp_start, req_comp_end, req_end); + }; + + auto batch = Batch(raw_requests, + request_count, + *(model_state->TritonMemoryManager()), + std::move(output_shape_fetcher), + std::move(statistics_reporter), + model_state->EnablePinnedInput(), + model_state->EnablePinnedOutput(), + max_batch_size, + model.get_stream()); auto predict_err = static_cast(nullptr); try { From b9a9ef6bf64d76851a37b654675972ce503d5d99 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 23 Sep 2021 14:33:22 -0400 Subject: [PATCH 089/199] Change const model to allow for stream-related state Allow `get_stream` method to change state within the model in order to allow for smart distribution across a pool of streams --- cpp/include/rapids_triton/triton/model_instance_state.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/triton/model_instance_state.hpp b/cpp/include/rapids_triton/triton/model_instance_state.hpp index ba05009..b2ebb91 100644 --- a/cpp/include/rapids_triton/triton/model_instance_state.hpp +++ b/cpp/include/rapids_triton/triton/model_instance_state.hpp @@ -40,7 +40,7 @@ struct ModelInstanceState : public BackendModelInstance { { } - auto& get_model() const { return model_; } + auto& get_model() { return model_; } void load() { model_.load(); } void unload() { model_.unload(); } From e3803c2c23056262eb105f9eee6974cc5931a019 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 27 Sep 2021 13:49:09 -0400 Subject: [PATCH 090/199] Revert "Change const model to allow for stream-related state" This reverts commit b9a9ef6bf64d76851a37b654675972ce503d5d99. Tracking of stream state should be controlled by explicitly-labeled mutable data in order to allow for sensible const usage --- cpp/include/rapids_triton/triton/model_instance_state.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/triton/model_instance_state.hpp b/cpp/include/rapids_triton/triton/model_instance_state.hpp index b2ebb91..ba05009 100644 --- a/cpp/include/rapids_triton/triton/model_instance_state.hpp +++ b/cpp/include/rapids_triton/triton/model_instance_state.hpp @@ -40,7 +40,7 @@ struct ModelInstanceState : public BackendModelInstance { { } - auto& get_model() { return model_; } + auto& get_model() const { return model_; } void load() { model_.load(); } void unload() { model_.unload(); } From 37f9879d4b7a83fd617fd75bcfebdc809193d008 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 27 Sep 2021 13:55:18 -0400 Subject: [PATCH 091/199] Make get_stream a const function for Model --- cpp/include/rapids_triton/model/model.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/model/model.hpp b/cpp/include/rapids_triton/model/model.hpp index 7c8a510..249ea89 100644 --- a/cpp/include/rapids_triton/model/model.hpp +++ b/cpp/include/rapids_triton/model/model.hpp @@ -71,7 +71,7 @@ struct Model { * several streams in order to distribute batches across them, but care * should be taken to ensure proper synchronization in this case. */ - virtual cudaStream_t get_stream() { return default_stream_; } + virtual cudaStream_t get_stream() const { return default_stream_; } /** * @brief Get input tensor of a particular named input for an entire batch From 35a1266a47eacce1219415ec04cc6729ef2564d9 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 27 Sep 2021 14:22:22 -0400 Subject: [PATCH 092/199] Allow construction of Buffer from Buffer --- cpp/include/rapids_triton/memory/buffer.hpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cpp/include/rapids_triton/memory/buffer.hpp b/cpp/include/rapids_triton/memory/buffer.hpp index c334495..bd6a78a 100644 --- a/cpp/include/rapids_triton/memory/buffer.hpp +++ b/cpp/include/rapids_triton/memory/buffer.hpp @@ -221,7 +221,10 @@ struct Buffer { // checked static void copy(data_ptr const& dst, data_ptr const& src, size_type len, cudaStream_t stream) { - auto raw_dst = get_raw_ptr(dst); + // This function will only be called in constructors, so we allow a + // const_cast here to perform the initial copy of data from a + // Buffer to a newly-created Buffer + auto raw_dst = const_cast*>get_raw_ptr(dst); auto raw_src = get_raw_ptr(src); auto dst_mem_type = dst.index() % 2 == 0 ? HostMemory : DeviceMemory; @@ -242,11 +245,6 @@ struct Buffer { * at which to begin copying from. * @param src_end The offset from the beginning of the source buffer * before which to end copying from. - * - * @warning This method is NOT thread-safe. If the stream of the src buffer - * changes while a copy is in progress, dst may receive incorrect data from - * src. Avoid interactions between buffers on different streams, *especially* - * when those buffers may be modified on different host threads as well. */ template void copy(Buffer& dst, From 0a533f1bd82adcf87744befa8f9c67f5ca4cd9fd Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 27 Sep 2021 14:28:59 -0400 Subject: [PATCH 093/199] Fix syntax error --- cpp/include/rapids_triton/memory/buffer.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/memory/buffer.hpp b/cpp/include/rapids_triton/memory/buffer.hpp index bd6a78a..6c7b1ca 100644 --- a/cpp/include/rapids_triton/memory/buffer.hpp +++ b/cpp/include/rapids_triton/memory/buffer.hpp @@ -224,7 +224,7 @@ struct Buffer { // This function will only be called in constructors, so we allow a // const_cast here to perform the initial copy of data from a // Buffer to a newly-created Buffer - auto raw_dst = const_cast*>get_raw_ptr(dst); + auto raw_dst = const_cast*>(get_raw_ptr(dst)); auto raw_src = get_raw_ptr(src); auto dst_mem_type = dst.index() % 2 == 0 ? HostMemory : DeviceMemory; From e8e4032ac1d22e67e35cb94e0e5397dbb075ddc3 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 28 Sep 2021 09:37:16 -0400 Subject: [PATCH 094/199] Update Greek characters for MD rendering --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b9a7d6a..a26d8c9 100644 --- a/README.md +++ b/README.md @@ -36,9 +36,9 @@ step-by-step how to create a backend with RAPIDS-Triton that, when given two vectors (**u** and **v**) as input will return a vector **r** according to the following equation: -**r** = \alpha * **u** + **v** + **c** +**r** = α * **u** + **v** + **c** -where \alpha is a scalar constant read from a configuration file and **c** is a +where α is a scalar constant read from a configuration file and **c** is a constant vector read from a "model" file. Along the way, we will illustrate a variety of useful operations in RAPIDS-Triton, including retrieving data from a configuration file and loading model resources. @@ -104,7 +104,7 @@ different things: 1. The name of the backend which will be used to serve this model (the name chosen in step 2) 2. The names of the input vectors -3. The value of \alpha +3. The value of α 4. Where to load **c** from With this in mind, an example configuration file for the `rapids_linear` @@ -215,7 +215,7 @@ manages state that is relevant to all instances. The most basic state that is shared among instances of a model is the model configuration itself (as defined in step 3), but we can also use it to share data that is specifically required by our backend. In our particular case, it -would be useful to cache the value of \alpha so that we do not have to retrieve +would be useful to cache the value of α so that we do not have to retrieve it from the configuration each time (which may involve additional parsing). All RAPIDS-Triton backends store their shared state in a class called @@ -240,7 +240,7 @@ change between backends. Take a look at `src/shared_state.h` to see this implementation in context. Note that we have added a public member variables to this class definition -which will be used to store \alpha. One could equally well have made +which will be used to store α. One could equally well have made these private members with getter functions or added arbitrarily complex logic to this class definition, but we will leave them as is for simplicity. @@ -248,7 +248,7 @@ to this class definition, but we will leave them as is for simplicity. Next, we need to actually load a value into this newly-defined member. We can do this by filling out the logic for our `load` method. For example, in order -to load \alpha, we could implement something like: +to load α, we could implement something like: ```cpp void load() { alpha = get_config_param("alpha"); } ``` From 420f6219adfaeb327b39f9d6db73c47cc22484b9 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 28 Sep 2021 10:43:26 -0400 Subject: [PATCH 095/199] Add option to squeeze output shape --- cpp/include/rapids_triton/model/shared_state.hpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cpp/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp index f8570e0..6016f54 100644 --- a/cpp/include/rapids_triton/model/shared_state.hpp +++ b/cpp/include/rapids_triton/model/shared_state.hpp @@ -41,10 +41,10 @@ struct SharedModelState { virtual void load() {} virtual void unload() {} - explicit SharedModelState(std::unique_ptr&& config) + explicit SharedModelState(std::unique_ptr&& config, bool squeeze_output=false) : config_{std::move(config)}, max_batch_size_{get_max_batch_size(*config_)}, - output_shapes_([this]() { + output_shapes_([this, squeeze_output]() { auto result = std::vector>>{}; auto output_entries = triton::common::TritonJson::Value{}; triton_check(config_->MemberAsArray("output", &output_entries)); @@ -66,6 +66,14 @@ struct SharedModelState { ParseShape(output_entry, "dims", &shape); } if (shape[0] != -1) { shape.insert(shape.begin(), -1); } + // The squeeze_output option was introduced to handle a bad choice of + // convention in the original FIL backend implementation. For legacy + // compatibility, we introduced this option into RAPIDS-Triton, but + // in general, new backends are advised to avoid using it and defer + // this sort of flattening operation to the consumer. + if (squeeze_output) { + std::remove(shape.begin(), shape.end(), std::int64_t{1}); + } result.insert( std::upper_bound(std::begin(output_shapes_), std::end(output_shapes_), From 11ff73bf6db8711900a839a5f94f30dfe16a0453 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 28 Sep 2021 11:18:01 -0400 Subject: [PATCH 096/199] Actually erase elements during squeeze --- cpp/include/rapids_triton/model/shared_state.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp index 6016f54..772e34d 100644 --- a/cpp/include/rapids_triton/model/shared_state.hpp +++ b/cpp/include/rapids_triton/model/shared_state.hpp @@ -72,7 +72,7 @@ struct SharedModelState { // in general, new backends are advised to avoid using it and defer // this sort of flattening operation to the consumer. if (squeeze_output) { - std::remove(shape.begin(), shape.end(), std::int64_t{1}); + shape.erase(std::remove(shape.begin(), shape.end(), std::int64_t{1}), shape.end()); } result.insert( std::upper_bound(std::begin(output_shapes_), From 9480c1b35b93f13556530265c186f15f1393be47 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 4 Oct 2021 10:24:08 -0400 Subject: [PATCH 097/199] Appropriately handle null response pointer --- cpp/include/rapids_triton/triton/responses.hpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cpp/include/rapids_triton/triton/responses.hpp b/cpp/include/rapids_triton/triton/responses.hpp index b4309f0..2bab46f 100644 --- a/cpp/include/rapids_triton/triton/responses.hpp +++ b/cpp/include/rapids_triton/triton/responses.hpp @@ -57,11 +57,15 @@ void send_responses(Iter begin, Iter end, TRITONSERVER_Error* err) err_copy = err; } - try { - triton_check( - TRITONBACKEND_ResponseSend(response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, err_copy)); - } catch (TritonException& err) { - log_error(__FILE__, __LINE__, err.what()); + if (response == nullptr) { + log_error(__FILE__, __LINE__) << "Failure in input or output collection"); + } else { + try { + triton_check( + TRITONBACKEND_ResponseSend(response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, err_copy)); + } catch (TritonException& err) { + log_error(__FILE__, __LINE__, err.what()); + } } }); } From a048a7284b0bccb3a4f5d5da20cd78928f47012a Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 4 Oct 2021 10:33:36 -0400 Subject: [PATCH 098/199] Correct syntax error in log statement --- cpp/include/rapids_triton/triton/responses.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/triton/responses.hpp b/cpp/include/rapids_triton/triton/responses.hpp index 2bab46f..20eb719 100644 --- a/cpp/include/rapids_triton/triton/responses.hpp +++ b/cpp/include/rapids_triton/triton/responses.hpp @@ -58,7 +58,7 @@ void send_responses(Iter begin, Iter end, TRITONSERVER_Error* err) } if (response == nullptr) { - log_error(__FILE__, __LINE__) << "Failure in input or output collection"); + log_error(__FILE__, __LINE__) << "Failure in input or output collection"; } else { try { triton_check( From 3f6c0cc66d04b0f274d4509eea38a95a21de4868 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 4 Oct 2021 17:23:55 -0400 Subject: [PATCH 099/199] Check for issues with input/output collation --- cpp/include/rapids_triton/batch/batch.hpp | 6 ++++++ cpp/include/rapids_triton/triton/responses.hpp | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/batch/batch.hpp b/cpp/include/rapids_triton/batch/batch.hpp index fed959d..5d4d6e5 100644 --- a/cpp/include/rapids_triton/batch/batch.hpp +++ b/cpp/include/rapids_triton/batch/batch.hpp @@ -150,6 +150,12 @@ struct Batch { &reported_mem_type, &reported_device_id)); + std::for_each(std::begin(responses_), std::end(responses_), [](auto* response) { + if (response == nullptr) { + throw TritonException(Error::Internal, "Input collection failed"); + } + }); + auto buffer = Buffer(reinterpret_cast(raw_buffer), reported_bytes / sizeof(T), reported_mem_type, diff --git a/cpp/include/rapids_triton/triton/responses.hpp b/cpp/include/rapids_triton/triton/responses.hpp index 20eb719..b361c5a 100644 --- a/cpp/include/rapids_triton/triton/responses.hpp +++ b/cpp/include/rapids_triton/triton/responses.hpp @@ -58,7 +58,7 @@ void send_responses(Iter begin, Iter end, TRITONSERVER_Error* err) } if (response == nullptr) { - log_error(__FILE__, __LINE__) << "Failure in input or output collection"; + log_error(__FILE__, __LINE__) << "Failure in response collation"; } else { try { triton_check( From edad156099531f16ea0d8fe1c90c2b257d23c99f Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 5 Oct 2021 12:13:37 -0400 Subject: [PATCH 100/199] Add warning about shared memory --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index efcbfa0..ad7b821 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,12 @@ into the NVIDIA Triton Inference Server. Originally developed to assist with the integration of RAPIDS algorithms, this library can be used by anyone to quickly get up and running with a custom backend for Triton. +**WARNING**: Due to an upstream bug in Triton, backends which make use of +RAPIDS-Triton may encounter issues with Triton's shared memory mode. +Specifically, backends deployed on the host or which request input on host +memory should not be given cuda shared memory inputs and vice versa. A fix for +this issue in Triton is expected soon. + ## Background ### Triton From ac4b36c37ee00c3e16d582d4c5f520548f31fe35 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 7 Oct 2021 15:16:50 -0400 Subject: [PATCH 101/199] Move to RMM for device memory management --- cpp/include/rapids_triton/memory/buffer.hpp | 41 +++++++++++-------- .../rapids_triton/memory/detail/allocate.hpp | 5 ++- cpp/include/rapids_triton/model/model.hpp | 6 +++ 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/cpp/include/rapids_triton/memory/buffer.hpp b/cpp/include/rapids_triton/memory/buffer.hpp index 6c7b1ca..f61f229 100644 --- a/cpp/include/rapids_triton/memory/buffer.hpp +++ b/cpp/include/rapids_triton/memory/buffer.hpp @@ -23,10 +23,11 @@ #include #include -#include +#include #include #include #include +#include namespace triton { namespace backend { @@ -36,11 +37,11 @@ struct Buffer { using size_type = std::size_t; using value_type = T; - using h_ptr = T*; - using d_ptr = T*; - using owned_h_ptr = std::unique_ptr; - using owned_d_ptr = std::unique_ptr>; - using data_ptr = std::variant; + using h_buffer = T*; + using d_buffer = T*; + using owned_h_buffer = std::unique_ptr; + using owned_d_buffer = rmm::device_buffer; + using data_store = std::variant; Buffer() noexcept : device_{}, data_{std::in_place_index<0>, nullptr}, size_{}, stream_{} {} @@ -74,11 +75,11 @@ struct Buffer { cudaStream_t stream = 0) : device_{device}, data_{[&memory_type, &input_data]() { - auto result = data_ptr{}; + auto result = data_store{}; if (memory_type == HostMemory) { - result = data_ptr{std::in_place_index<0>, input_data}; + result = data_store{std::in_place_index<0>, input_data}; } else { - result = data_ptr{std::in_place_index<1>, input_data}; + result = data_store{std::in_place_index<1>, input_data}; } return result; }()}, @@ -114,7 +115,7 @@ struct Buffer { Buffer(Buffer&& other, MemoryType memory_type) : device_{other.device()}, data_{[&other, memory_type]() { - data_ptr result; + data_store result; if (memory_type == other.mem_type()) { result = std::move(other.data_); } else { @@ -176,12 +177,13 @@ struct Buffer { private: device_id_t device_; - data_ptr data_; + data_store data_; size_type size_; cudaStream_t stream_; - // Helper function for accessing raw pointer to underlying data of data_ptr - static auto* get_raw_ptr(data_ptr const& ptr) noexcept + // Helper function for accessing raw pointer to underlying data of + // data_store + static auto* get_raw_ptr(data_store const& ptr) noexcept { /* Switch statement is an optimization relative to std::visit to avoid * vtable overhead for a small number of alternatives */ @@ -190,7 +192,7 @@ struct Buffer { case 0: result = std::get<0>(ptr); break; case 1: result = std::get<1>(ptr); break; case 2: result = std::get<2>(ptr).get(); break; - case 3: result = std::get<3>(ptr).get(); break; + case 3: result = reinterpret_cast(std::get<3>(ptr).data()); break; } return result; } @@ -201,11 +203,14 @@ struct Buffer { MemoryType memory_type = DeviceMemory, cudaStream_t stream = 0) { - auto result = data_ptr{}; + auto result = data_store{}; if (memory_type == DeviceMemory) { if constexpr (IS_GPU_BUILD) { - cuda_check(cudaSetDevice(device)); - result = data_ptr{owned_d_ptr{detail::dev_allocate(size, stream)}}; + result = data_store{owned_d_buffer{ + size * sizeof(T), + stream, + get_memory_resource(device) + }}; } else { throw TritonException(Error::Internal, "DeviceMemory requested in CPU-only build of FIL backend"); @@ -219,7 +224,7 @@ struct Buffer { // Helper function for copying memory in constructors, where there are // stronger guarantees on conditions that would otherwise need to be // checked - static void copy(data_ptr const& dst, data_ptr const& src, size_type len, cudaStream_t stream) + static void copy(data_store const& dst, data_store const& src, size_type len, cudaStream_t stream) { // This function will only be called in constructors, so we allow a // const_cast here to perform the initial copy of data from a diff --git a/cpp/include/rapids_triton/memory/detail/allocate.hpp b/cpp/include/rapids_triton/memory/detail/allocate.hpp index 1db6f64..fa3aa1d 100644 --- a/cpp/include/rapids_triton/memory/detail/allocate.hpp +++ b/cpp/include/rapids_triton/memory/detail/allocate.hpp @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -38,7 +39,7 @@ struct dev_deallocater { // in a RAII context. If we are deallocating this memory, we allocated it // and made it const. Removing the const qualifier allows the // deallocation to proceed. - cudaFree(reinterpret_cast(const_cast*>(d_ptr))); + get_memory_resource()->deallocate(reinterpret_cast(const_cast*>(d_ptr))); } else { log_error( __FILE__, __LINE__, "ERROR: device deallocation cannot be performed in non-GPU build!"); @@ -56,7 +57,7 @@ template throw TritonException(Error::Internal, "device allocation attempted in non-GPU build"); } auto* ptr_d = - static_cast(rmm::mr::get_current_device_resource()->allocate(sizeof(T) * count, stream)); + static_cast(get_memory_resource()->allocate(sizeof(T) * count, stream)); return ptr_d; } diff --git a/cpp/include/rapids_triton/model/model.hpp b/cpp/include/rapids_triton/model/model.hpp index 249ea89..954a051 100644 --- a/cpp/include/rapids_triton/model/model.hpp +++ b/cpp/include/rapids_triton/model/model.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -165,6 +166,11 @@ struct Model { deployment_type_{deployment_type}, filepath_{filepath} { + if constexpr (IS_GPU_BUILD) { + if (deplyment_type_ == GPUDeployment) { + detail::setup_memory_resource(device_id); + } + } } auto get_device_id() const { return device_id_; } From d7f26d4772b7b49422089f47e06ad0da9974445b Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 7 Oct 2021 15:30:27 -0400 Subject: [PATCH 102/199] Remove unused allocation code --- .../rapids_triton/memory/detail/allocate.hpp | 67 ------------------- cpp/test/memory/detail/allocate.cpp | 53 --------------- 2 files changed, 120 deletions(-) delete mode 100644 cpp/include/rapids_triton/memory/detail/allocate.hpp delete mode 100644 cpp/test/memory/detail/allocate.cpp diff --git a/cpp/include/rapids_triton/memory/detail/allocate.hpp b/cpp/include/rapids_triton/memory/detail/allocate.hpp deleted file mode 100644 index fa3aa1d..0000000 --- a/cpp/include/rapids_triton/memory/detail/allocate.hpp +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include - -#include - -#include -#include -#include -#include -#include - -namespace triton { -namespace backend { -namespace rapids { -namespace detail { -template -struct dev_deallocater { - void operator()(T* d_ptr) - { - if constexpr (IS_GPU_BUILD) { - // Note: We allow a const_cast here because this deallocator is only used - // in a RAII context. If we are deallocating this memory, we allocated it - // and made it const. Removing the const qualifier allows the - // deallocation to proceed. - get_memory_resource()->deallocate(reinterpret_cast(const_cast*>(d_ptr))); - } else { - log_error( - __FILE__, __LINE__, "ERROR: device deallocation cannot be performed in non-GPU build!"); - } - } -}; - -/** - * @brief Allocate given number of elements on GPU and return device pointer - */ -template -[[nodiscard]] T* dev_allocate(std::size_t count, cudaStream_t stream) -{ - if constexpr (!IS_GPU_BUILD) { - throw TritonException(Error::Internal, "device allocation attempted in non-GPU build"); - } - auto* ptr_d = - static_cast(get_memory_resource()->allocate(sizeof(T) * count, stream)); - return ptr_d; -} - -} // namespace detail -} // namespace rapids -} // namespace backend -} // namespace triton diff --git a/cpp/test/memory/detail/allocate.cpp b/cpp/test/memory/detail/allocate.cpp deleted file mode 100644 index ff315a5..0000000 --- a/cpp/test/memory/detail/allocate.cpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include - -#include -#include -#include -#include - -namespace triton { -namespace backend { -namespace rapids { -TEST(RapidsTriton, dev_allocate) -{ - auto data = std::vector{1, 2, 3}; - if constexpr (IS_GPU_BUILD) { - auto ptr_d = detail::dev_allocate(data.size(), 0); - auto data_out = std::vector(data.size()); - cudaMemcpy(static_cast(ptr_d), - static_cast(data.data()), - sizeof(int) * data.size(), - cudaMemcpyHostToDevice); - cudaMemcpy(static_cast(data_out.data()), - static_cast(ptr_d), - sizeof(int) * data.size(), - cudaMemcpyDeviceToHost); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); - detail::dev_deallocater{}(ptr_d); - } else { - EXPECT_THROW([&data]() { return detail::dev_allocate(data.size(), 0); }(), - TritonException); - } -} - -} // namespace rapids -} // namespace backend -} // namespace triton From db791ded9d1cebb22bdd8662f6abc71fd4bfc425 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 7 Oct 2021 15:30:56 -0400 Subject: [PATCH 103/199] Add code for handling memory managers --- .../rapids_triton/memory/detail/manager.hpp | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 cpp/include/rapids_triton/memory/detail/manager.hpp diff --git a/cpp/include/rapids_triton/memory/detail/manager.hpp b/cpp/include/rapids_triton/memory/detail/manager.hpp new file mode 100644 index 0000000..083d768 --- /dev/null +++ b/cpp/include/rapids_triton/memory/detail/manager.hpp @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace detail { + + inline auto& manager_lock() { + static auto lock = std::mutex{}; + return lock; + } + + struct manager_data { + manager_data() : base_mr_{}, + pool_mr_{&base_mr_} {} + auto* get_resource() { + return &pool_mr_; + } + private: + rmm::mr::cuda_memory_resource base_mr_; + rmm::mr::pool_memory_resource pool_mr_; + }; + + auto& all_device_managers() { + // This vector keeps the underlying memory resource objects in-scope until + // the backend is unloaded. This ensures that they will not be destroyed + // while a model is still making use of them. + static auto device_managers = std::vector{}; + return device_managers; + } + + auto is_default_resource (rmm::cuda_device_id const& device_id) { + return rmm::get_per_device_resource(device_id)->is_equal(rmm::cuda_memory_resource{}); + } + + auto setup_memory_resource(rmm::cuda_device_id device_id) { + auto lock = std::lock_guard{manager_lock()}; + if (is_default_resource(device_id)) { + auto& device_managers = all_device_managers(); + device_managers.push_back(manager_data{}); + rmm::mr::set_per_device_resource( + rmm::cuda_device_id{device_id}, device_managers.back()->get_resource()); + } + + return rmm::get_per_device_resource(rmm_device_id); + } +} // namespace detail + +auto* get_memory_resource(device_id_t device_id) { + auto result = static_cast(nullptr); + auto rmm_device_id = rmm::cuda_device_id{device_id}; + if (is_default_resource(rmm_device_id)) { + result = setup_memory_resource(rmm_device_id); + } else { + result = rmm::get_per_device_resource(rmm_device_id); + } + return result; +} + +auto* get_memory_resource() { + auto device_id = int{}; + cuda_check(cudaGetDevice(&device_id)); + + return get_memory_resource(device_id); +} + +} // namespace rapids +} // namespace backend +} // namespace triton From 43d89c6d858afab199f4405c059e77fc66847874 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 7 Oct 2021 15:35:21 -0400 Subject: [PATCH 104/199] Rename header for memory management --- cpp/include/rapids_triton/memory/buffer.hpp | 2 +- .../rapids_triton/memory/detail/{manager.hpp => resource.hpp} | 0 cpp/include/rapids_triton/model/model.hpp | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename cpp/include/rapids_triton/memory/detail/{manager.hpp => resource.hpp} (100%) diff --git a/cpp/include/rapids_triton/memory/buffer.hpp b/cpp/include/rapids_triton/memory/buffer.hpp index f61f229..b3ab62c 100644 --- a/cpp/include/rapids_triton/memory/buffer.hpp +++ b/cpp/include/rapids_triton/memory/buffer.hpp @@ -23,7 +23,7 @@ #include #include -#include +#include #include #include #include diff --git a/cpp/include/rapids_triton/memory/detail/manager.hpp b/cpp/include/rapids_triton/memory/detail/resource.hpp similarity index 100% rename from cpp/include/rapids_triton/memory/detail/manager.hpp rename to cpp/include/rapids_triton/memory/detail/resource.hpp diff --git a/cpp/include/rapids_triton/model/model.hpp b/cpp/include/rapids_triton/model/model.hpp index 954a051..8b90942 100644 --- a/cpp/include/rapids_triton/model/model.hpp +++ b/cpp/include/rapids_triton/model/model.hpp @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include #include From 7e21f780ddc0b41139cfac7636440899bc503ccf Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 7 Oct 2021 15:36:58 -0400 Subject: [PATCH 105/199] Adjust naming convention for memory management --- .../rapids_triton/memory/detail/resource.hpp | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cpp/include/rapids_triton/memory/detail/resource.hpp b/cpp/include/rapids_triton/memory/detail/resource.hpp index 083d768..d8a1984 100644 --- a/cpp/include/rapids_triton/memory/detail/resource.hpp +++ b/cpp/include/rapids_triton/memory/detail/resource.hpp @@ -28,13 +28,13 @@ namespace backend { namespace rapids { namespace detail { - inline auto& manager_lock() { + inline auto& resource_lock() { static auto lock = std::mutex{}; return lock; } - struct manager_data { - manager_data() : base_mr_{}, + struct resource_data { + resource_data() : base_mr_{}, pool_mr_{&base_mr_} {} auto* get_resource() { return &pool_mr_; @@ -44,12 +44,12 @@ namespace detail { rmm::mr::pool_memory_resource pool_mr_; }; - auto& all_device_managers() { + auto& all_device_resources() { // This vector keeps the underlying memory resource objects in-scope until // the backend is unloaded. This ensures that they will not be destroyed // while a model is still making use of them. - static auto device_managers = std::vector{}; - return device_managers; + static auto device_resources = std::vector{}; + return device_resources; } auto is_default_resource (rmm::cuda_device_id const& device_id) { @@ -57,12 +57,12 @@ namespace detail { } auto setup_memory_resource(rmm::cuda_device_id device_id) { - auto lock = std::lock_guard{manager_lock()}; + auto lock = std::lock_guard{resource_lock()}; if (is_default_resource(device_id)) { - auto& device_managers = all_device_managers(); - device_managers.push_back(manager_data{}); + auto& device_resources = all_device_resources(); + device_resources.push_back(resource_data{}); rmm::mr::set_per_device_resource( - rmm::cuda_device_id{device_id}, device_managers.back()->get_resource()); + rmm::cuda_device_id{device_id}, device_resources.back()->get_resource()); } return rmm::get_per_device_resource(rmm_device_id); From b523ee43427043812b03793846f2e8767ac543dd Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 7 Oct 2021 20:41:41 -0400 Subject: [PATCH 106/199] Refactor owned device pointer implementation --- cpp/include/rapids_triton/memory/buffer.hpp | 39 +++++++++++++-- .../rapids_triton/memory/detail/copy.hpp | 2 +- .../rapids_triton/memory/detail/resource.hpp | 50 +++++++++++-------- cpp/include/rapids_triton/model/model.hpp | 4 +- cpp/test/CMakeLists.txt | 2 +- cpp/test/memory/detail/resource.cpp | 42 ++++++++++++++++ 6 files changed, 108 insertions(+), 31 deletions(-) create mode 100644 cpp/test/memory/detail/resource.cpp diff --git a/cpp/include/rapids_triton/memory/buffer.hpp b/cpp/include/rapids_triton/memory/buffer.hpp index b3ab62c..930c6df 100644 --- a/cpp/include/rapids_triton/memory/buffer.hpp +++ b/cpp/include/rapids_triton/memory/buffer.hpp @@ -27,7 +27,6 @@ #include #include #include -#include namespace triton { namespace backend { @@ -40,7 +39,37 @@ struct Buffer { using h_buffer = T*; using d_buffer = T*; using owned_h_buffer = std::unique_ptr; - using owned_d_buffer = rmm::device_buffer; + struct owned_d_buffer { + using non_const_T = std::remove_const_t; + owned_d_buffer(device_id_t device_id, std::size_t size, cudaStream_t stream) : + device_{device_id}, + byte_size_{size * sizeof(non_const_T)}, + data_{[this, &stream]() { + auto* result = static_cast(nullptr); + try { + result = static_cast( + get_memory_resource(device_)->allocate(byte_size_, stream) + ); + } catch(rmm::bad_alloc const& err) { + throw TritonException(Error::Internal, err.what()); + } + return result; + }()} {} + ~owned_d_buffer() { + get_memory_resource(device_)->deallocate(reinterpret_cast(data_), byte_size_); + } + + owned_d_buffer(owned_d_buffer const& other) = delete; + owned_d_buffer(owned_d_buffer&& other) noexcept = default; + owned_d_buffer& operator=(owned_d_buffer const& other) = delete; + owned_d_buffer& operator=(owned_d_buffer&& other) = default; + + auto* get() const { return data_; } + private: + device_id_t device_; + std::size_t byte_size_; + non_const_T* data_; + }; using data_store = std::variant; Buffer() noexcept : device_{}, data_{std::in_place_index<0>, nullptr}, size_{}, stream_{} {} @@ -192,7 +221,7 @@ struct Buffer { case 0: result = std::get<0>(ptr); break; case 1: result = std::get<1>(ptr); break; case 2: result = std::get<2>(ptr).get(); break; - case 3: result = reinterpret_cast(std::get<3>(ptr).data()); break; + case 3: result = std::get<3>(ptr).get(); break; } return result; } @@ -207,9 +236,9 @@ struct Buffer { if (memory_type == DeviceMemory) { if constexpr (IS_GPU_BUILD) { result = data_store{owned_d_buffer{ - size * sizeof(T), + device, + size, stream, - get_memory_resource(device) }}; } else { throw TritonException(Error::Internal, diff --git a/cpp/include/rapids_triton/memory/detail/copy.hpp b/cpp/include/rapids_triton/memory/detail/copy.hpp index de0efe2..3211893 100644 --- a/cpp/include/rapids_triton/memory/detail/copy.hpp +++ b/cpp/include/rapids_triton/memory/detail/copy.hpp @@ -38,7 +38,7 @@ void dev_copy(T* dst, T const* src, std::size_t len, cudaStream_t stream) if constexpr (IS_GPU_BUILD) { try { raft::copy(dst, src, len, stream); - } catch (const raft::cuda_error& err) { + } catch (raft::cuda_error const& err) { throw TritonException(Error::Internal, err.what()); } } else { diff --git a/cpp/include/rapids_triton/memory/detail/resource.hpp b/cpp/include/rapids_triton/memory/detail/resource.hpp index d8a1984..5fa39a1 100644 --- a/cpp/include/rapids_triton/memory/detail/resource.hpp +++ b/cpp/include/rapids_triton/memory/detail/resource.hpp @@ -15,6 +15,7 @@ */ #pragma once +#include #include #include #include @@ -22,6 +23,7 @@ #include #include #include +#include namespace triton { namespace backend { @@ -33,54 +35,58 @@ namespace detail { return lock; } + /** A struct used solely to keep memory resources in-scope for the lifetime + * of the backend */ struct resource_data { resource_data() : base_mr_{}, - pool_mr_{&base_mr_} {} - auto* get_resource() { - return &pool_mr_; + pool_mrs_{} {} + auto* make_new_resource() { + pool_mrs_.emplace_back(&base_mr_); + return &(pool_mrs_.back()); } private: rmm::mr::cuda_memory_resource base_mr_; - rmm::mr::pool_memory_resource pool_mr_; + std::deque> pool_mrs_; }; - auto& all_device_resources() { - // This vector keeps the underlying memory resource objects in-scope until - // the backend is unloaded. This ensures that they will not be destroyed - // while a model is still making use of them. - static auto device_resources = std::vector{}; + inline auto& get_device_resources() { + static auto device_resources = resource_data{}; return device_resources; } - auto is_default_resource (rmm::cuda_device_id const& device_id) { - return rmm::get_per_device_resource(device_id)->is_equal(rmm::cuda_memory_resource{}); + inline auto is_default_resource (rmm::cuda_device_id const& device_id) { + return rmm::mr::get_per_device_resource(device_id)->is_equal(rmm::mr::cuda_memory_resource{}); } - auto setup_memory_resource(rmm::cuda_device_id device_id) { + inline auto setup_memory_resource(rmm::cuda_device_id device_id) { auto lock = std::lock_guard{resource_lock()}; if (is_default_resource(device_id)) { - auto& device_resources = all_device_resources(); - device_resources.push_back(resource_data{}); + auto& device_resources = get_device_resources(); rmm::mr::set_per_device_resource( - rmm::cuda_device_id{device_id}, device_resources.back()->get_resource()); + rmm::cuda_device_id{device_id}, device_resources.make_new_resource()); } - return rmm::get_per_device_resource(rmm_device_id); + return rmm::mr::get_per_device_resource(device_id); } } // namespace detail -auto* get_memory_resource(device_id_t device_id) { - auto result = static_cast(nullptr); +inline auto setup_memory_resource(device_id_t device_id) { auto rmm_device_id = rmm::cuda_device_id{device_id}; - if (is_default_resource(rmm_device_id)) { - result = setup_memory_resource(rmm_device_id); + return detail::setup_memory_resource(rmm_device_id); +} + +inline auto* get_memory_resource(device_id_t device_id) { + auto result = static_cast(nullptr); + auto rmm_device_id = rmm::cuda_device_id{device_id}; + if (detail::is_default_resource(rmm_device_id)) { + result = detail::setup_memory_resource(rmm_device_id); } else { - result = rmm::get_per_device_resource(rmm_device_id); + result = rmm::mr::get_per_device_resource(rmm_device_id); } return result; } -auto* get_memory_resource() { +inline auto* get_memory_resource() { auto device_id = int{}; cuda_check(cudaGetDevice(&device_id)); diff --git a/cpp/include/rapids_triton/model/model.hpp b/cpp/include/rapids_triton/model/model.hpp index 8b90942..0ad3b14 100644 --- a/cpp/include/rapids_triton/model/model.hpp +++ b/cpp/include/rapids_triton/model/model.hpp @@ -167,8 +167,8 @@ struct Model { filepath_{filepath} { if constexpr (IS_GPU_BUILD) { - if (deplyment_type_ == GPUDeployment) { - detail::setup_memory_resource(device_id); + if (deployment_type_ == GPUDeployment) { + setup_memory_resource(device_id); } } } diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 1af11bb..2f54f7c 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -20,8 +20,8 @@ add_executable(test_rapids_triton test/build_control.cpp test/exceptions.cpp test/memory/buffer.cpp - test/memory/detail/allocate.cpp test/memory/detail/copy.cpp + test/memory/detail/resource.cpp test/memory/types.cpp test/tensor/dtype.cpp test/tensor/tensor.cpp diff --git a/cpp/test/memory/detail/resource.cpp b/cpp/test/memory/detail/resource.cpp new file mode 100644 index 0000000..3331d08 --- /dev/null +++ b/cpp/test/memory/detail/resource.cpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +TEST(RapidsTriton, get_memory_resource) +{ + if constexpr(IS_GPU_BUILD) { + auto device_id = int{}; + cuda_check(cudaGetDevice(&device_id)); + auto rmm_device_id = rmm::cuda_device_id{device_id}; + EXPECT_EQ(get_memory_resource(), get_memory_resource(device_id)); + EXPECT_EQ(detail::is_default_resource(rmm_device_id), false); + } +} + +} // namespace rapids +} // namespace backend +} // namespace triton From 16885636a15aaf0de8238c37c796c6ecd237626b Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 8 Oct 2021 12:51:13 -0400 Subject: [PATCH 107/199] Introduce triton_memory_resource for management --- cpp/include/rapids_triton/memory/buffer.hpp | 14 ++- .../memory/{detail => }/resource.hpp | 63 ++++++------- cpp/include/rapids_triton/model/model.hpp | 6 +- .../rapids_triton/triton/api/initialize.hpp | 21 +++++ .../triton/api/instance_initialize.hpp | 3 + .../triton/triton_memory_resource.hpp | 89 +++++++++++++++++++ 6 files changed, 153 insertions(+), 43 deletions(-) rename cpp/include/rapids_triton/memory/{detail => }/resource.hpp (50%) create mode 100644 cpp/include/rapids_triton/triton/triton_memory_resource.hpp diff --git a/cpp/include/rapids_triton/memory/buffer.hpp b/cpp/include/rapids_triton/memory/buffer.hpp index 930c6df..9713bbb 100644 --- a/cpp/include/rapids_triton/memory/buffer.hpp +++ b/cpp/include/rapids_triton/memory/buffer.hpp @@ -17,14 +17,16 @@ #pragma once #include #include +#include #include #include #include #include -#include +#include #include +#include #include #include @@ -50,13 +52,19 @@ struct Buffer { result = static_cast( get_memory_resource(device_)->allocate(byte_size_, stream) ); - } catch(rmm::bad_alloc const& err) { + } catch(std::bad_alloc const& err) { throw TritonException(Error::Internal, err.what()); } return result; }()} {} ~owned_d_buffer() { - get_memory_resource(device_)->deallocate(reinterpret_cast(data_), byte_size_); + try { + get_memory_resource(device_)->deallocate(reinterpret_cast(data_), byte_size_); + } catch (TritonException const& err) { + log_error(__FILE__, __LINE__) << err.what(); + } catch (...) { + log_error(__FILE__, __LINE__) << "Unknown error in owned_d_buffer destructor!"; + } } owned_d_buffer(owned_d_buffer const& other) = delete; diff --git a/cpp/include/rapids_triton/memory/detail/resource.hpp b/cpp/include/rapids_triton/memory/resource.hpp similarity index 50% rename from cpp/include/rapids_triton/memory/detail/resource.hpp rename to cpp/include/rapids_triton/memory/resource.hpp index 5fa39a1..6656231 100644 --- a/cpp/include/rapids_triton/memory/detail/resource.hpp +++ b/cpp/include/rapids_triton/memory/resource.hpp @@ -20,10 +20,11 @@ #include #include #include +#include #include #include #include -#include +#include namespace triton { namespace backend { @@ -39,14 +40,17 @@ namespace detail { * of the backend */ struct resource_data { resource_data() : base_mr_{}, - pool_mrs_{} {} - auto* make_new_resource() { - pool_mrs_.emplace_back(&base_mr_); - return &(pool_mrs_.back()); + triton_mrs_{} {} + auto* make_new_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* manager) { + if (manager == nullptr && triton_mrs_.size() != 0) { + manager = triton_mrs_.back().get_triton_manager(); + } + triton_mrs_.emplace_back(manager, device_id, &base_mr_); + return &(triton_mrs_.back()); } private: rmm::mr::cuda_memory_resource base_mr_; - std::deque> pool_mrs_; + std::deque triton_mrs_; }; inline auto& get_device_resources() { @@ -54,44 +58,31 @@ namespace detail { return device_resources; } - inline auto is_default_resource (rmm::cuda_device_id const& device_id) { - return rmm::mr::get_per_device_resource(device_id)->is_equal(rmm::mr::cuda_memory_resource{}); + inline auto is_triton_resource (rmm::cuda_device_id const& device_id) { + auto* triton_mr = dynamic_cast( + rmm::mr::get_per_device_resource(device_id) + ); + return (triton_mr != nullptr && triton_mr->get_triton_manager() != nullptr); } +} // namespace detail + + inline auto* setup_memory_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* triton_manager=nullptr) { + auto lock = std::lock_guard{detail::resource_lock()}; + auto rmm_device_id = rmm::cuda_device_id{device_id}; - inline auto setup_memory_resource(rmm::cuda_device_id device_id) { - auto lock = std::lock_guard{resource_lock()}; - if (is_default_resource(device_id)) { - auto& device_resources = get_device_resources(); + if (!detail::is_triton_resource(rmm_device_id)) { + auto& device_resources = detail::get_device_resources(); rmm::mr::set_per_device_resource( - rmm::cuda_device_id{device_id}, device_resources.make_new_resource()); + rmm_device_id, device_resources.make_new_resource(device_id, triton_manager)); } - return rmm::mr::get_per_device_resource(device_id); + return rmm::mr::get_per_device_resource(rmm_device_id); } -} // namespace detail -inline auto setup_memory_resource(device_id_t device_id) { - auto rmm_device_id = rmm::cuda_device_id{device_id}; - return detail::setup_memory_resource(rmm_device_id); -} - -inline auto* get_memory_resource(device_id_t device_id) { - auto result = static_cast(nullptr); - auto rmm_device_id = rmm::cuda_device_id{device_id}; - if (detail::is_default_resource(rmm_device_id)) { - result = detail::setup_memory_resource(rmm_device_id); - } else { - result = rmm::mr::get_per_device_resource(rmm_device_id); + inline auto* get_memory_resource(device_id_t device_id) { + auto rmm_device_id = rmm::cuda_device_id{device_id}; + return rmm::mr::get_per_device_resource(rmm_device_id); } - return result; -} - -inline auto* get_memory_resource() { - auto device_id = int{}; - cuda_check(cudaGetDevice(&device_id)); - - return get_memory_resource(device_id); -} } // namespace rapids } // namespace backend diff --git a/cpp/include/rapids_triton/model/model.hpp b/cpp/include/rapids_triton/model/model.hpp index 0ad3b14..d2aab53 100644 --- a/cpp/include/rapids_triton/model/model.hpp +++ b/cpp/include/rapids_triton/model/model.hpp @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include #include @@ -167,9 +167,7 @@ struct Model { filepath_{filepath} { if constexpr (IS_GPU_BUILD) { - if (deployment_type_ == GPUDeployment) { - setup_memory_resource(device_id); - } + setup_memory_resource(device_id_); } } diff --git a/cpp/include/rapids_triton/triton/api/initialize.hpp b/cpp/include/rapids_triton/triton/api/initialize.hpp index cccc054..7e7f830 100644 --- a/cpp/include/rapids_triton/triton/api/initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/initialize.hpp @@ -15,11 +15,15 @@ */ #pragma once +#include #include +#include #include #include +#include #include +#include #include namespace triton { @@ -38,6 +42,23 @@ inline auto* initialize(TRITONBACKEND_Backend* backend) throw TritonException{Error::Unsupported, "triton backend API version does not support this backend"}; } + if constexpr (IS_GPU_BUILD) { + auto device_count = int{}; + auto cuda_err = cudaGetDeviceCount(&device_count); + if (device_count > 0 && cuda_err == cudaSuccess) { + auto device_id = int{}; + cuda_check(cudaGetDevice(&device_id)); + auto* triton_manager = static_cast( + nullptr + ); + triton_check(TRITONBACKEND_BackendMemoryManager(backend, &triton_manager)); + + setup_memory_resource( + static_cast(device_id), + triton_manager + ); + } + } } catch (TritonException& err) { result = err.error(); } diff --git a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp index 0efab9d..a4239a4 100644 --- a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp @@ -42,6 +42,9 @@ auto* instance_initialize(TRITONBACKEND_ModelInstance* instance) auto* triton_model = get_model_from_instance(*instance); auto* model_state = get_model_state(*triton_model); + if constexpr (IS_GPU_BUILD) { + setup_memory_resource(device_id, model_state->TritonMemoryManager()); + } auto rapids_model = std::make_unique(*model_state, instance); rapids_model->load(); diff --git a/cpp/include/rapids_triton/triton/triton_memory_resource.hpp b/cpp/include/rapids_triton/triton/triton_memory_resource.hpp new file mode 100644 index 0000000..72af258 --- /dev/null +++ b/cpp/include/rapids_triton/triton/triton_memory_resource.hpp @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +struct triton_memory_resource final : public rmm::mr::device_memory_resource { + triton_memory_resource( + TRITONBACKEND_MemoryManager* manager, + device_id_t device_id, + rmm::mr::device_memory_resource* fallback) : manager_{manager}, device_id_{device_id}, fallback_{fallback} {} + + bool supports_streams() const noexcept override { return false; } + bool supports_get_mem_info() const noexcept override { return false; } + auto* get_triton_manager() const noexcept { return manager_; } + + private: + TRITONBACKEND_MemoryManager* manager_; + std::int64_t device_id_; + rmm::mr::device_memory_resource* fallback_; + + void* do_allocate(std::size_t bytes, rmm::cuda_stream_view stream) override { + auto* ptr = static_cast(nullptr); + if (manager_ == nullptr) { + ptr = fallback_->allocate(bytes, stream); + } else { + triton_check(TRITONBACKEND_MemoryManagerAllocate( + manager_, + &ptr, + TRITONSERVER_MEMORY_GPU, + device_id_, + static_cast(bytes) + )); + } + return ptr; + } + + void do_deallocate(void* ptr, std::size_t bytes, rmm::cuda_stream_view stream) { + if (manager_ == nullptr) { + fallback_->deallocate(ptr, bytes, stream); + } else { + triton_check(TRITONBACKEND_MemoryManagerFree( + manager_, + ptr, + TRITONSERVER_MEMORY_GPU, + device_id_ + )); + } + } + + bool do_is_equal(rmm::mr::device_memory_resource const& other) const noexcept override + { + auto* other_triton_mr = dynamic_cast(&other); + return (other_triton_mr != nullptr && other_triton_mr->get_triton_manager() == manager_); + } + + std::pair do_get_mem_info(rmm::cuda_stream_view stream) const override { + throw std::runtime_error("Mem info API not supported by triton_memory_resource"); + } + +}; + +}}} From e8f0dd3cb487a83d98472c44dc5fbb9d8548f8eb Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 8 Oct 2021 16:00:58 -0400 Subject: [PATCH 108/199] Correct move implementations for owned_d_buffer --- cpp/include/rapids_triton/memory/buffer.hpp | 45 +++++++++++++++------ cpp/src/model.h | 2 +- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/cpp/include/rapids_triton/memory/buffer.hpp b/cpp/include/rapids_triton/memory/buffer.hpp index 9713bbb..0c0c12a 100644 --- a/cpp/include/rapids_triton/memory/buffer.hpp +++ b/cpp/include/rapids_triton/memory/buffer.hpp @@ -58,29 +58,48 @@ struct Buffer { return result; }()} {} ~owned_d_buffer() { - try { - get_memory_resource(device_)->deallocate(reinterpret_cast(data_), byte_size_); - } catch (TritonException const& err) { - log_error(__FILE__, __LINE__) << err.what(); - } catch (...) { - log_error(__FILE__, __LINE__) << "Unknown error in owned_d_buffer destructor!"; - } + free_memory(); } owned_d_buffer(owned_d_buffer const& other) = delete; - owned_d_buffer(owned_d_buffer&& other) noexcept = default; + owned_d_buffer(owned_d_buffer&& other) noexcept : device_{other.device_}, byte_size_{other.byte_size_}, data_{nullptr} { + data_ = other.data_; + other.data_ = nullptr; + } owned_d_buffer& operator=(owned_d_buffer const& other) = delete; - owned_d_buffer& operator=(owned_d_buffer&& other) = default; + owned_d_buffer& operator=(owned_d_buffer&& other) { + if (this != &other) { + device_ = other.device_; + byte_size_ = other.byte_size_; + free_memory(); + data_ = other.data_; + other.data_ = nullptr; + } + return *this; + } auto* get() const { return data_; } private: device_id_t device_; std::size_t byte_size_; non_const_T* data_; + void free_memory() { + if (data_ != nullptr) { + try { + get_memory_resource(device_)->deallocate(reinterpret_cast(data_), byte_size_); + } catch (TritonException const& err) { + log_error(__FILE__, __LINE__) << err.what(); + } catch (...) { + log_error(__FILE__, __LINE__) << "Unknown error in owned_d_buffer destructor!"; + } + } + data_ = nullptr; + } }; using data_store = std::variant; - Buffer() noexcept : device_{}, data_{std::in_place_index<0>, nullptr}, size_{}, stream_{} {} + Buffer() noexcept : device_{}, data_{std::in_place_index<0>, nullptr}, size_{}, stream_{} { + } /** * @brief Construct buffer of given size in given memory location (either @@ -147,7 +166,8 @@ struct Buffer { * @brief Create owning copy of existing buffer * The memory type of this new buffer will be the same as the original */ - Buffer(Buffer const& other) : Buffer(other, other.mem_type(), other.device()) {} + Buffer(Buffer const& other) : Buffer(other, other.mem_type(), other.device()) { + } Buffer(Buffer&& other, MemoryType memory_type) : device_{other.device()}, @@ -170,7 +190,8 @@ struct Buffer { Buffer& operator=(Buffer&& other) = default; - ~Buffer() = default; + ~Buffer() { + } /** * @brief Return where memory for this buffer is located (host or device) diff --git a/cpp/src/model.h b/cpp/src/model.h index 1232221..26d10c1 100644 --- a/cpp/src/model.h +++ b/cpp/src/model.h @@ -143,7 +143,7 @@ struct RapidsModel : rapids::Model { **************************************************************************/ std::optional preferred_mem_type(rapids::Batch& batch) const { - return std::nullopt; + return rapids::HostMemory; } }; From def5d861f557aa223f64a88eb9f601df0e98d264 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 8 Oct 2021 16:35:40 -0400 Subject: [PATCH 109/199] Update memory resource tests --- cpp/include/rapids_triton/memory/resource.hpp | 4 ++++ .../rapids_triton/triton/api/instance_initialize.hpp | 1 + cpp/src/model.h | 2 +- cpp/test/CMakeLists.txt | 2 +- cpp/test/memory/{detail => }/resource.cpp | 10 ++++++---- 5 files changed, 13 insertions(+), 6 deletions(-) rename cpp/test/memory/{detail => }/resource.cpp (73%) diff --git a/cpp/include/rapids_triton/memory/resource.hpp b/cpp/include/rapids_triton/memory/resource.hpp index 6656231..451d396 100644 --- a/cpp/include/rapids_triton/memory/resource.hpp +++ b/cpp/include/rapids_triton/memory/resource.hpp @@ -84,6 +84,10 @@ namespace detail { return rmm::mr::get_per_device_resource(rmm_device_id); } + inline auto* get_memory_resource() { + return rmm::mr::get_current_device_resource(); + } + } // namespace rapids } // namespace backend } // namespace triton diff --git a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp index a4239a4..e9e5a0c 100644 --- a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp @@ -18,6 +18,7 @@ #include #include +#include #include #include #include diff --git a/cpp/src/model.h b/cpp/src/model.h index 26d10c1..1232221 100644 --- a/cpp/src/model.h +++ b/cpp/src/model.h @@ -143,7 +143,7 @@ struct RapidsModel : rapids::Model { **************************************************************************/ std::optional preferred_mem_type(rapids::Batch& batch) const { - return rapids::HostMemory; + return std::nullopt; } }; diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 2f54f7c..467d273 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -21,7 +21,7 @@ add_executable(test_rapids_triton test/exceptions.cpp test/memory/buffer.cpp test/memory/detail/copy.cpp - test/memory/detail/resource.cpp + test/memory/resource.cpp test/memory/types.cpp test/tensor/dtype.cpp test/tensor/tensor.cpp diff --git a/cpp/test/memory/detail/resource.cpp b/cpp/test/memory/resource.cpp similarity index 73% rename from cpp/test/memory/detail/resource.cpp rename to cpp/test/memory/resource.cpp index 3331d08..6a0aeb1 100644 --- a/cpp/test/memory/detail/resource.cpp +++ b/cpp/test/memory/resource.cpp @@ -20,8 +20,9 @@ #include #include -#include +#include #include +#include namespace triton { namespace backend { @@ -31,9 +32,10 @@ TEST(RapidsTriton, get_memory_resource) if constexpr(IS_GPU_BUILD) { auto device_id = int{}; cuda_check(cudaGetDevice(&device_id)); - auto rmm_device_id = rmm::cuda_device_id{device_id}; - EXPECT_EQ(get_memory_resource(), get_memory_resource(device_id)); - EXPECT_EQ(detail::is_default_resource(rmm_device_id), false); + EXPECT_EQ(get_memory_resource(device_id)->is_equal(rmm::mr::cuda_memory_resource{}), true); + setup_memory_resource(device_id); + EXPECT_EQ(get_memory_resource(device_id)->is_equal(rmm::mr::cuda_memory_resource{}), false); + EXPECT_EQ(get_memory_resource(device_id), get_memory_resource()); } } From d81e6ef201a2e3326a0b65b207b72e57bfc2d75b Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 8 Oct 2021 16:46:11 -0400 Subject: [PATCH 110/199] Update behavior of do_get_mem_info Simply return zeros rather than throwing in accordance with typical RMM usage --- cpp/include/rapids_triton/triton/triton_memory_resource.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/triton/triton_memory_resource.hpp b/cpp/include/rapids_triton/triton/triton_memory_resource.hpp index 72af258..52861b6 100644 --- a/cpp/include/rapids_triton/triton/triton_memory_resource.hpp +++ b/cpp/include/rapids_triton/triton/triton_memory_resource.hpp @@ -81,7 +81,7 @@ struct triton_memory_resource final : public rmm::mr::device_memory_resource { } std::pair do_get_mem_info(rmm::cuda_stream_view stream) const override { - throw std::runtime_error("Mem info API not supported by triton_memory_resource"); + return {0, 0}; } }; From a08afc67fcc539535aa148e67e849179de496069 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 11 Oct 2021 13:11:23 -0400 Subject: [PATCH 111/199] Update to Triton 21.09 --- Dockerfile | 2 +- cpp/CMakeLists.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6be8a53..c6e1bfa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # Arguments for controlling build details ########################################################################################### # Version of Triton to use -ARG TRITON_VERSION=21.08 +ARG TRITON_VERSION=21.09 # Base container image ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 # Whether or not to build indicated components diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 217223a..e3e0ae0 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -50,9 +50,9 @@ option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(NVTX "Enable nvtx markers" OFF) option(TRITON_ENABLE_GPU "Enable GPU support in Triton" ON) option(TRITON_ENABLE_STATS "Enable statistics collection in Triton" ON) -set(TRITON_COMMON_REPO_TAG "r21.08" CACHE STRING "Tag for triton-inference-server/common repo") -set(TRITON_CORE_REPO_TAG "r21.08" CACHE STRING "Tag for triton-inference-server/core repo") -set(TRITON_BACKEND_REPO_TAG "r21.08" CACHE STRING "Tag for triton-inference-server/backend repo") +set(TRITON_COMMON_REPO_TAG "r21.09" CACHE STRING "Tag for triton-inference-server/common repo") +set(TRITON_CORE_REPO_TAG "r21.09" CACHE STRING "Tag for triton-inference-server/core repo") +set(TRITON_BACKEND_REPO_TAG "r21.09" CACHE STRING "Tag for triton-inference-server/backend repo") message(VERBOSE "RAPIDS_TRITON: Build RAPIDS_TRITON unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAPIDS_TRITON: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") From b4cd88964db3a8db45eaa676200487bab70aff03 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 11 Oct 2021 13:50:00 -0400 Subject: [PATCH 112/199] Update Triton bug warning --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ad7b821..65ef436 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,10 @@ the integration of RAPIDS algorithms, this library can be used by anyone to quickly get up and running with a custom backend for Triton. **WARNING**: Due to an upstream bug in Triton, backends which make use of -RAPIDS-Triton may encounter issues with Triton's shared memory mode. -Specifically, backends deployed on the host or which request input on host -memory should not be given cuda shared memory inputs and vice versa. A fix for -this issue in Triton is expected soon. +RAPIDS-Triton may encounter issues with Triton's shared memory mode. A fix is +available in Triton 21.10, which will be released around the end of October. At +that time, RAPIDS-Triton will be updated to default to Triton 21.10, and this +warning will be removed. ## Background From 334b504efad85ec0cc673779674eb925052c63a1 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 11 Oct 2021 13:58:49 -0400 Subject: [PATCH 113/199] Update rapids::copy to allow template param omission --- README.md | 2 +- cpp/include/rapids_triton/tensor/tensor.hpp | 4 ++-- cpp/src/model.h | 2 +- cpp/test/tensor/tensor.cpp | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 65ef436..c29e0e3 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ backend is defining the `predict` function for your model as shown below: rapids::Tensor input = get_input(batch, "input__0"); rapids::Tensor output = get_output(batch, "output__0"); - rapids::copy(output, input); + rapids::copy(output, input); output.finalize(); } diff --git a/cpp/include/rapids_triton/tensor/tensor.hpp b/cpp/include/rapids_triton/tensor/tensor.hpp index 17abc74..a4fc34d 100644 --- a/cpp/include/rapids_triton/tensor/tensor.hpp +++ b/cpp/include/rapids_triton/tensor/tensor.hpp @@ -171,8 +171,8 @@ struct OutputTensor final : BaseTensor { std::shared_ptr responder_; }; -template -void copy(BaseTensor>& dst, BaseTensor& src) +template , T>>> +void copy(BaseTensor& dst, BaseTensor& src) { copy(dst.buffer(), src.buffer()); } diff --git a/cpp/src/model.h b/cpp/src/model.h index 1232221..b9acf86 100644 --- a/cpp/src/model.h +++ b/cpp/src/model.h @@ -87,7 +87,7 @@ struct RapidsModel : rapids::Model { // 3. Perform inference. In this example, we simply copy the data from the // input to the output tensor. - rapids::copy(output, input); + rapids::copy(output, input); // 4. Call finalize on all output tensors. In this case, we have just one // output, so we call finalize on it. diff --git a/cpp/test/tensor/tensor.cpp b/cpp/test/tensor/tensor.cpp index d6b53f4..672402d 100644 --- a/cpp/test/tensor/tensor.cpp +++ b/cpp/test/tensor/tensor.cpp @@ -83,7 +83,7 @@ TEST(RapidsTriton, tensor_copy) auto data2 = std::vector(data1.size()); auto tensor2 = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); - copy(tensor2, tensor1); + rapids::copy(tensor2, tensor1); auto data_out = std::vector(tensor2.data(), tensor2.data() + tensor2.size()); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); @@ -93,7 +93,7 @@ TEST(RapidsTriton, tensor_copy) auto tensor3 = Tensor(small_shape, Buffer{small_data.data(), small_data.size(), HostMemory}); - EXPECT_THROW(copy(tensor3, tensor1), TritonException); + EXPECT_THROW(rapids::copy(tensor3, tensor1), TritonException); } TEST(RapidsTriton, tensor_multi_copy) From 54f1815f3a340e3b5760112dc0b433198601d0a6 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 11 Oct 2021 14:03:00 -0400 Subject: [PATCH 114/199] Fix style --- cpp/include/rapids_triton/memory/buffer.hpp | 59 ++++++----- cpp/include/rapids_triton/memory/resource.hpp | 98 ++++++++++--------- cpp/include/rapids_triton/model/model.hpp | 4 +- .../rapids_triton/model/shared_state.hpp | 3 +- cpp/include/rapids_triton/tensor/tensor.hpp | 4 +- .../rapids_triton/triton/api/initialize.hpp | 13 +-- .../triton/triton_memory_resource.hpp | 47 +++++---- cpp/test/memory/resource.cpp | 2 +- 8 files changed, 113 insertions(+), 117 deletions(-) diff --git a/cpp/include/rapids_triton/memory/buffer.hpp b/cpp/include/rapids_triton/memory/buffer.hpp index 0c0c12a..802ce72 100644 --- a/cpp/include/rapids_triton/memory/buffer.hpp +++ b/cpp/include/rapids_triton/memory/buffer.hpp @@ -24,11 +24,11 @@ #include #include -#include #include #include #include #include +#include namespace triton { namespace backend { @@ -43,47 +43,49 @@ struct Buffer { using owned_h_buffer = std::unique_ptr; struct owned_d_buffer { using non_const_T = std::remove_const_t; - owned_d_buffer(device_id_t device_id, std::size_t size, cudaStream_t stream) : - device_{device_id}, - byte_size_{size * sizeof(non_const_T)}, - data_{[this, &stream]() { - auto* result = static_cast(nullptr); - try { - result = static_cast( - get_memory_resource(device_)->allocate(byte_size_, stream) - ); - } catch(std::bad_alloc const& err) { - throw TritonException(Error::Internal, err.what()); - } - return result; - }()} {} - ~owned_d_buffer() { - free_memory(); + owned_d_buffer(device_id_t device_id, std::size_t size, cudaStream_t stream) + : device_{device_id}, byte_size_{size * sizeof(non_const_T)}, data_{[this, &stream]() { + auto* result = static_cast(nullptr); + try { + result = + static_cast(get_memory_resource(device_)->allocate(byte_size_, stream)); + } catch (std::bad_alloc const& err) { + throw TritonException(Error::Internal, err.what()); + } + return result; + }()} + { } + ~owned_d_buffer() { free_memory(); } owned_d_buffer(owned_d_buffer const& other) = delete; - owned_d_buffer(owned_d_buffer&& other) noexcept : device_{other.device_}, byte_size_{other.byte_size_}, data_{nullptr} { - data_ = other.data_; + owned_d_buffer(owned_d_buffer&& other) noexcept + : device_{other.device_}, byte_size_{other.byte_size_}, data_{nullptr} + { + data_ = other.data_; other.data_ = nullptr; } owned_d_buffer& operator=(owned_d_buffer const& other) = delete; - owned_d_buffer& operator=(owned_d_buffer&& other) { + owned_d_buffer& operator =(owned_d_buffer&& other) + { if (this != &other) { - device_ = other.device_; + device_ = other.device_; byte_size_ = other.byte_size_; free_memory(); - data_ = other.data_; + data_ = other.data_; other.data_ = nullptr; } return *this; } auto* get() const { return data_; } + private: device_id_t device_; std::size_t byte_size_; non_const_T* data_; - void free_memory() { + void free_memory() + { if (data_ != nullptr) { try { get_memory_resource(device_)->deallocate(reinterpret_cast(data_), byte_size_); @@ -96,10 +98,9 @@ struct Buffer { data_ = nullptr; } }; - using data_store = std::variant; + using data_store = std::variant; - Buffer() noexcept : device_{}, data_{std::in_place_index<0>, nullptr}, size_{}, stream_{} { - } + Buffer() noexcept : device_{}, data_{std::in_place_index<0>, nullptr}, size_{}, stream_{} {} /** * @brief Construct buffer of given size in given memory location (either @@ -166,8 +167,7 @@ struct Buffer { * @brief Create owning copy of existing buffer * The memory type of this new buffer will be the same as the original */ - Buffer(Buffer const& other) : Buffer(other, other.mem_type(), other.device()) { - } + Buffer(Buffer const& other) : Buffer(other, other.mem_type(), other.device()) {} Buffer(Buffer&& other, MemoryType memory_type) : device_{other.device()}, @@ -190,8 +190,7 @@ struct Buffer { Buffer& operator=(Buffer&& other) = default; - ~Buffer() { - } + ~Buffer() {} /** * @brief Return where memory for this buffer is located (host or device) diff --git a/cpp/include/rapids_triton/memory/resource.hpp b/cpp/include/rapids_triton/memory/resource.hpp index 451d396..ab843cf 100644 --- a/cpp/include/rapids_triton/memory/resource.hpp +++ b/cpp/include/rapids_triton/memory/resource.hpp @@ -15,6 +15,7 @@ */ #pragma once +#include #include #include #include @@ -24,69 +25,72 @@ #include #include #include -#include namespace triton { namespace backend { namespace rapids { namespace detail { - inline auto& resource_lock() { - static auto lock = std::mutex{}; - return lock; - } +inline auto& resource_lock() +{ + static auto lock = std::mutex{}; + return lock; +} - /** A struct used solely to keep memory resources in-scope for the lifetime - * of the backend */ - struct resource_data { - resource_data() : base_mr_{}, - triton_mrs_{} {} - auto* make_new_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* manager) { - if (manager == nullptr && triton_mrs_.size() != 0) { - manager = triton_mrs_.back().get_triton_manager(); - } - triton_mrs_.emplace_back(manager, device_id, &base_mr_); - return &(triton_mrs_.back()); +/** A struct used solely to keep memory resources in-scope for the lifetime + * of the backend */ +struct resource_data { + resource_data() : base_mr_{}, triton_mrs_{} {} + auto* make_new_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* manager) + { + if (manager == nullptr && triton_mrs_.size() != 0) { + manager = triton_mrs_.back().get_triton_manager(); } - private: - rmm::mr::cuda_memory_resource base_mr_; - std::deque triton_mrs_; - }; - - inline auto& get_device_resources() { - static auto device_resources = resource_data{}; - return device_resources; + triton_mrs_.emplace_back(manager, device_id, &base_mr_); + return &(triton_mrs_.back()); } - inline auto is_triton_resource (rmm::cuda_device_id const& device_id) { - auto* triton_mr = dynamic_cast( - rmm::mr::get_per_device_resource(device_id) - ); - return (triton_mr != nullptr && triton_mr->get_triton_manager() != nullptr); - } -} // namespace detail + private: + rmm::mr::cuda_memory_resource base_mr_; + std::deque triton_mrs_; +}; - inline auto* setup_memory_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* triton_manager=nullptr) { - auto lock = std::lock_guard{detail::resource_lock()}; - auto rmm_device_id = rmm::cuda_device_id{device_id}; +inline auto& get_device_resources() +{ + static auto device_resources = resource_data{}; + return device_resources; +} - if (!detail::is_triton_resource(rmm_device_id)) { - auto& device_resources = detail::get_device_resources(); - rmm::mr::set_per_device_resource( - rmm_device_id, device_resources.make_new_resource(device_id, triton_manager)); - } +inline auto is_triton_resource(rmm::cuda_device_id const& device_id) +{ + auto* triton_mr = + dynamic_cast(rmm::mr::get_per_device_resource(device_id)); + return (triton_mr != nullptr && triton_mr->get_triton_manager() != nullptr); +} +} // namespace detail - return rmm::mr::get_per_device_resource(rmm_device_id); - } +inline auto* setup_memory_resource(device_id_t device_id, + TRITONBACKEND_MemoryManager* triton_manager = nullptr) +{ + auto lock = std::lock_guard{detail::resource_lock()}; + auto rmm_device_id = rmm::cuda_device_id{device_id}; - inline auto* get_memory_resource(device_id_t device_id) { - auto rmm_device_id = rmm::cuda_device_id{device_id}; - return rmm::mr::get_per_device_resource(rmm_device_id); + if (!detail::is_triton_resource(rmm_device_id)) { + auto& device_resources = detail::get_device_resources(); + rmm::mr::set_per_device_resource(rmm_device_id, + device_resources.make_new_resource(device_id, triton_manager)); } - inline auto* get_memory_resource() { - return rmm::mr::get_current_device_resource(); - } + return rmm::mr::get_per_device_resource(rmm_device_id); +} + +inline auto* get_memory_resource(device_id_t device_id) +{ + auto rmm_device_id = rmm::cuda_device_id{device_id}; + return rmm::mr::get_per_device_resource(rmm_device_id); +} + +inline auto* get_memory_resource() { return rmm::mr::get_current_device_resource(); } } // namespace rapids } // namespace backend diff --git a/cpp/include/rapids_triton/model/model.hpp b/cpp/include/rapids_triton/model/model.hpp index d2aab53..a2abcf1 100644 --- a/cpp/include/rapids_triton/model/model.hpp +++ b/cpp/include/rapids_triton/model/model.hpp @@ -166,9 +166,7 @@ struct Model { deployment_type_{deployment_type}, filepath_{filepath} { - if constexpr (IS_GPU_BUILD) { - setup_memory_resource(device_id_); - } + if constexpr (IS_GPU_BUILD) { setup_memory_resource(device_id_); } } auto get_device_id() const { return device_id_; } diff --git a/cpp/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp index 772e34d..7927a59 100644 --- a/cpp/include/rapids_triton/model/shared_state.hpp +++ b/cpp/include/rapids_triton/model/shared_state.hpp @@ -41,7 +41,8 @@ struct SharedModelState { virtual void load() {} virtual void unload() {} - explicit SharedModelState(std::unique_ptr&& config, bool squeeze_output=false) + explicit SharedModelState(std::unique_ptr&& config, + bool squeeze_output = false) : config_{std::move(config)}, max_batch_size_{get_max_batch_size(*config_)}, output_shapes_([this, squeeze_output]() { diff --git a/cpp/include/rapids_triton/tensor/tensor.hpp b/cpp/include/rapids_triton/tensor/tensor.hpp index a4fc34d..3a7f1e0 100644 --- a/cpp/include/rapids_triton/tensor/tensor.hpp +++ b/cpp/include/rapids_triton/tensor/tensor.hpp @@ -171,7 +171,9 @@ struct OutputTensor final : BaseTensor { std::shared_ptr responder_; }; -template , T>>> +template , T>>> void copy(BaseTensor& dst, BaseTensor& src) { copy(dst.buffer(), src.buffer()); diff --git a/cpp/include/rapids_triton/triton/api/initialize.hpp b/cpp/include/rapids_triton/triton/api/initialize.hpp index 7e7f830..b46dd8e 100644 --- a/cpp/include/rapids_triton/triton/api/initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/initialize.hpp @@ -20,10 +20,10 @@ #include #include +#include #include #include #include -#include #include namespace triton { @@ -44,19 +44,14 @@ inline auto* initialize(TRITONBACKEND_Backend* backend) } if constexpr (IS_GPU_BUILD) { auto device_count = int{}; - auto cuda_err = cudaGetDeviceCount(&device_count); + auto cuda_err = cudaGetDeviceCount(&device_count); if (device_count > 0 && cuda_err == cudaSuccess) { auto device_id = int{}; cuda_check(cudaGetDevice(&device_id)); - auto* triton_manager = static_cast( - nullptr - ); + auto* triton_manager = static_cast(nullptr); triton_check(TRITONBACKEND_BackendMemoryManager(backend, &triton_manager)); - setup_memory_resource( - static_cast(device_id), - triton_manager - ); + setup_memory_resource(static_cast(device_id), triton_manager); } } } catch (TritonException& err) { diff --git a/cpp/include/rapids_triton/triton/triton_memory_resource.hpp b/cpp/include/rapids_triton/triton/triton_memory_resource.hpp index 52861b6..77a879e 100644 --- a/cpp/include/rapids_triton/triton/triton_memory_resource.hpp +++ b/cpp/include/rapids_triton/triton/triton_memory_resource.hpp @@ -16,25 +16,27 @@ #pragma once +#include +#include #include #include -#include -#include #include #include #include #include -#include -#include +#include +#include namespace triton { namespace backend { namespace rapids { struct triton_memory_resource final : public rmm::mr::device_memory_resource { - triton_memory_resource( - TRITONBACKEND_MemoryManager* manager, - device_id_t device_id, - rmm::mr::device_memory_resource* fallback) : manager_{manager}, device_id_{device_id}, fallback_{fallback} {} + triton_memory_resource(TRITONBACKEND_MemoryManager* manager, + device_id_t device_id, + rmm::mr::device_memory_resource* fallback) + : manager_{manager}, device_id_{device_id}, fallback_{fallback} + { + } bool supports_streams() const noexcept override { return false; } bool supports_get_mem_info() const noexcept override { return false; } @@ -45,32 +47,25 @@ struct triton_memory_resource final : public rmm::mr::device_memory_resource { std::int64_t device_id_; rmm::mr::device_memory_resource* fallback_; - void* do_allocate(std::size_t bytes, rmm::cuda_stream_view stream) override { + void* do_allocate(std::size_t bytes, rmm::cuda_stream_view stream) override + { auto* ptr = static_cast(nullptr); if (manager_ == nullptr) { ptr = fallback_->allocate(bytes, stream); } else { triton_check(TRITONBACKEND_MemoryManagerAllocate( - manager_, - &ptr, - TRITONSERVER_MEMORY_GPU, - device_id_, - static_cast(bytes) - )); + manager_, &ptr, TRITONSERVER_MEMORY_GPU, device_id_, static_cast(bytes))); } return ptr; } - void do_deallocate(void* ptr, std::size_t bytes, rmm::cuda_stream_view stream) { + void do_deallocate(void* ptr, std::size_t bytes, rmm::cuda_stream_view stream) + { if (manager_ == nullptr) { fallback_->deallocate(ptr, bytes, stream); } else { - triton_check(TRITONBACKEND_MemoryManagerFree( - manager_, - ptr, - TRITONSERVER_MEMORY_GPU, - device_id_ - )); + triton_check( + TRITONBACKEND_MemoryManagerFree(manager_, ptr, TRITONSERVER_MEMORY_GPU, device_id_)); } } @@ -80,10 +75,12 @@ struct triton_memory_resource final : public rmm::mr::device_memory_resource { return (other_triton_mr != nullptr && other_triton_mr->get_triton_manager() == manager_); } - std::pair do_get_mem_info(rmm::cuda_stream_view stream) const override { + std::pair do_get_mem_info(rmm::cuda_stream_view stream) const override + { return {0, 0}; } - }; -}}} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/test/memory/resource.cpp b/cpp/test/memory/resource.cpp index 6a0aeb1..46c4982 100644 --- a/cpp/test/memory/resource.cpp +++ b/cpp/test/memory/resource.cpp @@ -29,7 +29,7 @@ namespace backend { namespace rapids { TEST(RapidsTriton, get_memory_resource) { - if constexpr(IS_GPU_BUILD) { + if constexpr (IS_GPU_BUILD) { auto device_id = int{}; cuda_check(cudaGetDevice(&device_id)); EXPECT_EQ(get_memory_resource(device_id)->is_equal(rmm::mr::cuda_memory_resource{}), true); From 2c4c1d1068fa1e84a9bbcd98a3a60c6a6043bda7 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 12 Oct 2021 12:49:00 -0400 Subject: [PATCH 115/199] Provide basic usage information --- CONTRIBUTING.md | 15 ++- docs/usage.md | 319 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 325 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f73039e..dbb1fa4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,8 +35,8 @@ To contribute code to this project, please follow these steps: 1. Find an issue to work on or submit an issue documenting the problem you would like to work on. 2. Comment on the issue saying that you plan to work on it. -3. Review the implementation details section below for information to help you - make your changes in a way that is consistent with the rest of the codebase. +3. Review the conventions below for information to help you make your changes + in a way that is consistent with the rest of the codebase. 4. Code! 5. Create your pull request. 6. Wait for other developers to review your code and update your PR as needed. @@ -45,8 +45,15 @@ To contribute code to this project, please follow these steps: ### Coding Conventions * RAPIDS-Triton follows [Almost Always Auto (AAA)](https://herbsutter.com/2013/08/12/gotw-94-solution-aaa-style-almost-always-auto/) - style. Please maintain this style in any contributions. -* Avoid raw loops where possible + style. Please maintain this style in any contributions, with the possible + exception of some docs, where type information may be helpful for new users + trying to understand a snippet in isolation. +* Avoid raw loops where possible. +* C++ versions of types should be used instead of C versions except when + interfacing with C code (e.g. use `std::size_t` instead of `size_t`). +* Avoid using output pointers in function signatures. Prefer instead to + actually return the value computed by the function and take advantage of + return value optimization and move semantics. ### Signing Your Work * We require that all contributors "sign-off" on their commits. This certifies that the contribution is your original work, or you have rights to submit it under the same license, or a compatible license. diff --git a/docs/usage.md b/docs/usage.md index 3c39701..d36a040 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -16,12 +16,321 @@ limitations under the License. # Using RAPIDS-Triton +## Getting Started To begin developing a custom backend with RAPIDS-Triton, we strongly recommend that you take advantage of the [rapids-triton-template repo](https://github.com/rapidsai/rapids-triton-template), which provides a -basic template for your backend code. For a detailed example of how to make use -of this template, check out the [Linear Example -repo](https://github.com/rapidsai/rapids-triton-linear-example), which provides -a complete walkthrough of how to build a backend using RAPIDS-Triton +basic template for your backend code. If this is your first time developing a +backend with RAPIDS-Triton, the easiest way to get started is to follow the +[Linear Example](https://github.com/rapidsai/rapids-triton-linear-example). +This provides a detailed walkthrough of every step in the process of creating a +backend with example code. The rest of these usage docs will provide general +information on specific features you are likely to take advantage of when +building your backend. - +## Logging +To provide logging messages in your backend, RAPIDS-Triton provides `log_info`, +`log_warn`, `log_error`, and `log_debug`. During default Triton execution, all +logging messages up to (but not including) debug level will be visible. These +functions can be invoked in two ways and can optionally include file and line +information. To add a logging message to your code, use one of the following +invocations: +```cpp +#include + +void logging_example() { + rapids::log_info() << "This is a log message."; + rapids::log_info("This is an equivalent invocation."); + rapids::log_info(__FILE__, __LINE__) << "This one has file and line info."; + rapids::log_info(__FILE__, __LINE__, "And so does this one."); +} +``` + +## Error Handling +If you encounter an error condition at any point in your backend which cannot +be otherwise handled, you should throw a `TritonException`. In most cases, this +error will be gracefully handled and passed to the Triton server in a way that +will not interfere with execution of other backends, models, or requests. + +`TritonException` objects are constructed with an error type and a message +indicating what went wrong, as shown below: +```cpp +#include + +void error_example() { + throw rapids::TritonException(rapids::Error::Internal, "Something bad happened!"); +} +``` + +Available error types are: +* `Internal`: The most common error type. Used when an unexpected condition + which is not the result of bad user input (e.g. CUDA error). +* `NotFound`: An error type returned when a named resource (e.g. named CUDA IPC + memory block) cannot be found. +* `InvalidArg`: An error type returned when the user has provided invalid input + in a configuration file or request. +* `Unavailable`: An error returned when a resource exists but is currently + unavailable. +* `Unsupported`: An error which indicates that a requested functionality is not + implemented by this backend (e.g. GPU execution for a CPU-only backend). +* `AlreadyExists`: An error which indicates that a resource which is being + created has already been created. +* `Unknown`: The type of the error cannot be established. This type should be + avoided wherever possible. + +The `cuda_check` function is provided to help facilitate error handling of +direct invocations of the CUDA API. If such an invocation fails, `cuda_check` +will throw an appropriate `TritonException`: + +```cpp +#include + +void cuda_check_example() { + rapids::cuda_check(cudaSetDevice(0)); +} +``` + +If a `TritonException` is thrown while a backend is being loaded, Triton's +server logs will indicate the failure and include the error message. If a +`TritonException` is thrown while a model is being loaded, Triton's server logs +will display the error message in the loading logs for that model. If a +`TritonException` is thrown during handling of a request, the client will +receive an indication that the request failed along with the error message, but +the model can continue to process other requests. + +## CPU-Only Builds +Most Triton backends include support for builds intended to support only CPU +execution. While this is not required, RAPIDS-Triton includes a compile-time +constant which can be useful for facilitating this: + +```cpp +#include + +void do_a_gpu_thing() { + if constexpr (rapids::IS_GPU_BUILD) { + rapids::log_info("Executing on GPU..."); + } else { + rapids::log_error("Can't do that! This is a CPU-only build."); + } +} +``` + +You can also make use of the preprocessor identifier `TRITON_ENABLE_GPU` for +conditional inclusion of headers: +```cpp +#ifdef TRITON_ENABLE_GPU +#include +#endif +``` + +## Buffers +Within a backend, it is often useful to process data in a way that is agnostic +to whether the underlying memory is on the host or on device and whether that +memory is owned by the backend or provided by Triton. For instance, a backend +may receive input data from Triton on the host and conditionally transfer it to +the GPU before processing. In this case, owned memory must be allocated on the +GPU to store the data, but after that point, the backend will treat the data +exactly the same as if Triton had provided it on device in the first place. + +In order to handle such situations, RAPIDS-Triton provides the `Buffer` object. +When the `Buffer` is non-owning, it provides a lightweight wrapper to the +underlying memory. When it is owning, `Buffer` will handle any necessary +deallocation (on host or device). These objects can also be extremely useful +for passing data back and forth between host and device. The following examples +show ways in which `Buffer` objects can be constructed and used: + +```cpp +#include +#include +#include // rapids::HostMemory and rapids::DeviceMemory +#include // rapids::Buffer + +void buffer_examples() { + auto data = std::vector{0, 1, 2, 3, 4}; + + // This buffer is a lightweight wrapper around the data stored in the `data` + // vector. Because this constructor takes an `int*` pointer as its first + // argument, it is assumed that the lifecycle of the underlying memory is + // separately managed. + auto non_owning_host_buffer = rapids::Buffer(data.data(), rapids::HostMemory); + + // This buffer owns its own memory on the host, with space for 5 ints. When + // it goes out of scope, the memory will be appropriately deallocated. + auto owning_host_buffer = rapids::Buffer(5, rapids::HostMemory); + + // This buffer is constructed as a copy of `non_owning_host_buffer`. Because + // its requested memory type is `DeviceMemory`, the data will be copied to a + // new (owned) GPU allocation. Device and stream can also be specified in the + // constructor. + auto owning_device_buffer = rapids::Buffer(non_owning_host_buffer, rapids::DeviceMemory); + + // Once again, because this constructor takes an `int*` pointer, it will + // simply be a lightweight wrapper around the memory that is actually managed + // by `owning_device_buffer`. Here we have omitted the memory type argument, + // since it defaults to `DeviceMemory`. This constructor can also accept + // device and stream arguments, and care should be taken to ensure that the + // right device is specified when the buffer does not allocate its own + // memory. + auto non_owning_device_buffer = rapids::Buffer(owning_host_buffer.data()); + + auto base_buffer1 = rapids::Buffer(data.data(), rapids::HostMemory); + // Because this buffer is on the host, just like the (moved-from) buffer it + // is being constructed from, it remains non-owning + auto non_owning_moved_buffer = rapids::Buffer(std::move(base_buffer1), rapids::HostMemory); + + auto base_buffer2 = rapids::Buffer(data.data(), rapids::HostMemory); + // Because this buffer is on the device, unlike the (moved-from) buffer it is + // being constructed from, memory must be allocated on-device, and the new + // buffer becomes owning. + auto owning_moved_buffer = rapids::Buffer(std::move(base_buffer2), rapids::DeviceMemory); +} +``` + +### Useful Methods +* `data()`: Return a raw pointer to the buffer's data +* `size()`: Return the number of elements contained by the buffer +* `mem_type()`: Return the type of memory (`HostMemory` or `DeviceMemory`) + contained by the buffer +* `device()`: Return the id of the device on which this buffer resides (always + 0 for host buffers) +* `stream()`: Return the CUDA stream associated with this buffer. +* `stream_synchronize()`: Perform a stream synchronization on this buffer's + stream. +* `set_stream(cudaStream_t new_stream)`: Synchronize on the current stream and + then switch buffer to the new stream. + +## Tensors +`Tensor` objects are wrappers around `Buffers` with some additional metadata +and functionality. All `Tensor` objects have a shape which can be retrieved as +a `std::vector` using the `shape()` method. A reference to the underlying +buffer can also be retrieved with the `buffer()` method. + +`OutputTensor` objects are used to store data which will eventually be returned +as part of Triton's response to a client request. Their `finalize` methods are +used to actually marshal their underlying data into a response. + +In general, `OutputTensor` objects should not be constructed directly but +should instead be retrieved using the `get_output` method of a `Model` +(described later). + +## Moving Data: `rapids::copy` +Moving data around between host and device or simply between buffers of the +same type can be one of the more error-prone tasks outside of actual model +execution in a backend. To help make this process easier, RAPIDS-Triton +provides a number of overrides of the `rapids::copy` function, which provides a +safe way to mode data between buffers or tensors. Assuming the size attribute +of the buffer or tensor has not been corrupted, `rapids::copy` should never +result in segfaults or invalid memory access on device. + +Additional overrides of `rapids::copy` exist, but we will describe the most +common uses of it here. Note that you need not worry about where the underlying +data is located (on host or device) when invoking `rapids::copy`. The function +will take care of detecting and handling this. `Tensor` overrides are in +`rapids_triton/tensor/tensor.hpp` and `Buffer` overrides are in +`rapids_triton/memory/buffer.hpp`. + +### Between two buffers or tensors... +If you wish to simply copy the entire contents of one buffer into another or +one tensor into another, `rapids::copy` can be invoked as follows: +```cpp +rapids::copy(destination_buffer, source_buffer); +rapids::copy(destination_tensor, source_tensor); +``` +If the destination is too small to contain the data from the source, a +`TritonException` will be thrown. + +### From one tensor to many... +To distribute data from one tensor to many, the following override is +available: +```cpp +rapids::copy(iterator_to_first_destination, iterator_to_last_destination, source); +``` +Note that destination tensors can be of different sizes. If the destination +buffers cannot contain all data from the source, a `TritonException` will be +thrown. Destination tensors can also be a mixture of device and host tensors if +desired. + +### From part of one buffer to part of another... +To move data from part of one buffer to part of another, you can use another +override as in the following example: +```cpp +rapids::copy(destination_buffer, source_buffer, 10, 3, 6); +``` +The extra arguments here provide the offset from the beginning of the +destination buffer to which data should be copied, the index of the beginning +element to be copied from the source, and the index one past the final element to +be copied from the source. Thus, this invocation will copy the third, fourth, +and fifth elements of the source buffer to the tenth, eleventh, and twelfth +elements of the destination. If the destination buffer only had room for (e.g.) +eleven elements, a `TritonException` would be thrown. + +## `Model` +For a thorough introduction to developing a RAPIDS-Triton `Model` for your +backend, see the [Linear Example +repo](https://github.com/rapidsai/rapids-triton-linear-example). Here, we will +just briefly summarize some of the useful methods of `Model` objects. + +### Non-Virtual Methods +* `get_input`: Used to retrieve an input tensor of a particular name from + Triton +* `get_output`: Used to retrieve an output tensor of a particular name from + Triton +* `get_config_param`: Used to retrieve a named parameter from the configuration + file for this model +* `get_device_id`: The device on which this model is deployed (0 for host + deployments) +* `get_deployment_type`: One of `GPUDeployment` or `CPUDeployment` depending on + whether this model is configured to be deployed on device or host + +### Virtual Methods +* `predict`: The method which performs actual inference on input data and + stores it to the output location +* `load`: A method which can be overridden to load resources that will be used + for the lifetime of the model +* `unload`: A method used to unload any resources loaded in `load` if + necessary +* `preferred_mem_type`, `preferred_mem_type_in`, and `preferred_mem_type_out`: + The location (device or host) where input and output data should be stored. + The latter two methods can be overridden if input and output data should be + stored differently. Otherwise, `preferred_mem_type` will be used for both. +* `get_stream`: A method which can be overridden to provide different streams + for handling successive batches. Otherwise, the default stream associated + with this model will be used. + +## `SharedState` +Multiple instances of a RAPIDS-Triton model may need to share some data between +them (or may choose to do so for efficiency). `SharedState` objects facilitate +this. For a thorough introduction to developing a RAPIDS-Triton `SharedState` +for your backend, see the [Linear Example +repo](https://github.com/rapidsai/rapids-triton-linear-example). Just like the +`Model` objects which share a particular `SharedState` object, configuration +parameters can be retrieved using `SharedState`'s `get_config_param` method. +Otherwise, most additional functionality is defined by the backend +implementation, including `load` and `unload` methods for any necessary +loading/unloading of resources that will be used for the lifetime of the shared +state. + +Note that just one shared state is constructed by the server regardless of how +many instances of a given model are created. + +## Other Memory Allocations +For most device memory allocations, it is strongly recommended that you simply +construct a `Buffer` of the correct size and type. However, if you do not wish +to use a `Buffer` in a particular context, you are encouraged to allocate and +deallocate device memory using [RMM](https://github.com/rapidsai/rmm). Any +memory managed in this way will make use of Triton's CUDA memory pool, which +will be faster than performing individual allocations. Memory can be allocated +and deallocated using RMM as follows: + +```cpp +#include + +void rmm_example(std::size_t number_of_bytes, cudaStream_t stream) { + void* data = rmm::get_current_device_resource()->allocate(number_of_bytes, stream); + rmm::get_current_device_resource()->deallocate(data, number_of_bytes); +} +``` + +It is strongly recommended that you not change the RMM device resource in your +backend, since doing so will cause allocations to no longer make use of +Triton's memory pool. From 6ba3e8e4e304788db93e2c461cd7a397c789379d Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 12 Oct 2021 12:54:22 -0400 Subject: [PATCH 116/199] Update default install prefix --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9f0c1f9..f8293a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,7 +58,7 @@ ENV BUILD_TESTS=$BUILD_TESTS ARG BUILD_EXAMPLE ENV BUILD_EXAMPLE=$BUILD_EXAMPLE -RUN mkdir /rapids_triton/build +RUN mkdir /rapids_triton/build /rapids_triton/install WORKDIR /rapids_triton/build @@ -66,6 +66,7 @@ RUN cmake \ -GNinja \ -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ -DBUILD_TESTS="${BUILD_TESTS}" \ + -DCMAKE_INSTALL_PREFIX=/rapids_triton/install \ .. RUN ninja install From 9cbfe049136d595a7426c7e1b8d1d98293a0bee2 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 12 Oct 2021 13:47:51 -0400 Subject: [PATCH 117/199] Upgrade to Triton 21.09 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f8293a5..6bf8c99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # Arguments for controlling build details ########################################################################################### # Version of Triton to use -ARG TRITON_VERSION=21.08 +ARG TRITON_VERSION=21.09 # Base container image ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 # Whether or not to build indicated components From b7b51650f7de44bbb428dff68e3c20240e4b7c27 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 12 Oct 2021 13:48:19 -0400 Subject: [PATCH 118/199] Use AAA style in loop --- src/model.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/model.h b/src/model.h index d1107e8..8465485 100644 --- a/src/model.h +++ b/src/model.h @@ -56,7 +56,7 @@ struct RapidsModel : rapids::Model { auto alpha = get_shared_state()->alpha; if (u.mem_type() == rapids::HostMemory) { - for (std::size_t i{}; i < u.size(); ++i) { + for (auto i = std::size_t{}; i < u.size(); ++i) { r.data()[i] = alpha * u.data()[i] + v.data()[i] + c.data()[i % c.size()]; } From 81846b0f9e77f250a4821d1e08e4e114e45eaa27 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 12 Oct 2021 13:48:53 -0400 Subject: [PATCH 119/199] Disable shared memory tests until after 21.10 --- qa/L0_e2e/test_model.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/qa/L0_e2e/test_model.py b/qa/L0_e2e/test_model.py index 2ce0ccf..bb9b87c 100644 --- a/qa/L0_e2e/test_model.py +++ b/qa/L0_e2e/test_model.py @@ -48,9 +48,9 @@ def get_ground_truth(inputs): def test_model(model_name, model_inputs, model_output_sizes): client = Client() result = client.predict(model_name, model_inputs, model_output_sizes) - shm_result = client.predict( - model_name, model_inputs, model_output_sizes, shared_mem='cuda' - ) + # shm_result = client.predict( + # model_name, model_inputs, model_output_sizes, shared_mem='cuda' + # ) ground_truth = get_ground_truth(model_inputs) for output_name in sorted(ground_truth.keys()): @@ -60,9 +60,9 @@ def test_model(model_name, model_inputs, model_output_sizes): atol=1e-5, assert_close=True ) - arrays_close( - shm_result[output_name], - ground_truth[output_name], - atol=1e-5, - assert_close=True - ) + # arrays_close( + # shm_result[output_name], + # ground_truth[output_name], + # atol=1e-5, + # assert_close=True + # ) From eba34298606faaf53952a22fcf48afb203674f7b Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 12 Oct 2021 13:57:30 -0400 Subject: [PATCH 120/199] Specify Triton repo versions in Dockerfile --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 6bf8c99..c1d575a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,6 +67,9 @@ RUN cmake \ -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ -DBUILD_TESTS="${BUILD_TESTS}" \ -DCMAKE_INSTALL_PREFIX=/rapids_triton/install \ + -DTRITON_COMMON_REPO_TAG="r${TRITON_VERSION}" \ + -DTRITON_CORE_REPO_TAG="r${TRITON_VERSION}" \ + -DTRITON_BACKEND_REPO_TAG="r${TRITON_VERSION}" \ .. RUN ninja install From 8c67ff40f5de6a488ceeed36fff9b0382aa8dae4 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 12 Oct 2021 14:35:05 -0400 Subject: [PATCH 121/199] Correct stream handling --- README.md | 8 +++++--- src/gpu_infer.cu | 5 +++-- src/gpu_infer.h | 3 ++- src/model.h | 15 +++++++++++---- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index a26d8c9..3d66f93 100644 --- a/README.md +++ b/README.md @@ -404,20 +404,22 @@ __global__ void cu_gpu_infer(float* r, float const* u, float const* v, r[id] = alpha * u[id] + v[id] + c[id % features]; } } + void gpu_infer(float* r, float const* u, float const* v, float* c, float alpha, - std::size_t features, std::size_t length) { + std::size_t features, std::size_t length, cudaStream_t stream) { auto constexpr block_size = 1024; auto grid_size = static_cast(std::max(1.0f, std::ceil(length / static_cast(block_size))));; - cu_gpu_infer<<>>(r, u, v, c, alpha, features, length); + cu_gpu_infer<<>>(r, u, v, c, alpha, features, length); } ``` and then call it within our RapidsModel `predict` method via: ```cpp +rapids::cuda_check(cudaSetDevice(get_device_id())); gpu_infer(r.data(), u.data(), v.data(), c.data(), alpha, c.size(), - u.size()); + u.size(), r.stream()); ``` After the actual inference has been performed, the one remaining task is to diff --git a/src/gpu_infer.cu b/src/gpu_infer.cu index dba8383..059341e 100644 --- a/src/gpu_infer.cu +++ b/src/gpu_infer.cu @@ -16,6 +16,7 @@ #include +#include #include #include #include @@ -35,11 +36,11 @@ __global__ void cu_gpu_infer(float* r, float const* u, float const* v, } void gpu_infer(float* r, float const* u, float const* v, float* c, float alpha, - std::size_t features, std::size_t length) { + std::size_t features, std::size_t length, cudaStream_t stream) { auto constexpr block_size = 1024; auto grid_size = static_cast(std::max(1.0f, std::ceil(length / static_cast(block_size))));; - cu_gpu_infer<<>>(r, u, v, c, alpha, features, length); + cu_gpu_infer<<>>(r, u, v, c, alpha, features, length); } }}} diff --git a/src/gpu_infer.h b/src/gpu_infer.h index 9eed9ef..b188282 100644 --- a/src/gpu_infer.h +++ b/src/gpu_infer.h @@ -16,6 +16,7 @@ #pragma once +#include #include #include @@ -27,7 +28,7 @@ namespace backend { namespace NAMESPACE { void gpu_infer(float* r, float const* u, float const* v, float* c, float alpha, - std::size_t features, std::size_t length); + std::size_t features, std::size_t length, cudaStream_t stream); } } // namespace backend } // namespace triton diff --git a/src/model.h b/src/model.h index 8465485..4cd2c5f 100644 --- a/src/model.h +++ b/src/model.h @@ -61,8 +61,9 @@ struct RapidsModel : rapids::Model { alpha * u.data()[i] + v.data()[i] + c.data()[i % c.size()]; } } else { + rapids::cuda_check(cudaSetDevice(get_device_id())); gpu_infer(r.data(), u.data(), v.data(), c.data(), alpha, c.size(), - u.size()); + u.size(), r.stream()); } r.finalize(); @@ -92,13 +93,19 @@ struct RapidsModel : rapids::Model { // Construct buffer to hold c based on details of this model deployment auto memory_type = rapids::MemoryType{}; - if (get_deployment_type() == rapids::GPUDeployment) { - memory_type = rapids::DeviceMemory; + if constexpr (rapids::IS_GPU_BUILD) { + if (get_deployment_type() == rapids::GPUDeployment) { + memory_type = rapids::DeviceMemory; + rapids::cuda_check(cudaSetDevice(get_device_id())); + } else { + memory_type = rapids::HostMemory; + } } else { memory_type = rapids::HostMemory; } - c = rapids::Buffer(model_vec.size(), memory_type, get_device_id()); + c = rapids::Buffer(model_vec.size(), memory_type, get_device_id(), + get_stream()); /* Use a Buffer view on model_vec to safely copy data to its final * location. Making use of rapids::copy here provides additional safety From c04d920098d7281f281fdba4ca9d0b5e35f389b5 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 12 Oct 2021 18:03:46 -0400 Subject: [PATCH 122/199] Always set device for execution and initialization --- cpp/include/rapids_triton/triton/api/execute.hpp | 6 ++++++ .../rapids_triton/triton/api/instance_initialize.hpp | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/cpp/include/rapids_triton/triton/api/execute.hpp b/cpp/include/rapids_triton/triton/api/execute.hpp index bce6bc6..f324065 100644 --- a/cpp/include/rapids_triton/triton/api/execute.hpp +++ b/cpp/include/rapids_triton/triton/api/execute.hpp @@ -89,6 +89,12 @@ auto* execute(TRITONBACKEND_ModelInstance* instance, max_batch_size, model.get_stream()); + if constexpr (IS_GPU_BUILD) { + if (model.get_deployment_type() == GPUDeployment) { + cuda_check(cudaSetDevice(model.get_device_id())); + } + } + auto predict_err = static_cast(nullptr); try { model.predict(batch); diff --git a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp index e9e5a0c..e60a9ab 100644 --- a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -48,6 +49,12 @@ auto* instance_initialize(TRITONBACKEND_ModelInstance* instance) } auto rapids_model = std::make_unique(*model_state, instance); + if constexpr (IS_GPU_BUILD) { + auto& model = rapids_model->get_model(); + if (model.get_deployment_type() == GPUDeployment) { + cuda_check(cudaSetDevice(model.get_device_id())); + } + } rapids_model->load(); set_instance_state(*instance, std::move(rapids_model)); From 1fe80cbcfed84847210338c11c37a9443b15de58 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 12 Oct 2021 18:17:16 -0400 Subject: [PATCH 123/199] Revert "Update to Triton 21.09" This reverts commit a08afc67fcc539535aa148e67e849179de496069. --- Dockerfile | 2 +- cpp/CMakeLists.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index c6e1bfa..6be8a53 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # Arguments for controlling build details ########################################################################################### # Version of Triton to use -ARG TRITON_VERSION=21.09 +ARG TRITON_VERSION=21.08 # Base container image ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 # Whether or not to build indicated components diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index e3e0ae0..217223a 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -50,9 +50,9 @@ option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(NVTX "Enable nvtx markers" OFF) option(TRITON_ENABLE_GPU "Enable GPU support in Triton" ON) option(TRITON_ENABLE_STATS "Enable statistics collection in Triton" ON) -set(TRITON_COMMON_REPO_TAG "r21.09" CACHE STRING "Tag for triton-inference-server/common repo") -set(TRITON_CORE_REPO_TAG "r21.09" CACHE STRING "Tag for triton-inference-server/core repo") -set(TRITON_BACKEND_REPO_TAG "r21.09" CACHE STRING "Tag for triton-inference-server/backend repo") +set(TRITON_COMMON_REPO_TAG "r21.08" CACHE STRING "Tag for triton-inference-server/common repo") +set(TRITON_CORE_REPO_TAG "r21.08" CACHE STRING "Tag for triton-inference-server/core repo") +set(TRITON_BACKEND_REPO_TAG "r21.08" CACHE STRING "Tag for triton-inference-server/backend repo") message(VERBOSE "RAPIDS_TRITON: Build RAPIDS_TRITON unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAPIDS_TRITON: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") From 9086c3400c6d483f2cb1da75f93209eb4398850a Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 13 Oct 2021 09:44:32 -0400 Subject: [PATCH 124/199] Revert to Triton 21.08 and pin RAPIDS-Triton to main --- Dockerfile | 2 +- cmake/thirdparty/get_rapids-triton.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c1d575a..fb8f478 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # Arguments for controlling build details ########################################################################################### # Version of Triton to use -ARG TRITON_VERSION=21.09 +ARG TRITON_VERSION=21.08 # Base container image ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 # Whether or not to build indicated components diff --git a/cmake/thirdparty/get_rapids-triton.cmake b/cmake/thirdparty/get_rapids-triton.cmake index 64ba57d..3403621 100644 --- a/cmake/thirdparty/get_rapids-triton.cmake +++ b/cmake/thirdparty/get_rapids-triton.cmake @@ -42,5 +42,5 @@ endfunction() # CPM_raft_SOURCE=/path/to/local/raft find_and_configure_rapids_triton(VERSION 21.10 FORK rapidsai - PINNED_TAG fea-initial + PINNED_TAG main ) From 1a0c2b7044a4ef57ddc09925770f09fda9692f0c Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 13 Oct 2021 09:51:50 -0400 Subject: [PATCH 125/199] Update docs based on RAPIDS-Triton main --- README.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3d66f93..84244bc 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ configuration file and loading model resources. This example is intended to provide a fair amount of depth about backend development with RAPIDS-Triton. For a simpler example, check out the pass-through backend in the main [RAPIDS-Triton -repo](https://github.com/rapidsai/rapids-triton/tree/fea-initial#simple-example). +repo](https://github.com/rapidsai/rapids-triton#simple-example). For even more detail on RAPIDS-Triton features introduced here, check out the API documentation. @@ -473,7 +473,7 @@ environment as follows: ```bash conda env create -f conda/environments/rapids_triton_test.yml conda activate rapids_triton_test -python -m pip install git+https://github.com/rapidsai/rapids-triton.git@fea-initial#subdirectory=python +python -m pip install git+https://github.com/rapidsai/rapids-triton.git#subdirectory=python ``` To use it for a basic test, we might execute something like the following @@ -529,12 +529,14 @@ steps, you should be able to integrate almost any algorithm for deployment with Triton. While we have tried to cover a wide variety of possible use cases with this -example, there is much more to explore in the RAPIDS-Triton API documentation. -If there is something you would like to do with RAPIDS-Triton which does not -seem to be covered by the available API or if something is not working as -expected, please submit a feature request or bug report to the [RAPIDS-Triton -issue tracker](https://github.com/rapidsai/rapids-triton/issues). If you think -this example could be improved or expanded in some way, please [submit a pull +example, there is more to explore in the [RAPIDS-Triton +documentation](https://github.com/rapidsai/rapids-triton/blob/main/docs/usage.md) +itself. If there is something you would like to do with RAPIDS-Triton which +does not seem to be covered by the available API or if something is not working +as expected, please submit a feature request or bug report to the +[RAPIDS-Triton issue +tracker](https://github.com/rapidsai/rapids-triton/issues). If you think this +example could be improved or expanded in some way, please [submit a pull request](https://github.com/rapidsai/rapids-triton-linear-example/pulls) or [issue](https://github.com/rapidsai/rapids-triton-linear-example/issues) to this repo. From f963b462fa34bb99b56c2e155aafb457e016dab3 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 19 Oct 2021 12:57:16 -0400 Subject: [PATCH 126/199] Update batch logic for new pinned memory default --- Dockerfile | 2 +- cpp/include/rapids_triton/batch/batch.hpp | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6be8a53..c6e1bfa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # Arguments for controlling build details ########################################################################################### # Version of Triton to use -ARG TRITON_VERSION=21.08 +ARG TRITON_VERSION=21.09 # Base container image ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 # Whether or not to build indicated components diff --git a/cpp/include/rapids_triton/batch/batch.hpp b/cpp/include/rapids_triton/batch/batch.hpp index 5d4d6e5..cf29bd5 100644 --- a/cpp/include/rapids_triton/batch/batch.hpp +++ b/cpp/include/rapids_triton/batch/batch.hpp @@ -26,6 +26,8 @@ #include #include #include +#include +#include #include #include #include @@ -150,6 +152,14 @@ struct Batch { &reported_mem_type, &reported_device_id)); + if(collector_.Finalize()){ + if constexpr (IS_GPU_BUILD) { + cuda_check(cudaStreamSynchronize(stream_)); + } else { + throw TritonException(Error::Internal, "stream synchronization required in non-GPU build"); + } + } + std::for_each(std::begin(responses_), std::end(responses_), [](auto* response) { if (response == nullptr) { throw TritonException(Error::Internal, "Input collection failed"); From dd2b509aaefc3864fc1487ff7e111257cff8a023 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 21 Oct 2021 16:09:30 -0400 Subject: [PATCH 127/199] Pin to RAPIDS-Triton 21.10 --- cmake/thirdparty/get_rapids-triton.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/thirdparty/get_rapids-triton.cmake b/cmake/thirdparty/get_rapids-triton.cmake index 3403621..c63b906 100644 --- a/cmake/thirdparty/get_rapids-triton.cmake +++ b/cmake/thirdparty/get_rapids-triton.cmake @@ -42,5 +42,5 @@ endfunction() # CPM_raft_SOURCE=/path/to/local/raft find_and_configure_rapids_triton(VERSION 21.10 FORK rapidsai - PINNED_TAG main + PINNED_TAG branch-21.10 ) From 6d3d7cc659a3d6dd101fc11ee80c28cb06ab0762 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 26 Oct 2021 17:54:58 -0400 Subject: [PATCH 128/199] Remove unnecessary cudaSetDevice calls --- README.md | 1 - src/model.h | 2 -- 2 files changed, 3 deletions(-) diff --git a/README.md b/README.md index 84244bc..ba666a9 100644 --- a/README.md +++ b/README.md @@ -417,7 +417,6 @@ void gpu_infer(float* r, float const* u, float const* v, float* c, float alpha, and then call it within our RapidsModel `predict` method via: ```cpp -rapids::cuda_check(cudaSetDevice(get_device_id())); gpu_infer(r.data(), u.data(), v.data(), c.data(), alpha, c.size(), u.size(), r.stream()); ``` diff --git a/src/model.h b/src/model.h index 4cd2c5f..4a9f9ee 100644 --- a/src/model.h +++ b/src/model.h @@ -61,7 +61,6 @@ struct RapidsModel : rapids::Model { alpha * u.data()[i] + v.data()[i] + c.data()[i % c.size()]; } } else { - rapids::cuda_check(cudaSetDevice(get_device_id())); gpu_infer(r.data(), u.data(), v.data(), c.data(), alpha, c.size(), u.size(), r.stream()); } @@ -96,7 +95,6 @@ struct RapidsModel : rapids::Model { if constexpr (rapids::IS_GPU_BUILD) { if (get_deployment_type() == rapids::GPUDeployment) { memory_type = rapids::DeviceMemory; - rapids::cuda_check(cudaSetDevice(get_device_id())); } else { memory_type = rapids::HostMemory; } From 6522ce62134351de8db2791f10a83be3a3d23ca3 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 15 Nov 2021 15:48:39 -0500 Subject: [PATCH 129/199] Provide initial CPU-only implementation --- Dockerfile | 7 +- cpp/CMakeLists.txt | 39 +++++--- cpp/include/rapids_triton/batch/batch.hpp | 4 + .../cpu_only/cuda_runtime_replacement.hpp | 53 ++++++++++ cpp/include/rapids_triton/exceptions.hpp | 15 ++- cpp/include/rapids_triton/memory/buffer.hpp | 67 ++----------- .../memory/detail/cpu_only/copy.hpp | 49 ++++++++++ .../detail/cpu_only/owned_device_buffer.hpp | 43 ++++++++ .../memory/detail/cpu_only/resource.hpp | 41 ++++++++ .../memory/detail/{ => gpu_only}/copy.hpp | 49 +++------- .../detail/gpu_only/owned_device_buffer.hpp | 49 ++++++++++ .../memory/detail/gpu_only/resource.hpp | 97 +++++++++++++++++++ .../memory/detail/owned_device_buffer.hpp | 30 ++++++ .../rapids_triton/memory/detail/resource.hpp | 35 +++++++ cpp/include/rapids_triton/memory/resource.hpp | 78 ++------------- cpp/include/rapids_triton/model/model.hpp | 4 + .../rapids_triton/model/shared_state.hpp | 4 + cpp/include/rapids_triton/tensor/tensor.hpp | 4 + .../rapids_triton/triton/api/initialize.hpp | 4 + .../triton/api/instance_initialize.hpp | 5 + .../rapids_triton/utils/device_setter.hpp | 50 ++++++++++ cpp/src/CMakeLists.txt | 59 +++++++---- cpp/src/model.h | 4 + cpp/test/memory/resource.cpp | 7 +- docs/usage.md | 24 ++--- 25 files changed, 603 insertions(+), 218 deletions(-) create mode 100644 cpp/include/rapids_triton/cpu_only/cuda_runtime_replacement.hpp create mode 100644 cpp/include/rapids_triton/memory/detail/cpu_only/copy.hpp create mode 100644 cpp/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp create mode 100644 cpp/include/rapids_triton/memory/detail/cpu_only/resource.hpp rename cpp/include/rapids_triton/memory/detail/{ => gpu_only}/copy.hpp (60%) create mode 100644 cpp/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp create mode 100644 cpp/include/rapids_triton/memory/detail/gpu_only/resource.hpp create mode 100644 cpp/include/rapids_triton/memory/detail/owned_device_buffer.hpp create mode 100644 cpp/include/rapids_triton/memory/detail/resource.hpp create mode 100644 cpp/include/rapids_triton/utils/device_setter.hpp diff --git a/Dockerfile b/Dockerfile index c6e1bfa..9b6e8b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,12 +2,14 @@ # Arguments for controlling build details ########################################################################################### # Version of Triton to use -ARG TRITON_VERSION=21.09 +ARG TRITON_VERSION=21.10 # Base container image ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 # Whether or not to build indicated components ARG BUILD_TESTS=OFF ARG BUILD_EXAMPLE=ON +# Whether or not to enable GPU build +ARG TRITON_ENABLE_GPU=ON FROM ${BASE_IMAGE} as base @@ -53,6 +55,8 @@ ARG BUILD_TESTS ENV BUILD_TESTS=$BUILD_TESTS ARG BUILD_EXAMPLE ENV BUILD_EXAMPLE=$BUILD_EXAMPLE +ARG TRITON_ENABLE_GPU +ENV TRITON_ENABLE_GPU=$TRITON_ENABLE_GPU RUN mkdir /rapids_triton/build @@ -63,6 +67,7 @@ RUN cmake \ -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ -DBUILD_TESTS="${BUILD_TESTS}" \ -DBUILD_EXAMPLE="${BUILD_EXAMPLE}" \ + -DTRITON_ENABLE_GPU="${TRITON_ENABLE_GPU}" \ .. RUN ninja install diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 217223a..aedd871 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -24,7 +24,11 @@ include(rapids-cuda) include(rapids-export) include(rapids-find) -rapids_cuda_init_architectures(RAPIDS_TRITON) +option(TRITON_ENABLE_GPU "Enable GPU support in Triton" ON) + +if(TRITON_ENABLE_GPU) + rapids_cuda_init_architectures(RAPIDS_TRITON) +endif() project(RAPIDS_TRITON VERSION 21.10.00 LANGUAGES CXX CUDA) @@ -48,7 +52,6 @@ option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF) option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" ON) option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(NVTX "Enable nvtx markers" OFF) -option(TRITON_ENABLE_GPU "Enable GPU support in Triton" ON) option(TRITON_ENABLE_STATS "Enable statistics collection in Triton" ON) set(TRITON_COMMON_REPO_TAG "r21.08" CACHE STRING "Tag for triton-inference-server/common repo") set(TRITON_CORE_REPO_TAG "r21.08" CACHE STRING "Tag for triton-inference-server/core repo") @@ -102,8 +105,11 @@ include(cmake/modules/ConfigureCUDA.cmake) # add third party dependencies using CPM rapids_cpm_init() -include(cmake/thirdparty/get_rmm.cmake) -include(cmake/thirdparty/get_raft.cmake) +if(TRITON_ENABLE_GPU) + include(cmake/thirdparty/get_rmm.cmake) + include(cmake/thirdparty/get_raft.cmake) +endif() + include(cmake/thirdparty/get_rapidjson.cmake) include(cmake/thirdparty/get_triton.cmake) @@ -119,15 +125,26 @@ add_library(rapids_triton::rapids_triton ALIAS rapids_triton) target_include_directories(rapids_triton INTERFACE "$" "$") -target_link_libraries(rapids_triton -INTERFACE - rmm::rmm - raft::raft - triton-core-serverstub - triton-backend-utils +if(TRITON_ENABLE_GPU) + target_link_libraries(rapids_triton + INTERFACE + rmm::rmm + raft::raft + triton-core-serverstub + triton-backend-utils ) +else() + target_link_libraries(rapids_triton + INTERFACE + triton-core-serverstub + triton-backend-utils + ) +endif() -target_compile_features(rapids_triton INTERFACE cxx_std_17 $) +target_compile_features( + rapids_triton INTERFACE cxx_std_17 + $<$:cuda_std_17> +) rapids_cmake_install_lib_dir(lib_dir) install(TARGETS rapids_triton diff --git a/cpp/include/rapids_triton/batch/batch.hpp b/cpp/include/rapids_triton/batch/batch.hpp index cf29bd5..db54df8 100644 --- a/cpp/include/rapids_triton/batch/batch.hpp +++ b/cpp/include/rapids_triton/batch/batch.hpp @@ -16,7 +16,11 @@ #pragma once +#ifdef TRITON_ENABLE_GPU #include +#else +#include +#endif #include #include #include diff --git a/cpp/include/rapids_triton/cpu_only/cuda_runtime_replacement.hpp b/cpp/include/rapids_triton/cpu_only/cuda_runtime_replacement.hpp new file mode 100644 index 0000000..148e38d --- /dev/null +++ b/cpp/include/rapids_triton/cpu_only/cuda_runtime_replacement.hpp @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifdef TRITON_ENABLE_GPU +#include +#else + +namespace triton { +namespace backend { +namespace rapids { + +using cudaStream_t = void*; + +enum struct cudaError_t {cudaSuccess, cudaNonGpuBuildError}; +auto constexpr cudaSuccess = cudaError_t::cudaSuccess; + +inline void cudaGetLastError() {} + +inline auto const * cudaGetErrorString(cudaError_t err) { + return "CUDA function used in non-GPU build"; +} + +inline auto cudaStreamSynchronize(cudaStream_t stream) { + return cudaError_t::cudaNonGpuBuildError; +} + +inline auto cudaGetDevice(int* device_id) { + return cudaError_t::cudaNonGpuBuildError; +} + +inline auto cudaGetDeviceCount(int* count) { + return cudaError_t::cudaNonGpuBuildError; +} + + +} // namespace rapids +} // namespace backend +} // namespace triton +#endif diff --git a/cpp/include/rapids_triton/exceptions.hpp b/cpp/include/rapids_triton/exceptions.hpp index dfdab76..e2d2bc2 100644 --- a/cpp/include/rapids_triton/exceptions.hpp +++ b/cpp/include/rapids_triton/exceptions.hpp @@ -15,9 +15,14 @@ */ #pragma once +#ifdef TRITON_ENABLE_GPU #include +#else +#include +#endif #include #include +#include #include namespace triton { @@ -74,9 +79,13 @@ inline void triton_check(TRITONSERVER_Error* err) inline void cuda_check(cudaError_t const& err) { - if (err != cudaSuccess) { - cudaGetLastError(); - throw TritonException(Error::Internal, cudaGetErrorString(err)); + if constexpr (IS_GPU_BUILD) { + if (err != cudaSuccess) { + cudaGetLastError(); + throw TritonException(Error::Internal, cudaGetErrorString(err)); + } + } else { + throw TritonException(Error::Internal, "cuda_check used in non-GPU build"); } } diff --git a/cpp/include/rapids_triton/memory/buffer.hpp b/cpp/include/rapids_triton/memory/buffer.hpp index 802ce72..8b7cbba 100644 --- a/cpp/include/rapids_triton/memory/buffer.hpp +++ b/cpp/include/rapids_triton/memory/buffer.hpp @@ -20,11 +20,18 @@ #include #include +#ifdef TRITON_ENABLE_GPU #include +#include +#include +#else +#include +#include +#include +#endif #include #include -#include #include #include #include @@ -41,63 +48,7 @@ struct Buffer { using h_buffer = T*; using d_buffer = T*; using owned_h_buffer = std::unique_ptr; - struct owned_d_buffer { - using non_const_T = std::remove_const_t; - owned_d_buffer(device_id_t device_id, std::size_t size, cudaStream_t stream) - : device_{device_id}, byte_size_{size * sizeof(non_const_T)}, data_{[this, &stream]() { - auto* result = static_cast(nullptr); - try { - result = - static_cast(get_memory_resource(device_)->allocate(byte_size_, stream)); - } catch (std::bad_alloc const& err) { - throw TritonException(Error::Internal, err.what()); - } - return result; - }()} - { - } - ~owned_d_buffer() { free_memory(); } - - owned_d_buffer(owned_d_buffer const& other) = delete; - owned_d_buffer(owned_d_buffer&& other) noexcept - : device_{other.device_}, byte_size_{other.byte_size_}, data_{nullptr} - { - data_ = other.data_; - other.data_ = nullptr; - } - owned_d_buffer& operator=(owned_d_buffer const& other) = delete; - owned_d_buffer& operator =(owned_d_buffer&& other) - { - if (this != &other) { - device_ = other.device_; - byte_size_ = other.byte_size_; - free_memory(); - data_ = other.data_; - other.data_ = nullptr; - } - return *this; - } - - auto* get() const { return data_; } - - private: - device_id_t device_; - std::size_t byte_size_; - non_const_T* data_; - void free_memory() - { - if (data_ != nullptr) { - try { - get_memory_resource(device_)->deallocate(reinterpret_cast(data_), byte_size_); - } catch (TritonException const& err) { - log_error(__FILE__, __LINE__) << err.what(); - } catch (...) { - log_error(__FILE__, __LINE__) << "Unknown error in owned_d_buffer destructor!"; - } - } - data_ = nullptr; - } - }; + using owned_d_buffer = detail::owned_device_buffer; using data_store = std::variant; Buffer() noexcept : device_{}, data_{std::in_place_index<0>, nullptr}, size_{}, stream_{} {} diff --git a/cpp/include/rapids_triton/memory/detail/cpu_only/copy.hpp b/cpp/include/rapids_triton/memory/detail/cpu_only/copy.hpp new file mode 100644 index 0000000..edfedba --- /dev/null +++ b/cpp/include/rapids_triton/memory/detail/cpu_only/copy.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include + +#ifndef TRITON_ENABLE_GPU +#include +#endif +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace detail { + +template +void copy(T* dst, + T const* src, + std::size_t len, + cudaStream_t stream, + MemoryType dst_type, + MemoryType src_type) +{ + if (dst_type == DeviceMemory || src_type == DeviceMemory) { + throw TritonException(Error::Internal, "Cannot copy device memory in non-GPU build"); + } else { + std::memcpy(dst, src, len * sizeof(T)); + } +} + +} // namespace detail +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp b/cpp/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp new file mode 100644 index 0000000..a866e3c --- /dev/null +++ b/cpp/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace detail { + +template +struct owned_device_buffer { + using non_const_T = std::remove_const_t; + owned_device_buffer(device_id_t device_id, std::size_t size, cudaStream_t stream) + { + throw TritonException(Error::Internal, + "Attempted to use device buffer in non-GPU build"); + } + + auto* get() const { return static_cast(nullptr); } +}; + +} // namespace detail +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/memory/detail/cpu_only/resource.hpp b/cpp/include/rapids_triton/memory/detail/cpu_only/resource.hpp new file mode 100644 index 0000000..6d134cf --- /dev/null +++ b/cpp/include/rapids_triton/memory/detail/cpu_only/resource.hpp @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace detail { + +template<> +inline void setup_memory_resource(device_id_t device_id, + TRITONBACKEND_MemoryManager* triton_manager) { } + +/* namespace rmm { +namespace mr { +using device_memory_resource = void; +} +} */ + +} // namespace detail +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/memory/detail/copy.hpp b/cpp/include/rapids_triton/memory/detail/gpu_only/copy.hpp similarity index 60% rename from cpp/include/rapids_triton/memory/detail/copy.hpp rename to cpp/include/rapids_triton/memory/detail/gpu_only/copy.hpp index 3211893..030b473 100644 --- a/cpp/include/rapids_triton/memory/detail/copy.hpp +++ b/cpp/include/rapids_triton/memory/detail/gpu_only/copy.hpp @@ -15,48 +15,21 @@ */ #pragma once -#include -#include - +#ifdef TRITON_ENABLE_GPU #include - #include +#endif + +#include +#include #include +#include namespace triton { namespace backend { namespace rapids { namespace detail { -/** - * @brief Copy given number of elements from one place to another, with either - * source or destination on device - */ -template -void dev_copy(T* dst, T const* src, std::size_t len, cudaStream_t stream) -{ - if constexpr (IS_GPU_BUILD) { - try { - raft::copy(dst, src, len, stream); - } catch (raft::cuda_error const& err) { - throw TritonException(Error::Internal, err.what()); - } - } else { - throw TritonException(Error::Internal, - "copy to or from device memory cannot be used in CPU-only builds"); - } -} - -/** - * @brief Copy given number of elements from one place to another, with either - * source or destination on device - */ -template -void host_copy(T* dst, T const* src, std::size_t len) -{ - std::memcpy(dst, src, len * sizeof(T)); -} - template void copy(T* dst, T const* src, @@ -66,13 +39,13 @@ void copy(T* dst, MemoryType src_type) { if (dst_type == DeviceMemory || src_type == DeviceMemory) { - if constexpr (IS_GPU_BUILD) { - dev_copy(dst, src, len, stream); - } else { - throw TritonException(Error::Internal, "DeviceMemory copy cannot be used in CPU-only builds"); + try { + raft::copy(dst, src, len, stream); + } catch (raft::cuda_error const& err) { + throw TritonException(Error::Internal, err.what()); } } else { - host_copy(dst, src, len); + std::memcpy(dst, src, len * sizeof(T)); } } diff --git a/cpp/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp b/cpp/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp new file mode 100644 index 0000000..c02d42a --- /dev/null +++ b/cpp/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace detail { + +template +struct owned_device_buffer { + using non_const_T = std::remove_const_t; + owned_device_buffer(device_id_t device_id, std::size_t size, cudaStream_t stream) + : data_{[this, &device_id, &size, &stream]() { + auto device_context = device_setter{device_id}; + return rmm::device_buffer{size, rmm::cuda_stream_view{stream}}; + }()} + { + } + + auto* get() const { return reinterpret_cast(data_.data()); } + + private: + mutable rmm::device_buffer data_; +}; + +} // namespace detail +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/memory/detail/gpu_only/resource.hpp b/cpp/include/rapids_triton/memory/detail/gpu_only/resource.hpp new file mode 100644 index 0000000..f2ea115 --- /dev/null +++ b/cpp/include/rapids_triton/memory/detail/gpu_only/resource.hpp @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace detail { + +inline auto& resource_lock() +{ + static auto lock = std::mutex{}; + return lock; +} + +/** A struct used solely to keep memory resources in-scope for the lifetime + * of the backend */ +struct resource_data { + resource_data() : base_mr_{}, triton_mrs_{} {} + auto* make_new_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* manager) + { + if (manager == nullptr && triton_mrs_.size() != 0) { + manager = triton_mrs_.back().get_triton_manager(); + } + triton_mrs_.emplace_back(manager, device_id, &base_mr_); + return &(triton_mrs_.back()); + } + + private: + rmm::mr::cuda_memory_resource base_mr_; + std::deque triton_mrs_; +}; + +inline auto& get_device_resources() +{ + static auto device_resources = resource_data{}; + return device_resources; +} + +inline auto is_triton_resource(rmm::cuda_device_id const& device_id) +{ + auto* triton_mr = + dynamic_cast(rmm::mr::get_per_device_resource(device_id)); + return (triton_mr != nullptr && triton_mr->get_triton_manager() != nullptr); +} + +template<> +inline void setup_memory_resource(device_id_t device_id, + TRITONBACKEND_MemoryManager* triton_manager) +{ + auto lock = std::lock_guard{detail::resource_lock()}; + auto rmm_device_id = rmm::cuda_device_id{device_id}; + + if (!detail::is_triton_resource(rmm_device_id)) { + auto& device_resources = detail::get_device_resources(); + rmm::mr::set_per_device_resource(rmm_device_id, + device_resources.make_new_resource(device_id, triton_manager)); + } +} + +/* inline auto* get_memory_resource(device_id_t device_id) +{ + auto rmm_device_id = rmm::cuda_device_id{device_id}; + return rmm::mr::get_per_device_resource(rmm_device_id); +} + +inline auto* get_memory_resource() { return rmm::mr::get_current_device_resource(); } */ + +} // namespace detail +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/memory/detail/owned_device_buffer.hpp b/cpp/include/rapids_triton/memory/detail/owned_device_buffer.hpp new file mode 100644 index 0000000..ca25af9 --- /dev/null +++ b/cpp/include/rapids_triton/memory/detail/owned_device_buffer.hpp @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +namespace triton { +namespace backend { +namespace rapids { +namespace detail { + +template +struct owned_device_buffer { +}; + +} // namespace detail +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/memory/detail/resource.hpp b/cpp/include/rapids_triton/memory/detail/resource.hpp new file mode 100644 index 0000000..cfc4bc4 --- /dev/null +++ b/cpp/include/rapids_triton/memory/detail/resource.hpp @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace detail { + +template +inline void setup_memory_resource(device_id_t device_id, + TRITONBACKEND_MemoryManager* triton_manager = nullptr) { +} + +} // namespace detail +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/include/rapids_triton/memory/resource.hpp b/cpp/include/rapids_triton/memory/resource.hpp index ab843cf..afd7f2a 100644 --- a/cpp/include/rapids_triton/memory/resource.hpp +++ b/cpp/include/rapids_triton/memory/resource.hpp @@ -16,81 +16,23 @@ #pragma once #include -#include -#include -#include -#include + +#include +#include #include -#include -#include -#include -#include +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif namespace triton { namespace backend { namespace rapids { -namespace detail { - -inline auto& resource_lock() -{ - static auto lock = std::mutex{}; - return lock; -} - -/** A struct used solely to keep memory resources in-scope for the lifetime - * of the backend */ -struct resource_data { - resource_data() : base_mr_{}, triton_mrs_{} {} - auto* make_new_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* manager) - { - if (manager == nullptr && triton_mrs_.size() != 0) { - manager = triton_mrs_.back().get_triton_manager(); - } - triton_mrs_.emplace_back(manager, device_id, &base_mr_); - return &(triton_mrs_.back()); - } - - private: - rmm::mr::cuda_memory_resource base_mr_; - std::deque triton_mrs_; -}; - -inline auto& get_device_resources() -{ - static auto device_resources = resource_data{}; - return device_resources; -} -inline auto is_triton_resource(rmm::cuda_device_id const& device_id) -{ - auto* triton_mr = - dynamic_cast(rmm::mr::get_per_device_resource(device_id)); - return (triton_mr != nullptr && triton_mr->get_triton_manager() != nullptr); +inline void setup_memory_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* triton_manager = nullptr) { + detail::setup_memory_resource(device_id, triton_manager); } -} // namespace detail - -inline auto* setup_memory_resource(device_id_t device_id, - TRITONBACKEND_MemoryManager* triton_manager = nullptr) -{ - auto lock = std::lock_guard{detail::resource_lock()}; - auto rmm_device_id = rmm::cuda_device_id{device_id}; - - if (!detail::is_triton_resource(rmm_device_id)) { - auto& device_resources = detail::get_device_resources(); - rmm::mr::set_per_device_resource(rmm_device_id, - device_resources.make_new_resource(device_id, triton_manager)); - } - - return rmm::mr::get_per_device_resource(rmm_device_id); -} - -inline auto* get_memory_resource(device_id_t device_id) -{ - auto rmm_device_id = rmm::cuda_device_id{device_id}; - return rmm::mr::get_per_device_resource(rmm_device_id); -} - -inline auto* get_memory_resource() { return rmm::mr::get_current_device_resource(); } } // namespace rapids } // namespace backend diff --git a/cpp/include/rapids_triton/model/model.hpp b/cpp/include/rapids_triton/model/model.hpp index a2abcf1..3413fca 100644 --- a/cpp/include/rapids_triton/model/model.hpp +++ b/cpp/include/rapids_triton/model/model.hpp @@ -15,7 +15,11 @@ */ #pragma once +#ifdef TRITON_ENABLE_GPU #include +#else +#include +#endif #include #include #include diff --git a/cpp/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp index 7927a59..a3826ee 100644 --- a/cpp/include/rapids_triton/model/shared_state.hpp +++ b/cpp/include/rapids_triton/model/shared_state.hpp @@ -15,7 +15,11 @@ */ #pragma once +#ifdef TRITON_ENABLE_GPU #include +#else +#include +#endif #include #include #include diff --git a/cpp/include/rapids_triton/tensor/tensor.hpp b/cpp/include/rapids_triton/tensor/tensor.hpp index 3a7f1e0..80f2ba4 100644 --- a/cpp/include/rapids_triton/tensor/tensor.hpp +++ b/cpp/include/rapids_triton/tensor/tensor.hpp @@ -24,7 +24,11 @@ #include #include +#ifdef TRITON_ENABLE_GPU #include +#else +#include +#endif #include #include diff --git a/cpp/include/rapids_triton/triton/api/initialize.hpp b/cpp/include/rapids_triton/triton/api/initialize.hpp index b46dd8e..37824dd 100644 --- a/cpp/include/rapids_triton/triton/api/initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/initialize.hpp @@ -15,7 +15,11 @@ */ #pragma once +#ifdef TRITON_ENABLE_GPU #include +#else +#include +#endif #include #include diff --git a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp index e60a9ab..922152f 100644 --- a/cpp/include/rapids_triton/triton/api/instance_initialize.hpp +++ b/cpp/include/rapids_triton/triton/api/instance_initialize.hpp @@ -37,6 +37,11 @@ auto* instance_initialize(TRITONBACKEND_ModelInstance* instance) auto name = get_model_instance_name(*instance); auto device_id = get_device_id(*instance); auto deployment_type = get_deployment_type(*instance); + if constexpr (!IS_GPU_BUILD) { + if (deployment_type == GPUDeployment) { + throw TritonException(Error::Unsupported, "KIND_GPU cannot be used in CPU-only build"); + } + } log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInstanceInitialize: " << name << " (" << TRITONSERVER_InstanceGroupKindString(deployment_type) diff --git a/cpp/include/rapids_triton/utils/device_setter.hpp b/cpp/include/rapids_triton/utils/device_setter.hpp new file mode 100644 index 0000000..d8aae9b --- /dev/null +++ b/cpp/include/rapids_triton/utils/device_setter.hpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifdef TRITON_ENABLE_GPU +#include +#endif +#include +#include + +namespace triton { +namespace backend { +namespace rapids { + +/** Struct for setting cuda device within a code block */ +struct device_setter { + device_setter(device_id_t device) : prev_device_{} { + if constexpr(IS_GPU_BUILD) { + cuda_check(cudaGetDevice(&prev_device_)); + cuda_check(cudaSetDevice(device)); + } else { + throw TritonException(Error::Internal, "Device setter used in non-GPU build"); + } + } + + ~device_setter() { + if constexpr(IS_GPU_BUILD) { + cudaSetDevice(prev_device_); + } + } + private: + device_id_t prev_device_; +}; + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 32e2863..e69f350 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -20,16 +20,27 @@ add_library( src/api.cc ) -set_target_properties(triton_rapids-identity -PROPERTIES BUILD_RPATH "\$ORIGIN" - # set target compile options - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON -) +if(TRITON_ENABLE_GPU) + set_target_properties(triton_rapids-identity + PROPERTIES BUILD_RPATH "\$ORIGIN" + # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) +else() + set_target_properties(triton_rapids-identity + PROPERTIES BUILD_RPATH "\$ORIGIN" + # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) +endif() target_compile_options(triton_rapids-identity PRIVATE "$<$:${RAPIDS_TRITON_CXX_FLAGS}>" @@ -41,15 +52,25 @@ target_include_directories(triton_rapids-identity "${CMAKE_CURRENT_SOURCE_DIR}/src" ) -target_link_libraries(triton_rapids-identity -PRIVATE - rmm::rmm - raft::raft - triton-core-serverstub - triton-backend-utils - "${TRITONSERVER_LIB}" - $ -) +if(TRITON_ENABLE_GPU) + target_link_libraries(triton_rapids-identity + PRIVATE + rmm::rmm + raft::raft + triton-core-serverstub + triton-backend-utils + "${TRITONSERVER_LIB}" + $ + ) +else() + target_link_libraries(triton_rapids-identity + PRIVATE + triton-core-serverstub + triton-backend-utils + "${TRITONSERVER_LIB}" + $ + ) +endif() install( TARGETS triton_rapids-identity diff --git a/cpp/src/model.h b/cpp/src/model.h index b9acf86..5ba95f1 100644 --- a/cpp/src/model.h +++ b/cpp/src/model.h @@ -16,7 +16,11 @@ #pragma once +#ifdef TRITON_ENABLE_GPU #include +#else +#include +#endif #include #include diff --git a/cpp/test/memory/resource.cpp b/cpp/test/memory/resource.cpp index 46c4982..b786090 100644 --- a/cpp/test/memory/resource.cpp +++ b/cpp/test/memory/resource.cpp @@ -14,9 +14,12 @@ * limitations under the License. */ +// TODO(wphicks) <<<<< #include +#else #include #include +#include #include #include @@ -30,12 +33,12 @@ namespace rapids { TEST(RapidsTriton, get_memory_resource) { if constexpr (IS_GPU_BUILD) { - auto device_id = int{}; + /* auto device_id = int{}; cuda_check(cudaGetDevice(&device_id)); EXPECT_EQ(get_memory_resource(device_id)->is_equal(rmm::mr::cuda_memory_resource{}), true); setup_memory_resource(device_id); EXPECT_EQ(get_memory_resource(device_id)->is_equal(rmm::mr::cuda_memory_resource{}), false); - EXPECT_EQ(get_memory_resource(device_id), get_memory_resource()); + EXPECT_EQ(get_memory_resource(device_id), get_memory_resource()); */ } } diff --git a/docs/usage.md b/docs/usage.md index d36a040..f16bb00 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -315,22 +315,10 @@ many instances of a given model are created. ## Other Memory Allocations For most device memory allocations, it is strongly recommended that you simply -construct a `Buffer` of the correct size and type. However, if you do not wish -to use a `Buffer` in a particular context, you are encouraged to allocate and -deallocate device memory using [RMM](https://github.com/rapidsai/rmm). Any +construct a `Buffer` of the correct size and type. However, if you absolutely +cannot use a `Buffer` in a particular context, you are encouraged to allocate +and deallocate device memory using [RMM](https://github.com/rapidsai/rmm). Any memory managed in this way will make use of Triton's CUDA memory pool, which -will be faster than performing individual allocations. Memory can be allocated -and deallocated using RMM as follows: - -```cpp -#include - -void rmm_example(std::size_t number_of_bytes, cudaStream_t stream) { - void* data = rmm::get_current_device_resource()->allocate(number_of_bytes, stream); - rmm::get_current_device_resource()->deallocate(data, number_of_bytes); -} -``` - -It is strongly recommended that you not change the RMM device resource in your -backend, since doing so will cause allocations to no longer make use of -Triton's memory pool. +will be faster than performing individual allocations. It is strongly +recommended that you not change the RMM device resource in your backend, since +doing so will cause allocations to no longer make use of Triton's memory pool. From c43b35f821d71f962944e70f14c30a0af92c1927 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 16 Nov 2021 13:05:22 -0500 Subject: [PATCH 130/199] Update unit tests for CPU-only build --- .../cpu_only/cuda_runtime_replacement.hpp | 9 +- .../memory/detail/cpu_only/copy.hpp | 1 + cpp/test/CMakeLists.txt | 71 ++++++++---- cpp/test/exceptions.cpp | 8 ++ cpp/test/memory/buffer.cpp | 106 +++++++++--------- cpp/test/memory/detail/copy.cpp | 71 ++++-------- cpp/test/memory/resource.cpp | 33 +++--- cpp/test/tensor/dtype.cpp | 1 - cpp/test/tensor/tensor.cpp | 6 + cpp/test/utils/narrow.cpp | 1 - 10 files changed, 163 insertions(+), 144 deletions(-) diff --git a/cpp/include/rapids_triton/cpu_only/cuda_runtime_replacement.hpp b/cpp/include/rapids_triton/cpu_only/cuda_runtime_replacement.hpp index 148e38d..7cb6691 100644 --- a/cpp/include/rapids_triton/cpu_only/cuda_runtime_replacement.hpp +++ b/cpp/include/rapids_triton/cpu_only/cuda_runtime_replacement.hpp @@ -25,7 +25,8 @@ namespace rapids { using cudaStream_t = void*; -enum struct cudaError_t {cudaSuccess, cudaNonGpuBuildError}; +enum struct cudaError_t {cudaSuccess, cudaErrorNonGpuBuild}; +using cudaError = cudaError_t; auto constexpr cudaSuccess = cudaError_t::cudaSuccess; inline void cudaGetLastError() {} @@ -35,15 +36,15 @@ inline auto const * cudaGetErrorString(cudaError_t err) { } inline auto cudaStreamSynchronize(cudaStream_t stream) { - return cudaError_t::cudaNonGpuBuildError; + return cudaError_t::cudaErrorNonGpuBuild; } inline auto cudaGetDevice(int* device_id) { - return cudaError_t::cudaNonGpuBuildError; + return cudaError_t::cudaErrorNonGpuBuild; } inline auto cudaGetDeviceCount(int* count) { - return cudaError_t::cudaNonGpuBuildError; + return cudaError_t::cudaErrorNonGpuBuild; } diff --git a/cpp/include/rapids_triton/memory/detail/cpu_only/copy.hpp b/cpp/include/rapids_triton/memory/detail/cpu_only/copy.hpp index edfedba..beb5cb4 100644 --- a/cpp/include/rapids_triton/memory/detail/cpu_only/copy.hpp +++ b/cpp/include/rapids_triton/memory/detail/cpu_only/copy.hpp @@ -21,6 +21,7 @@ #ifndef TRITON_ENABLE_GPU #include #endif +#include #include namespace triton { diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 467d273..4c96fa0 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -47,16 +47,27 @@ add_executable(test_rapids_triton test/utils/narrow.cpp ) -set_target_properties(test_rapids_triton -PROPERTIES BUILD_RPATH "\$ORIGIN" - # set target compile options - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON -) +IF(TRITON_ENABLE_GPU) + set_target_properties(test_rapids_triton + PROPERTIES BUILD_RPATH "\$ORIGIN" + # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) +else() + set_target_properties(test_rapids_triton + PROPERTIES BUILD_RPATH "\$ORIGIN" + # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) +endif() target_compile_options(test_rapids_triton PRIVATE "$<$:${RAPIDS_TRITON_CXX_FLAGS}>" @@ -75,16 +86,30 @@ find_library( PATHS /opt/tritonserver/lib ) -target_link_libraries(test_rapids_triton -PRIVATE - rmm::rmm - raft::raft - triton-core-serverstub - triton-backend-utils - gmock - gmock_main - GTest::gtest - GTest::gtest_main - "${TRITONSERVER_LIB}" - $ -) +IF(TRITON_ENABLE_GPU) + target_link_libraries(test_rapids_triton + PRIVATE + rmm::rmm + raft::raft + triton-core-serverstub + triton-backend-utils + gmock + gmock_main + GTest::gtest + GTest::gtest_main + "${TRITONSERVER_LIB}" + $ + ) +else() + target_link_libraries(test_rapids_triton + PRIVATE + triton-core-serverstub + triton-backend-utils + gmock + gmock_main + GTest::gtest + GTest::gtest_main + "${TRITONSERVER_LIB}" + $ + ) +endif() diff --git a/cpp/test/exceptions.cpp b/cpp/test/exceptions.cpp index bc0fc59..022d184 100644 --- a/cpp/test/exceptions.cpp +++ b/cpp/test/exceptions.cpp @@ -14,7 +14,11 @@ * limitations under the License. */ +#ifdef TRITON_ENABLE_GPU #include +#else +#include +#endif #include #include @@ -66,8 +70,12 @@ TEST(RapidsTriton, triton_check) TEST(RapidsTriton, cuda_check) { +#ifdef TRITON_ENABLE_GPU EXPECT_THROW(cuda_check(cudaError::cudaErrorMissingConfiguration), TritonException); cuda_check(cudaError::cudaSuccess); +#else + EXPECT_THROW(cuda_check(cudaError::cudaErrorNonGpuBuild), TritonException); +#endif } } // namespace rapids diff --git a/cpp/test/memory/buffer.cpp b/cpp/test/memory/buffer.cpp index 1867f52..052ab1f 100644 --- a/cpp/test/memory/buffer.cpp +++ b/cpp/test/memory/buffer.cpp @@ -14,7 +14,9 @@ * limitations under the License. */ +#ifdef TRITON_ENABLE_GPU #include +#endif #include #include @@ -36,68 +38,68 @@ TEST(RapidsTriton, default_buffer) EXPECT_EQ(buffer.data(), nullptr); EXPECT_EQ(buffer.device(), 0); EXPECT_EQ(buffer.stream(), cudaStream_t{}); - if constexpr (IS_GPU_BUILD) { - auto stream = cudaStream_t{}; - cudaStreamCreate(&stream); - buffer.set_stream(stream); - EXPECT_EQ(buffer.stream(), stream); - cudaStreamDestroy(stream); - } +#ifdef TRITON_ENABLE_GPU + auto stream = cudaStream_t{}; + cudaStreamCreate(&stream); + buffer.set_stream(stream); + EXPECT_EQ(buffer.stream(), stream); + cudaStreamDestroy(stream); +#endif } TEST(RapidsTriton, device_buffer) { auto data = std::vector{1, 2, 3}; - if (IS_GPU_BUILD) { - auto buffer = Buffer(data.size(), DeviceMemory, 0, 0); - - ASSERT_EQ(buffer.mem_type(), DeviceMemory); - ASSERT_EQ(buffer.size(), data.size()); - ASSERT_NE(buffer.data(), nullptr); - - auto data_out = std::vector(data.size()); - cudaMemcpy(static_cast(buffer.data()), - static_cast(data.data()), - sizeof(int) * data.size(), - cudaMemcpyHostToDevice); - cudaMemcpy(static_cast(data_out.data()), - static_cast(buffer.data()), - sizeof(int) * data.size(), - cudaMemcpyDeviceToHost); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); - - } else { - EXPECT_THROW(Buffer(data.size(), DeviceMemory, 0, 0), TritonException); - } +#ifdef TRITON_ENABLE_GPU + auto buffer = Buffer(data.size(), DeviceMemory, 0, 0); + + ASSERT_EQ(buffer.mem_type(), DeviceMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_NE(buffer.data(), nullptr); + + auto data_out = std::vector(data.size()); + cudaMemcpy(static_cast(buffer.data()), + static_cast(data.data()), + sizeof(int) * data.size(), + cudaMemcpyHostToDevice); + cudaMemcpy(static_cast(data_out.data()), + static_cast(buffer.data()), + sizeof(int) * data.size(), + cudaMemcpyDeviceToHost); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + +#else + EXPECT_THROW(Buffer(data.size(), DeviceMemory, 0, 0), TritonException); +#endif } TEST(RapidsTriton, non_owning_device_buffer) { auto data = std::vector{1, 2, 3}; - if constexpr (IS_GPU_BUILD) { - auto* ptr_d = static_cast(nullptr); - cudaMalloc(reinterpret_cast(&ptr_d), sizeof(int) * data.size()); - cudaMemcpy(static_cast(ptr_d), - static_cast(data.data()), - sizeof(int) * data.size(), - cudaMemcpyHostToDevice); - auto buffer = Buffer(ptr_d, data.size(), DeviceMemory); - - ASSERT_EQ(buffer.mem_type(), DeviceMemory); - ASSERT_EQ(buffer.size(), data.size()); - ASSERT_EQ(buffer.data(), ptr_d); - - auto data_out = std::vector(data.size()); - cudaMemcpy(static_cast(data_out.data()), - static_cast(buffer.data()), - sizeof(int) * data.size(), - cudaMemcpyDeviceToHost); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); - - cudaFree(reinterpret_cast(ptr_d)); - } else { - ASSERT_THROW(Buffer(data.data(), data.size(), DeviceMemory), TritonException); - } +#ifdef TRITON_ENABLE_GPU + auto* ptr_d = static_cast(nullptr); + cudaMalloc(reinterpret_cast(&ptr_d), sizeof(int) * data.size()); + cudaMemcpy(static_cast(ptr_d), + static_cast(data.data()), + sizeof(int) * data.size(), + cudaMemcpyHostToDevice); + auto buffer = Buffer(ptr_d, data.size(), DeviceMemory); + + ASSERT_EQ(buffer.mem_type(), DeviceMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_EQ(buffer.data(), ptr_d); + + auto data_out = std::vector(data.size()); + cudaMemcpy(static_cast(data_out.data()), + static_cast(buffer.data()), + sizeof(int) * data.size(), + cudaMemcpyDeviceToHost); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + + cudaFree(reinterpret_cast(ptr_d)); +#else + ASSERT_THROW(Buffer(data.data(), data.size(), DeviceMemory), TritonException); +#endif } TEST(RapidsTriton, host_buffer) diff --git a/cpp/test/memory/detail/copy.cpp b/cpp/test/memory/detail/copy.cpp index ec29e22..d669d6b 100644 --- a/cpp/test/memory/detail/copy.cpp +++ b/cpp/test/memory/detail/copy.cpp @@ -14,47 +14,22 @@ * limitations under the License. */ +#ifdef TRITON_ENABLE_GPU #include +#include +#else +#include +#endif #include #include #include -#include -#include #include #include namespace triton { namespace backend { namespace rapids { -TEST(RapidsTriton, dev_copy) -{ - auto data = std::vector{1, 2, 3}; - auto data_out = std::vector(data.size()); - if constexpr (IS_GPU_BUILD) { - auto* ptr_d = static_cast(nullptr); - cudaMalloc(reinterpret_cast(&ptr_d), sizeof(int) * data.size()); - detail::dev_copy(ptr_d, data.data(), data.size(), 0); - - cudaMemcpy(static_cast(data_out.data()), - static_cast(ptr_d), - sizeof(int) * data.size(), - cudaMemcpyDeviceToHost); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); - cudaFree(reinterpret_cast(ptr_d)); - } else { - ASSERT_THROW(detail::dev_copy(data_out.data(), data.data(), data.size(), 0), TritonException); - } -} - -TEST(RapidsTriton, host_copy) -{ - auto data = std::vector{1, 2, 3}; - auto data_out = std::vector(data.size()); - detail::host_copy(data_out.data(), data.data(), data.size()); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); -} - TEST(RapidsTriton, copy) { auto data = std::vector{1, 2, 3}; @@ -63,25 +38,23 @@ TEST(RapidsTriton, copy) EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); data_out = std::vector(data.size()); - if constexpr (IS_GPU_BUILD) { - auto* ptr_d = static_cast(nullptr); - cudaMalloc(reinterpret_cast(&ptr_d), sizeof(int) * data.size()); - detail::copy(ptr_d, data.data(), data.size(), 0, DeviceMemory, HostMemory); - - cudaMemcpy(static_cast(data_out.data()), - static_cast(ptr_d), - sizeof(int) * data.size(), - cudaMemcpyDeviceToHost); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); - cudaFree(reinterpret_cast(ptr_d)); - } else { - EXPECT_THROW( - detail::copy(data_out.data(), data.data(), data.size(), 0, HostMemory, DeviceMemory), - TritonException); - EXPECT_THROW( - detail::copy(data_out.data(), data.data(), data.size(), 0, DeviceMemory, HostMemory), - TritonException); - } +#ifdef TRITON_ENABLE_GPU + auto* ptr_d = static_cast(nullptr); + cudaMalloc(reinterpret_cast(&ptr_d), sizeof(int) * data.size()); + detail::copy(ptr_d, data.data(), data.size(), 0, DeviceMemory, HostMemory); + + cudaMemcpy(static_cast(data_out.data()), + static_cast(ptr_d), + sizeof(int) * data.size(), + cudaMemcpyDeviceToHost); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + cudaFree(reinterpret_cast(ptr_d)); +#else + EXPECT_THROW(detail::copy(data_out.data(), data.data(), data.size(), 0, HostMemory, DeviceMemory), + TritonException); + EXPECT_THROW(detail::copy(data_out.data(), data.data(), data.size(), 0, DeviceMemory, HostMemory), + TritonException); +#endif } } // namespace rapids diff --git a/cpp/test/memory/resource.cpp b/cpp/test/memory/resource.cpp index b786090..5571ef7 100644 --- a/cpp/test/memory/resource.cpp +++ b/cpp/test/memory/resource.cpp @@ -14,32 +14,37 @@ * limitations under the License. */ -// TODO(wphicks) <<<<< +#ifdef TRITON_ENABLE_GPU #include -#else +#include +#include +#include +#endif + #include #include -#include #include #include #include -#include -#include namespace triton { namespace backend { namespace rapids { -TEST(RapidsTriton, get_memory_resource) + +TEST(RapidsTriton, set_memory_resource) { - if constexpr (IS_GPU_BUILD) { - /* auto device_id = int{}; - cuda_check(cudaGetDevice(&device_id)); - EXPECT_EQ(get_memory_resource(device_id)->is_equal(rmm::mr::cuda_memory_resource{}), true); - setup_memory_resource(device_id); - EXPECT_EQ(get_memory_resource(device_id)->is_equal(rmm::mr::cuda_memory_resource{}), false); - EXPECT_EQ(get_memory_resource(device_id), get_memory_resource()); */ - } +#ifdef TRITON_ENABLE_GPU + auto device_id = int{}; + cuda_check(cudaGetDevice(&device_id)); + EXPECT_EQ(rmm::mr::get_current_device_resource()->is_equal(rmm::mr::cuda_memory_resource{}), + true); + setup_memory_resource(device_id); + EXPECT_EQ(rmm::mr::get_current_device_resource()->is_equal(rmm::mr::cuda_memory_resource{}), + false); +#else + setup_memory_resource(0); +#endif } } // namespace rapids diff --git a/cpp/test/tensor/dtype.cpp b/cpp/test/tensor/dtype.cpp index ef1a9ca..2e80c60 100644 --- a/cpp/test/tensor/dtype.cpp +++ b/cpp/test/tensor/dtype.cpp @@ -14,7 +14,6 @@ * limitations under the License. */ -#include #include #include diff --git a/cpp/test/tensor/tensor.cpp b/cpp/test/tensor/tensor.cpp index 672402d..8bb6c9b 100644 --- a/cpp/test/tensor/tensor.cpp +++ b/cpp/test/tensor/tensor.cpp @@ -14,7 +14,11 @@ * limitations under the License. */ +#ifdef TRITON_ENABLE_GPU #include +#else +#include +#endif #include #include @@ -62,6 +66,7 @@ TEST(RapidsTriton, multi_buffer_tensor) std::transform(data.begin(), data.end(), std::back_inserter(all_buffers), [](auto& elem) { return Buffer{&elem, 1, DeviceMemory}; }); +#ifdef TRITON_ENABLE_GPU auto tensor = Tensor(shape, all_buffers.begin(), all_buffers.end(), DeviceMemory, 0, cudaStream_t{}); @@ -71,6 +76,7 @@ TEST(RapidsTriton, multi_buffer_tensor) sizeof(int) * tensor.size(), cudaMemcpyDeviceToHost); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +#endif } TEST(RapidsTriton, tensor_copy) diff --git a/cpp/test/utils/narrow.cpp b/cpp/test/utils/narrow.cpp index 9263e6e..d9919ca 100644 --- a/cpp/test/utils/narrow.cpp +++ b/cpp/test/utils/narrow.cpp @@ -14,7 +14,6 @@ * limitations under the License. */ -#include #include #include From 394e2a7e77637c5523f2c80c4becaf0870fc9926 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 16 Nov 2021 13:29:31 -0500 Subject: [PATCH 131/199] Add usage information on CPU-only helpers --- docs/usage.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/usage.md b/docs/usage.md index f16bb00..65468f0 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -123,6 +123,26 @@ conditional inclusion of headers: #endif ``` +Sometimes, having a CUDA symbol available in a CPU-only build can avoid layers +of indirection which would otherwise be required to allow for compilation of +both GPU and CPU versions of particular code. RAPIDS-Triton includes a header +which has some placeholders for CUDA symbols used internally by the library, +and which may be useful for backends which implement CPU-only builds as well. +Note that all placeholder symbols are namespaced within +`triton::backend::rapids`. Note that not all symbols from the CUDA runtime API +are included, but additional symbols will be added over time. All placeholder +symbols will be implemented in a way that is consistent with similar +placeholders in the main Triton codebase. A typical usage is shown below: +```cpp +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +// E.g. cudaStream_t is now defined regardless of whether or not this is a +// CPU-only build. +``` + ## Buffers Within a backend, it is often useful to process data in a way that is agnostic to whether the underlying memory is on the host or on device and whether that From e9cf10d7235c6beca0241fe1ef50e39437d24f95 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 16 Nov 2021 18:34:07 -0500 Subject: [PATCH 132/199] Add build script --- build.sh | 147 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100755 build.sh diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..9fb16b2 --- /dev/null +++ b/build.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +REPODIR=$(cd $(dirname $0); pwd) + +NUMARGS=$# +ARGS=$* +VALIDTARGETS="example tests" +VALIDFLAGS="--cpu-only -g -h --help" +VALIDARGS="${VALIDTARGETS} ${VALIDFLAGS}" +HELP="$0 [ ...] [ ...] + where is: + example - build the identity backend example + tests - build container(s) with unit tests + and is: + -g - build for debug + -h - print this text + --cpu-only - build CPU-only versions of targets + --tag-commit - tag docker images based on current git commit + + default action (no args) is to build all targets + The following environment variables are also accepted to allow further customization: + BASE_IMAGE - Base image for Docker images + TRITON_VERSION - Triton version to use for build +" + +BUILD_TYPE=Release +IMAGE_TAG="" +TRITON_ENABLE_GPU=OFF +DOCKER_ARGS="" + +function hasArg { + (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") +} + +function completeBuild { + (( ${NUMARGS} == 0 )) && return + for a in ${ARGS}; do + if (echo " ${VALIDTARGETS} " | grep -q " ${a} "); then + false; return + fi + done + true +} + +if hasArg -h || hasArg --help; then + echo "${HELP}" + exit 0 +fi + +# Long arguments +LONG_ARGUMENT_LIST=( + "cpu-only" + "tag-commit" +) + +# Short arguments +ARGUMENT_LIST=( + "g" +) + +# read arguments +opts=$(getopt \ + --longoptions "$(printf "%s," "${LONG_ARGUMENT_LIST[@]}")" \ + --name "$(basename "$0")" \ + --options "$(printf "%s" "${ARGUMENT_LIST[@]}")" \ + -- "$@" +) + +if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi + +eval set -- "$opts" + +while true +do + case "$1" in + -g | --debug ) + BUILD_TYPE=Debug + ;; + --cpu-only ) + TRITON_ENABLE_GPU=OFF + ;; + --tag-commit ) + IMAGE_TAG=":$(cd $REPODIR; git rev-parse --short HEAD)" + ;; + --) + shift + break + ;; + esac + shift +done + +DOCKER_ARGS="$DOCKER_ARGS --build-arg BUILD_TYPE=${BUILD_TYPE}" +DOCKER_ARGS="$DOCKER_ARGS --build-arg TRITON_ENABLE_GPU=${TRITON_ENABLE_GPU}" + +if [ ! -z $BASE_IMAGE ] +then + DOCKER_ARGS="$DOCKER_ARGS --build-arg BASE_IMAGE=${BASE_IMAGE}" +fi +if [ ! -z $TRITON_VERSION ] +then + DOCKER_ARGS="$DOCKER_ARGS --build-arg TRITON_VERSION=${TRITON_VERSION}" +fi + +if completeBuild || hasArg example +then + BACKEND=1 + DOCKER_ARGS="$DOCKER_ARGS --build-arg BUILD_EXAMPLE=ON" +fi + +if completeBuild || hasArg tests +then + TESTS=1 + DOCKER_ARGS="$DOCKER_ARGS --build-arg BUILD_TESTS=ON" +fi + +if [ $BACKEND -eq 1 ] +then + docker build \ + $DOCKER_ARGS \ + -t "rapids_triton_identity${IMAGE_TAG}" \ + $REPODIR +fi + +if [ $TESTS -eq 1 ] +then + docker build \ + $DOCKER_ARGS \ + --target build-stage \ + -t "rapids_triton_identity_test${IMAGE_TAG}" \ + $REPODIR +fi From 5b876344d770a4c685e62953abd431ecab0df740 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 18 Nov 2021 11:52:55 -0500 Subject: [PATCH 133/199] Provide working build/tests for CPU-only mode --- Dockerfile | 26 ++++- build.sh | 29 ++++- ci/local/build.sh | 19 ++++ conda/environments/rapids_triton_test.yml | 5 +- cpp/include/rapids_triton/memory/buffer.hpp | 14 +++ .../detail/cpu_only/owned_device_buffer.hpp | 2 + .../detail/gpu_only/owned_device_buffer.hpp | 2 +- cpp/test/CMakeLists.txt | 1 + cpp/test/memory/buffer.cpp | 4 + .../memory/detail/owned_device_buffer.cpp | 64 +++++++++++ cpp/test/tensor/tensor.cpp | 16 ++- .../identity/config.pbtxt | 21 ++++ .../model_repository/identity/config.pbtxt | 21 ++++ qa/L0_e2e/test_model.py | 69 ++++++++++++ qa/entrypoint.sh | 28 +++++ ...ab1-73ba-4c54-80ee-e2812d146191-server.log | 75 +++++++++++++ ...952-d091-4805-80dc-729dddba0368-server.log | 77 +++++++++++++ ...f1d-b4d6-40f0-8333-0d1b8d11414a-server.log | 75 +++++++++++++ ...3ab-4d1b-4048-85c9-0b44e32b78d0-server.log | 77 +++++++++++++ ...7bb-edaf-493c-a2c1-32673a6a5bc1-server.log | 71 ++++++++++++ ...c2e-2cc4-4848-9eca-dd17f2b93686-server.log | 74 ++++++++++++ ...cf4-e0b3-48ed-9772-69853e7833b3-server.log | 5 + ...73b-5709-47da-9c80-a09db7116498-server.log | 75 +++++++++++++ ...6fc-1a84-4f57-a798-fa3d1be97409-server.log | 77 +++++++++++++ ...053-82c2-41c8-8bde-209499a5ec47-server.log | 77 +++++++++++++ ...b94-6959-43a0-a972-7bd52289b21e-server.log | 71 ++++++++++++ ...c20-ba41-4c04-ac7b-ec324fcb0754-server.log | 74 ++++++++++++ qa/run_tests.sh | 105 ++++++++++++++++++ 28 files changed, 1235 insertions(+), 19 deletions(-) create mode 100755 ci/local/build.sh create mode 100644 cpp/test/memory/detail/owned_device_buffer.cpp create mode 100644 qa/L0_e2e/cpu_model_repository/identity/config.pbtxt create mode 100644 qa/L0_e2e/model_repository/identity/config.pbtxt create mode 100644 qa/L0_e2e/test_model.py create mode 100755 qa/entrypoint.sh create mode 100644 qa/logs/1d745ab1-73ba-4c54-80ee-e2812d146191-server.log create mode 100644 qa/logs/22810952-d091-4805-80dc-729dddba0368-server.log create mode 100644 qa/logs/28aa9f1d-b4d6-40f0-8333-0d1b8d11414a-server.log create mode 100644 qa/logs/4592e3ab-4d1b-4048-85c9-0b44e32b78d0-server.log create mode 100644 qa/logs/6c0cf7bb-edaf-493c-a2c1-32673a6a5bc1-server.log create mode 100644 qa/logs/6c2c5c2e-2cc4-4848-9eca-dd17f2b93686-server.log create mode 100644 qa/logs/73be3cf4-e0b3-48ed-9772-69853e7833b3-server.log create mode 100644 qa/logs/7f6ed73b-5709-47da-9c80-a09db7116498-server.log create mode 100644 qa/logs/933cd6fc-1a84-4f57-a798-fa3d1be97409-server.log create mode 100644 qa/logs/974b0053-82c2-41c8-8bde-209499a5ec47-server.log create mode 100644 qa/logs/da517b94-6959-43a0-a972-7bd52289b21e-server.log create mode 100644 qa/logs/f09aec20-ba41-4c04-ac7b-ec324fcb0754-server.log create mode 100755 qa/run_tests.sh diff --git a/Dockerfile b/Dockerfile index 9b6e8b7..49401b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,14 +38,12 @@ RUN conda env update -f /environment.yml \ ENV PYTHONDONTWRITEBYTECODE=false -COPY ./cpp /rapids_triton - -WORKDIR /rapids_triton - SHELL ["conda", "run", "--no-capture-output", "-n", "rapids_triton_dev", "/bin/bash", "-c"] FROM base as build-stage +COPY ./cpp /rapids_triton + ARG TRITON_VERSION ENV TRITON_VERSION=$TRITON_VERSION @@ -72,7 +70,25 @@ RUN cmake \ RUN ninja install -ENTRYPOINT ["/rapids_triton/build/test_rapids_triton"] +FROM base as test-install + +COPY ./conda/environments/rapids_triton_test.yml /environment.yml + +RUN conda env update -f /environment.yml \ + && rm /environment.yml \ + && conda clean -afy \ + && find /root/miniconda3/ -follow -type f -name '*.pyc' -delete \ + && find /root/miniconda3/ -follow -type f -name '*.js.map' -delete + +FROM build-stage as test-stage + +COPY --from=test-install /root/miniconda3 /root/miniconda3 + +ENV TEST_EXE=/rapids_triton/build/test_rapids_triton + +COPY qa /qa + +ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "rapids_triton_test", "/bin/bash", "/qa/entrypoint.sh"] FROM ${BASE_IMAGE} diff --git a/build.sh b/build.sh index 9fb16b2..583afb6 100755 --- a/build.sh +++ b/build.sh @@ -36,13 +36,16 @@ HELP="$0 [ ...] [ ...] The following environment variables are also accepted to allow further customization: BASE_IMAGE - Base image for Docker images TRITON_VERSION - Triton version to use for build + EXAMPLE_TAG - The tag to use for the server image + TEST_TAG - The tag to use for the test image " BUILD_TYPE=Release -IMAGE_TAG="" -TRITON_ENABLE_GPU=OFF +TRITON_ENABLE_GPU=ON DOCKER_ARGS="" +export DOCKER_BUILDKIT=1 + function hasArg { (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") } @@ -95,7 +98,12 @@ do TRITON_ENABLE_GPU=OFF ;; --tag-commit ) - IMAGE_TAG=":$(cd $REPODIR; git rev-parse --short HEAD)" + [ -z $EXAMPLE_TAG ] \ + && EXAMPLE_TAG="rapids_triton_identity:$(cd $REPODIR; git rev-parse --short HEAD)" \ + || true + [ -z $TEST_TAG ] \ + && TEST_TAG="rapids_triton_identity_test:$(cd $REPODIR; git rev-parse --short HEAD)" \ + || true ;; --) shift @@ -105,6 +113,15 @@ do shift done +if [ -z $EXAMPLE_TAG ] +then + EXAMPLE_TAG='rapids_triton_identity' +fi +if [ -z $TEST_TAG ] +then + TEST_TAG='rapids_triton_identity_test' +fi + DOCKER_ARGS="$DOCKER_ARGS --build-arg BUILD_TYPE=${BUILD_TYPE}" DOCKER_ARGS="$DOCKER_ARGS --build-arg TRITON_ENABLE_GPU=${TRITON_ENABLE_GPU}" @@ -133,7 +150,7 @@ if [ $BACKEND -eq 1 ] then docker build \ $DOCKER_ARGS \ - -t "rapids_triton_identity${IMAGE_TAG}" \ + -t "$EXAMPLE_TAG" \ $REPODIR fi @@ -141,7 +158,7 @@ if [ $TESTS -eq 1 ] then docker build \ $DOCKER_ARGS \ - --target build-stage \ - -t "rapids_triton_identity_test${IMAGE_TAG}" \ + --target test-stage \ + -t "$TEST_TAG" \ $REPODIR fi diff --git a/ci/local/build.sh b/ci/local/build.sh new file mode 100755 index 0000000..b13f9cb --- /dev/null +++ b/ci/local/build.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +REPODIR=$(cd $(dirname $0)/../../; pwd) + +EXAMPLE_TAG=rapids_triton_identity \ + TEST_TAG=rapids_triton_identity_test \ + $REPODIR/build.sh +if [ -z $CUDA_VISIBLE_DEVICES ] +then + docker run --gpus all --rm rapids_triton_identity_test +else + docker run --gpus $CUDA_VISIBLE_DEVICES --rm rapids_triton_identity_test +fi +EXAMPLE_TAG=rapids_triton_identity:cpu \ + TEST_TAG=rapids_triton_identity_test:cpu \ + $REPODIR/build.sh --cpu-only +docker run -v "${REPODIR}/qa/logs:/qa/logs" --gpus all --rm rapids_triton_identity_test:cpu diff --git a/conda/environments/rapids_triton_test.yml b/conda/environments/rapids_triton_test.yml index ded958f..7f78bff 100644 --- a/conda/environments/rapids_triton_test.yml +++ b/conda/environments/rapids_triton_test.yml @@ -3,10 +3,11 @@ name: rapids_triton_test channels: - conda-forge dependencies: - - clang-tools=11.0.0 - flake8 - pip - python + - pytest - numpy - pip: - - nvidia-pyindex + - tritonclient[all] + - git+https://github.com/rapidsai/rapids-triton.git#subdirectory=python diff --git a/cpp/include/rapids_triton/memory/buffer.hpp b/cpp/include/rapids_triton/memory/buffer.hpp index 8b7cbba..124437c 100644 --- a/cpp/include/rapids_triton/memory/buffer.hpp +++ b/cpp/include/rapids_triton/memory/buffer.hpp @@ -68,6 +68,14 @@ struct Buffer { size_{size}, stream_{stream} { + if constexpr (!IS_GPU_BUILD) { + if (memory_type == DeviceMemory) { + throw TritonException( + Error::Internal, + "Cannot use device buffer in non-GPU build" + ); + } + } } /** @@ -87,6 +95,12 @@ struct Buffer { if (memory_type == HostMemory) { result = data_store{std::in_place_index<0>, input_data}; } else { + if constexpr (!IS_GPU_BUILD) { + throw TritonException( + Error::Internal, + "Cannot use device buffer in non-GPU build" + ); + } result = data_store{std::in_place_index<1>, input_data}; } return result; diff --git a/cpp/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp b/cpp/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp index a866e3c..4006a70 100644 --- a/cpp/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp +++ b/cpp/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp @@ -17,6 +17,8 @@ #pragma once #include #include +#include +#include #include #include diff --git a/cpp/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp b/cpp/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp index c02d42a..29edacd 100644 --- a/cpp/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp +++ b/cpp/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp @@ -32,7 +32,7 @@ struct owned_device_buffer { owned_device_buffer(device_id_t device_id, std::size_t size, cudaStream_t stream) : data_{[this, &device_id, &size, &stream]() { auto device_context = device_setter{device_id}; - return rmm::device_buffer{size, rmm::cuda_stream_view{stream}}; + return rmm::device_buffer{size * sizeof(T), rmm::cuda_stream_view{stream}}; }()} { } diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 4c96fa0..0f90912 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -21,6 +21,7 @@ add_executable(test_rapids_triton test/exceptions.cpp test/memory/buffer.cpp test/memory/detail/copy.cpp + test/memory/detail/owned_device_buffer.cpp test/memory/resource.cpp test/memory/types.cpp test/tensor/dtype.cpp diff --git a/cpp/test/memory/buffer.cpp b/cpp/test/memory/buffer.cpp index 052ab1f..1181560 100644 --- a/cpp/test/memory/buffer.cpp +++ b/cpp/test/memory/buffer.cpp @@ -162,7 +162,11 @@ TEST(RapidsTriton, move_assignment_buffer) { auto data = std::vector{1, 2, 3}; +#ifdef TRITON_ENABLE_GPU auto buffer = Buffer{data.data(), data.size() - 1, DeviceMemory}; +#else + auto buffer = Buffer{data.data(), data.size() - 1, HostMemory}; +#endif buffer = Buffer{data.size(), HostMemory}; ASSERT_EQ(buffer.mem_type(), HostMemory); diff --git a/cpp/test/memory/detail/owned_device_buffer.cpp b/cpp/test/memory/detail/owned_device_buffer.cpp new file mode 100644 index 0000000..9bd613f --- /dev/null +++ b/cpp/test/memory/detail/owned_device_buffer.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef TRITON_ENABLE_GPU +#include +#include +#else +#include +#endif +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +TEST(RapidsTriton, owned_device_buffer) +{ + auto data = std::vector{1, 2, 3}; +#ifdef TRITON_ENABLE_GPU + auto device_id = 0; + cudaGetDevice(&device_id); + auto stream = cudaStream_t{}; + cudaStreamCreate(&stream); + + auto buffer = detail::owned_device_buffer(device_id, data.size(), stream); + auto data_out = std::vector(data.size()); + cudaMemcpy(static_cast(buffer.get()), + static_cast(data.data()), + sizeof(int) * data.size(), + cudaMemcpyHostToDevice); + cudaMemcpy(static_cast(data_out.data()), + static_cast(buffer.get()), + sizeof(int) * data.size(), + cudaMemcpyDeviceToHost); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + cudaStreamDestroy(stream); +#else + // Workaround for ungraceful handling of multiple template parameters in + // EXPECT_THROW + using dev_buffer = detail::owned_device_buffer; + EXPECT_THROW(dev_buffer(0, data.size(), 0), TritonException); +#endif +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/cpp/test/tensor/tensor.cpp b/cpp/test/tensor/tensor.cpp index 8bb6c9b..3a1c5c2 100644 --- a/cpp/test/tensor/tensor.cpp +++ b/cpp/test/tensor/tensor.cpp @@ -63,20 +63,26 @@ TEST(RapidsTriton, multi_buffer_tensor) auto all_buffers = std::vector>{}; all_buffers.reserve(data.size()); - std::transform(data.begin(), data.end(), std::back_inserter(all_buffers), [](auto& elem) { - return Buffer{&elem, 1, DeviceMemory}; + auto mem_type = HostMemory; + if constexpr (IS_GPU_BUILD) { mem_type = DeviceMemory; } + std::transform(data.begin(), data.end(), std::back_inserter(all_buffers), [mem_type](auto& elem) { + return Buffer{&elem, 1, mem_type}; }); -#ifdef TRITON_ENABLE_GPU auto tensor = - Tensor(shape, all_buffers.begin(), all_buffers.end(), DeviceMemory, 0, cudaStream_t{}); + Tensor(shape, all_buffers.begin(), all_buffers.end(), mem_type, 0, cudaStream_t{}); auto data_out = std::vector(data.size()); +#ifdef TRITON_ENABLE_GPU cudaMemcpy(static_cast(data_out.data()), static_cast(tensor.data()), sizeof(int) * tensor.size(), cudaMemcpyDeviceToHost); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +#else + std::memcpy(static_cast(data_out.data()), + static_cast(tensor.data()), + sizeof(int) * tensor.size()); #endif + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } TEST(RapidsTriton, tensor_copy) diff --git a/qa/L0_e2e/cpu_model_repository/identity/config.pbtxt b/qa/L0_e2e/cpu_model_repository/identity/config.pbtxt new file mode 100644 index 0000000..5479199 --- /dev/null +++ b/qa/L0_e2e/cpu_model_repository/identity/config.pbtxt @@ -0,0 +1,21 @@ +backend: "rapids-identity" +max_batch_size: 32768 +input [ + { + name: "input__0" + data_type: TYPE_FP32 + dims: [ 1 ] + } +] +output [ + { + name: "output__0" + data_type: TYPE_FP32 + dims: [ 1 ] + } +] +instance_group [{ kind: KIND_CPU }] +parameters [ ] +dynamic_batching { + max_queue_delay_microseconds: 50 +} diff --git a/qa/L0_e2e/model_repository/identity/config.pbtxt b/qa/L0_e2e/model_repository/identity/config.pbtxt new file mode 100644 index 0000000..5fcba28 --- /dev/null +++ b/qa/L0_e2e/model_repository/identity/config.pbtxt @@ -0,0 +1,21 @@ +backend: "rapids-identity" +max_batch_size: 32768 +input [ + { + name: "input__0" + data_type: TYPE_FP32 + dims: [ 1 ] + } +] +output [ + { + name: "output__0" + data_type: TYPE_FP32 + dims: [ 1 ] + } +] +instance_group [{ kind: KIND_GPU }] +parameters [ ] +dynamic_batching { + max_queue_delay_microseconds: 50 +} diff --git a/qa/L0_e2e/test_model.py b/qa/L0_e2e/test_model.py new file mode 100644 index 0000000..9379ff5 --- /dev/null +++ b/qa/L0_e2e/test_model.py @@ -0,0 +1,69 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import numpy as np +import pytest + +from rapids_triton import Client +from rapids_triton.testing import get_random_seed, arrays_close + +TOTAL_SAMPLES = 8192 + +def valid_shm_modes(): + modes = [None] + if os.environ.get('CPU_ONLY', 0) == 0: + modes.append('cuda') + return modes + + +@pytest.fixture(scope='session') +def client(): + client = Client() + client.wait_for_server(60) + return client + +@pytest.fixture +def model_inputs(): + np.random.seed(get_random_seed()) + return { + input_name: + np.random.rand(TOTAL_SAMPLES, 1).astype('float32') + for input_name in ('input__0',) + } + +@pytest.fixture +def model_output_sizes(): + return {'output__0': TOTAL_SAMPLES * np.dtype('float32').itemsize} + +def get_ground_truth(inputs): + return {'output__0': inputs['input__0']} + + +@pytest.mark.parametrize("model_name", ['identity']) +@pytest.mark.parametrize("shared_mem", valid_shm_modes()) +def test_model(client, model_name, shared_mem, model_inputs, model_output_sizes): + result = client.predict( + model_name, model_inputs, model_output_sizes, shared_mem=shared_mem + ) + ground_truth = get_ground_truth(model_inputs) + + for output_name in sorted(ground_truth.keys()): + arrays_close( + result[output_name], + ground_truth[output_name], + atol=1e-5, + assert_close=True + ) diff --git a/qa/entrypoint.sh b/qa/entrypoint.sh new file mode 100755 index 0000000..046f8a0 --- /dev/null +++ b/qa/entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +QA_DIR=$(cd $(dirname $0); pwd) +TEST_SCRIPT="$QA_DIR/run_tests.sh" + +if [[ $TRITON_ENABLE_GPU != "OFF" ]] +then + MODEL_REPO="${QA_DIR}/L0_e2e/model_repository" "$TEST_SCRIPT" + export TEST_EXE='' # Only run unit tests once + MODEL_REPO="${QA_DIR}/L0_e2e/cpu_model_repository" "$TEST_SCRIPT" +fi + +CPU_ONLY=1 MODEL_REPO="${QA_DIR}/L0_e2e/cpu_model_repository" "$TEST_SCRIPT" diff --git a/qa/logs/1d745ab1-73ba-4c54-80ee-e2812d146191-server.log b/qa/logs/1d745ab1-73ba-4c54-80ee-e2812d146191-server.log new file mode 100644 index 0000000..074fb59 --- /dev/null +++ b/qa/logs/1d745ab1-73ba-4c54-80ee-e2812d146191-server.log @@ -0,0 +1,75 @@ +I1117 22:05:03.848968 1 metrics.cc:298] Collecting metrics for GPU 0: Quadro RTX 8000 +I1117 22:05:03.849179 1 metrics.cc:298] Collecting metrics for GPU 1: Quadro RTX 8000 +I1117 22:05:04.140935 1 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch +I1117 22:05:04.140964 1 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 +I1117 22:05:04.140969 1 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 +2021-11-17 22:05:04.417430: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 +I1117 22:05:04.453291 1 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow +I1117 22:05:04.453355 1 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 +I1117 22:05:04.453370 1 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 +I1117 22:05:04.453383 1 tensorflow.cc:2210] backend configuration: +{} +I1117 22:05:04.455331 1 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime +I1117 22:05:04.455351 1 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 +I1117 22:05:04.455356 1 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 +I1117 22:05:04.472805 1 openvino.cc:1193] TRITONBACKEND_Initialize: openvino +I1117 22:05:04.472822 1 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 +I1117 22:05:04.472826 1 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 +I1117 22:05:04.627788 1 pinned_memory_manager.cc:240] Pinned memory pool is created at '0x7f0680000000' with size 268435456 +I1117 22:05:04.628579 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 0 with size 67108864 +I1117 22:05:04.628590 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 1 with size 67108864 +I1117 22:05:04.709460 1 model_repository_manager.cc:1022] loading: identity:1 +I1117 22:05:04.813427 1 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity +I1117 22:05:04.813468 1 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 +I1117 22:05:04.813482 1 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 +I1117 22:05:04.824910 1 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) +I1117 22:05:04.829878 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (GPU device 0) +I1117 22:05:08.995377 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (GPU device 1) +I1117 22:05:12.926306 1 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 +I1117 22:05:12.926485 1 server.cc:522] ++------------------+------+ +| Repository Agent | Path | ++------------------+------+ ++------------------+------+ + +I1117 22:05:12.926630 1 server.cc:549] ++-----------------+-------------------------------------------------------------------------+--------+ +| Backend | Path | Config | ++-----------------+-------------------------------------------------------------------------+--------+ +| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | +| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | +| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | +| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | +| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | ++-----------------+-------------------------------------------------------------------------+--------+ + +I1117 22:05:12.926691 1 server.cc:592] ++----------+---------+--------+ +| Model | Version | Status | ++----------+---------+--------+ +| identity | 1 | READY | ++----------+---------+--------+ + +I1117 22:05:12.926944 1 tritonserver.cc:1920] ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Option | Value | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| server_id | triton | +| server_version | 2.15.0 | +| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | +| model_repository_path[0] | /models | +| model_control_mode | MODE_NONE | +| strict_model_config | 1 | +| rate_limit | OFF | +| pinned_memory_pool_byte_size | 268435456 | +| cuda_memory_pool_byte_size{0} | 67108864 | +| cuda_memory_pool_byte_size{1} | 67108864 | +| response_cache_byte_size | 0 | +| min_supported_compute_capability | 6.0 | +| strict_readiness | 1 | +| exit_timeout | 30 | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +I1117 22:05:12.928671 1 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 +I1117 22:05:12.929159 1 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 +I1117 22:05:12.988127 1 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 diff --git a/qa/logs/22810952-d091-4805-80dc-729dddba0368-server.log b/qa/logs/22810952-d091-4805-80dc-729dddba0368-server.log new file mode 100644 index 0000000..8ca1ba8 --- /dev/null +++ b/qa/logs/22810952-d091-4805-80dc-729dddba0368-server.log @@ -0,0 +1,77 @@ +W1118 16:37:43.979175 59 metrics.cc:282] Cannot get CUDA device count, GPU metrics will not be available +I1118 16:37:44.225688 59 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch +I1118 16:37:44.225720 59 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 +I1118 16:37:44.225728 59 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 +2021-11-18 16:37:44.506800: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 +I1118 16:37:44.621643 59 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow +I1118 16:37:44.621689 59 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 +I1118 16:37:44.621696 59 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 +I1118 16:37:44.621702 59 tensorflow.cc:2210] backend configuration: +{} +I1118 16:37:44.623264 59 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime +I1118 16:37:44.623283 59 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 +I1118 16:37:44.623289 59 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 +I1118 16:37:44.645064 59 openvino.cc:1193] TRITONBACKEND_Initialize: openvino +I1118 16:37:44.645082 59 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 +I1118 16:37:44.645088 59 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 +W1118 16:37:44.645107 59 pinned_memory_manager.cc:236] Unable to allocate pinned system memory, pinned memory pool will not be available: no CUDA-capable device is detected +I1118 16:37:44.645120 59 cuda_memory_manager.cc:115] CUDA memory pool disabled +I1118 16:37:44.645843 59 model_repository_manager.cc:1022] loading: identity:1 +I1118 16:37:44.747655 59 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity +I1118 16:37:44.747707 59 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 +I1118 16:37:44.747727 59 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 +I1118 16:37:44.747788 59 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) +I1118 16:37:44.751099 59 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (CPU device 0) +I1118 16:37:44.751549 59 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 +I1118 16:37:44.751697 59 server.cc:522] ++------------------+------+ +| Repository Agent | Path | ++------------------+------+ ++------------------+------+ + +I1118 16:37:44.751845 59 server.cc:549] ++-----------------+-------------------------------------------------------------------------+--------+ +| Backend | Path | Config | ++-----------------+-------------------------------------------------------------------------+--------+ +| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | +| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | +| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | +| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | +| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | ++-----------------+-------------------------------------------------------------------------+--------+ + +I1118 16:37:44.751905 59 server.cc:592] ++----------+---------+--------+ +| Model | Version | Status | ++----------+---------+--------+ +| identity | 1 | READY | ++----------+---------+--------+ + +I1118 16:37:44.752133 59 tritonserver.cc:1920] ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Option | Value | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| server_id | triton | +| server_version | 2.15.0 | +| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | +| model_repository_path[0] | /qa/L0_e2e/cpu_model_repository | +| model_control_mode | MODE_NONE | +| strict_model_config | 1 | +| rate_limit | OFF | +| pinned_memory_pool_byte_size | 268435456 | +| response_cache_byte_size | 0 | +| min_supported_compute_capability | 6.0 | +| strict_readiness | 1 | +| exit_timeout | 30 | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +I1118 16:37:44.754118 59 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 +I1118 16:37:44.754645 59 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 +I1118 16:37:44.796860 59 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 +Signal (15) received. +I1118 16:37:45.303881 59 server.cc:252] Waiting for in-flight requests to complete. +I1118 16:37:45.303923 59 model_repository_manager.cc:1055] unloading: identity:1 +I1118 16:37:45.304111 59 server.cc:267] Timeout 30: Found 1 live models and 0 in-flight non-inference requests +I1118 16:37:45.304341 59 instance_finalize.hpp:36] TRITONBACKEND_ModelInstanceFinalize: delete instance state +I1118 16:37:45.304389 59 model_finalize.hpp:36] TRITONBACKEND_ModelFinalize: delete model state +I1118 16:37:45.304418 59 model_repository_manager.cc:1166] successfully unloaded 'identity' version 1 diff --git a/qa/logs/28aa9f1d-b4d6-40f0-8333-0d1b8d11414a-server.log b/qa/logs/28aa9f1d-b4d6-40f0-8333-0d1b8d11414a-server.log new file mode 100644 index 0000000..492a0ff --- /dev/null +++ b/qa/logs/28aa9f1d-b4d6-40f0-8333-0d1b8d11414a-server.log @@ -0,0 +1,75 @@ +I1117 22:20:14.588016 1 metrics.cc:298] Collecting metrics for GPU 0: Quadro RTX 8000 +I1117 22:20:14.588219 1 metrics.cc:298] Collecting metrics for GPU 1: Quadro RTX 8000 +I1117 22:20:14.878472 1 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch +I1117 22:20:14.878499 1 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 +I1117 22:20:14.878503 1 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 +2021-11-17 22:20:15.156769: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 +I1117 22:20:15.203663 1 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow +I1117 22:20:15.203694 1 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 +I1117 22:20:15.203699 1 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 +I1117 22:20:15.203704 1 tensorflow.cc:2210] backend configuration: +{} +I1117 22:20:15.205319 1 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime +I1117 22:20:15.205340 1 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 +I1117 22:20:15.205346 1 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 +I1117 22:20:15.222921 1 openvino.cc:1193] TRITONBACKEND_Initialize: openvino +I1117 22:20:15.222940 1 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 +I1117 22:20:15.222944 1 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 +I1117 22:20:15.372426 1 pinned_memory_manager.cc:240] Pinned memory pool is created at '0x7fc19c000000' with size 268435456 +I1117 22:20:15.373217 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 0 with size 67108864 +I1117 22:20:15.373226 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 1 with size 67108864 +I1117 22:20:15.454591 1 model_repository_manager.cc:1022] loading: identity:1 +I1117 22:20:15.558651 1 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity +I1117 22:20:15.558699 1 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 +I1117 22:20:15.558715 1 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 +I1117 22:20:15.569766 1 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) +I1117 22:20:15.574496 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (GPU device 0) +I1117 22:20:19.810052 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (GPU device 1) +I1117 22:20:23.841843 1 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 +I1117 22:20:23.842022 1 server.cc:522] ++------------------+------+ +| Repository Agent | Path | ++------------------+------+ ++------------------+------+ + +I1117 22:20:23.842223 1 server.cc:549] ++-----------------+-------------------------------------------------------------------------+--------+ +| Backend | Path | Config | ++-----------------+-------------------------------------------------------------------------+--------+ +| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | +| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | +| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | +| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | +| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | ++-----------------+-------------------------------------------------------------------------+--------+ + +I1117 22:20:23.842304 1 server.cc:592] ++----------+---------+--------+ +| Model | Version | Status | ++----------+---------+--------+ +| identity | 1 | READY | ++----------+---------+--------+ + +I1117 22:20:23.842586 1 tritonserver.cc:1920] ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Option | Value | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| server_id | triton | +| server_version | 2.15.0 | +| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | +| model_repository_path[0] | /models | +| model_control_mode | MODE_NONE | +| strict_model_config | 1 | +| rate_limit | OFF | +| pinned_memory_pool_byte_size | 268435456 | +| cuda_memory_pool_byte_size{0} | 67108864 | +| cuda_memory_pool_byte_size{1} | 67108864 | +| response_cache_byte_size | 0 | +| min_supported_compute_capability | 6.0 | +| strict_readiness | 1 | +| exit_timeout | 30 | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +I1117 22:20:23.844401 1 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 +I1117 22:20:23.844905 1 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 +I1117 22:20:23.908017 1 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 diff --git a/qa/logs/4592e3ab-4d1b-4048-85c9-0b44e32b78d0-server.log b/qa/logs/4592e3ab-4d1b-4048-85c9-0b44e32b78d0-server.log new file mode 100644 index 0000000..542931a --- /dev/null +++ b/qa/logs/4592e3ab-4d1b-4048-85c9-0b44e32b78d0-server.log @@ -0,0 +1,77 @@ +W1118 01:10:36.673941 60 metrics.cc:282] Cannot get CUDA device count, GPU metrics will not be available +I1118 01:10:37.100238 60 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch +I1118 01:10:37.100281 60 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 +I1118 01:10:37.100290 60 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 +2021-11-18 01:10:37.603689: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 +I1118 01:10:37.670577 60 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow +I1118 01:10:37.670690 60 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 +I1118 01:10:37.670700 60 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 +I1118 01:10:37.670708 60 tensorflow.cc:2210] backend configuration: +{} +I1118 01:10:37.672734 60 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime +I1118 01:10:37.672773 60 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 +I1118 01:10:37.672781 60 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 +I1118 01:10:37.699687 60 openvino.cc:1193] TRITONBACKEND_Initialize: openvino +I1118 01:10:37.699728 60 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 +I1118 01:10:37.699736 60 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 +W1118 01:10:37.699765 60 pinned_memory_manager.cc:236] Unable to allocate pinned system memory, pinned memory pool will not be available: no CUDA-capable device is detected +I1118 01:10:37.699781 60 cuda_memory_manager.cc:115] CUDA memory pool disabled +I1118 01:10:37.700810 60 model_repository_manager.cc:1022] loading: identity:1 +I1118 01:10:37.806669 60 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity +I1118 01:10:37.806712 60 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 +I1118 01:10:37.806721 60 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 +I1118 01:10:37.806759 60 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) +I1118 01:10:37.808506 60 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (CPU device 0) +I1118 01:10:37.821798 60 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 +I1118 01:10:37.825712 60 server.cc:522] ++------------------+------+ +| Repository Agent | Path | ++------------------+------+ ++------------------+------+ + +I1118 01:10:37.825803 60 server.cc:549] ++-----------------+-------------------------------------------------------------------------+--------+ +| Backend | Path | Config | ++-----------------+-------------------------------------------------------------------------+--------+ +| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | +| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | +| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | +| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | +| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | ++-----------------+-------------------------------------------------------------------------+--------+ + +I1118 01:10:37.825835 60 server.cc:592] ++----------+---------+--------+ +| Model | Version | Status | ++----------+---------+--------+ +| identity | 1 | READY | ++----------+---------+--------+ + +I1118 01:10:37.825941 60 tritonserver.cc:1920] ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Option | Value | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| server_id | triton | +| server_version | 2.15.0 | +| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | +| model_repository_path[0] | /qa/L0_e2e/cpu_model_repository | +| model_control_mode | MODE_NONE | +| strict_model_config | 1 | +| rate_limit | OFF | +| pinned_memory_pool_byte_size | 268435456 | +| response_cache_byte_size | 0 | +| min_supported_compute_capability | 6.0 | +| strict_readiness | 1 | +| exit_timeout | 30 | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +I1118 01:10:37.865670 60 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 +I1118 01:10:37.865993 60 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 +I1118 01:10:37.929867 60 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 +Signal (15) received. +I1118 01:10:38.693724 60 server.cc:252] Waiting for in-flight requests to complete. +I1118 01:10:38.693746 60 model_repository_manager.cc:1055] unloading: identity:1 +I1118 01:10:38.693843 60 server.cc:267] Timeout 30: Found 1 live models and 0 in-flight non-inference requests +I1118 01:10:38.701807 60 instance_finalize.hpp:36] TRITONBACKEND_ModelInstanceFinalize: delete instance state +I1118 01:10:38.701873 60 model_finalize.hpp:36] TRITONBACKEND_ModelFinalize: delete model state +I1118 01:10:38.701908 60 model_repository_manager.cc:1166] successfully unloaded 'identity' version 1 diff --git a/qa/logs/6c0cf7bb-edaf-493c-a2c1-32673a6a5bc1-server.log b/qa/logs/6c0cf7bb-edaf-493c-a2c1-32673a6a5bc1-server.log new file mode 100644 index 0000000..5ca2c6e --- /dev/null +++ b/qa/logs/6c0cf7bb-edaf-493c-a2c1-32673a6a5bc1-server.log @@ -0,0 +1,71 @@ +Error: Failed to initialize NVML +W1117 22:05:21.949102 1 metrics.cc:221] DCGM unable to start: DCGM initialization error +I1117 22:05:22.246064 1 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch +I1117 22:05:22.246092 1 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 +I1117 22:05:22.246097 1 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 +2021-11-17 22:05:22.518006: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 +I1117 22:05:22.565361 1 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow +I1117 22:05:22.565421 1 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 +I1117 22:05:22.565437 1 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 +I1117 22:05:22.565450 1 tensorflow.cc:2210] backend configuration: +{} +I1117 22:05:22.567123 1 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime +I1117 22:05:22.567143 1 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 +I1117 22:05:22.567149 1 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 +I1117 22:05:22.584726 1 openvino.cc:1193] TRITONBACKEND_Initialize: openvino +I1117 22:05:22.584743 1 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 +I1117 22:05:22.584748 1 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 +W1117 22:05:22.584877 1 pinned_memory_manager.cc:236] Unable to allocate pinned system memory, pinned memory pool will not be available: CUDA driver version is insufficient for CUDA runtime version +I1117 22:05:22.584899 1 cuda_memory_manager.cc:115] CUDA memory pool disabled +I1117 22:05:22.585623 1 model_repository_manager.cc:1022] loading: identity:1 +I1117 22:05:22.687643 1 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity +I1117 22:05:22.687659 1 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 +I1117 22:05:22.687666 1 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 +I1117 22:05:22.687759 1 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) +I1117 22:05:22.688967 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (CPU device 0) +I1117 22:05:22.689244 1 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 +I1117 22:05:22.689344 1 server.cc:522] ++------------------+------+ +| Repository Agent | Path | ++------------------+------+ ++------------------+------+ + +I1117 22:05:22.689500 1 server.cc:549] ++-----------------+-------------------------------------------------------------------------+--------+ +| Backend | Path | Config | ++-----------------+-------------------------------------------------------------------------+--------+ +| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | +| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | +| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | +| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | +| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | ++-----------------+-------------------------------------------------------------------------+--------+ + +I1117 22:05:22.689564 1 server.cc:592] ++----------+---------+--------+ +| Model | Version | Status | ++----------+---------+--------+ +| identity | 1 | READY | ++----------+---------+--------+ + +I1117 22:05:22.689830 1 tritonserver.cc:1920] ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Option | Value | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| server_id | triton | +| server_version | 2.15.0 | +| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | +| model_repository_path[0] | /models | +| model_control_mode | MODE_NONE | +| strict_model_config | 1 | +| rate_limit | OFF | +| pinned_memory_pool_byte_size | 268435456 | +| response_cache_byte_size | 0 | +| min_supported_compute_capability | 6.0 | +| strict_readiness | 1 | +| exit_timeout | 30 | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +I1117 22:05:22.691777 1 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 +I1117 22:05:22.692280 1 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 +I1117 22:05:22.734784 1 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 diff --git a/qa/logs/6c2c5c2e-2cc4-4848-9eca-dd17f2b93686-server.log b/qa/logs/6c2c5c2e-2cc4-4848-9eca-dd17f2b93686-server.log new file mode 100644 index 0000000..0eeb095 --- /dev/null +++ b/qa/logs/6c2c5c2e-2cc4-4848-9eca-dd17f2b93686-server.log @@ -0,0 +1,74 @@ +I1117 22:20:26.459837 1 metrics.cc:298] Collecting metrics for GPU 0: Quadro RTX 8000 +I1117 22:20:26.460015 1 metrics.cc:298] Collecting metrics for GPU 1: Quadro RTX 8000 +I1117 22:20:26.748086 1 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch +I1117 22:20:26.748111 1 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 +I1117 22:20:26.748116 1 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 +2021-11-17 22:20:27.015263: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 +I1117 22:20:27.051650 1 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow +I1117 22:20:27.051697 1 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 +I1117 22:20:27.051712 1 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 +I1117 22:20:27.051726 1 tensorflow.cc:2210] backend configuration: +{} +I1117 22:20:27.054447 1 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime +I1117 22:20:27.054481 1 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 +I1117 22:20:27.054492 1 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 +I1117 22:20:27.077779 1 openvino.cc:1193] TRITONBACKEND_Initialize: openvino +I1117 22:20:27.077797 1 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 +I1117 22:20:27.077802 1 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 +I1117 22:20:27.221995 1 pinned_memory_manager.cc:240] Pinned memory pool is created at '0x7f138c000000' with size 268435456 +I1117 22:20:27.222780 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 0 with size 67108864 +I1117 22:20:27.222790 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 1 with size 67108864 +I1117 22:20:27.301985 1 model_repository_manager.cc:1022] loading: identity:1 +I1117 22:20:27.406019 1 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity +I1117 22:20:27.406057 1 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 +I1117 22:20:27.406072 1 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 +I1117 22:20:27.417493 1 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) +I1117 22:20:27.419533 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (CPU device 0) +I1117 22:20:27.419880 1 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 +I1117 22:20:27.420013 1 server.cc:522] ++------------------+------+ +| Repository Agent | Path | ++------------------+------+ ++------------------+------+ + +I1117 22:20:27.420150 1 server.cc:549] ++-----------------+-------------------------------------------------------------------------+--------+ +| Backend | Path | Config | ++-----------------+-------------------------------------------------------------------------+--------+ +| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | +| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | +| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | +| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | +| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | ++-----------------+-------------------------------------------------------------------------+--------+ + +I1117 22:20:27.420209 1 server.cc:592] ++----------+---------+--------+ +| Model | Version | Status | ++----------+---------+--------+ +| identity | 1 | READY | ++----------+---------+--------+ + +I1117 22:20:27.420458 1 tritonserver.cc:1920] ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Option | Value | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| server_id | triton | +| server_version | 2.15.0 | +| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | +| model_repository_path[0] | /models | +| model_control_mode | MODE_NONE | +| strict_model_config | 1 | +| rate_limit | OFF | +| pinned_memory_pool_byte_size | 268435456 | +| cuda_memory_pool_byte_size{0} | 67108864 | +| cuda_memory_pool_byte_size{1} | 67108864 | +| response_cache_byte_size | 0 | +| min_supported_compute_capability | 6.0 | +| strict_readiness | 1 | +| exit_timeout | 30 | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +I1117 22:20:27.422263 1 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 +I1117 22:20:27.422765 1 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 +I1117 22:20:27.484088 1 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 diff --git a/qa/logs/73be3cf4-e0b3-48ed-9772-69853e7833b3-server.log b/qa/logs/73be3cf4-e0b3-48ed-9772-69853e7833b3-server.log new file mode 100644 index 0000000..6d54eb2 --- /dev/null +++ b/qa/logs/73be3cf4-e0b3-48ed-9772-69853e7833b3-server.log @@ -0,0 +1,5 @@ +I1117 22:19:53.638353 1 metrics.cc:298] Collecting metrics for GPU 0: Quadro RTX 8000 +I1117 22:19:53.638528 1 metrics.cc:298] Collecting metrics for GPU 1: Quadro RTX 8000 +I1117 22:19:53.928328 1 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch +I1117 22:19:53.928356 1 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 +I1117 22:19:53.928361 1 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 diff --git a/qa/logs/7f6ed73b-5709-47da-9c80-a09db7116498-server.log b/qa/logs/7f6ed73b-5709-47da-9c80-a09db7116498-server.log new file mode 100644 index 0000000..73bbe80 --- /dev/null +++ b/qa/logs/7f6ed73b-5709-47da-9c80-a09db7116498-server.log @@ -0,0 +1,75 @@ +I1117 21:10:16.302656 1 metrics.cc:298] Collecting metrics for GPU 0: Quadro RTX 8000 +I1117 21:10:16.302838 1 metrics.cc:298] Collecting metrics for GPU 1: Quadro RTX 8000 +I1117 21:10:16.587588 1 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch +I1117 21:10:16.587613 1 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 +I1117 21:10:16.587618 1 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 +2021-11-17 21:10:16.852727: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 +I1117 21:10:16.898886 1 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow +I1117 21:10:16.898931 1 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 +I1117 21:10:16.898946 1 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 +I1117 21:10:16.898959 1 tensorflow.cc:2210] backend configuration: +{} +I1117 21:10:16.900957 1 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime +I1117 21:10:16.900977 1 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 +I1117 21:10:16.900982 1 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 +I1117 21:10:16.918482 1 openvino.cc:1193] TRITONBACKEND_Initialize: openvino +I1117 21:10:16.918499 1 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 +I1117 21:10:16.918504 1 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 +I1117 21:10:17.065889 1 pinned_memory_manager.cc:240] Pinned memory pool is created at '0x7f81a8000000' with size 268435456 +I1117 21:10:17.066675 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 0 with size 67108864 +I1117 21:10:17.066684 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 1 with size 67108864 +I1117 21:10:17.147338 1 model_repository_manager.cc:1022] loading: identity:1 +I1117 21:10:17.251342 1 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity +I1117 21:10:17.251382 1 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 +I1117 21:10:17.251397 1 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 +I1117 21:10:17.262707 1 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) +I1117 21:10:17.267809 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (GPU device 0) +I1117 21:10:21.458378 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (GPU device 1) +I1117 21:10:25.471393 1 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 +I1117 21:10:25.471564 1 server.cc:522] ++------------------+------+ +| Repository Agent | Path | ++------------------+------+ ++------------------+------+ + +I1117 21:10:25.471706 1 server.cc:549] ++-----------------+-------------------------------------------------------------------------+--------+ +| Backend | Path | Config | ++-----------------+-------------------------------------------------------------------------+--------+ +| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | +| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | +| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | +| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | +| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | ++-----------------+-------------------------------------------------------------------------+--------+ + +I1117 21:10:25.471767 1 server.cc:592] ++----------+---------+--------+ +| Model | Version | Status | ++----------+---------+--------+ +| identity | 1 | READY | ++----------+---------+--------+ + +I1117 21:10:25.472022 1 tritonserver.cc:1920] ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Option | Value | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| server_id | triton | +| server_version | 2.15.0 | +| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | +| model_repository_path[0] | /models | +| model_control_mode | MODE_NONE | +| strict_model_config | 1 | +| rate_limit | OFF | +| pinned_memory_pool_byte_size | 268435456 | +| cuda_memory_pool_byte_size{0} | 67108864 | +| cuda_memory_pool_byte_size{1} | 67108864 | +| response_cache_byte_size | 0 | +| min_supported_compute_capability | 6.0 | +| strict_readiness | 1 | +| exit_timeout | 30 | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +I1117 21:10:25.473753 1 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 +I1117 21:10:25.474247 1 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 +I1117 21:10:25.548082 1 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 diff --git a/qa/logs/933cd6fc-1a84-4f57-a798-fa3d1be97409-server.log b/qa/logs/933cd6fc-1a84-4f57-a798-fa3d1be97409-server.log new file mode 100644 index 0000000..7222233 --- /dev/null +++ b/qa/logs/933cd6fc-1a84-4f57-a798-fa3d1be97409-server.log @@ -0,0 +1,77 @@ +W1118 16:52:22.254276 61 metrics.cc:282] Cannot get CUDA device count, GPU metrics will not be available +I1118 16:52:22.502441 61 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch +I1118 16:52:22.502472 61 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 +I1118 16:52:22.502479 61 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 +2021-11-18 16:52:22.775842: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 +I1118 16:52:22.841043 61 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow +I1118 16:52:22.841092 61 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 +I1118 16:52:22.841110 61 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 +I1118 16:52:22.841121 61 tensorflow.cc:2210] backend configuration: +{} +I1118 16:52:22.843563 61 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime +I1118 16:52:22.843598 61 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 +I1118 16:52:22.843609 61 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 +I1118 16:52:22.879216 61 openvino.cc:1193] TRITONBACKEND_Initialize: openvino +I1118 16:52:22.879235 61 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 +I1118 16:52:22.879242 61 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 +W1118 16:52:22.879259 61 pinned_memory_manager.cc:236] Unable to allocate pinned system memory, pinned memory pool will not be available: no CUDA-capable device is detected +I1118 16:52:22.879271 61 cuda_memory_manager.cc:115] CUDA memory pool disabled +I1118 16:52:22.879994 61 model_repository_manager.cc:1022] loading: identity:1 +I1118 16:52:22.981315 61 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity +I1118 16:52:22.981350 61 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 +I1118 16:52:22.981362 61 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 +I1118 16:52:22.981401 61 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) +I1118 16:52:22.983424 61 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (CPU device 0) +I1118 16:52:22.983753 61 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 +I1118 16:52:22.983895 61 server.cc:522] ++------------------+------+ +| Repository Agent | Path | ++------------------+------+ ++------------------+------+ + +I1118 16:52:22.984037 61 server.cc:549] ++-----------------+-------------------------------------------------------------------------+--------+ +| Backend | Path | Config | ++-----------------+-------------------------------------------------------------------------+--------+ +| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | +| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | +| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | +| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | +| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | ++-----------------+-------------------------------------------------------------------------+--------+ + +I1118 16:52:22.984105 61 server.cc:592] ++----------+---------+--------+ +| Model | Version | Status | ++----------+---------+--------+ +| identity | 1 | READY | ++----------+---------+--------+ + +I1118 16:52:22.984323 61 tritonserver.cc:1920] ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Option | Value | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| server_id | triton | +| server_version | 2.15.0 | +| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | +| model_repository_path[0] | /qa/L0_e2e/cpu_model_repository | +| model_control_mode | MODE_NONE | +| strict_model_config | 1 | +| rate_limit | OFF | +| pinned_memory_pool_byte_size | 268435456 | +| response_cache_byte_size | 0 | +| min_supported_compute_capability | 6.0 | +| strict_readiness | 1 | +| exit_timeout | 30 | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +I1118 16:52:22.986177 61 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 +I1118 16:52:22.986702 61 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 +I1118 16:52:23.028884 61 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 +Signal (15) received. +I1118 16:52:23.576444 61 server.cc:252] Waiting for in-flight requests to complete. +I1118 16:52:23.576465 61 model_repository_manager.cc:1055] unloading: identity:1 +I1118 16:52:23.576550 61 server.cc:267] Timeout 30: Found 1 live models and 0 in-flight non-inference requests +I1118 16:52:23.576801 61 instance_finalize.hpp:36] TRITONBACKEND_ModelInstanceFinalize: delete instance state +I1118 16:52:23.576862 61 model_finalize.hpp:36] TRITONBACKEND_ModelFinalize: delete model state +I1118 16:52:23.576900 61 model_repository_manager.cc:1166] successfully unloaded 'identity' version 1 diff --git a/qa/logs/974b0053-82c2-41c8-8bde-209499a5ec47-server.log b/qa/logs/974b0053-82c2-41c8-8bde-209499a5ec47-server.log new file mode 100644 index 0000000..d7425fb --- /dev/null +++ b/qa/logs/974b0053-82c2-41c8-8bde-209499a5ec47-server.log @@ -0,0 +1,77 @@ +W1118 16:42:09.768091 60 metrics.cc:282] Cannot get CUDA device count, GPU metrics will not be available +I1118 16:42:10.014849 60 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch +I1118 16:42:10.014882 60 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 +I1118 16:42:10.014889 60 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 +2021-11-18 16:42:10.291590: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 +I1118 16:42:10.362424 60 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow +I1118 16:42:10.362466 60 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 +I1118 16:42:10.362476 60 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 +I1118 16:42:10.362486 60 tensorflow.cc:2210] backend configuration: +{} +I1118 16:42:10.364743 60 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime +I1118 16:42:10.364771 60 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 +I1118 16:42:10.364781 60 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 +I1118 16:42:10.396832 60 openvino.cc:1193] TRITONBACKEND_Initialize: openvino +I1118 16:42:10.396850 60 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 +I1118 16:42:10.396856 60 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 +W1118 16:42:10.396872 60 pinned_memory_manager.cc:236] Unable to allocate pinned system memory, pinned memory pool will not be available: no CUDA-capable device is detected +I1118 16:42:10.396885 60 cuda_memory_manager.cc:115] CUDA memory pool disabled +I1118 16:42:10.397603 60 model_repository_manager.cc:1022] loading: identity:1 +I1118 16:42:10.499429 60 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity +I1118 16:42:10.499483 60 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 +I1118 16:42:10.499503 60 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 +I1118 16:42:10.499565 60 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) +I1118 16:42:10.502831 60 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (CPU device 0) +I1118 16:42:10.503271 60 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 +I1118 16:42:10.503404 60 server.cc:522] ++------------------+------+ +| Repository Agent | Path | ++------------------+------+ ++------------------+------+ + +I1118 16:42:10.503516 60 server.cc:549] ++-----------------+-------------------------------------------------------------------------+--------+ +| Backend | Path | Config | ++-----------------+-------------------------------------------------------------------------+--------+ +| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | +| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | +| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | +| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | +| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | ++-----------------+-------------------------------------------------------------------------+--------+ + +I1118 16:42:10.503570 60 server.cc:592] ++----------+---------+--------+ +| Model | Version | Status | ++----------+---------+--------+ +| identity | 1 | READY | ++----------+---------+--------+ + +I1118 16:42:10.503732 60 tritonserver.cc:1920] ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Option | Value | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| server_id | triton | +| server_version | 2.15.0 | +| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | +| model_repository_path[0] | /qa/L0_e2e/cpu_model_repository | +| model_control_mode | MODE_NONE | +| strict_model_config | 1 | +| rate_limit | OFF | +| pinned_memory_pool_byte_size | 268435456 | +| response_cache_byte_size | 0 | +| min_supported_compute_capability | 6.0 | +| strict_readiness | 1 | +| exit_timeout | 30 | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +I1118 16:42:10.505348 60 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 +I1118 16:42:10.505873 60 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 +I1118 16:42:10.548117 60 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 +Signal (15) received. +I1118 16:42:11.073757 60 server.cc:252] Waiting for in-flight requests to complete. +I1118 16:42:11.073800 60 model_repository_manager.cc:1055] unloading: identity:1 +I1118 16:42:11.073979 60 server.cc:267] Timeout 30: Found 1 live models and 0 in-flight non-inference requests +I1118 16:42:11.074100 60 instance_finalize.hpp:36] TRITONBACKEND_ModelInstanceFinalize: delete instance state +I1118 16:42:11.074149 60 model_finalize.hpp:36] TRITONBACKEND_ModelFinalize: delete model state +I1118 16:42:11.074176 60 model_repository_manager.cc:1166] successfully unloaded 'identity' version 1 diff --git a/qa/logs/da517b94-6959-43a0-a972-7bd52289b21e-server.log b/qa/logs/da517b94-6959-43a0-a972-7bd52289b21e-server.log new file mode 100644 index 0000000..89888c2 --- /dev/null +++ b/qa/logs/da517b94-6959-43a0-a972-7bd52289b21e-server.log @@ -0,0 +1,71 @@ +Error: Failed to initialize NVML +W1117 22:20:33.435468 1 metrics.cc:221] DCGM unable to start: DCGM initialization error +I1117 22:20:33.736191 1 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch +I1117 22:20:33.736219 1 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 +I1117 22:20:33.736224 1 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 +2021-11-17 22:20:34.008207: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 +I1117 22:20:34.055124 1 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow +I1117 22:20:34.055178 1 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 +I1117 22:20:34.055193 1 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 +I1117 22:20:34.055206 1 tensorflow.cc:2210] backend configuration: +{} +I1117 22:20:34.056747 1 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime +I1117 22:20:34.056768 1 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 +I1117 22:20:34.056774 1 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 +I1117 22:20:34.074190 1 openvino.cc:1193] TRITONBACKEND_Initialize: openvino +I1117 22:20:34.074207 1 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 +I1117 22:20:34.074212 1 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 +W1117 22:20:34.074308 1 pinned_memory_manager.cc:236] Unable to allocate pinned system memory, pinned memory pool will not be available: CUDA driver version is insufficient for CUDA runtime version +I1117 22:20:34.074332 1 cuda_memory_manager.cc:115] CUDA memory pool disabled +I1117 22:20:34.075044 1 model_repository_manager.cc:1022] loading: identity:1 +I1117 22:20:34.178827 1 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity +I1117 22:20:34.178865 1 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 +I1117 22:20:34.178879 1 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 +I1117 22:20:34.179076 1 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) +I1117 22:20:34.182329 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (CPU device 0) +I1117 22:20:34.182694 1 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 +I1117 22:20:34.182808 1 server.cc:522] ++------------------+------+ +| Repository Agent | Path | ++------------------+------+ ++------------------+------+ + +I1117 22:20:34.182950 1 server.cc:549] ++-----------------+-------------------------------------------------------------------------+--------+ +| Backend | Path | Config | ++-----------------+-------------------------------------------------------------------------+--------+ +| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | +| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | +| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | +| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | +| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | ++-----------------+-------------------------------------------------------------------------+--------+ + +I1117 22:20:34.183010 1 server.cc:592] ++----------+---------+--------+ +| Model | Version | Status | ++----------+---------+--------+ +| identity | 1 | READY | ++----------+---------+--------+ + +I1117 22:20:34.183238 1 tritonserver.cc:1920] ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Option | Value | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| server_id | triton | +| server_version | 2.15.0 | +| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | +| model_repository_path[0] | /models | +| model_control_mode | MODE_NONE | +| strict_model_config | 1 | +| rate_limit | OFF | +| pinned_memory_pool_byte_size | 268435456 | +| response_cache_byte_size | 0 | +| min_supported_compute_capability | 6.0 | +| strict_readiness | 1 | +| exit_timeout | 30 | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +I1117 22:20:34.185149 1 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 +I1117 22:20:34.185658 1 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 +I1117 22:20:34.228187 1 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 diff --git a/qa/logs/f09aec20-ba41-4c04-ac7b-ec324fcb0754-server.log b/qa/logs/f09aec20-ba41-4c04-ac7b-ec324fcb0754-server.log new file mode 100644 index 0000000..dfc2198 --- /dev/null +++ b/qa/logs/f09aec20-ba41-4c04-ac7b-ec324fcb0754-server.log @@ -0,0 +1,74 @@ +I1117 22:05:15.108220 1 metrics.cc:298] Collecting metrics for GPU 0: Quadro RTX 8000 +I1117 22:05:15.108397 1 metrics.cc:298] Collecting metrics for GPU 1: Quadro RTX 8000 +I1117 22:05:15.394902 1 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch +I1117 22:05:15.394929 1 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 +I1117 22:05:15.394933 1 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 +2021-11-17 22:05:15.669411: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 +I1117 22:05:15.705319 1 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow +I1117 22:05:15.705374 1 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 +I1117 22:05:15.705389 1 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 +I1117 22:05:15.705402 1 tensorflow.cc:2210] backend configuration: +{} +I1117 22:05:15.708421 1 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime +I1117 22:05:15.708460 1 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 +I1117 22:05:15.708470 1 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 +I1117 22:05:15.733935 1 openvino.cc:1193] TRITONBACKEND_Initialize: openvino +I1117 22:05:15.733952 1 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 +I1117 22:05:15.733957 1 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 +I1117 22:05:15.879810 1 pinned_memory_manager.cc:240] Pinned memory pool is created at '0x7f7c74000000' with size 268435456 +I1117 22:05:15.880601 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 0 with size 67108864 +I1117 22:05:15.880611 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 1 with size 67108864 +I1117 22:05:15.964300 1 model_repository_manager.cc:1022] loading: identity:1 +I1117 22:05:16.068298 1 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity +I1117 22:05:16.068350 1 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 +I1117 22:05:16.068379 1 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 +I1117 22:05:16.079181 1 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) +I1117 22:05:16.081280 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (CPU device 0) +I1117 22:05:16.081659 1 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 +I1117 22:05:16.081804 1 server.cc:522] ++------------------+------+ +| Repository Agent | Path | ++------------------+------+ ++------------------+------+ + +I1117 22:05:16.081942 1 server.cc:549] ++-----------------+-------------------------------------------------------------------------+--------+ +| Backend | Path | Config | ++-----------------+-------------------------------------------------------------------------+--------+ +| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | +| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | +| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | +| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | +| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | ++-----------------+-------------------------------------------------------------------------+--------+ + +I1117 22:05:16.082004 1 server.cc:592] ++----------+---------+--------+ +| Model | Version | Status | ++----------+---------+--------+ +| identity | 1 | READY | ++----------+---------+--------+ + +I1117 22:05:16.082246 1 tritonserver.cc:1920] ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Option | Value | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| server_id | triton | +| server_version | 2.15.0 | +| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | +| model_repository_path[0] | /models | +| model_control_mode | MODE_NONE | +| strict_model_config | 1 | +| rate_limit | OFF | +| pinned_memory_pool_byte_size | 268435456 | +| cuda_memory_pool_byte_size{0} | 67108864 | +| cuda_memory_pool_byte_size{1} | 67108864 | +| response_cache_byte_size | 0 | +| min_supported_compute_capability | 6.0 | +| strict_readiness | 1 | +| exit_timeout | 30 | ++----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +I1117 22:05:16.084027 1 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 +I1117 22:05:16.084514 1 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 +I1117 22:05:16.140121 1 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 diff --git a/qa/run_tests.sh b/qa/run_tests.sh new file mode 100755 index 0000000..592b413 --- /dev/null +++ b/qa/run_tests.sh @@ -0,0 +1,105 @@ +#!/bin/bash +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +QA_DIR=$(cd $(dirname $0); pwd) +SERVER_ARGS="" +UUID="$(cat /proc/sys/kernel/random/uuid)" +CONTAINER_NAME="rapids_triton-ci-$UUID" +DOCKER_RUN=0 +DOCKER_ARGS="-d -p 8000:8000 -p 8001:8001 -p 8002:8002 --name ${CONTAINER_NAME}" +TRITON_PID="" +LOG_DIR="${QA_DIR}/logs" +SERVER_LOG="${LOG_DIR}/${UUID}-server.log" + +if [ ! -d "${LOG_DIR}" ] +then + mkdir -p "${LOG_DIR}" +fi + +if [ -z $MODEL_REPO ] +then + MODEL_REPO="${QA_DIR}/L0_e2e/model_repository" +fi +MODEL_REPO="$(readlink -f $MODEL_REPO)" + +DOCKER_ARGS="${DOCKER_ARGS} -v ${MODEL_REPO}:/models" + +if [ -z $CPU_ONLY ] || [ $CPU_ONLY -eq 0 ] +then + if [[ -v CUDA_VISIBLE_DEVICES ]] + then + if [ -z $CUDA_VISIBLE_DEVICES ] + then + CPU_ONLY=1 + else + DOCKER_ARGS="${DOCKER_ARGS} --gpus ${CUDA_VISIBLE_DEVICES}" + fi + else + DOCKER_ARGS="${DOCKER_ARGS} --gpus all" + fi +else + export CUDA_VISIBLE_DEVICES="" +fi + +# If a Triton Docker image has been provided or no tritonserver executable is +# available, run the server via Docker +if [ ! -z $TRITON_IMAGE ] || ! command -v tritonserver +then + DOCKER_RUN=1 + TRITON_IMAGE=${TRITON_IMAGE:-rapids_triton_identity} + SERVER_ARGS="${SERVER_ARGS} --model-repository=/models" +else + SERVER_ARGS="${SERVER_ARGS} --model-repository=${MODEL_REPO}" +fi + +start_server() { + if [ $DOCKER_RUN -eq 1 ] + then + docker run $DOCKER_ARGS $TRITON_IMAGE > /dev/null + else + tritonserver $SERVER_ARGS > $SERVER_LOG 2>&1 & + TRITON_PID="$!" + fi +} + +start_server + +if [ -z $TEST_EXE ] +then + echo 'No TEST_EXE variable defined; skipping unit tests' +else + if [ $DOCKER_RUN -eq 1 ] + then + docker exec $CONTAINER_NAME "$TEST_EXE" + else + "$TEST_EXE" + fi +fi + +finally() { + if [ -z $TRITON_PID ] + then + docker logs $CONTAINER_NAME > $SERVER_LOG 2>&1 + docker rm -f $CONTAINER_NAME > /dev/null 2>&1 + else + kill -15 $TRITON_PID + fi +} + +trap finally EXIT + +pytest "$QA_DIR" From 30bb810a624f7d63721ff2ac5a4e459a96590c7b Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 18 Nov 2021 12:13:59 -0500 Subject: [PATCH 134/199] Remove logs directory --- .gitignore | 1 + ...ab1-73ba-4c54-80ee-e2812d146191-server.log | 75 ------------------ ...952-d091-4805-80dc-729dddba0368-server.log | 77 ------------------- ...f1d-b4d6-40f0-8333-0d1b8d11414a-server.log | 75 ------------------ ...3ab-4d1b-4048-85c9-0b44e32b78d0-server.log | 77 ------------------- ...7bb-edaf-493c-a2c1-32673a6a5bc1-server.log | 71 ----------------- ...c2e-2cc4-4848-9eca-dd17f2b93686-server.log | 74 ------------------ ...cf4-e0b3-48ed-9772-69853e7833b3-server.log | 5 -- ...73b-5709-47da-9c80-a09db7116498-server.log | 75 ------------------ ...6fc-1a84-4f57-a798-fa3d1be97409-server.log | 77 ------------------- ...053-82c2-41c8-8bde-209499a5ec47-server.log | 77 ------------------- ...b94-6959-43a0-a972-7bd52289b21e-server.log | 71 ----------------- ...c20-ba41-4c04-ac7b-ec324fcb0754-server.log | 74 ------------------ 13 files changed, 1 insertion(+), 828 deletions(-) delete mode 100644 qa/logs/1d745ab1-73ba-4c54-80ee-e2812d146191-server.log delete mode 100644 qa/logs/22810952-d091-4805-80dc-729dddba0368-server.log delete mode 100644 qa/logs/28aa9f1d-b4d6-40f0-8333-0d1b8d11414a-server.log delete mode 100644 qa/logs/4592e3ab-4d1b-4048-85c9-0b44e32b78d0-server.log delete mode 100644 qa/logs/6c0cf7bb-edaf-493c-a2c1-32673a6a5bc1-server.log delete mode 100644 qa/logs/6c2c5c2e-2cc4-4848-9eca-dd17f2b93686-server.log delete mode 100644 qa/logs/73be3cf4-e0b3-48ed-9772-69853e7833b3-server.log delete mode 100644 qa/logs/7f6ed73b-5709-47da-9c80-a09db7116498-server.log delete mode 100644 qa/logs/933cd6fc-1a84-4f57-a798-fa3d1be97409-server.log delete mode 100644 qa/logs/974b0053-82c2-41c8-8bde-209499a5ec47-server.log delete mode 100644 qa/logs/da517b94-6959-43a0-a972-7bd52289b21e-server.log delete mode 100644 qa/logs/f09aec20-ba41-4c04-ac7b-ec324fcb0754-server.log diff --git a/.gitignore b/.gitignore index a2d3658..59c558f 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ cpp/build +qa/logs diff --git a/qa/logs/1d745ab1-73ba-4c54-80ee-e2812d146191-server.log b/qa/logs/1d745ab1-73ba-4c54-80ee-e2812d146191-server.log deleted file mode 100644 index 074fb59..0000000 --- a/qa/logs/1d745ab1-73ba-4c54-80ee-e2812d146191-server.log +++ /dev/null @@ -1,75 +0,0 @@ -I1117 22:05:03.848968 1 metrics.cc:298] Collecting metrics for GPU 0: Quadro RTX 8000 -I1117 22:05:03.849179 1 metrics.cc:298] Collecting metrics for GPU 1: Quadro RTX 8000 -I1117 22:05:04.140935 1 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch -I1117 22:05:04.140964 1 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 -I1117 22:05:04.140969 1 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 -2021-11-17 22:05:04.417430: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 -I1117 22:05:04.453291 1 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow -I1117 22:05:04.453355 1 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 -I1117 22:05:04.453370 1 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 -I1117 22:05:04.453383 1 tensorflow.cc:2210] backend configuration: -{} -I1117 22:05:04.455331 1 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime -I1117 22:05:04.455351 1 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 -I1117 22:05:04.455356 1 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 -I1117 22:05:04.472805 1 openvino.cc:1193] TRITONBACKEND_Initialize: openvino -I1117 22:05:04.472822 1 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 -I1117 22:05:04.472826 1 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 -I1117 22:05:04.627788 1 pinned_memory_manager.cc:240] Pinned memory pool is created at '0x7f0680000000' with size 268435456 -I1117 22:05:04.628579 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 0 with size 67108864 -I1117 22:05:04.628590 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 1 with size 67108864 -I1117 22:05:04.709460 1 model_repository_manager.cc:1022] loading: identity:1 -I1117 22:05:04.813427 1 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity -I1117 22:05:04.813468 1 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 -I1117 22:05:04.813482 1 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 -I1117 22:05:04.824910 1 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) -I1117 22:05:04.829878 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (GPU device 0) -I1117 22:05:08.995377 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (GPU device 1) -I1117 22:05:12.926306 1 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 -I1117 22:05:12.926485 1 server.cc:522] -+------------------+------+ -| Repository Agent | Path | -+------------------+------+ -+------------------+------+ - -I1117 22:05:12.926630 1 server.cc:549] -+-----------------+-------------------------------------------------------------------------+--------+ -| Backend | Path | Config | -+-----------------+-------------------------------------------------------------------------+--------+ -| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | -| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | -| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | -| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | -| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | -+-----------------+-------------------------------------------------------------------------+--------+ - -I1117 22:05:12.926691 1 server.cc:592] -+----------+---------+--------+ -| Model | Version | Status | -+----------+---------+--------+ -| identity | 1 | READY | -+----------+---------+--------+ - -I1117 22:05:12.926944 1 tritonserver.cc:1920] -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Option | Value | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| server_id | triton | -| server_version | 2.15.0 | -| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | -| model_repository_path[0] | /models | -| model_control_mode | MODE_NONE | -| strict_model_config | 1 | -| rate_limit | OFF | -| pinned_memory_pool_byte_size | 268435456 | -| cuda_memory_pool_byte_size{0} | 67108864 | -| cuda_memory_pool_byte_size{1} | 67108864 | -| response_cache_byte_size | 0 | -| min_supported_compute_capability | 6.0 | -| strict_readiness | 1 | -| exit_timeout | 30 | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - -I1117 22:05:12.928671 1 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 -I1117 22:05:12.929159 1 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 -I1117 22:05:12.988127 1 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 diff --git a/qa/logs/22810952-d091-4805-80dc-729dddba0368-server.log b/qa/logs/22810952-d091-4805-80dc-729dddba0368-server.log deleted file mode 100644 index 8ca1ba8..0000000 --- a/qa/logs/22810952-d091-4805-80dc-729dddba0368-server.log +++ /dev/null @@ -1,77 +0,0 @@ -W1118 16:37:43.979175 59 metrics.cc:282] Cannot get CUDA device count, GPU metrics will not be available -I1118 16:37:44.225688 59 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch -I1118 16:37:44.225720 59 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 -I1118 16:37:44.225728 59 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 -2021-11-18 16:37:44.506800: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 -I1118 16:37:44.621643 59 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow -I1118 16:37:44.621689 59 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 -I1118 16:37:44.621696 59 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 -I1118 16:37:44.621702 59 tensorflow.cc:2210] backend configuration: -{} -I1118 16:37:44.623264 59 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime -I1118 16:37:44.623283 59 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 -I1118 16:37:44.623289 59 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 -I1118 16:37:44.645064 59 openvino.cc:1193] TRITONBACKEND_Initialize: openvino -I1118 16:37:44.645082 59 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 -I1118 16:37:44.645088 59 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 -W1118 16:37:44.645107 59 pinned_memory_manager.cc:236] Unable to allocate pinned system memory, pinned memory pool will not be available: no CUDA-capable device is detected -I1118 16:37:44.645120 59 cuda_memory_manager.cc:115] CUDA memory pool disabled -I1118 16:37:44.645843 59 model_repository_manager.cc:1022] loading: identity:1 -I1118 16:37:44.747655 59 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity -I1118 16:37:44.747707 59 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 -I1118 16:37:44.747727 59 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 -I1118 16:37:44.747788 59 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) -I1118 16:37:44.751099 59 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (CPU device 0) -I1118 16:37:44.751549 59 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 -I1118 16:37:44.751697 59 server.cc:522] -+------------------+------+ -| Repository Agent | Path | -+------------------+------+ -+------------------+------+ - -I1118 16:37:44.751845 59 server.cc:549] -+-----------------+-------------------------------------------------------------------------+--------+ -| Backend | Path | Config | -+-----------------+-------------------------------------------------------------------------+--------+ -| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | -| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | -| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | -| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | -| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | -+-----------------+-------------------------------------------------------------------------+--------+ - -I1118 16:37:44.751905 59 server.cc:592] -+----------+---------+--------+ -| Model | Version | Status | -+----------+---------+--------+ -| identity | 1 | READY | -+----------+---------+--------+ - -I1118 16:37:44.752133 59 tritonserver.cc:1920] -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Option | Value | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| server_id | triton | -| server_version | 2.15.0 | -| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | -| model_repository_path[0] | /qa/L0_e2e/cpu_model_repository | -| model_control_mode | MODE_NONE | -| strict_model_config | 1 | -| rate_limit | OFF | -| pinned_memory_pool_byte_size | 268435456 | -| response_cache_byte_size | 0 | -| min_supported_compute_capability | 6.0 | -| strict_readiness | 1 | -| exit_timeout | 30 | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - -I1118 16:37:44.754118 59 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 -I1118 16:37:44.754645 59 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 -I1118 16:37:44.796860 59 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 -Signal (15) received. -I1118 16:37:45.303881 59 server.cc:252] Waiting for in-flight requests to complete. -I1118 16:37:45.303923 59 model_repository_manager.cc:1055] unloading: identity:1 -I1118 16:37:45.304111 59 server.cc:267] Timeout 30: Found 1 live models and 0 in-flight non-inference requests -I1118 16:37:45.304341 59 instance_finalize.hpp:36] TRITONBACKEND_ModelInstanceFinalize: delete instance state -I1118 16:37:45.304389 59 model_finalize.hpp:36] TRITONBACKEND_ModelFinalize: delete model state -I1118 16:37:45.304418 59 model_repository_manager.cc:1166] successfully unloaded 'identity' version 1 diff --git a/qa/logs/28aa9f1d-b4d6-40f0-8333-0d1b8d11414a-server.log b/qa/logs/28aa9f1d-b4d6-40f0-8333-0d1b8d11414a-server.log deleted file mode 100644 index 492a0ff..0000000 --- a/qa/logs/28aa9f1d-b4d6-40f0-8333-0d1b8d11414a-server.log +++ /dev/null @@ -1,75 +0,0 @@ -I1117 22:20:14.588016 1 metrics.cc:298] Collecting metrics for GPU 0: Quadro RTX 8000 -I1117 22:20:14.588219 1 metrics.cc:298] Collecting metrics for GPU 1: Quadro RTX 8000 -I1117 22:20:14.878472 1 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch -I1117 22:20:14.878499 1 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 -I1117 22:20:14.878503 1 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 -2021-11-17 22:20:15.156769: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 -I1117 22:20:15.203663 1 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow -I1117 22:20:15.203694 1 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 -I1117 22:20:15.203699 1 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 -I1117 22:20:15.203704 1 tensorflow.cc:2210] backend configuration: -{} -I1117 22:20:15.205319 1 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime -I1117 22:20:15.205340 1 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 -I1117 22:20:15.205346 1 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 -I1117 22:20:15.222921 1 openvino.cc:1193] TRITONBACKEND_Initialize: openvino -I1117 22:20:15.222940 1 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 -I1117 22:20:15.222944 1 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 -I1117 22:20:15.372426 1 pinned_memory_manager.cc:240] Pinned memory pool is created at '0x7fc19c000000' with size 268435456 -I1117 22:20:15.373217 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 0 with size 67108864 -I1117 22:20:15.373226 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 1 with size 67108864 -I1117 22:20:15.454591 1 model_repository_manager.cc:1022] loading: identity:1 -I1117 22:20:15.558651 1 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity -I1117 22:20:15.558699 1 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 -I1117 22:20:15.558715 1 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 -I1117 22:20:15.569766 1 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) -I1117 22:20:15.574496 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (GPU device 0) -I1117 22:20:19.810052 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (GPU device 1) -I1117 22:20:23.841843 1 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 -I1117 22:20:23.842022 1 server.cc:522] -+------------------+------+ -| Repository Agent | Path | -+------------------+------+ -+------------------+------+ - -I1117 22:20:23.842223 1 server.cc:549] -+-----------------+-------------------------------------------------------------------------+--------+ -| Backend | Path | Config | -+-----------------+-------------------------------------------------------------------------+--------+ -| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | -| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | -| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | -| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | -| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | -+-----------------+-------------------------------------------------------------------------+--------+ - -I1117 22:20:23.842304 1 server.cc:592] -+----------+---------+--------+ -| Model | Version | Status | -+----------+---------+--------+ -| identity | 1 | READY | -+----------+---------+--------+ - -I1117 22:20:23.842586 1 tritonserver.cc:1920] -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Option | Value | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| server_id | triton | -| server_version | 2.15.0 | -| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | -| model_repository_path[0] | /models | -| model_control_mode | MODE_NONE | -| strict_model_config | 1 | -| rate_limit | OFF | -| pinned_memory_pool_byte_size | 268435456 | -| cuda_memory_pool_byte_size{0} | 67108864 | -| cuda_memory_pool_byte_size{1} | 67108864 | -| response_cache_byte_size | 0 | -| min_supported_compute_capability | 6.0 | -| strict_readiness | 1 | -| exit_timeout | 30 | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - -I1117 22:20:23.844401 1 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 -I1117 22:20:23.844905 1 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 -I1117 22:20:23.908017 1 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 diff --git a/qa/logs/4592e3ab-4d1b-4048-85c9-0b44e32b78d0-server.log b/qa/logs/4592e3ab-4d1b-4048-85c9-0b44e32b78d0-server.log deleted file mode 100644 index 542931a..0000000 --- a/qa/logs/4592e3ab-4d1b-4048-85c9-0b44e32b78d0-server.log +++ /dev/null @@ -1,77 +0,0 @@ -W1118 01:10:36.673941 60 metrics.cc:282] Cannot get CUDA device count, GPU metrics will not be available -I1118 01:10:37.100238 60 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch -I1118 01:10:37.100281 60 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 -I1118 01:10:37.100290 60 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 -2021-11-18 01:10:37.603689: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 -I1118 01:10:37.670577 60 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow -I1118 01:10:37.670690 60 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 -I1118 01:10:37.670700 60 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 -I1118 01:10:37.670708 60 tensorflow.cc:2210] backend configuration: -{} -I1118 01:10:37.672734 60 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime -I1118 01:10:37.672773 60 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 -I1118 01:10:37.672781 60 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 -I1118 01:10:37.699687 60 openvino.cc:1193] TRITONBACKEND_Initialize: openvino -I1118 01:10:37.699728 60 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 -I1118 01:10:37.699736 60 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 -W1118 01:10:37.699765 60 pinned_memory_manager.cc:236] Unable to allocate pinned system memory, pinned memory pool will not be available: no CUDA-capable device is detected -I1118 01:10:37.699781 60 cuda_memory_manager.cc:115] CUDA memory pool disabled -I1118 01:10:37.700810 60 model_repository_manager.cc:1022] loading: identity:1 -I1118 01:10:37.806669 60 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity -I1118 01:10:37.806712 60 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 -I1118 01:10:37.806721 60 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 -I1118 01:10:37.806759 60 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) -I1118 01:10:37.808506 60 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (CPU device 0) -I1118 01:10:37.821798 60 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 -I1118 01:10:37.825712 60 server.cc:522] -+------------------+------+ -| Repository Agent | Path | -+------------------+------+ -+------------------+------+ - -I1118 01:10:37.825803 60 server.cc:549] -+-----------------+-------------------------------------------------------------------------+--------+ -| Backend | Path | Config | -+-----------------+-------------------------------------------------------------------------+--------+ -| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | -| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | -| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | -| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | -| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | -+-----------------+-------------------------------------------------------------------------+--------+ - -I1118 01:10:37.825835 60 server.cc:592] -+----------+---------+--------+ -| Model | Version | Status | -+----------+---------+--------+ -| identity | 1 | READY | -+----------+---------+--------+ - -I1118 01:10:37.825941 60 tritonserver.cc:1920] -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Option | Value | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| server_id | triton | -| server_version | 2.15.0 | -| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | -| model_repository_path[0] | /qa/L0_e2e/cpu_model_repository | -| model_control_mode | MODE_NONE | -| strict_model_config | 1 | -| rate_limit | OFF | -| pinned_memory_pool_byte_size | 268435456 | -| response_cache_byte_size | 0 | -| min_supported_compute_capability | 6.0 | -| strict_readiness | 1 | -| exit_timeout | 30 | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - -I1118 01:10:37.865670 60 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 -I1118 01:10:37.865993 60 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 -I1118 01:10:37.929867 60 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 -Signal (15) received. -I1118 01:10:38.693724 60 server.cc:252] Waiting for in-flight requests to complete. -I1118 01:10:38.693746 60 model_repository_manager.cc:1055] unloading: identity:1 -I1118 01:10:38.693843 60 server.cc:267] Timeout 30: Found 1 live models and 0 in-flight non-inference requests -I1118 01:10:38.701807 60 instance_finalize.hpp:36] TRITONBACKEND_ModelInstanceFinalize: delete instance state -I1118 01:10:38.701873 60 model_finalize.hpp:36] TRITONBACKEND_ModelFinalize: delete model state -I1118 01:10:38.701908 60 model_repository_manager.cc:1166] successfully unloaded 'identity' version 1 diff --git a/qa/logs/6c0cf7bb-edaf-493c-a2c1-32673a6a5bc1-server.log b/qa/logs/6c0cf7bb-edaf-493c-a2c1-32673a6a5bc1-server.log deleted file mode 100644 index 5ca2c6e..0000000 --- a/qa/logs/6c0cf7bb-edaf-493c-a2c1-32673a6a5bc1-server.log +++ /dev/null @@ -1,71 +0,0 @@ -Error: Failed to initialize NVML -W1117 22:05:21.949102 1 metrics.cc:221] DCGM unable to start: DCGM initialization error -I1117 22:05:22.246064 1 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch -I1117 22:05:22.246092 1 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 -I1117 22:05:22.246097 1 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 -2021-11-17 22:05:22.518006: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 -I1117 22:05:22.565361 1 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow -I1117 22:05:22.565421 1 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 -I1117 22:05:22.565437 1 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 -I1117 22:05:22.565450 1 tensorflow.cc:2210] backend configuration: -{} -I1117 22:05:22.567123 1 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime -I1117 22:05:22.567143 1 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 -I1117 22:05:22.567149 1 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 -I1117 22:05:22.584726 1 openvino.cc:1193] TRITONBACKEND_Initialize: openvino -I1117 22:05:22.584743 1 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 -I1117 22:05:22.584748 1 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 -W1117 22:05:22.584877 1 pinned_memory_manager.cc:236] Unable to allocate pinned system memory, pinned memory pool will not be available: CUDA driver version is insufficient for CUDA runtime version -I1117 22:05:22.584899 1 cuda_memory_manager.cc:115] CUDA memory pool disabled -I1117 22:05:22.585623 1 model_repository_manager.cc:1022] loading: identity:1 -I1117 22:05:22.687643 1 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity -I1117 22:05:22.687659 1 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 -I1117 22:05:22.687666 1 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 -I1117 22:05:22.687759 1 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) -I1117 22:05:22.688967 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (CPU device 0) -I1117 22:05:22.689244 1 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 -I1117 22:05:22.689344 1 server.cc:522] -+------------------+------+ -| Repository Agent | Path | -+------------------+------+ -+------------------+------+ - -I1117 22:05:22.689500 1 server.cc:549] -+-----------------+-------------------------------------------------------------------------+--------+ -| Backend | Path | Config | -+-----------------+-------------------------------------------------------------------------+--------+ -| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | -| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | -| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | -| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | -| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | -+-----------------+-------------------------------------------------------------------------+--------+ - -I1117 22:05:22.689564 1 server.cc:592] -+----------+---------+--------+ -| Model | Version | Status | -+----------+---------+--------+ -| identity | 1 | READY | -+----------+---------+--------+ - -I1117 22:05:22.689830 1 tritonserver.cc:1920] -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Option | Value | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| server_id | triton | -| server_version | 2.15.0 | -| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | -| model_repository_path[0] | /models | -| model_control_mode | MODE_NONE | -| strict_model_config | 1 | -| rate_limit | OFF | -| pinned_memory_pool_byte_size | 268435456 | -| response_cache_byte_size | 0 | -| min_supported_compute_capability | 6.0 | -| strict_readiness | 1 | -| exit_timeout | 30 | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - -I1117 22:05:22.691777 1 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 -I1117 22:05:22.692280 1 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 -I1117 22:05:22.734784 1 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 diff --git a/qa/logs/6c2c5c2e-2cc4-4848-9eca-dd17f2b93686-server.log b/qa/logs/6c2c5c2e-2cc4-4848-9eca-dd17f2b93686-server.log deleted file mode 100644 index 0eeb095..0000000 --- a/qa/logs/6c2c5c2e-2cc4-4848-9eca-dd17f2b93686-server.log +++ /dev/null @@ -1,74 +0,0 @@ -I1117 22:20:26.459837 1 metrics.cc:298] Collecting metrics for GPU 0: Quadro RTX 8000 -I1117 22:20:26.460015 1 metrics.cc:298] Collecting metrics for GPU 1: Quadro RTX 8000 -I1117 22:20:26.748086 1 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch -I1117 22:20:26.748111 1 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 -I1117 22:20:26.748116 1 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 -2021-11-17 22:20:27.015263: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 -I1117 22:20:27.051650 1 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow -I1117 22:20:27.051697 1 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 -I1117 22:20:27.051712 1 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 -I1117 22:20:27.051726 1 tensorflow.cc:2210] backend configuration: -{} -I1117 22:20:27.054447 1 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime -I1117 22:20:27.054481 1 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 -I1117 22:20:27.054492 1 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 -I1117 22:20:27.077779 1 openvino.cc:1193] TRITONBACKEND_Initialize: openvino -I1117 22:20:27.077797 1 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 -I1117 22:20:27.077802 1 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 -I1117 22:20:27.221995 1 pinned_memory_manager.cc:240] Pinned memory pool is created at '0x7f138c000000' with size 268435456 -I1117 22:20:27.222780 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 0 with size 67108864 -I1117 22:20:27.222790 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 1 with size 67108864 -I1117 22:20:27.301985 1 model_repository_manager.cc:1022] loading: identity:1 -I1117 22:20:27.406019 1 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity -I1117 22:20:27.406057 1 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 -I1117 22:20:27.406072 1 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 -I1117 22:20:27.417493 1 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) -I1117 22:20:27.419533 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (CPU device 0) -I1117 22:20:27.419880 1 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 -I1117 22:20:27.420013 1 server.cc:522] -+------------------+------+ -| Repository Agent | Path | -+------------------+------+ -+------------------+------+ - -I1117 22:20:27.420150 1 server.cc:549] -+-----------------+-------------------------------------------------------------------------+--------+ -| Backend | Path | Config | -+-----------------+-------------------------------------------------------------------------+--------+ -| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | -| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | -| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | -| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | -| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | -+-----------------+-------------------------------------------------------------------------+--------+ - -I1117 22:20:27.420209 1 server.cc:592] -+----------+---------+--------+ -| Model | Version | Status | -+----------+---------+--------+ -| identity | 1 | READY | -+----------+---------+--------+ - -I1117 22:20:27.420458 1 tritonserver.cc:1920] -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Option | Value | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| server_id | triton | -| server_version | 2.15.0 | -| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | -| model_repository_path[0] | /models | -| model_control_mode | MODE_NONE | -| strict_model_config | 1 | -| rate_limit | OFF | -| pinned_memory_pool_byte_size | 268435456 | -| cuda_memory_pool_byte_size{0} | 67108864 | -| cuda_memory_pool_byte_size{1} | 67108864 | -| response_cache_byte_size | 0 | -| min_supported_compute_capability | 6.0 | -| strict_readiness | 1 | -| exit_timeout | 30 | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - -I1117 22:20:27.422263 1 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 -I1117 22:20:27.422765 1 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 -I1117 22:20:27.484088 1 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 diff --git a/qa/logs/73be3cf4-e0b3-48ed-9772-69853e7833b3-server.log b/qa/logs/73be3cf4-e0b3-48ed-9772-69853e7833b3-server.log deleted file mode 100644 index 6d54eb2..0000000 --- a/qa/logs/73be3cf4-e0b3-48ed-9772-69853e7833b3-server.log +++ /dev/null @@ -1,5 +0,0 @@ -I1117 22:19:53.638353 1 metrics.cc:298] Collecting metrics for GPU 0: Quadro RTX 8000 -I1117 22:19:53.638528 1 metrics.cc:298] Collecting metrics for GPU 1: Quadro RTX 8000 -I1117 22:19:53.928328 1 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch -I1117 22:19:53.928356 1 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 -I1117 22:19:53.928361 1 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 diff --git a/qa/logs/7f6ed73b-5709-47da-9c80-a09db7116498-server.log b/qa/logs/7f6ed73b-5709-47da-9c80-a09db7116498-server.log deleted file mode 100644 index 73bbe80..0000000 --- a/qa/logs/7f6ed73b-5709-47da-9c80-a09db7116498-server.log +++ /dev/null @@ -1,75 +0,0 @@ -I1117 21:10:16.302656 1 metrics.cc:298] Collecting metrics for GPU 0: Quadro RTX 8000 -I1117 21:10:16.302838 1 metrics.cc:298] Collecting metrics for GPU 1: Quadro RTX 8000 -I1117 21:10:16.587588 1 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch -I1117 21:10:16.587613 1 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 -I1117 21:10:16.587618 1 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 -2021-11-17 21:10:16.852727: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 -I1117 21:10:16.898886 1 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow -I1117 21:10:16.898931 1 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 -I1117 21:10:16.898946 1 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 -I1117 21:10:16.898959 1 tensorflow.cc:2210] backend configuration: -{} -I1117 21:10:16.900957 1 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime -I1117 21:10:16.900977 1 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 -I1117 21:10:16.900982 1 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 -I1117 21:10:16.918482 1 openvino.cc:1193] TRITONBACKEND_Initialize: openvino -I1117 21:10:16.918499 1 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 -I1117 21:10:16.918504 1 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 -I1117 21:10:17.065889 1 pinned_memory_manager.cc:240] Pinned memory pool is created at '0x7f81a8000000' with size 268435456 -I1117 21:10:17.066675 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 0 with size 67108864 -I1117 21:10:17.066684 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 1 with size 67108864 -I1117 21:10:17.147338 1 model_repository_manager.cc:1022] loading: identity:1 -I1117 21:10:17.251342 1 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity -I1117 21:10:17.251382 1 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 -I1117 21:10:17.251397 1 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 -I1117 21:10:17.262707 1 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) -I1117 21:10:17.267809 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (GPU device 0) -I1117 21:10:21.458378 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (GPU device 1) -I1117 21:10:25.471393 1 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 -I1117 21:10:25.471564 1 server.cc:522] -+------------------+------+ -| Repository Agent | Path | -+------------------+------+ -+------------------+------+ - -I1117 21:10:25.471706 1 server.cc:549] -+-----------------+-------------------------------------------------------------------------+--------+ -| Backend | Path | Config | -+-----------------+-------------------------------------------------------------------------+--------+ -| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | -| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | -| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | -| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | -| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | -+-----------------+-------------------------------------------------------------------------+--------+ - -I1117 21:10:25.471767 1 server.cc:592] -+----------+---------+--------+ -| Model | Version | Status | -+----------+---------+--------+ -| identity | 1 | READY | -+----------+---------+--------+ - -I1117 21:10:25.472022 1 tritonserver.cc:1920] -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Option | Value | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| server_id | triton | -| server_version | 2.15.0 | -| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | -| model_repository_path[0] | /models | -| model_control_mode | MODE_NONE | -| strict_model_config | 1 | -| rate_limit | OFF | -| pinned_memory_pool_byte_size | 268435456 | -| cuda_memory_pool_byte_size{0} | 67108864 | -| cuda_memory_pool_byte_size{1} | 67108864 | -| response_cache_byte_size | 0 | -| min_supported_compute_capability | 6.0 | -| strict_readiness | 1 | -| exit_timeout | 30 | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - -I1117 21:10:25.473753 1 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 -I1117 21:10:25.474247 1 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 -I1117 21:10:25.548082 1 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 diff --git a/qa/logs/933cd6fc-1a84-4f57-a798-fa3d1be97409-server.log b/qa/logs/933cd6fc-1a84-4f57-a798-fa3d1be97409-server.log deleted file mode 100644 index 7222233..0000000 --- a/qa/logs/933cd6fc-1a84-4f57-a798-fa3d1be97409-server.log +++ /dev/null @@ -1,77 +0,0 @@ -W1118 16:52:22.254276 61 metrics.cc:282] Cannot get CUDA device count, GPU metrics will not be available -I1118 16:52:22.502441 61 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch -I1118 16:52:22.502472 61 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 -I1118 16:52:22.502479 61 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 -2021-11-18 16:52:22.775842: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 -I1118 16:52:22.841043 61 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow -I1118 16:52:22.841092 61 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 -I1118 16:52:22.841110 61 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 -I1118 16:52:22.841121 61 tensorflow.cc:2210] backend configuration: -{} -I1118 16:52:22.843563 61 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime -I1118 16:52:22.843598 61 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 -I1118 16:52:22.843609 61 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 -I1118 16:52:22.879216 61 openvino.cc:1193] TRITONBACKEND_Initialize: openvino -I1118 16:52:22.879235 61 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 -I1118 16:52:22.879242 61 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 -W1118 16:52:22.879259 61 pinned_memory_manager.cc:236] Unable to allocate pinned system memory, pinned memory pool will not be available: no CUDA-capable device is detected -I1118 16:52:22.879271 61 cuda_memory_manager.cc:115] CUDA memory pool disabled -I1118 16:52:22.879994 61 model_repository_manager.cc:1022] loading: identity:1 -I1118 16:52:22.981315 61 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity -I1118 16:52:22.981350 61 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 -I1118 16:52:22.981362 61 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 -I1118 16:52:22.981401 61 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) -I1118 16:52:22.983424 61 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (CPU device 0) -I1118 16:52:22.983753 61 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 -I1118 16:52:22.983895 61 server.cc:522] -+------------------+------+ -| Repository Agent | Path | -+------------------+------+ -+------------------+------+ - -I1118 16:52:22.984037 61 server.cc:549] -+-----------------+-------------------------------------------------------------------------+--------+ -| Backend | Path | Config | -+-----------------+-------------------------------------------------------------------------+--------+ -| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | -| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | -| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | -| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | -| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | -+-----------------+-------------------------------------------------------------------------+--------+ - -I1118 16:52:22.984105 61 server.cc:592] -+----------+---------+--------+ -| Model | Version | Status | -+----------+---------+--------+ -| identity | 1 | READY | -+----------+---------+--------+ - -I1118 16:52:22.984323 61 tritonserver.cc:1920] -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Option | Value | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| server_id | triton | -| server_version | 2.15.0 | -| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | -| model_repository_path[0] | /qa/L0_e2e/cpu_model_repository | -| model_control_mode | MODE_NONE | -| strict_model_config | 1 | -| rate_limit | OFF | -| pinned_memory_pool_byte_size | 268435456 | -| response_cache_byte_size | 0 | -| min_supported_compute_capability | 6.0 | -| strict_readiness | 1 | -| exit_timeout | 30 | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - -I1118 16:52:22.986177 61 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 -I1118 16:52:22.986702 61 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 -I1118 16:52:23.028884 61 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 -Signal (15) received. -I1118 16:52:23.576444 61 server.cc:252] Waiting for in-flight requests to complete. -I1118 16:52:23.576465 61 model_repository_manager.cc:1055] unloading: identity:1 -I1118 16:52:23.576550 61 server.cc:267] Timeout 30: Found 1 live models and 0 in-flight non-inference requests -I1118 16:52:23.576801 61 instance_finalize.hpp:36] TRITONBACKEND_ModelInstanceFinalize: delete instance state -I1118 16:52:23.576862 61 model_finalize.hpp:36] TRITONBACKEND_ModelFinalize: delete model state -I1118 16:52:23.576900 61 model_repository_manager.cc:1166] successfully unloaded 'identity' version 1 diff --git a/qa/logs/974b0053-82c2-41c8-8bde-209499a5ec47-server.log b/qa/logs/974b0053-82c2-41c8-8bde-209499a5ec47-server.log deleted file mode 100644 index d7425fb..0000000 --- a/qa/logs/974b0053-82c2-41c8-8bde-209499a5ec47-server.log +++ /dev/null @@ -1,77 +0,0 @@ -W1118 16:42:09.768091 60 metrics.cc:282] Cannot get CUDA device count, GPU metrics will not be available -I1118 16:42:10.014849 60 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch -I1118 16:42:10.014882 60 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 -I1118 16:42:10.014889 60 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 -2021-11-18 16:42:10.291590: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 -I1118 16:42:10.362424 60 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow -I1118 16:42:10.362466 60 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 -I1118 16:42:10.362476 60 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 -I1118 16:42:10.362486 60 tensorflow.cc:2210] backend configuration: -{} -I1118 16:42:10.364743 60 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime -I1118 16:42:10.364771 60 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 -I1118 16:42:10.364781 60 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 -I1118 16:42:10.396832 60 openvino.cc:1193] TRITONBACKEND_Initialize: openvino -I1118 16:42:10.396850 60 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 -I1118 16:42:10.396856 60 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 -W1118 16:42:10.396872 60 pinned_memory_manager.cc:236] Unable to allocate pinned system memory, pinned memory pool will not be available: no CUDA-capable device is detected -I1118 16:42:10.396885 60 cuda_memory_manager.cc:115] CUDA memory pool disabled -I1118 16:42:10.397603 60 model_repository_manager.cc:1022] loading: identity:1 -I1118 16:42:10.499429 60 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity -I1118 16:42:10.499483 60 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 -I1118 16:42:10.499503 60 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 -I1118 16:42:10.499565 60 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) -I1118 16:42:10.502831 60 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (CPU device 0) -I1118 16:42:10.503271 60 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 -I1118 16:42:10.503404 60 server.cc:522] -+------------------+------+ -| Repository Agent | Path | -+------------------+------+ -+------------------+------+ - -I1118 16:42:10.503516 60 server.cc:549] -+-----------------+-------------------------------------------------------------------------+--------+ -| Backend | Path | Config | -+-----------------+-------------------------------------------------------------------------+--------+ -| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | -| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | -| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | -| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | -| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | -+-----------------+-------------------------------------------------------------------------+--------+ - -I1118 16:42:10.503570 60 server.cc:592] -+----------+---------+--------+ -| Model | Version | Status | -+----------+---------+--------+ -| identity | 1 | READY | -+----------+---------+--------+ - -I1118 16:42:10.503732 60 tritonserver.cc:1920] -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Option | Value | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| server_id | triton | -| server_version | 2.15.0 | -| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | -| model_repository_path[0] | /qa/L0_e2e/cpu_model_repository | -| model_control_mode | MODE_NONE | -| strict_model_config | 1 | -| rate_limit | OFF | -| pinned_memory_pool_byte_size | 268435456 | -| response_cache_byte_size | 0 | -| min_supported_compute_capability | 6.0 | -| strict_readiness | 1 | -| exit_timeout | 30 | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - -I1118 16:42:10.505348 60 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 -I1118 16:42:10.505873 60 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 -I1118 16:42:10.548117 60 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 -Signal (15) received. -I1118 16:42:11.073757 60 server.cc:252] Waiting for in-flight requests to complete. -I1118 16:42:11.073800 60 model_repository_manager.cc:1055] unloading: identity:1 -I1118 16:42:11.073979 60 server.cc:267] Timeout 30: Found 1 live models and 0 in-flight non-inference requests -I1118 16:42:11.074100 60 instance_finalize.hpp:36] TRITONBACKEND_ModelInstanceFinalize: delete instance state -I1118 16:42:11.074149 60 model_finalize.hpp:36] TRITONBACKEND_ModelFinalize: delete model state -I1118 16:42:11.074176 60 model_repository_manager.cc:1166] successfully unloaded 'identity' version 1 diff --git a/qa/logs/da517b94-6959-43a0-a972-7bd52289b21e-server.log b/qa/logs/da517b94-6959-43a0-a972-7bd52289b21e-server.log deleted file mode 100644 index 89888c2..0000000 --- a/qa/logs/da517b94-6959-43a0-a972-7bd52289b21e-server.log +++ /dev/null @@ -1,71 +0,0 @@ -Error: Failed to initialize NVML -W1117 22:20:33.435468 1 metrics.cc:221] DCGM unable to start: DCGM initialization error -I1117 22:20:33.736191 1 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch -I1117 22:20:33.736219 1 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 -I1117 22:20:33.736224 1 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 -2021-11-17 22:20:34.008207: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 -I1117 22:20:34.055124 1 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow -I1117 22:20:34.055178 1 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 -I1117 22:20:34.055193 1 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 -I1117 22:20:34.055206 1 tensorflow.cc:2210] backend configuration: -{} -I1117 22:20:34.056747 1 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime -I1117 22:20:34.056768 1 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 -I1117 22:20:34.056774 1 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 -I1117 22:20:34.074190 1 openvino.cc:1193] TRITONBACKEND_Initialize: openvino -I1117 22:20:34.074207 1 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 -I1117 22:20:34.074212 1 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 -W1117 22:20:34.074308 1 pinned_memory_manager.cc:236] Unable to allocate pinned system memory, pinned memory pool will not be available: CUDA driver version is insufficient for CUDA runtime version -I1117 22:20:34.074332 1 cuda_memory_manager.cc:115] CUDA memory pool disabled -I1117 22:20:34.075044 1 model_repository_manager.cc:1022] loading: identity:1 -I1117 22:20:34.178827 1 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity -I1117 22:20:34.178865 1 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 -I1117 22:20:34.178879 1 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 -I1117 22:20:34.179076 1 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) -I1117 22:20:34.182329 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (CPU device 0) -I1117 22:20:34.182694 1 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 -I1117 22:20:34.182808 1 server.cc:522] -+------------------+------+ -| Repository Agent | Path | -+------------------+------+ -+------------------+------+ - -I1117 22:20:34.182950 1 server.cc:549] -+-----------------+-------------------------------------------------------------------------+--------+ -| Backend | Path | Config | -+-----------------+-------------------------------------------------------------------------+--------+ -| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | -| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | -| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | -| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | -| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | -+-----------------+-------------------------------------------------------------------------+--------+ - -I1117 22:20:34.183010 1 server.cc:592] -+----------+---------+--------+ -| Model | Version | Status | -+----------+---------+--------+ -| identity | 1 | READY | -+----------+---------+--------+ - -I1117 22:20:34.183238 1 tritonserver.cc:1920] -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Option | Value | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| server_id | triton | -| server_version | 2.15.0 | -| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | -| model_repository_path[0] | /models | -| model_control_mode | MODE_NONE | -| strict_model_config | 1 | -| rate_limit | OFF | -| pinned_memory_pool_byte_size | 268435456 | -| response_cache_byte_size | 0 | -| min_supported_compute_capability | 6.0 | -| strict_readiness | 1 | -| exit_timeout | 30 | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - -I1117 22:20:34.185149 1 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 -I1117 22:20:34.185658 1 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 -I1117 22:20:34.228187 1 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 diff --git a/qa/logs/f09aec20-ba41-4c04-ac7b-ec324fcb0754-server.log b/qa/logs/f09aec20-ba41-4c04-ac7b-ec324fcb0754-server.log deleted file mode 100644 index dfc2198..0000000 --- a/qa/logs/f09aec20-ba41-4c04-ac7b-ec324fcb0754-server.log +++ /dev/null @@ -1,74 +0,0 @@ -I1117 22:05:15.108220 1 metrics.cc:298] Collecting metrics for GPU 0: Quadro RTX 8000 -I1117 22:05:15.108397 1 metrics.cc:298] Collecting metrics for GPU 1: Quadro RTX 8000 -I1117 22:05:15.394902 1 libtorch.cc:1092] TRITONBACKEND_Initialize: pytorch -I1117 22:05:15.394929 1 libtorch.cc:1102] Triton TRITONBACKEND API version: 1.6 -I1117 22:05:15.394933 1 libtorch.cc:1108] 'pytorch' TRITONBACKEND API version: 1.6 -2021-11-17 22:05:15.669411: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0 -I1117 22:05:15.705319 1 tensorflow.cc:2170] TRITONBACKEND_Initialize: tensorflow -I1117 22:05:15.705374 1 tensorflow.cc:2180] Triton TRITONBACKEND API version: 1.6 -I1117 22:05:15.705389 1 tensorflow.cc:2186] 'tensorflow' TRITONBACKEND API version: 1.6 -I1117 22:05:15.705402 1 tensorflow.cc:2210] backend configuration: -{} -I1117 22:05:15.708421 1 onnxruntime.cc:1999] TRITONBACKEND_Initialize: onnxruntime -I1117 22:05:15.708460 1 onnxruntime.cc:2009] Triton TRITONBACKEND API version: 1.6 -I1117 22:05:15.708470 1 onnxruntime.cc:2015] 'onnxruntime' TRITONBACKEND API version: 1.6 -I1117 22:05:15.733935 1 openvino.cc:1193] TRITONBACKEND_Initialize: openvino -I1117 22:05:15.733952 1 openvino.cc:1203] Triton TRITONBACKEND API version: 1.6 -I1117 22:05:15.733957 1 openvino.cc:1209] 'openvino' TRITONBACKEND API version: 1.6 -I1117 22:05:15.879810 1 pinned_memory_manager.cc:240] Pinned memory pool is created at '0x7f7c74000000' with size 268435456 -I1117 22:05:15.880601 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 0 with size 67108864 -I1117 22:05:15.880611 1 cuda_memory_manager.cc:105] CUDA memory pool is created on device 1 with size 67108864 -I1117 22:05:15.964300 1 model_repository_manager.cc:1022] loading: identity:1 -I1117 22:05:16.068298 1 initialize.hpp:43] TRITONBACKEND_Initialize: rapids-identity -I1117 22:05:16.068350 1 backend.hpp:47] Triton TRITONBACKEND API version: 1.6 -I1117 22:05:16.068379 1 backend.hpp:52] 'rapids-identity' TRITONBACKEND API version: 1.4 -I1117 22:05:16.079181 1 model_initialize.hpp:37] TRITONBACKEND_ModelInitialize: identity (version 1) -I1117 22:05:16.081280 1 instance_initialize.hpp:46] TRITONBACKEND_ModelInstanceInitialize: identity_0 (CPU device 0) -I1117 22:05:16.081659 1 model_repository_manager.cc:1183] successfully loaded 'identity' version 1 -I1117 22:05:16.081804 1 server.cc:522] -+------------------+------+ -| Repository Agent | Path | -+------------------+------+ -+------------------+------+ - -I1117 22:05:16.081942 1 server.cc:549] -+-----------------+-------------------------------------------------------------------------+--------+ -| Backend | Path | Config | -+-----------------+-------------------------------------------------------------------------+--------+ -| pytorch | /opt/tritonserver/backends/pytorch/libtriton_pytorch.so | {} | -| tensorflow | /opt/tritonserver/backends/tensorflow1/libtriton_tensorflow1.so | {} | -| onnxruntime | /opt/tritonserver/backends/onnxruntime/libtriton_onnxruntime.so | {} | -| openvino | /opt/tritonserver/backends/openvino/libtriton_openvino.so | {} | -| rapids-identity | /opt/tritonserver/backends/rapids-identity/libtriton_rapids-identity.so | {} | -+-----------------+-------------------------------------------------------------------------+--------+ - -I1117 22:05:16.082004 1 server.cc:592] -+----------+---------+--------+ -| Model | Version | Status | -+----------+---------+--------+ -| identity | 1 | READY | -+----------+---------+--------+ - -I1117 22:05:16.082246 1 tritonserver.cc:1920] -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Option | Value | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| server_id | triton | -| server_version | 2.15.0 | -| server_extensions | classification sequence model_repository model_repository(unload_dependents) schedule_policy model_configuration system_shared_memory cuda_shared_memory binary_tensor_data statistics | -| model_repository_path[0] | /models | -| model_control_mode | MODE_NONE | -| strict_model_config | 1 | -| rate_limit | OFF | -| pinned_memory_pool_byte_size | 268435456 | -| cuda_memory_pool_byte_size{0} | 67108864 | -| cuda_memory_pool_byte_size{1} | 67108864 | -| response_cache_byte_size | 0 | -| min_supported_compute_capability | 6.0 | -| strict_readiness | 1 | -| exit_timeout | 30 | -+----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - -I1117 22:05:16.084027 1 grpc_server.cc:4117] Started GRPCInferenceService at 0.0.0.0:8001 -I1117 22:05:16.084514 1 http_server.cc:2815] Started HTTPService at 0.0.0.0:8000 -I1117 22:05:16.140121 1 http_server.cc:167] Started Metrics Service at 0.0.0.0:8002 From f6b02cda8962524f5d9be053ff5ddfe2c21ad28e Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 18 Nov 2021 13:44:16 -0500 Subject: [PATCH 135/199] Enable ccache --- Dockerfile | 6 ++++-- ...s_triton_dev_cuda11.2.yml => rapids_triton_dev.yml} | 3 +-- conda/environments/rapids_triton_dev_cuda11.4.yml | 10 ---------- cpp/CMakeLists.txt | 3 +++ 4 files changed, 8 insertions(+), 14 deletions(-) rename conda/environments/{rapids_triton_dev_cuda11.2.yml => rapids_triton_dev.yml} (77%) delete mode 100644 conda/environments/rapids_triton_dev_cuda11.4.yml diff --git a/Dockerfile b/Dockerfile index 49401b4..e0bcb45 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ RUN wget \ && bash Miniconda3-latest-Linux-x86_64.sh -b \ && rm -f Miniconda3-latest-Linux-x86_64.sh -COPY ./conda/environments/rapids_triton_dev_cuda11.4.yml /environment.yml +COPY ./conda/environments/rapids_triton_dev.yml /environment.yml RUN conda env update -f /environment.yml \ && rm /environment.yml \ @@ -68,7 +68,9 @@ RUN cmake \ -DTRITON_ENABLE_GPU="${TRITON_ENABLE_GPU}" \ .. -RUN ninja install +ENV CCACHE_DIR=/ccache + +RUN --mount=type=cache,target=/ccache/ ninja install && ccache -s FROM base as test-install diff --git a/conda/environments/rapids_triton_dev_cuda11.2.yml b/conda/environments/rapids_triton_dev.yml similarity index 77% rename from conda/environments/rapids_triton_dev_cuda11.2.yml rename to conda/environments/rapids_triton_dev.yml index 0755643..1edf8a9 100644 --- a/conda/environments/rapids_triton_dev_cuda11.2.yml +++ b/conda/environments/rapids_triton_dev.yml @@ -1,10 +1,9 @@ --- name: rapids_triton_dev channels: - - nvidia - conda-forge dependencies: + - ccache - cmake>=3.21 - - cudatoolkit=11.2 - ninja - rapidjson diff --git a/conda/environments/rapids_triton_dev_cuda11.4.yml b/conda/environments/rapids_triton_dev_cuda11.4.yml deleted file mode 100644 index 409c7bc..0000000 --- a/conda/environments/rapids_triton_dev_cuda11.4.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: rapids_triton_dev -channels: - - nvidia - - conda-forge -dependencies: - - cmake>=3.21 - - cudatoolkit=11.4 - - ninja - - rapidjson diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index aedd871..0341051 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -88,6 +88,9 @@ endif() ############################################################################## # - compiler options --------------------------------------------------------- +set(CMAKE_C_COMPILER_LAUNCHER ccache) +set(CMAKE_CXX_COMPILER_LAUNCHER ccache) +set(CMAKE_CUDA_COMPILER_LAUNCHER ccache) # * find CUDAToolkit package # * determine GPU architectures From f01823c72e9d5cd86d571a621e2e58aa05b80d3c Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 22 Nov 2021 10:04:00 -0500 Subject: [PATCH 136/199] Make cleaner use of cmake generator expressions --- cpp/CMakeLists.txt | 58 ++++++++++++++++++----------------------- cpp/src/CMakeLists.txt | 28 +++++++------------- cpp/test/CMakeLists.txt | 40 +++++++++------------------- 3 files changed, 48 insertions(+), 78 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 0341051..f3247fe 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -24,26 +24,10 @@ include(rapids-cuda) include(rapids-export) include(rapids-find) -option(TRITON_ENABLE_GPU "Enable GPU support in Triton" ON) - -if(TRITON_ENABLE_GPU) - rapids_cuda_init_architectures(RAPIDS_TRITON) -endif() - -project(RAPIDS_TRITON VERSION 21.10.00 LANGUAGES CXX CUDA) - -############################################################################## -# - build type --------------------------------------------------------------- - -# Set a default build type if none was specified -rapids_cmake_build_type(Release) - -# this is needed for clang-tidy runs -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - ############################################################################## # - User Options ------------------------------------------------------------ +option(TRITON_ENABLE_GPU "Enable GPU support in Triton" ON) option(BUILD_TESTS "Build rapids_triton unit-tests" ON) option(BUILD_EXAMPLE "Build rapids_identity example backend" OFF) option(CUDA_ENABLE_KERNELINFO "Enable kernel resource usage info" OFF) @@ -70,6 +54,24 @@ message(VERBOSE "RAPIDS_TRITON: Triton common repo tag: ${TRITON_COMMON_REPO_TAG message(VERBOSE "RAPIDS_TRITON: Triton core repo tag: ${TRITON_CORE_REPO_TAG}") message(VERBOSE "RAPIDS_TRITON: Triton backend repo tag: ${TRITON_BACKEND_REPO_TAG}") +############################################################################## +# - Project Initialization --------------------------------------------------- + +if(TRITON_ENABLE_GPU) + rapids_cuda_init_architectures(RAPIDS_TRITON) +endif() + +project(RAPIDS_TRITON VERSION 21.10.00 LANGUAGES CXX CUDA) + +############################################################################## +# - build type --------------------------------------------------------------- + +# Set a default build type if none was specified +rapids_cmake_build_type(Release) + +# this is needed for clang-tidy runs +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + # Set RMM logging level set(RMM_LOGGING_LEVEL "INFO" CACHE STRING "Choose the logging level.") set_property(CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF") @@ -128,21 +130,13 @@ add_library(rapids_triton::rapids_triton ALIAS rapids_triton) target_include_directories(rapids_triton INTERFACE "$" "$") -if(TRITON_ENABLE_GPU) - target_link_libraries(rapids_triton - INTERFACE - rmm::rmm - raft::raft - triton-core-serverstub - triton-backend-utils - ) -else() - target_link_libraries(rapids_triton - INTERFACE - triton-core-serverstub - triton-backend-utils - ) -endif() +target_link_libraries(rapids_triton +INTERFACE + $<$:rmm::rmm> + $<$:raft::raft> + triton-core-serverstub + triton-backend-utils +) target_compile_features( rapids_triton INTERFACE cxx_std_17 diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index e69f350..1d5904c 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -52,25 +52,15 @@ target_include_directories(triton_rapids-identity "${CMAKE_CURRENT_SOURCE_DIR}/src" ) -if(TRITON_ENABLE_GPU) - target_link_libraries(triton_rapids-identity - PRIVATE - rmm::rmm - raft::raft - triton-core-serverstub - triton-backend-utils - "${TRITONSERVER_LIB}" - $ - ) -else() - target_link_libraries(triton_rapids-identity - PRIVATE - triton-core-serverstub - triton-backend-utils - "${TRITONSERVER_LIB}" - $ - ) -endif() +target_link_libraries(triton_rapids-identity +PRIVATE + $<$:rmm::rmm> + $<$:raft::raft> + triton-core-serverstub + triton-backend-utils + "${TRITONSERVER_LIB}" + $ +) install( TARGETS triton_rapids-identity diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 0f90912..dbe92b2 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -87,30 +87,16 @@ find_library( PATHS /opt/tritonserver/lib ) -IF(TRITON_ENABLE_GPU) - target_link_libraries(test_rapids_triton - PRIVATE - rmm::rmm - raft::raft - triton-core-serverstub - triton-backend-utils - gmock - gmock_main - GTest::gtest - GTest::gtest_main - "${TRITONSERVER_LIB}" - $ - ) -else() - target_link_libraries(test_rapids_triton - PRIVATE - triton-core-serverstub - triton-backend-utils - gmock - gmock_main - GTest::gtest - GTest::gtest_main - "${TRITONSERVER_LIB}" - $ - ) -endif() +target_link_libraries(test_rapids_triton +PRIVATE + $<$:rmm::rmm> + $<$:raft::raft> + triton-core-serverstub + triton-backend-utils + gmock + gmock_main + GTest::gtest + GTest::gtest_main + "${TRITONSERVER_LIB}" + $ +) From 2aed622c1dd3108774fdfb8f31877fd291c36825 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 22 Nov 2021 15:01:48 -0500 Subject: [PATCH 137/199] Remove debug ccache reporting --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e0bcb45..d16ab81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,7 +70,7 @@ RUN cmake \ ENV CCACHE_DIR=/ccache -RUN --mount=type=cache,target=/ccache/ ninja install && ccache -s +RUN --mount=type=cache,target=/ccache/ ninja install FROM base as test-install From 47c808d95f89af166a86acbdf0d03853e3a71c1d Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 23 Nov 2021 11:22:35 -0500 Subject: [PATCH 138/199] Remove GPU restriction on C++ standard --- cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f3247fe..14fb9bb 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -140,7 +140,7 @@ INTERFACE target_compile_features( rapids_triton INTERFACE cxx_std_17 - $<$:cuda_std_17> + $ ) rapids_cmake_install_lib_dir(lib_dir) From 4d01da3423fe31427850577fc2f944e1b7429308 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 1 Dec 2021 09:50:28 -0500 Subject: [PATCH 139/199] Remove old commented-out code --- .../rapids_triton/memory/detail/cpu_only/resource.hpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cpp/include/rapids_triton/memory/detail/cpu_only/resource.hpp b/cpp/include/rapids_triton/memory/detail/cpu_only/resource.hpp index 6d134cf..04bb1c5 100644 --- a/cpp/include/rapids_triton/memory/detail/cpu_only/resource.hpp +++ b/cpp/include/rapids_triton/memory/detail/cpu_only/resource.hpp @@ -29,12 +29,6 @@ template<> inline void setup_memory_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* triton_manager) { } -/* namespace rmm { -namespace mr { -using device_memory_resource = void; -} -} */ - } // namespace detail } // namespace rapids } // namespace backend From 2009803a91ad3d960ed786515ce6a1f0f4bf9dfa Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 1 Dec 2021 10:00:05 -0500 Subject: [PATCH 140/199] Remove unnecessary capture from lambda --- .../memory/detail/gpu_only/owned_device_buffer.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp b/cpp/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp index 29edacd..34095f7 100644 --- a/cpp/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp +++ b/cpp/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp @@ -30,7 +30,7 @@ template struct owned_device_buffer { using non_const_T = std::remove_const_t; owned_device_buffer(device_id_t device_id, std::size_t size, cudaStream_t stream) - : data_{[this, &device_id, &size, &stream]() { + : data_{[&device_id, &size, &stream]() { auto device_context = device_setter{device_id}; return rmm::device_buffer{size * sizeof(T), rmm::cuda_stream_view{stream}}; }()} From 5a7f8f6924a0361e1958249882ba6ba15ebf18f7 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 8 Dec 2021 14:19:18 -0500 Subject: [PATCH 141/199] Do not require nvcc in CPU-only environments --- cpp/CMakeLists.txt | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 14fb9bb..7cf26bb 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -59,9 +59,11 @@ message(VERBOSE "RAPIDS_TRITON: Triton backend repo tag: ${TRITON_BACKEND_REPO_T if(TRITON_ENABLE_GPU) rapids_cuda_init_architectures(RAPIDS_TRITON) + project(RAPIDS_TRITON VERSION 21.10.00 LANGUAGES CXX CUDA) +else() + project(RAPIDS_TRITON VERSION 21.10.00 LANGUAGES CXX) endif() -project(RAPIDS_TRITON VERSION 21.10.00 LANGUAGES CXX CUDA) ############################################################################## # - build type --------------------------------------------------------------- @@ -92,17 +94,19 @@ endif() # - compiler options --------------------------------------------------------- set(CMAKE_C_COMPILER_LAUNCHER ccache) set(CMAKE_CXX_COMPILER_LAUNCHER ccache) -set(CMAKE_CUDA_COMPILER_LAUNCHER ccache) - -# * find CUDAToolkit package -# * determine GPU architectures -# * enable the CMake CUDA language -# * set other CUDA compilation flags -rapids_find_package(CUDAToolkit REQUIRED - BUILD_EXPORT_SET rapids_triton-exports - INSTALL_EXPORT_SET rapids_triton-exports - ) -include(cmake/modules/ConfigureCUDA.cmake) +if(TRITON_ENABLE_GPU) + set(CMAKE_CUDA_COMPILER_LAUNCHER ccache) + + # * find CUDAToolkit package + # * determine GPU architectures + # * enable the CMake CUDA language + # * set other CUDA compilation flags + rapids_find_package(CUDAToolkit REQUIRED + BUILD_EXPORT_SET rapids_triton-exports + INSTALL_EXPORT_SET rapids_triton-exports + ) + include(cmake/modules/ConfigureCUDA.cmake) +endif() ############################################################################## # - Requirements ------------------------------------------------------------- From 60fe5e8ebd8db9a8b03877ae78cc7b3049effb59 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 8 Dec 2021 16:08:02 -0500 Subject: [PATCH 142/199] Appropriately guard cuda_std command in CPU-only build --- cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 7cf26bb..a8526bc 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -144,7 +144,7 @@ INTERFACE target_compile_features( rapids_triton INTERFACE cxx_std_17 - $ + $<$>:cuda_std_17> ) rapids_cmake_install_lib_dir(lib_dir) From beb5526d6abe5ff35919cfc044dcd2c2d9d7b89a Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 8 Dec 2021 16:18:58 -0500 Subject: [PATCH 143/199] Correct CMake generator expression --- cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index a8526bc..7f1d90b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -144,7 +144,7 @@ INTERFACE target_compile_features( rapids_triton INTERFACE cxx_std_17 - $<$>:cuda_std_17> + $<$,$>:cuda_std_17> ) rapids_cmake_install_lib_dir(lib_dir) From 75f9b222f551fe4b9207c9012e897ebb60408783 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 8 Dec 2021 16:29:21 -0500 Subject: [PATCH 144/199] Disable cuda build standard differently --- cpp/CMakeLists.txt | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 7f1d90b..3bb556e 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -142,10 +142,16 @@ INTERFACE triton-backend-utils ) -target_compile_features( - rapids_triton INTERFACE cxx_std_17 - $<$,$>:cuda_std_17> -) +if (TRITON_ENABLE_GPU) + target_compile_features( + rapids_triton INTERFACE cxx_std_17 + $:cuda_std_17> + ) +else() + target_compile_features( + rapids_triton INTERFACE cxx_std_17 + ) +endif() rapids_cmake_install_lib_dir(lib_dir) install(TARGETS rapids_triton From 6f915e648f9be1360c5c89791fb0cefa1971346b Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 8 Dec 2021 17:17:46 -0500 Subject: [PATCH 145/199] Correct generator expression --- cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3bb556e..f2b1041 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -145,7 +145,7 @@ INTERFACE if (TRITON_ENABLE_GPU) target_compile_features( rapids_triton INTERFACE cxx_std_17 - $:cuda_std_17> + $ ) else() target_compile_features( From 47782dc39dcba667573d0f75b179b5c9395c14d2 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 9 Dec 2021 15:26:35 -0500 Subject: [PATCH 146/199] Guard against CUDA shared memory import on CPU-only machine --- python/rapids_triton/triton/io.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/rapids_triton/triton/io.py b/python/rapids_triton/triton/io.py index c3bdedc..75c7661 100644 --- a/python/rapids_triton/triton/io.py +++ b/python/rapids_triton/triton/io.py @@ -17,7 +17,10 @@ import tritonclient.http as triton_http import tritonclient.grpc as triton_grpc -import tritonclient.utils.cuda_shared_memory as shm +try: + import tritonclient.utils.cuda_shared_memory as shm +except OSError: # CUDA libraries not available + shm = None from tritonclient import utils as triton_utils From 03a5977ab1e2fce28d1a0755b668253a2b87e713 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 9 Dec 2021 17:49:30 -0500 Subject: [PATCH 147/199] Guard against missed cuda_shared_memory import --- python/rapids_triton/triton/response.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/rapids_triton/triton/response.py b/python/rapids_triton/triton/response.py index af8c511..965f1a4 100644 --- a/python/rapids_triton/triton/response.py +++ b/python/rapids_triton/triton/response.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import tritonclient.utils.cuda_shared_memory as shm +try: + import tritonclient.utils.cuda_shared_memory as shm +except OSError: + shm = None from tritonclient import utils as triton_utils from rapids_triton.triton.message import TritonMessage From 3a05c3b2c992aed57de2c1ab1e7d0eabf68d1a3c Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 10 Dec 2021 15:25:33 -0500 Subject: [PATCH 148/199] Provide better error messages for import failures --- python/rapids_triton/triton/io.py | 5 +++-- python/rapids_triton/triton/response.py | 9 +++++---- python/rapids_triton/utils/__init__.py | 0 python/rapids_triton/utils/safe_import.py | 22 ++++++++++++++++++++++ 4 files changed, 30 insertions(+), 6 deletions(-) create mode 100644 python/rapids_triton/utils/__init__.py create mode 100644 python/rapids_triton/utils/safe_import.py diff --git a/python/rapids_triton/triton/io.py b/python/rapids_triton/triton/io.py index 75c7661..5126871 100644 --- a/python/rapids_triton/triton/io.py +++ b/python/rapids_triton/triton/io.py @@ -17,11 +17,12 @@ import tritonclient.http as triton_http import tritonclient.grpc as triton_grpc +from rapids_triton.utils.safe_import import ImportReplacement +from tritonclient import utils as triton_utils try: import tritonclient.utils.cuda_shared_memory as shm except OSError: # CUDA libraries not available - shm = None -from tritonclient import utils as triton_utils + shm = ImportReplacement('tritonclient.utils.cuda_shared_memory') TritonInput = namedtuple('TritonInput', ('name', 'input')) diff --git a/python/rapids_triton/triton/response.py b/python/rapids_triton/triton/response.py index 965f1a4..47332ce 100644 --- a/python/rapids_triton/triton/response.py +++ b/python/rapids_triton/triton/response.py @@ -12,13 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -try: - import tritonclient.utils.cuda_shared_memory as shm -except OSError: - shm = None from tritonclient import utils as triton_utils from rapids_triton.triton.message import TritonMessage +from rapids_triton.utils.safe_import import ImportReplacement +try: + import tritonclient.utils.cuda_shared_memory as shm +except OSError: # CUDA libraries not available + shm = ImportReplacement('tritonclient.utils.cuda_shared_memory') def get_response_data(response, output_handle, output_name): diff --git a/python/rapids_triton/utils/__init__.py b/python/rapids_triton/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/rapids_triton/utils/safe_import.py b/python/rapids_triton/utils/safe_import.py new file mode 100644 index 0000000..e29e263 --- /dev/null +++ b/python/rapids_triton/utils/safe_import.py @@ -0,0 +1,22 @@ +class UnavailableError(Exception): + '''Error thrown if a symbol is unavailable due to an issue importing it''' + +class ImportReplacement: + """A class to be used in place of an importable symbol if that symbol + cannot be imported + + Parameters + ---------- + symbol: str + The name or import path to be used in error messages when attempting to + make use of this symbol. E.g. "some_pkg.func" would result in an + exception with message "some_pkg.func could not be imported" + """ + def __init__(self, symbol): + self._msg = f'{symbol} could not be imported' + + def __getattr__(self, name): + raise UnavailableError(self._msg) + + def __call__(self, *args, **kwargs): + raise UnavailableError(self._msg) From ec154f1061a84fe5e4a6714586ae4ba00d94a08f Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 13 Dec 2021 16:55:18 -0500 Subject: [PATCH 149/199] Update version numbers for 21.12 release --- Dockerfile | 2 +- README.md | 6 ------ cpp/CMakeLists.txt | 12 ++++++------ python/setup.py | 2 +- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index d16ab81..3d26926 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # Arguments for controlling build details ########################################################################################### # Version of Triton to use -ARG TRITON_VERSION=21.10 +ARG TRITON_VERSION=21.11 # Base container image ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 # Whether or not to build indicated components diff --git a/README.md b/README.md index c29e0e3..59a4973 100644 --- a/README.md +++ b/README.md @@ -23,12 +23,6 @@ into the NVIDIA Triton Inference Server. Originally developed to assist with the integration of RAPIDS algorithms, this library can be used by anyone to quickly get up and running with a custom backend for Triton. -**WARNING**: Due to an upstream bug in Triton, backends which make use of -RAPIDS-Triton may encounter issues with Triton's shared memory mode. A fix is -available in Triton 21.10, which will be released around the end of October. At -that time, RAPIDS-Triton will be updated to default to Triton 21.10, and this -warning will be removed. - ## Background ### Triton diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f2b1041..9cd8a6c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -15,7 +15,7 @@ #============================================================================= cmake_minimum_required(VERSION 3.21 FATAL_ERROR) -file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-21.10/RAPIDS.cmake +file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-21.12/RAPIDS.cmake ${CMAKE_BINARY_DIR}/RAPIDS.cmake) include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) include(rapids-cmake) @@ -37,9 +37,9 @@ option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(NVTX "Enable nvtx markers" OFF) option(TRITON_ENABLE_STATS "Enable statistics collection in Triton" ON) -set(TRITON_COMMON_REPO_TAG "r21.08" CACHE STRING "Tag for triton-inference-server/common repo") -set(TRITON_CORE_REPO_TAG "r21.08" CACHE STRING "Tag for triton-inference-server/core repo") -set(TRITON_BACKEND_REPO_TAG "r21.08" CACHE STRING "Tag for triton-inference-server/backend repo") +set(TRITON_COMMON_REPO_TAG "r21.11" CACHE STRING "Tag for triton-inference-server/common repo") +set(TRITON_CORE_REPO_TAG "r21.11" CACHE STRING "Tag for triton-inference-server/core repo") +set(TRITON_BACKEND_REPO_TAG "r21.11" CACHE STRING "Tag for triton-inference-server/backend repo") message(VERBOSE "RAPIDS_TRITON: Build RAPIDS_TRITON unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAPIDS_TRITON: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") @@ -59,9 +59,9 @@ message(VERBOSE "RAPIDS_TRITON: Triton backend repo tag: ${TRITON_BACKEND_REPO_T if(TRITON_ENABLE_GPU) rapids_cuda_init_architectures(RAPIDS_TRITON) - project(RAPIDS_TRITON VERSION 21.10.00 LANGUAGES CXX CUDA) + project(RAPIDS_TRITON VERSION 21.12.00 LANGUAGES CXX CUDA) else() - project(RAPIDS_TRITON VERSION 21.10.00 LANGUAGES CXX) + project(RAPIDS_TRITON VERSION 21.12.00 LANGUAGES CXX) endif() diff --git a/python/setup.py b/python/setup.py index 9b35976..3fdd214 100644 --- a/python/setup.py +++ b/python/setup.py @@ -17,7 +17,7 @@ setup( name='rapids_triton', description="Tools for clients to RAPIDS-Triton backends", - version='21.10.00', # TODO(wphicks): versioneer + version='21.12.00', # TODO(wphicks): versioneer author='NVIDIA Corporation', license='Apache', packages=find_packages(), From 1b953bab568c9e2bcb05466b6a3cfadc5274ab33 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 21 Jan 2022 12:38:33 -0500 Subject: [PATCH 150/199] Update versions of upstream RAPIDS libraries --- Dockerfile | 16 +++++++++++++++- build.sh | 3 +++ cpp/CMakeLists.txt | 14 +++++++------- cpp/cmake/thirdparty/get_raft.cmake | 1 + python/setup.py | 4 ++-- 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3d26926..5c73d4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,22 @@ +# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + ########################################################################################### # Arguments for controlling build details ########################################################################################### # Version of Triton to use -ARG TRITON_VERSION=21.11 +ARG TRITON_VERSION=21.12 # Base container image ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 # Whether or not to build indicated components diff --git a/build.sh b/build.sh index 583afb6..1b7f871 100755 --- a/build.sh +++ b/build.sh @@ -150,6 +150,7 @@ if [ $BACKEND -eq 1 ] then docker build \ $DOCKER_ARGS \ + --no-cache \ -t "$EXAMPLE_TAG" \ $REPODIR fi @@ -158,6 +159,8 @@ if [ $TESTS -eq 1 ] then docker build \ $DOCKER_ARGS \ + --no-cache \ + -t "$EXAMPLE_TAG" \ --target test-stage \ -t "$TEST_TAG" \ $REPODIR diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 9cd8a6c..280997f 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,5 +1,5 @@ #============================================================================= -# Copyright (c) 2021, NVIDIA CORPORATION. +# Copyright (c) 2021-2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ #============================================================================= cmake_minimum_required(VERSION 3.21 FATAL_ERROR) -file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-21.12/RAPIDS.cmake +file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.02/RAPIDS.cmake ${CMAKE_BINARY_DIR}/RAPIDS.cmake) include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) include(rapids-cmake) @@ -37,9 +37,9 @@ option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(NVTX "Enable nvtx markers" OFF) option(TRITON_ENABLE_STATS "Enable statistics collection in Triton" ON) -set(TRITON_COMMON_REPO_TAG "r21.11" CACHE STRING "Tag for triton-inference-server/common repo") -set(TRITON_CORE_REPO_TAG "r21.11" CACHE STRING "Tag for triton-inference-server/core repo") -set(TRITON_BACKEND_REPO_TAG "r21.11" CACHE STRING "Tag for triton-inference-server/backend repo") +set(TRITON_COMMON_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-server/common repo") +set(TRITON_CORE_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-server/core repo") +set(TRITON_BACKEND_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-server/backend repo") message(VERBOSE "RAPIDS_TRITON: Build RAPIDS_TRITON unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAPIDS_TRITON: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") @@ -59,9 +59,9 @@ message(VERBOSE "RAPIDS_TRITON: Triton backend repo tag: ${TRITON_BACKEND_REPO_T if(TRITON_ENABLE_GPU) rapids_cuda_init_architectures(RAPIDS_TRITON) - project(RAPIDS_TRITON VERSION 21.12.00 LANGUAGES CXX CUDA) + project(RAPIDS_TRITON VERSION 22.02.00 LANGUAGES CXX CUDA) else() - project(RAPIDS_TRITON VERSION 21.12.00 LANGUAGES CXX) + project(RAPIDS_TRITON VERSION 22.02.00 LANGUAGES CXX) endif() diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index 1dc0876..6a37d7e 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -30,6 +30,7 @@ function(find_and_configure_raft) SOURCE_SUBDIR cpp OPTIONS "BUILD_TESTS OFF" + "RAFT_COMPILE_LIBRARIES OFF" ) message(VERBOSE "RAPIDS_TRITON: Using RAFT located in ${raft_SOURCE_DIR}") diff --git a/python/setup.py b/python/setup.py index 3fdd214..2cfa713 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. +# Copyright (c) 2021-2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ setup( name='rapids_triton', description="Tools for clients to RAPIDS-Triton backends", - version='21.12.00', # TODO(wphicks): versioneer + version='22.02.00', # TODO(wphicks): versioneer author='NVIDIA Corporation', license='Apache', packages=find_packages(), From 4aac0054bafc5913dd5f2c44457802674da3e8ab Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 21 Jan 2022 12:43:50 -0500 Subject: [PATCH 151/199] Remove no-cache debug lines --- build.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/build.sh b/build.sh index 1b7f871..ab15d97 100755 --- a/build.sh +++ b/build.sh @@ -150,7 +150,6 @@ if [ $BACKEND -eq 1 ] then docker build \ $DOCKER_ARGS \ - --no-cache \ -t "$EXAMPLE_TAG" \ $REPODIR fi @@ -159,7 +158,6 @@ if [ $TESTS -eq 1 ] then docker build \ $DOCKER_ARGS \ - --no-cache \ -t "$EXAMPLE_TAG" \ --target test-stage \ -t "$TEST_TAG" \ From c77b633f3b6446c22bf7d926cd879fcf98ee9685 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 26 Jan 2022 19:23:21 -0500 Subject: [PATCH 152/199] Add asynchronous prediction to RAPIDS-Triton client --- Dockerfile | 5 ++ conda/environments/rapids_triton_test.yml | 1 - python/rapids_triton/client.py | 69 +++++++++++++++++++++++ qa/L0_e2e/test_model.py | 38 +++++++++++++ 4 files changed, 112 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5c73d4e..748dc6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -96,6 +96,11 @@ RUN conda env update -f /environment.yml \ && find /root/miniconda3/ -follow -type f -name '*.pyc' -delete \ && find /root/miniconda3/ -follow -type f -name '*.js.map' -delete +COPY ./python /rapids_triton + +RUN conda run -n rapids_triton_test pip install /rapids_triton \ + && rm -rf /rapids_triton + FROM build-stage as test-stage COPY --from=test-install /root/miniconda3 /root/miniconda3 diff --git a/conda/environments/rapids_triton_test.yml b/conda/environments/rapids_triton_test.yml index 7f78bff..4a86579 100644 --- a/conda/environments/rapids_triton_test.yml +++ b/conda/environments/rapids_triton_test.yml @@ -10,4 +10,3 @@ dependencies: - numpy - pip: - tritonclient[all] - - git+https://github.com/rapidsai/rapids-triton.git#subdirectory=python diff --git a/python/rapids_triton/client.py b/python/rapids_triton/client.py index 7a70a00..afa0fb6 100644 --- a/python/rapids_triton/client.py +++ b/python/rapids_triton/client.py @@ -13,6 +13,7 @@ # limitations under the License. import time +from concurrent.futures import Future from rapids_triton.triton.client import get_triton_client from rapids_triton.triton.io import create_triton_input, create_triton_output @@ -139,3 +140,71 @@ def predict( name=output_.name ) return result + + def predict_async( + self, + model_name, + input_data, + output_sizes, + model_version='1', + shared_mem=None, + attempts=1): + + model_version = str(model_version) + + inputs = [ + self.create_input( + arr, + name, + dtype_to_triton_name(arr.dtype), + shared_mem=shared_mem + ) + for name, arr in input_data.items() + ] + + outputs = { + name: self.create_output(size, name, shared_mem=shared_mem) + for name, size in output_sizes.items() + } + + future_result = Future() + def callback(result, error): + if error is None: + output_arrays = { + name: get_response_data(result, handle, name) + for name, (_, handle, _) in outputs.items() + } + + future_result.set_result(output_arrays) + + for input_ in inputs: + if input_.name is not None: + self.triton_client.unregister_cuda_shared_memory( + name=input_.name + ) + for output_ in outputs.values(): + if output_.name is not None: + self.triton_client.unregister_cuda_shared_memory( + name=output_.name + ) + else: + if isinstance(error, triton_utils.InferenceServerException): + if attempts > 1: + return self.predict( + model_name, + input_data, + output_sizes, + model_version=model_version, + shared_mem=shared_mem, + attempts=attempts - 1 + ) + future_result.set_exception(error) + + self.triton_client.async_infer( + model_name, + model_version=model_version, + inputs=[input_.input for input_ in inputs], + outputs=[output_.output for output_ in outputs.values()], + callback=callback + ) + return future_result diff --git a/qa/L0_e2e/test_model.py b/qa/L0_e2e/test_model.py index 9379ff5..08f98cc 100644 --- a/qa/L0_e2e/test_model.py +++ b/qa/L0_e2e/test_model.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import concurrent import os import numpy as np @@ -67,3 +68,40 @@ def test_model(client, model_name, shared_mem, model_inputs, model_output_sizes) atol=1e-5, assert_close=True ) + + +@pytest.mark.parametrize("model_name", ['identity']) +@pytest.mark.parametrize("shared_mem", valid_shm_modes()) +@pytest.mark.parametrize( + "batch_size", + [1, TOTAL_SAMPLES // 3, TOTAL_SAMPLES // 2] +) +def test_model_async(client, model_name, shared_mem, model_inputs, batch_size): + results = [] + gt_results = [] + for i in range( + 0, + TOTAL_SAMPLES // batch_size + int(bool(TOTAL_SAMPLES % batch_size)) + ): + min_index = i * batch_size + max_index = min((i + 1) * batch_size, TOTAL_SAMPLES) + cur_input = {name: arr[min_index: max_index] for name, arr in + model_inputs.items()} + cur_output_size = { + 'output__0': (max_index - min_index) * np.dtype('float32').itemsize + } + results.append(client.predict_async( + model_name, cur_input, cur_output_size, shared_mem=shared_mem + )) + gt_results.append(get_ground_truth(cur_input)) + concurrent.futures.wait(results, timeout=60) + results = [result_.result() for result_ in results] + + for result, ground_truth in zip(results, gt_results): + for output_name in sorted(ground_truth.keys()): + arrays_close( + result[output_name], + ground_truth[output_name], + atol=1e-5, + assert_close=True + ) From 081731322ce49e28b9d476ee278b51e02b671cbf Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 2 Feb 2022 14:51:44 -0500 Subject: [PATCH 153/199] Add multimodel evaluation helper --- python/rapids_triton/client.py | 208 ++++++++++++++-------- python/rapids_triton/exceptions.py | 18 ++ python/rapids_triton/triton/io.py | 39 +++- python/rapids_triton/utils/safe_import.py | 20 ++- 4 files changed, 201 insertions(+), 84 deletions(-) create mode 100644 python/rapids_triton/exceptions.py diff --git a/python/rapids_triton/client.py b/python/rapids_triton/client.py index afa0fb6..91c5ebd 100644 --- a/python/rapids_triton/client.py +++ b/python/rapids_triton/client.py @@ -12,24 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. +import concurrent.futures import time -from concurrent.futures import Future from rapids_triton.triton.client import get_triton_client -from rapids_triton.triton.io import create_triton_input, create_triton_output +from rapids_triton.triton.io import ( + create_triton_input, create_triton_output, destroy_shared_memory_region +) from rapids_triton.triton.dtype import dtype_to_triton_name from rapids_triton.triton.response import get_response_data from tritonclient import utils as triton_utils +# TODO(wphicks): Propagate device ids for cuda shared memory + + class Client(object): def __init__( self, protocol='grpc', host='localhost', port=None, - concurrency=4): + concurrency=4, + reuse_shared_memory=False): self.triton_client = get_triton_client( protocol=protocol, host=host, @@ -42,24 +48,29 @@ def __init__( def protocol(self): return self._protocol - def create_input(self, data, name, dtype, shared_mem=None): - return create_triton_input( - self.triton_client, - data, - name, - dtype, - protocol=self.protocol, - shared_mem=shared_mem - ) + def create_inputs(self, array_inputs, shared_mem=None): + return [ + create_triton_input( + self.triton_client, + arr, + name, + dtype_to_triton_name(arr.dtype), + protocol=self.protocol, + shared_mem=shared_mem + ) + for name, arr in array_inputs.items() + ] - def create_output(self, size, name, shared_mem=None): - return create_triton_output( - self.triton_client, - size, - name, - protocol=self.protocol, - shared_mem=shared_mem - ) + def create_outputs(self, output_sizes, shared_mem=None): + return { + name: create_triton_output( + self.triton_client, + size, + name, + protocol=self.protocol, + shared_mem=shared_mem + ) for name, size in output_sizes.items() + } def wait_for_server(self, timeout): server_wait_start = time.time() @@ -77,6 +88,16 @@ def clear_shared_memory(self): self.triton_client.unregister_cuda_shared_memory() self.triton_client.unregister_system_shared_memory() + def release_io(self, io_objs): + for io_ in io_objs: + if io_.name is not None: + self.triton_client.unregister_cuda_shared_memory( + name=io_.name + ) + destroy_shared_memory_region( + io_.handle, shared_mem='cuda' + ) + def get_model_config(self, model_name): return self.triton_client.get_model_config(model_name).config @@ -87,24 +108,13 @@ def predict( output_sizes, model_version='1', shared_mem=None, + device_id=0, attempts=1): model_version = str(model_version) try: - inputs = [ - self.create_input( - arr, - name, - dtype_to_triton_name(arr.dtype), - shared_mem=shared_mem - ) - for name, arr in input_data.items() - ] - - outputs = { - name: self.create_output(size, name, shared_mem=shared_mem) - for name, size in output_sizes.items() - } + inputs = self.create_inputs(input_data, shared_mem=shared_mem) + outputs = self.create_outputs(output_sizes, shared_mem=shared_mem) response = self.triton_client.infer( model_name, @@ -112,6 +122,12 @@ def predict( inputs=[input_.input for input_ in inputs], outputs=[output_.output for output_ in outputs.values()] ) + result = { + name: get_response_data(response, handle, name) + for name, (_, handle, _) in outputs.items() + } + self.release_io(inputs) + self.release_io(outputs.values()) except triton_utils.InferenceServerException: if attempts > 1: @@ -124,21 +140,6 @@ def predict( attempts=attempts - 1 ) raise - result = { - name: get_response_data(response, handle, name) - for name, (_, handle, _) in outputs.items() - } - - for input_ in inputs: - if input_.name is not None: - self.triton_client.unregister_cuda_shared_memory( - name=input_.name - ) - for output_ in outputs.values(): - if output_.name is not None: - self.triton_client.unregister_cuda_shared_memory( - name=output_.name - ) return result def predict_async( @@ -148,26 +149,15 @@ def predict_async( output_sizes, model_version='1', shared_mem=None, + device_id=0, attempts=1): model_version = str(model_version) - inputs = [ - self.create_input( - arr, - name, - dtype_to_triton_name(arr.dtype), - shared_mem=shared_mem - ) - for name, arr in input_data.items() - ] - - outputs = { - name: self.create_output(size, name, shared_mem=shared_mem) - for name, size in output_sizes.items() - } + inputs = self.create_inputs(input_data, shared_mem=shared_mem) + outputs = self.create_outputs(output_sizes, shared_mem=shared_mem) - future_result = Future() + future_result = concurrent.futures.Future() def callback(result, error): if error is None: output_arrays = { @@ -177,27 +167,18 @@ def callback(result, error): future_result.set_result(output_arrays) - for input_ in inputs: - if input_.name is not None: - self.triton_client.unregister_cuda_shared_memory( - name=input_.name - ) - for output_ in outputs.values(): - if output_.name is not None: - self.triton_client.unregister_cuda_shared_memory( - name=output_.name - ) + self.release_io(outputs.values()) else: if isinstance(error, triton_utils.InferenceServerException): if attempts > 1: - return self.predict( + future_result.set_result(self.predict( model_name, input_data, output_sizes, model_version=model_version, shared_mem=shared_mem, attempts=attempts - 1 - ) + )) future_result.set_exception(error) self.triton_client.async_infer( @@ -207,4 +188,79 @@ def callback(result, error): outputs=[output_.output for output_ in outputs.values()], callback=callback ) + + def release_callback(fut): + self.release_io(inputs) + + future_result.add_done_callback(release_callback) return future_result + + def predict_multimodel( + self, + model_names, + input_data, + output_sizes, + model_versions=('1',), + shared_mem=None, + device_ids=(0,), + executor=None, + attempts=1): + + all_models = [ + (name, str(version)) + for name in model_names + for version in model_versions + ] + + inputs = self.create_inputs(input_data, shared_mem=shared_mem) + + all_future_results = [] + for model_name, version in all_models: + outputs = self.create_outputs(output_sizes, shared_mem=shared_mem) + + def create_callback(future_result, outputs): + def callback(result, error): + if error is None: + output_arrays = { + name: get_response_data(result, handle, name) + for name, (_, handle, _) in outputs.items() + } + + future_result.set_result( + (model_name, version, output_arrays) + ) + + self.release_io(outputs.values()) + else: + if isinstance(error, triton_utils.InferenceServerException): + if attempts > 1: + future_result.set_result(self.predict( + model_name, + input_data, + output_sizes, + model_version=version, + shared_mem=shared_mem, + attempts=attempts - 1 + )) + future_result.set_exception(error) + return callback + + all_future_results.append(concurrent.futures.Future()) + self.triton_client.async_infer( + model_name, + model_version=version, + inputs=[input_.input for input_ in inputs], + outputs=[output_.output for output_ in outputs.values()], + callback=create_callback(all_future_results[-1], outputs) + ) + + def wait_for_all(future_results, releasable_inputs): + concurrent.futures.wait(future_results) + self.release_io(releasable_inputs) + return [fut.result() for fut in future_results] + + if executor is None: + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + return executor.submit(wait_for_all, all_future_results, inputs) + else: + return executor.submit(wait_for_all, all_future_results, inputs) diff --git a/python/rapids_triton/exceptions.py b/python/rapids_triton/exceptions.py new file mode 100644 index 0000000..dc7f8c5 --- /dev/null +++ b/python/rapids_triton/exceptions.py @@ -0,0 +1,18 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class IncompatibleSharedMemory(Exception): + """Error thrown if operation cannot be completed with given shared memory + type""" diff --git a/python/rapids_triton/triton/io.py b/python/rapids_triton/triton/io.py index 5126871..b86097d 100644 --- a/python/rapids_triton/triton/io.py +++ b/python/rapids_triton/triton/io.py @@ -18,6 +18,7 @@ import tritonclient.http as triton_http import tritonclient.grpc as triton_grpc from rapids_triton.utils.safe_import import ImportReplacement +from rapids_triton.exceptions import IncompatibleSharedMemory from tritonclient import utils as triton_utils try: import tritonclient.utils.cuda_shared_memory as shm @@ -25,7 +26,7 @@ shm = ImportReplacement('tritonclient.utils.cuda_shared_memory') -TritonInput = namedtuple('TritonInput', ('name', 'input')) +TritonInput = namedtuple('TritonInput', ('name', 'handle', 'input')) TritonOutput = namedtuple('TritonOutput', ('name', 'handle', 'output')) def set_unshared_input_data(triton_input, data, protocol='grpc'): @@ -34,7 +35,7 @@ def set_unshared_input_data(triton_input, data, protocol='grpc'): else: triton_input.set_data_from_numpy(data, binary_data=True) - return None + return TritonInput(None, None, triton_input) def set_shared_input_data(triton_client, triton_input, data, protocol='grpc'): @@ -54,7 +55,7 @@ def set_shared_input_data(triton_client, triton_input, data, protocol='grpc'): triton_input.set_shared_memory(input_name, input_size) - return input_name + return TritonInput(input_name, input_handle, triton_input) def set_input_data( @@ -81,7 +82,7 @@ def create_triton_input( else: triton_input = triton_http.InferInput(name, data.shape, dtype) - input_name = set_input_data( + return set_input_data( triton_client, triton_input, data, @@ -89,8 +90,6 @@ def create_triton_input( shared_mem=shared_mem ) - return TritonInput(name=input_name, input=triton_input) - def create_output_handle(triton_client, triton_output, size, shared_mem=None): if shared_mem is None: @@ -141,3 +140,31 @@ def create_triton_output( handle=output_handle, output=triton_output ) + + +def destroy_shared_memory_region(handle, shared_mem='cuda'): + """Release memory from a given shared memory handle + + Parameters + ---------- + handle : c_void_p + The handle (as returned by the Triton client) for the region to be + released. + shared_mem : 'cuda' or 'system' or None + The type of shared memory region to release. If None, an exception will + be thrown. + """ + if shared_mem is None: + raise IncompatibleSharedMemory( + "Attempting to release non-shared memory" + ) + elif shared_mem == 'system': + raise NotImplementedError( + "System shared memory not yet supported" + ) + elif shared_mem == 'cuda': + shm.destroy_shared_memory_region(handle) + else: + raise NotImplementedError( + f"Unrecognized memory type {shared_mem}" + ) diff --git a/python/rapids_triton/utils/safe_import.py b/python/rapids_triton/utils/safe_import.py index e29e263..4e2317c 100644 --- a/python/rapids_triton/utils/safe_import.py +++ b/python/rapids_triton/utils/safe_import.py @@ -1,6 +1,22 @@ -class UnavailableError(Exception): +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class ImportUnavailableError(Exception): '''Error thrown if a symbol is unavailable due to an issue importing it''' + class ImportReplacement: """A class to be used in place of an importable symbol if that symbol cannot be imported @@ -19,4 +35,4 @@ def __getattr__(self, name): raise UnavailableError(self._msg) def __call__(self, *args, **kwargs): - raise UnavailableError(self._msg) + raise ImportUnavailableError(self._msg) From 037770e36e2c3d63f0c274217ce0363fbdf2d314 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 2 Feb 2022 15:41:56 -0500 Subject: [PATCH 154/199] Add multimodel tests --- python/rapids_triton/client.py | 7 ++++- .../identity/config.pbtxt | 1 + .../model_repository/identity/config.pbtxt | 1 + qa/L0_e2e/test_model.py | 28 ++++++++++++++++++- 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/python/rapids_triton/client.py b/python/rapids_triton/client.py index 91c5ebd..51f2cbd 100644 --- a/python/rapids_triton/client.py +++ b/python/rapids_triton/client.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from collections import namedtuple import concurrent.futures import time @@ -27,6 +28,8 @@ # TODO(wphicks): Propagate device ids for cuda shared memory +MultiModelOutput = namedtuple('MultiModelOutput', ('name', 'version', 'output')) + class Client(object): def __init__( @@ -227,7 +230,9 @@ def callback(result, error): } future_result.set_result( - (model_name, version, output_arrays) + MultiModelOutput( + name=model_name, version=version, output=output_arrays + ) ) self.release_io(outputs.values()) diff --git a/qa/L0_e2e/cpu_model_repository/identity/config.pbtxt b/qa/L0_e2e/cpu_model_repository/identity/config.pbtxt index 5479199..33f4eed 100644 --- a/qa/L0_e2e/cpu_model_repository/identity/config.pbtxt +++ b/qa/L0_e2e/cpu_model_repository/identity/config.pbtxt @@ -14,6 +14,7 @@ output [ dims: [ 1 ] } ] +version_policy: { all { }} instance_group [{ kind: KIND_CPU }] parameters [ ] dynamic_batching { diff --git a/qa/L0_e2e/model_repository/identity/config.pbtxt b/qa/L0_e2e/model_repository/identity/config.pbtxt index 5fcba28..572434a 100644 --- a/qa/L0_e2e/model_repository/identity/config.pbtxt +++ b/qa/L0_e2e/model_repository/identity/config.pbtxt @@ -14,6 +14,7 @@ output [ dims: [ 1 ] } ] +version_policy: { all { }} instance_group [{ kind: KIND_GPU }] parameters [ ] dynamic_batching { diff --git a/qa/L0_e2e/test_model.py b/qa/L0_e2e/test_model.py index 08f98cc..35aa2f5 100644 --- a/qa/L0_e2e/test_model.py +++ b/qa/L0_e2e/test_model.py @@ -30,6 +30,10 @@ def valid_shm_modes(): return modes +@pytest.fixture(scope='session') +def model_versions(): + return (1, 2) + @pytest.fixture(scope='session') def client(): client = Client() @@ -76,7 +80,7 @@ def test_model(client, model_name, shared_mem, model_inputs, model_output_sizes) "batch_size", [1, TOTAL_SAMPLES // 3, TOTAL_SAMPLES // 2] ) -def test_model_async(client, model_name, shared_mem, model_inputs, batch_size): +def test_predict_async(client, model_name, shared_mem, model_inputs, batch_size): results = [] gt_results = [] for i in range( @@ -105,3 +109,25 @@ def test_model_async(client, model_name, shared_mem, model_inputs, batch_size): atol=1e-5, assert_close=True ) + + +@pytest.mark.parametrize("model_name", ['identity']) +@pytest.mark.parametrize("shared_mem", valid_shm_modes()) +def test_predict_multimodel( + client, model_name, shared_mem, model_inputs, model_output_sizes, + model_versions): + all_results = client.predict_multimodel( + [model_name], model_inputs, model_output_sizes, + model_versions=model_versions, shared_mem=shared_mem + ) + ground_truth = get_ground_truth(model_inputs) + + all_results = all_results.result(timeout=60) + for result in all_results: + for output_name in sorted(ground_truth.keys()): + arrays_close( + result.output[output_name], + ground_truth[output_name], + atol=1e-5, + assert_close=True + ) From c44ff36890b57f701c6c0051b1ac2febb0572643 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 2 Feb 2022 15:50:02 -0500 Subject: [PATCH 155/199] Add placeholder files for model directories --- qa/L0_e2e/cpu_model_repository/identity/1/empty.model | 0 qa/L0_e2e/cpu_model_repository/identity/2/empty.model | 0 qa/L0_e2e/model_repository/identity/1/empty.model | 0 qa/L0_e2e/model_repository/identity/2/empty.model | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 qa/L0_e2e/cpu_model_repository/identity/1/empty.model create mode 100644 qa/L0_e2e/cpu_model_repository/identity/2/empty.model create mode 100644 qa/L0_e2e/model_repository/identity/1/empty.model create mode 100644 qa/L0_e2e/model_repository/identity/2/empty.model diff --git a/qa/L0_e2e/cpu_model_repository/identity/1/empty.model b/qa/L0_e2e/cpu_model_repository/identity/1/empty.model new file mode 100644 index 0000000..e69de29 diff --git a/qa/L0_e2e/cpu_model_repository/identity/2/empty.model b/qa/L0_e2e/cpu_model_repository/identity/2/empty.model new file mode 100644 index 0000000..e69de29 diff --git a/qa/L0_e2e/model_repository/identity/1/empty.model b/qa/L0_e2e/model_repository/identity/1/empty.model new file mode 100644 index 0000000..e69de29 diff --git a/qa/L0_e2e/model_repository/identity/2/empty.model b/qa/L0_e2e/model_repository/identity/2/empty.model new file mode 100644 index 0000000..e69de29 From af10cf7c4d9b51449755130ff125906951f05fd0 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 2 Feb 2022 15:57:42 -0500 Subject: [PATCH 156/199] Remove unused parameter --- python/rapids_triton/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/rapids_triton/client.py b/python/rapids_triton/client.py index 51f2cbd..9906ddd 100644 --- a/python/rapids_triton/client.py +++ b/python/rapids_triton/client.py @@ -37,8 +37,7 @@ def __init__( protocol='grpc', host='localhost', port=None, - concurrency=4, - reuse_shared_memory=False): + concurrency=4) self.triton_client = get_triton_client( protocol=protocol, host=host, From 5bc516479ed8f326cf15e9d85c93f35d14b7f61c Mon Sep 17 00:00:00 2001 From: William Hicks Date: Wed, 2 Feb 2022 16:07:11 -0500 Subject: [PATCH 157/199] Remove unused device_id parameters --- python/rapids_triton/client.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/rapids_triton/client.py b/python/rapids_triton/client.py index 9906ddd..c2bbba7 100644 --- a/python/rapids_triton/client.py +++ b/python/rapids_triton/client.py @@ -110,7 +110,6 @@ def predict( output_sizes, model_version='1', shared_mem=None, - device_id=0, attempts=1): model_version = str(model_version) @@ -151,7 +150,6 @@ def predict_async( output_sizes, model_version='1', shared_mem=None, - device_id=0, attempts=1): model_version = str(model_version) @@ -204,7 +202,6 @@ def predict_multimodel( output_sizes, model_versions=('1',), shared_mem=None, - device_ids=(0,), executor=None, attempts=1): From 34c9def4c3737f029da29599b378bd54921bdc25 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 4 Feb 2022 15:19:42 -0500 Subject: [PATCH 158/199] Only add callback when necessary to release I/O --- python/rapids_triton/client.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/rapids_triton/client.py b/python/rapids_triton/client.py index c2bbba7..8136d78 100644 --- a/python/rapids_triton/client.py +++ b/python/rapids_triton/client.py @@ -189,10 +189,11 @@ def callback(result, error): callback=callback ) - def release_callback(fut): - self.release_io(inputs) + if shared_mem is not None: + def release_callback(fut): + self.release_io(inputs) - future_result.add_done_callback(release_callback) + future_result.add_done_callback(release_callback) return future_result def predict_multimodel( From 775eadcd9c9d0574c6a6e5496e34d446a3068e0e Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 4 Feb 2022 15:33:21 -0500 Subject: [PATCH 159/199] Fix syntax error --- python/rapids_triton/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/rapids_triton/client.py b/python/rapids_triton/client.py index 8136d78..5fdb480 100644 --- a/python/rapids_triton/client.py +++ b/python/rapids_triton/client.py @@ -37,7 +37,7 @@ def __init__( protocol='grpc', host='localhost', port=None, - concurrency=4) + concurrency=4): self.triton_client = get_triton_client( protocol=protocol, host=host, From 44c57a85045cbe3bbe898643ce4a07809a31e3b2 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 7 Feb 2022 17:11:05 -0500 Subject: [PATCH 160/199] Update python/rapids_triton/utils/safe_import.py Co-authored-by: Micka <9810050+lowener@users.noreply.github.com> --- python/rapids_triton/utils/safe_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/rapids_triton/utils/safe_import.py b/python/rapids_triton/utils/safe_import.py index 4e2317c..1738600 100644 --- a/python/rapids_triton/utils/safe_import.py +++ b/python/rapids_triton/utils/safe_import.py @@ -32,7 +32,7 @@ def __init__(self, symbol): self._msg = f'{symbol} could not be imported' def __getattr__(self, name): - raise UnavailableError(self._msg) + raise ImportUnavailableError(self._msg) def __call__(self, *args, **kwargs): raise ImportUnavailableError(self._msg) From 301cdd5d955c794b734c53d7c2ba9c6f9282f1f2 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 7 Feb 2022 17:12:04 -0500 Subject: [PATCH 161/199] Update function name to reflect asynchronicity --- python/rapids_triton/client.py | 2 +- qa/L0_e2e/test_model.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/rapids_triton/client.py b/python/rapids_triton/client.py index 5fdb480..5da497b 100644 --- a/python/rapids_triton/client.py +++ b/python/rapids_triton/client.py @@ -196,7 +196,7 @@ def release_callback(fut): future_result.add_done_callback(release_callback) return future_result - def predict_multimodel( + def predict_multimodel_async( self, model_names, input_data, diff --git a/qa/L0_e2e/test_model.py b/qa/L0_e2e/test_model.py index 35aa2f5..cd13d62 100644 --- a/qa/L0_e2e/test_model.py +++ b/qa/L0_e2e/test_model.py @@ -113,10 +113,10 @@ def test_predict_async(client, model_name, shared_mem, model_inputs, batch_size) @pytest.mark.parametrize("model_name", ['identity']) @pytest.mark.parametrize("shared_mem", valid_shm_modes()) -def test_predict_multimodel( +def test_predict_multimodel_async( client, model_name, shared_mem, model_inputs, model_output_sizes, model_versions): - all_results = client.predict_multimodel( + all_results = client.predict_multimodel_async( [model_name], model_inputs, model_output_sizes, model_versions=model_versions, shared_mem=shared_mem ) From 43771e90894e478d050321008a3bf2eecd3d0778 Mon Sep 17 00:00:00 2001 From: divyegala Date: Tue, 8 Feb 2022 19:11:07 -0800 Subject: [PATCH 162/199] Initial code --- cpp/include/rapids_triton/model/shared_state.hpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cpp/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp index a3826ee..9ee7d15 100644 --- a/cpp/include/rapids_triton/model/shared_state.hpp +++ b/cpp/include/rapids_triton/model/shared_state.hpp @@ -119,6 +119,22 @@ struct SharedModelState { } } + auto get_output_names() const { + auto output_names = std::vector{output_shapes_.size()}; + std::for_each(std::begin(output_shapes_), std::end(output_shapes_), [&output_names](auto& output_shape) { + output_names.push_back(output_shape.first); + }); + return output_names; + } + + auto check_output_name(std::string const& name) const -> bool { + auto cached_shape = std::lower_bound( + std::begin(output_shapes_), std::end(output_shapes_), name, [](auto& entry, auto& value) { + return entry.first < value; + }); + return cached_shape != std::end(output_shapes_); + } + private: std::unique_ptr config_; Batch::size_type max_batch_size_; From 8631a187b9ca5f522e668c8a4529a147814f34e1 Mon Sep 17 00:00:00 2001 From: Divye Gala Date: Tue, 22 Feb 2022 12:27:19 -0500 Subject: [PATCH 163/199] Apply suggestions from code review Co-authored-by: William Hicks --- cpp/include/rapids_triton/model/shared_state.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp index 9ee7d15..3aa2a12 100644 --- a/cpp/include/rapids_triton/model/shared_state.hpp +++ b/cpp/include/rapids_triton/model/shared_state.hpp @@ -120,19 +120,19 @@ struct SharedModelState { } auto get_output_names() const { - auto output_names = std::vector{output_shapes_.size()}; - std::for_each(std::begin(output_shapes_), std::end(output_shapes_), [&output_names](auto& output_shape) { - output_names.push_back(output_shape.first); + auto output_names = std::vector{}; + output_names.reserve(output_shapes_.size()); + std::transform(std::begin(output_shapes_), std::end(output_shapes_), std::back_inserter(output_names), [](auto& output_shape) { + return output_shape.first; }); return output_names; } auto check_output_name(std::string const& name) const -> bool { - auto cached_shape = std::lower_bound( + return std::binary_search( std::begin(output_shapes_), std::end(output_shapes_), name, [](auto& entry, auto& value) { return entry.first < value; }); - return cached_shape != std::end(output_shapes_); } private: From 60f5e8b9816771ac63fa8c1c102ef8ae5d0e5ebf Mon Sep 17 00:00:00 2001 From: divyegala Date: Tue, 22 Feb 2022 10:07:23 -0800 Subject: [PATCH 164/199] name check in get output shape --- cpp/include/rapids_triton/model/shared_state.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp index 9ee7d15..5f7dbaf 100644 --- a/cpp/include/rapids_triton/model/shared_state.hpp +++ b/cpp/include/rapids_triton/model/shared_state.hpp @@ -110,7 +110,7 @@ struct SharedModelState { std::begin(output_shapes_), std::end(output_shapes_), name, [](auto& entry, auto& value) { return entry.first < value; }); - if (cached_shape == std::end(output_shapes_)) { + if (cached_shape == std::end(output_shapes_) || name != cached_shape->first) { auto log_stream = std::stringstream{}; log_stream << "No output with name " << name << " in configuration."; throw TritonException(Error::Internal, log_stream.str()); From e18e9f4a6d357ba3fe4e49ca8e91a27e8bfc431f Mon Sep 17 00:00:00 2001 From: divyegala Date: Tue, 22 Feb 2022 15:14:19 -0800 Subject: [PATCH 165/199] temporarily using lower bound instead of binary search --- cpp/include/rapids_triton/model/shared_state.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp index 16707b8..9bbf47d 100644 --- a/cpp/include/rapids_triton/model/shared_state.hpp +++ b/cpp/include/rapids_triton/model/shared_state.hpp @@ -129,10 +129,11 @@ struct SharedModelState { } auto check_output_name(std::string const& name) const -> bool { - return std::binary_search( + auto cached_shape = std::lower_bound( std::begin(output_shapes_), std::end(output_shapes_), name, [](auto& entry, auto& value) { return entry.first < value; }); + return cached_shape != std::end(output_shapes_) && name == cached_shape->first; } private: From 3bd5f1e641038a6d6f856bca5d9494f4f91b5089 Mon Sep 17 00:00:00 2001 From: divyegala Date: Thu, 24 Feb 2022 10:00:58 -0800 Subject: [PATCH 166/199] review comments --- cpp/include/rapids_triton/model/shared_state.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/include/rapids_triton/model/shared_state.hpp b/cpp/include/rapids_triton/model/shared_state.hpp index 9bbf47d..aa0d76d 100644 --- a/cpp/include/rapids_triton/model/shared_state.hpp +++ b/cpp/include/rapids_triton/model/shared_state.hpp @@ -128,7 +128,8 @@ struct SharedModelState { return output_names; } - auto check_output_name(std::string const& name) const -> bool { + auto check_output_name(std::string const& name) const { + // #TODO: Figure out a way to use std::binary_search here auto cached_shape = std::lower_bound( std::begin(output_shapes_), std::end(output_shapes_), name, [](auto& entry, auto& value) { return entry.first < value; From 3f61d1aa9bf78109a1320516b108b9a2abac99af Mon Sep 17 00:00:00 2001 From: GuanLuo Date: Wed, 31 Aug 2022 10:25:53 -0700 Subject: [PATCH 167/199] Upstream rapid-json --- backend/.clang-format | 164 +++++++++ backend/CMakeLists.txt | 218 ++++++++++++ backend/cmake/doxygen.cmake | 33 ++ backend/cmake/modules/ConfigureCUDA.cmake | 43 +++ backend/cmake/thirdparty/get_gtest.cmake | 22 ++ backend/cmake/thirdparty/get_raft.cmake | 49 +++ backend/cmake/thirdparty/get_rapidjson.cmake | 37 ++ backend/cmake/thirdparty/get_rmm.cmake | 22 ++ backend/cmake/thirdparty/get_triton.cmake | 36 ++ backend/include/rapids_triton.hpp | 31 ++ backend/include/rapids_triton/batch/batch.hpp | 273 +++++++++++++++ .../include/rapids_triton/build_control.hpp | 32 ++ .../cpu_only/cuda_runtime_replacement.hpp | 54 +++ backend/include/rapids_triton/exceptions.hpp | 94 ++++++ .../include/rapids_triton/memory/buffer.hpp | 317 ++++++++++++++++++ .../memory/detail/cpu_only/copy.hpp | 50 +++ .../detail/cpu_only/owned_device_buffer.hpp | 45 +++ .../memory/detail/cpu_only/resource.hpp | 35 ++ .../memory/detail/gpu_only/copy.hpp | 55 +++ .../detail/gpu_only/owned_device_buffer.hpp | 49 +++ .../memory/detail/gpu_only/resource.hpp | 97 ++++++ .../memory/detail/owned_device_buffer.hpp | 30 ++ .../rapids_triton/memory/detail/resource.hpp | 35 ++ .../include/rapids_triton/memory/resource.hpp | 39 +++ .../include/rapids_triton/memory/types.hpp | 28 ++ backend/include/rapids_triton/model/model.hpp | 197 +++++++++++ .../rapids_triton/model/shared_state.hpp | 189 +++++++++++ .../include/rapids_triton/tensor/dtype.hpp | 168 ++++++++++ .../include/rapids_triton/tensor/tensor.hpp | 205 +++++++++++ .../rapids_triton/triton/api/execute.hpp | 121 +++++++ .../rapids_triton/triton/api/initialize.hpp | 69 ++++ .../triton/api/instance_finalize.hpp | 49 +++ .../triton/api/instance_initialize.hpp | 74 ++++ .../triton/api/model_finalize.hpp | 48 +++ .../triton/api/model_initialize.hpp | 53 +++ .../include/rapids_triton/triton/backend.hpp | 61 ++++ .../include/rapids_triton/triton/config.hpp | 37 ++ .../rapids_triton/triton/deployment.hpp | 31 ++ .../include/rapids_triton/triton/device.hpp | 26 ++ .../include/rapids_triton/triton/input.hpp | 80 +++++ .../include/rapids_triton/triton/logging.hpp | 159 +++++++++ .../include/rapids_triton/triton/model.hpp | 90 +++++ .../rapids_triton/triton/model_instance.hpp | 97 ++++++ .../triton/model_instance_state.hpp | 54 +++ .../rapids_triton/triton/model_state.hpp | 44 +++ .../include/rapids_triton/triton/output.hpp | 72 ++++ .../include/rapids_triton/triton/requests.hpp | 43 +++ .../rapids_triton/triton/responses.hpp | 75 +++++ .../rapids_triton/triton/statistics.hpp | 89 +++++ .../triton/triton_memory_resource.hpp | 86 +++++ .../rapids_triton/utils/const_agnostic.hpp | 30 ++ .../rapids_triton/utils/device_setter.hpp | 50 +++ .../include/rapids_triton/utils/narrow.hpp | 41 +++ backend/scripts/run-clang-format.py | 148 ++++++++ backend/src/CMakeLists.txt | 68 ++++ backend/src/api.cc | 80 +++++ backend/src/model.h | 156 +++++++++ backend/src/names.h | 29 ++ backend/src/shared_state.h | 50 +++ backend/test/CMakeLists.txt | 102 ++++++ backend/test/batch/batch.cpp | 17 + backend/test/build_control.cpp | 35 ++ backend/test/exceptions.cpp | 83 +++++ backend/test/memory/buffer.cpp | 178 ++++++++++ backend/test/memory/detail/copy.cpp | 62 ++++ .../memory/detail/owned_device_buffer.cpp | 64 ++++ backend/test/memory/resource.cpp | 52 +++ backend/test/memory/types.cpp | 17 + backend/test/tensor/dtype.cpp | 51 +++ backend/test/tensor/tensor.cpp | 145 ++++++++ backend/test/test.cpp | 28 ++ backend/test/triton/api/execute.cpp | 17 + backend/test/triton/api/initialize.cpp | 17 + backend/test/triton/api/instance_finalize.cpp | 17 + .../test/triton/api/instance_initialize.cpp | 17 + backend/test/triton/api/model_finalize.cpp | 17 + backend/test/triton/api/model_initialize.cpp | 17 + backend/test/triton/backend.cpp | 17 + backend/test/triton/config.cpp | 17 + backend/test/triton/deployment.cpp | 17 + backend/test/triton/device.cpp | 17 + backend/test/triton/input.cpp | 17 + backend/test/triton/logging.cpp | 43 +++ backend/test/triton/model.cpp | 17 + backend/test/triton/model_instance.cpp | 17 + backend/test/triton/requests.cpp | 17 + backend/test/triton/responses.cpp | 17 + backend/test/triton/statistics.cpp | 17 + backend/test/utils/const_agnostic.cpp | 33 ++ backend/test/utils/narrow.cpp | 35 ++ 90 files changed, 5974 insertions(+) create mode 100644 backend/.clang-format create mode 100644 backend/CMakeLists.txt create mode 100644 backend/cmake/doxygen.cmake create mode 100644 backend/cmake/modules/ConfigureCUDA.cmake create mode 100644 backend/cmake/thirdparty/get_gtest.cmake create mode 100644 backend/cmake/thirdparty/get_raft.cmake create mode 100644 backend/cmake/thirdparty/get_rapidjson.cmake create mode 100644 backend/cmake/thirdparty/get_rmm.cmake create mode 100644 backend/cmake/thirdparty/get_triton.cmake create mode 100644 backend/include/rapids_triton.hpp create mode 100644 backend/include/rapids_triton/batch/batch.hpp create mode 100644 backend/include/rapids_triton/build_control.hpp create mode 100644 backend/include/rapids_triton/cpu_only/cuda_runtime_replacement.hpp create mode 100644 backend/include/rapids_triton/exceptions.hpp create mode 100644 backend/include/rapids_triton/memory/buffer.hpp create mode 100644 backend/include/rapids_triton/memory/detail/cpu_only/copy.hpp create mode 100644 backend/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp create mode 100644 backend/include/rapids_triton/memory/detail/cpu_only/resource.hpp create mode 100644 backend/include/rapids_triton/memory/detail/gpu_only/copy.hpp create mode 100644 backend/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp create mode 100644 backend/include/rapids_triton/memory/detail/gpu_only/resource.hpp create mode 100644 backend/include/rapids_triton/memory/detail/owned_device_buffer.hpp create mode 100644 backend/include/rapids_triton/memory/detail/resource.hpp create mode 100644 backend/include/rapids_triton/memory/resource.hpp create mode 100644 backend/include/rapids_triton/memory/types.hpp create mode 100644 backend/include/rapids_triton/model/model.hpp create mode 100644 backend/include/rapids_triton/model/shared_state.hpp create mode 100644 backend/include/rapids_triton/tensor/dtype.hpp create mode 100644 backend/include/rapids_triton/tensor/tensor.hpp create mode 100644 backend/include/rapids_triton/triton/api/execute.hpp create mode 100644 backend/include/rapids_triton/triton/api/initialize.hpp create mode 100644 backend/include/rapids_triton/triton/api/instance_finalize.hpp create mode 100644 backend/include/rapids_triton/triton/api/instance_initialize.hpp create mode 100644 backend/include/rapids_triton/triton/api/model_finalize.hpp create mode 100644 backend/include/rapids_triton/triton/api/model_initialize.hpp create mode 100644 backend/include/rapids_triton/triton/backend.hpp create mode 100644 backend/include/rapids_triton/triton/config.hpp create mode 100644 backend/include/rapids_triton/triton/deployment.hpp create mode 100644 backend/include/rapids_triton/triton/device.hpp create mode 100644 backend/include/rapids_triton/triton/input.hpp create mode 100644 backend/include/rapids_triton/triton/logging.hpp create mode 100644 backend/include/rapids_triton/triton/model.hpp create mode 100644 backend/include/rapids_triton/triton/model_instance.hpp create mode 100644 backend/include/rapids_triton/triton/model_instance_state.hpp create mode 100644 backend/include/rapids_triton/triton/model_state.hpp create mode 100644 backend/include/rapids_triton/triton/output.hpp create mode 100644 backend/include/rapids_triton/triton/requests.hpp create mode 100644 backend/include/rapids_triton/triton/responses.hpp create mode 100644 backend/include/rapids_triton/triton/statistics.hpp create mode 100644 backend/include/rapids_triton/triton/triton_memory_resource.hpp create mode 100644 backend/include/rapids_triton/utils/const_agnostic.hpp create mode 100644 backend/include/rapids_triton/utils/device_setter.hpp create mode 100644 backend/include/rapids_triton/utils/narrow.hpp create mode 100755 backend/scripts/run-clang-format.py create mode 100644 backend/src/CMakeLists.txt create mode 100644 backend/src/api.cc create mode 100644 backend/src/model.h create mode 100644 backend/src/names.h create mode 100644 backend/src/shared_state.h create mode 100644 backend/test/CMakeLists.txt create mode 100644 backend/test/batch/batch.cpp create mode 100644 backend/test/build_control.cpp create mode 100644 backend/test/exceptions.cpp create mode 100644 backend/test/memory/buffer.cpp create mode 100644 backend/test/memory/detail/copy.cpp create mode 100644 backend/test/memory/detail/owned_device_buffer.cpp create mode 100644 backend/test/memory/resource.cpp create mode 100644 backend/test/memory/types.cpp create mode 100644 backend/test/tensor/dtype.cpp create mode 100644 backend/test/tensor/tensor.cpp create mode 100644 backend/test/test.cpp create mode 100644 backend/test/triton/api/execute.cpp create mode 100644 backend/test/triton/api/initialize.cpp create mode 100644 backend/test/triton/api/instance_finalize.cpp create mode 100644 backend/test/triton/api/instance_initialize.cpp create mode 100644 backend/test/triton/api/model_finalize.cpp create mode 100644 backend/test/triton/api/model_initialize.cpp create mode 100644 backend/test/triton/backend.cpp create mode 100644 backend/test/triton/config.cpp create mode 100644 backend/test/triton/deployment.cpp create mode 100644 backend/test/triton/device.cpp create mode 100644 backend/test/triton/input.cpp create mode 100644 backend/test/triton/logging.cpp create mode 100644 backend/test/triton/model.cpp create mode 100644 backend/test/triton/model_instance.cpp create mode 100644 backend/test/triton/requests.cpp create mode 100644 backend/test/triton/responses.cpp create mode 100644 backend/test/triton/statistics.cpp create mode 100644 backend/test/utils/const_agnostic.cpp create mode 100644 backend/test/utils/narrow.cpp diff --git a/backend/.clang-format b/backend/.clang-format new file mode 100644 index 0000000..0c05436 --- /dev/null +++ b/backend/.clang-format @@ -0,0 +1,164 @@ +--- +# Refer to the following link for the explanation of each params: +# http://releases.llvm.org/8.0.0/tools/clang/docs/ClangFormatStyleOptions.html +Language: Cpp +# BasedOnStyle: Google +AccessModifierOffset: -1 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: true +AlignConsecutiveBitFields: true +AlignConsecutiveDeclarations: false +AlignConsecutiveMacros: true +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: true +AllowShortCaseLabelsOnASingleLine: true +AllowShortEnumsOnASingleLine: true +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: true +AllowShortLambdasOnASingleLine: true +AllowShortLoopsOnASingleLine: false +# This is deprecated +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + # disabling the below splits, else, they'll just add to the vertical length of source files! + SplitEmptyFunction: false + SplitEmptyRecord: false + SplitEmptyNamespace: false +BreakAfterJavaFieldAnnotations: false +BreakBeforeBinaryOperators: None +BreakBeforeBraces: WebKit +BreakBeforeInheritanceComma: false +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakInheritanceList: BeforeColon +BreakStringLiterals: true +ColumnLimit: 100 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +# Kept the below 2 to be the same as `IndentWidth` to keep everything uniform +ConstructorInitializerIndentWidth: 2 +ContinuationIndentWidth: 2 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^' + Priority: 2 + - Regex: '^<.*\.h>' + Priority: 1 + - Regex: '^<.*' + Priority: 2 + - Regex: '.*' + Priority: 3 +IncludeIsMainRegex: '([-_](test|unittest))?$' +IndentCaseLabels: true +IndentPPDirectives: None +IndentWidth: 2 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Never +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +RawStringFormats: + - Language: Cpp + Delimiters: + - cc + - CC + - cpp + - Cpp + - CPP + - 'c++' + - 'C++' + CanonicalDelimiter: '' + - Language: TextProto + Delimiters: + - pb + - PB + - proto + - PROTO + EnclosingFunctions: + - EqualsProto + - EquivToProto + - PARSE_PARTIAL_TEXT_PROTO + - PARSE_TEST_PROTO + - PARSE_TEXT_PROTO + - ParseTextOrDie + - ParseTextProtoOrDie + CanonicalDelimiter: '' + BasedOnStyle: google +# Enabling comment reflow causes doxygen comments to be messed up in their formats! +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceBeforeSquareBrackets: false +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: c++17 +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +# Be consistent with indent-width, even for people who use tab for indentation! +TabWidth: 2 +UseTab: Never diff --git a/backend/CMakeLists.txt b/backend/CMakeLists.txt new file mode 100644 index 0000000..136e032 --- /dev/null +++ b/backend/CMakeLists.txt @@ -0,0 +1,218 @@ +#============================================================================= +# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +cmake_minimum_required(VERSION 3.21 FATAL_ERROR) +file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.02/RAPIDS.cmake + ${CMAKE_BINARY_DIR}/RAPIDS.cmake) +include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) +include(rapids-cmake) +include(rapids-cpm) +include(rapids-cuda) +include(rapids-export) +include(rapids-find) + +############################################################################## +# - User Options ------------------------------------------------------------ + +option(TRITON_ENABLE_GPU "Enable GPU support in Triton" ON) +option(BUILD_TESTS "Build rapids_triton unit-tests" ON) +option(BUILD_EXAMPLE "Build rapids_identity example backend" OFF) +option(CUDA_ENABLE_KERNELINFO "Enable kernel resource usage info" OFF) +option(CUDA_ENABLE_LINEINFO "Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler)" OFF) +option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF) +option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" ON) +option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) +option(NVTX "Enable nvtx markers" OFF) +option(TRITON_ENABLE_STATS "Enable statistics collection in Triton" ON) +set(TRITON_COMMON_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-server/common repo") +set(TRITON_CORE_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-server/core repo") +set(TRITON_BACKEND_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-server/backend repo") + +message(VERBOSE "RAPIDS_TRITON: Build RAPIDS_TRITON unit-tests: ${BUILD_TESTS}") +message(VERBOSE "RAPIDS_TRITON: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") +message(VERBOSE "RAPIDS_TRITON: Disable depreaction warnings " ${DISABLE_DEPRECATION_WARNINGS}) +message(VERBOSE "RAPIDS_TRITON: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELINFO}") +message(VERBOSE "RAPIDS_TRITON: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") +message(VERBOSE "RAPIDS_TRITON: Enable nvtx markers: ${NVTX}") +message(VERBOSE "RAPIDS_TRITON: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") +message(VERBOSE "RAPIDS_TRITON: Enable GPU support: ${TRITON_ENABLE_GPU}") +message(VERBOSE "RAPIDS_TRITON: Enable statistics collection in Triton: ${TRITON_ENABLE_STATS}") +message(VERBOSE "RAPIDS_TRITON: Triton common repo tag: ${TRITON_COMMON_REPO_TAG}") +message(VERBOSE "RAPIDS_TRITON: Triton core repo tag: ${TRITON_CORE_REPO_TAG}") +message(VERBOSE "RAPIDS_TRITON: Triton backend repo tag: ${TRITON_BACKEND_REPO_TAG}") + +############################################################################## +# - Project Initialization --------------------------------------------------- + +if(TRITON_ENABLE_GPU) + rapids_cuda_init_architectures(RAPIDS_TRITON) + project(RAPIDS_TRITON VERSION 22.02.00 LANGUAGES CXX CUDA) +else() + project(RAPIDS_TRITON VERSION 22.02.00 LANGUAGES CXX) +endif() + + +############################################################################## +# - build type --------------------------------------------------------------- + +# Set a default build type if none was specified +rapids_cmake_build_type(Release) + +# this is needed for clang-tidy runs +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +# Set RMM logging level +set(RMM_LOGGING_LEVEL "INFO" CACHE STRING "Choose the logging level.") +set_property(CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF") +message(VERBOSE "RAPIDS_TRITON: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") + +############################################################################## +# - Conda environment detection ---------------------------------------------- + +if(DETECT_CONDA_ENV) + rapids_cmake_support_conda_env( conda_env MODIFY_PREFIX_PATH ) + if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED ENV{CONDA_PREFIX}) + message(STATUS "RAPIDS_TRITON: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}") + set(CMAKE_INSTALL_PREFIX "$ENV{CONDA_PREFIX}") + endif() +endif() + +############################################################################## +# - compiler options --------------------------------------------------------- +if(TRITON_ENABLE_GPU) + # * find CUDAToolkit package + # * determine GPU architectures + # * enable the CMake CUDA language + # * set other CUDA compilation flags + rapids_find_package(CUDAToolkit REQUIRED + BUILD_EXPORT_SET rapids_triton-exports + INSTALL_EXPORT_SET rapids_triton-exports + ) + include(cmake/modules/ConfigureCUDA.cmake) +endif() + +############################################################################## +# - Requirements ------------------------------------------------------------- + +# add third party dependencies using CPM +rapids_cpm_init() + +if(TRITON_ENABLE_GPU) + include(cmake/thirdparty/get_rmm.cmake) + include(cmake/thirdparty/get_raft.cmake) +endif() + +include(cmake/thirdparty/get_rapidjson.cmake) +include(cmake/thirdparty/get_triton.cmake) + +if(BUILD_TESTS) + include(cmake/thirdparty/get_gtest.cmake) +endif() + +############################################################################## +# - install targets----------------------------------------------------------- + +add_library(rapids_triton INTERFACE) +add_library(rapids_triton::rapids_triton ALIAS rapids_triton) +target_include_directories(rapids_triton INTERFACE "$" + "$") + +target_link_libraries(rapids_triton +INTERFACE + $<$:rmm::rmm> + $<$:raft::raft> + triton-core-serverstub + triton-backend-utils +) + +if (TRITON_ENABLE_GPU) + target_compile_features( + rapids_triton INTERFACE cxx_std_17 + $ + ) +else() + target_compile_features( + rapids_triton INTERFACE cxx_std_17 + ) +endif() + +rapids_cmake_install_lib_dir(lib_dir) +install(TARGETS rapids_triton + DESTINATION ${lib_dir} + EXPORT rapids_triton-exports + ) + +include(GNUInstallDirs) +install(DIRECTORY include/rapids_triton/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rapids_triton + ) + +# Temporary install of rapids_triton.hpp while the file is removed +install(FILES include/rapids_triton.hpp + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rapids_triton + ) + +############################################################################## +# - install export ----------------------------------------------------------- +set(doc_string +[=[ +Provide targets for RAPIDS_TRITON. + +RAPIDS_TRITON is a header-only library designed to make it easier and faster +to integrate RAPIDS algorithms as Triton backends. + +]=]) + + rapids_export(INSTALL rapids_triton + EXPORT_SET rapids_triton-exports + GLOBAL_TARGETS rapids_triton # since we can't hook into EXPORT SETS + NAMESPACE rapids_triton:: + DOCUMENTATION doc_string + ) + +############################################################################## +# - build export ------------------------------------------------------------- + +rapids_export(BUILD rapids_triton + EXPORT_SET rapids_triton-exports + GLOBAL_TARGETS rapids_triton # since we can't hook into EXPORT SETS + LANGUAGES CUDA + DOCUMENTATION doc_string + NAMESPACE rapids_triton:: + ) + +############################################################################## +# - build test executable ---------------------------------------------------- + +if(BUILD_TESTS) + include(test/CMakeLists.txt) +endif() + +############################################################################## +# - build example backend ---------------------------------------------------- + +if(BUILD_EXAMPLE) + include(src/CMakeLists.txt) +endif() + +############################################################################## +# - doxygen targets ---------------------------------------------------------- + +# TODO(wphicks) +# include(cmake/doxygen.cmake) +# add_doxygen_target(IN_DOXYFILE Doxyfile.in +# OUT_DOXYFILE ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile +# CWD ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/backend/cmake/doxygen.cmake b/backend/cmake/doxygen.cmake new file mode 100644 index 0000000..061981f --- /dev/null +++ b/backend/cmake/doxygen.cmake @@ -0,0 +1,33 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +find_package(Doxygen 1.8.11) + +function(add_doxygen_target) + if(Doxygen_FOUND) + set(options "") + set(oneValueArgs IN_DOXYFILE OUT_DOXYFILE CWD) + set(multiValueArgs "") + cmake_parse_arguments(dox "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + configure_file(${dox_IN_DOXYFILE} ${dox_OUT_DOXYFILE} @ONLY) + add_custom_target(doc + ${DOXYGEN_EXECUTABLE} ${dox_OUT_DOXYFILE} + WORKING_DIRECTORY ${dox_CWD} + VERBATIM + COMMENT "Generate doxygen docs") + else() + message("add_doxygen_target: doxygen exe not found") + endif() +endfunction(add_doxygen_target) diff --git a/backend/cmake/modules/ConfigureCUDA.cmake b/backend/cmake/modules/ConfigureCUDA.cmake new file mode 100644 index 0000000..20826ab --- /dev/null +++ b/backend/cmake/modules/ConfigureCUDA.cmake @@ -0,0 +1,43 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +if(DISABLE_DEPRECATION_WARNINGS) + list(APPEND RAPIDS_TRITON_CXX_FLAGS -Wno-deprecated-declarations) + list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Xcompiler=-Wno-deprecated-declarations) +endif() + +if(CMAKE_COMPILER_IS_GNUCXX) + list(APPEND RAPIDS_TRITON_CXX_FLAGS -Wall -Werror -Wno-unknown-pragmas -Wno-error=deprecated-declarations) +endif() + +list(APPEND RAPIDS_TRITON_CUDA_FLAGS --expt-extended-lambda --expt-relaxed-constexpr) + +# set warnings as errors +if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 11.2.0) + list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Werror=all-warnings) +endif() +list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Xcompiler=-Wall,-Werror,-Wno-error=deprecated-declarations) + +# Option to enable line info in CUDA device compilation to allow introspection when profiling / memchecking +if(CUDA_ENABLE_LINEINFO) + list(APPEND RAPIDS_TRITON_CUDA_FLAGS -lineinfo) +endif() + +# Debug options +if(CMAKE_BUILD_TYPE MATCHES Debug) + message(VERBOSE "RAPIDS_TRITON: Building with debugging flags") + list(APPEND RAPIDS_TRITON_CUDA_FLAGS -G -Xcompiler=-rdynamic) +endif() diff --git a/backend/cmake/thirdparty/get_gtest.cmake b/backend/cmake/thirdparty/get_gtest.cmake new file mode 100644 index 0000000..a29af07 --- /dev/null +++ b/backend/cmake/thirdparty/get_gtest.cmake @@ -0,0 +1,22 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_gtest) + include(${rapids-cmake-dir}/cpm/gtest.cmake) + rapids_cpm_gtest() +endfunction() + +find_and_configure_gtest() diff --git a/backend/cmake/thirdparty/get_raft.cmake b/backend/cmake/thirdparty/get_raft.cmake new file mode 100644 index 0000000..6a37d7e --- /dev/null +++ b/backend/cmake/thirdparty/get_raft.cmake @@ -0,0 +1,49 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_raft) + + set(oneValueArgs VERSION FORK PINNED_TAG) + cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN} ) + + rapids_cpm_find(raft ${PKG_VERSION} + GLOBAL_TARGETS raft::raft + BUILD_EXPORT_SET rapids_triton-exports + INSTALL_EXPORT_SET rapids_triton-exports + CPM_ARGS + GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git + GIT_TAG ${PKG_PINNED_TAG} + SOURCE_SUBDIR cpp + OPTIONS + "BUILD_TESTS OFF" + "RAFT_COMPILE_LIBRARIES OFF" + ) + + message(VERBOSE "RAPIDS_TRITON: Using RAFT located in ${raft_SOURCE_DIR}") + +endfunction() + +set(RAPIDS_TRITON_MIN_VERSION_raft "${RAPIDS_TRITON_VERSION_MAJOR}.${RAPIDS_TRITON_VERSION_MINOR}.00") +set(RAPIDS_TRITON_BRANCH_VERSION_raft "${RAPIDS_TRITON_VERSION_MAJOR}.${RAPIDS_TRITON_VERSION_MINOR}") + +# Change pinned tag here to test a commit in CI +# To use a different RAFT locally, set the CMake variable +# CPM_raft_SOURCE=/path/to/local/raft +find_and_configure_raft(VERSION ${RAPIDS_TRITON_MIN_VERSION_raft} + FORK rapidsai + PINNED_TAG branch-${RAPIDS_TRITON_BRANCH_VERSION_raft} + ) diff --git a/backend/cmake/thirdparty/get_rapidjson.cmake b/backend/cmake/thirdparty/get_rapidjson.cmake new file mode 100644 index 0000000..e9051c6 --- /dev/null +++ b/backend/cmake/thirdparty/get_rapidjson.cmake @@ -0,0 +1,37 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +# TODO(wphicks): Pass in version +function(find_and_configure_rapidjson VERSION) + + rapids_cpm_find(rapidjson ${VERSION} + GLOBAL_TARGETS rapidjson::rapidjson + BUILD_EXPORT_SET rapids_triton-exports + INSTALL_EXPORT_SET rapids_triton-exports + CPM_ARGS + GIT_REPOSITORY https://github.com/Tencent/rapidjson + GIT_TAG "v${VERSION}" + GIT_SHALLOW ON + OPTIONS + "RAPIDJSON_BUILD_DOC OFF" + "RAPIDJSON_BUILD_EXAMPLES OFF" + "RAPIDJSON_BUILD_TESTS OFF" + "RAPIDJSON_BUILD_THIRDPARTY_GTEST OFF" + ) + +endfunction() + +find_and_configure_rapidjson("1.1.0") diff --git a/backend/cmake/thirdparty/get_rmm.cmake b/backend/cmake/thirdparty/get_rmm.cmake new file mode 100644 index 0000000..4a3ca4a --- /dev/null +++ b/backend/cmake/thirdparty/get_rmm.cmake @@ -0,0 +1,22 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_rmm) + include(${rapids-cmake-dir}/cpm/rmm.cmake) + rapids_cpm_rmm() +endfunction() + +find_and_configure_rmm() diff --git a/backend/cmake/thirdparty/get_triton.cmake b/backend/cmake/thirdparty/get_triton.cmake new file mode 100644 index 0000000..e5353ec --- /dev/null +++ b/backend/cmake/thirdparty/get_triton.cmake @@ -0,0 +1,36 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= +include(FetchContent) + +FetchContent_Declare( + repo-common + GIT_REPOSITORY https://github.com/triton-inference-server/common.git + GIT_TAG ${TRITON_COMMON_REPO_TAG} + GIT_SHALLOW ON +) +FetchContent_Declare( + repo-core + GIT_REPOSITORY https://github.com/triton-inference-server/core.git + GIT_TAG ${TRITON_CORE_REPO_TAG} + GIT_SHALLOW ON +) +FetchContent_Declare( + repo-backend + GIT_REPOSITORY https://github.com/triton-inference-server/backend.git + GIT_TAG ${TRITON_BACKEND_REPO_TAG} + GIT_SHALLOW ON +) +FetchContent_MakeAvailable(repo-common repo-core repo-backend) diff --git a/backend/include/rapids_triton.hpp b/backend/include/rapids_triton.hpp new file mode 100644 index 0000000..16cf368 --- /dev/null +++ b/backend/include/rapids_triton.hpp @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { +namespace backend { +namespace rapids { + +/* Function for testing rapids_triton include + * + * @return message indicating rapids_triton has been included succesfully*/ +inline auto test_install() { return std::string("rapids_triton set up successfully"); } + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/batch/batch.hpp b/backend/include/rapids_triton/batch/batch.hpp new file mode 100644 index 0000000..db54df8 --- /dev/null +++ b/backend/include/rapids_triton/batch/batch.hpp @@ -0,0 +1,273 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +/** + * @brief A representation of all data about a single batch of inference + * requests + * + * Batch objects are the primary interface point between rapids_triton Models + * and the Triton server itself. By calling the `get_input` and `get_output` + * methods of a batch, Model implementations can retrieve the input Tensors + * necessary for prediction and the output Tensors where results can be + * stored. + * + * Batch objects also handle a variety of other tasks necessary for + * processing a batch in the Triton model. This includes reporting statistics + * on how long it took to process requests and sending responses to the + * client via the Triton server once processing is complete. + * + * It is not recommended that developers of rapids_triton backends try to + * construct Batch objects directly. Instead, you should make use of the + * rapids::triton_api::execute template, which will construct the Batch for + * you. + */ +struct Batch { + using size_type = std::size_t; + + Batch(TRITONBACKEND_Request** raw_requests, + request_size_t count, + TRITONBACKEND_MemoryManager& triton_mem_manager, + std::function(std::string const&, size_type)>&& get_output_shape, + std::function&& report_request_statistics, + bool use_pinned_input, + bool use_pinned_output, + size_type max_batch_size, + cudaStream_t stream) + : requests_{raw_requests, raw_requests + count}, + responses_{construct_responses(requests_.begin(), requests_.end())}, + get_output_shape_{std::move(get_output_shape)}, + report_statistics_{std::move(report_request_statistics)}, + collector_(raw_requests, count, &responses_, &triton_mem_manager, use_pinned_input, stream), + responder_{std::make_shared(raw_requests, + count, + &responses_, + max_batch_size, + &triton_mem_manager, + use_pinned_output, + stream)}, + stream_{stream}, + start_time_{std::chrono::steady_clock::now()}, + compute_start_time_{std::chrono::steady_clock::now()}, + batch_size_{} + { + } + + template + auto get_input_shape(std::string const& name) + { + auto result = std::vector{}; + if (!requests_.empty()) { + result = get_triton_input_shape(std::begin(requests_), std::end(requests_), name); + + auto input_batch_dim = size_type{}; + if (result.size() > 0) { input_batch_dim = result[0]; } + + if (batch_size_.has_value()) { + if (batch_size_.value() != input_batch_dim) { + throw TritonException(Error::Internal, + "all input tensors must have same batch dimension"); + } + } else { + batch_size_ = input_batch_dim; + } + } + return result; + } + + template + auto get_input(std::string const& name, + std::optional const& memory_type, + device_id_t device_id, + cudaStream_t stream) + { + auto shape = get_input_shape(name); + auto size_bytes = + sizeof(T) * std::reduce(shape.begin(), shape.end(), std::size_t{1}, std::multiplies<>()); + auto allowed_memory_configs = std::vector>{}; + if (memory_type.has_value()) { + allowed_memory_configs.emplace_back(memory_type.value(), device_id); + } else { + allowed_memory_configs.emplace_back(HostMemory, int64_t{}); + allowed_memory_configs.emplace_back(DeviceMemory, device_id); + } + + auto const* raw_buffer = static_cast(nullptr); + auto reported_bytes = std::size_t{}; + auto reported_mem_type = MemoryType{}; + auto reported_device_id = int64_t{}; + + triton_check( + collector_.ProcessTensor(name.c_str(), + static_cast(nullptr), // Return data without copy if possible + size_bytes, + allowed_memory_configs, + &raw_buffer, + &reported_bytes, + &reported_mem_type, + &reported_device_id)); + + if(collector_.Finalize()){ + if constexpr (IS_GPU_BUILD) { + cuda_check(cudaStreamSynchronize(stream_)); + } else { + throw TritonException(Error::Internal, "stream synchronization required in non-GPU build"); + } + } + + std::for_each(std::begin(responses_), std::end(responses_), [](auto* response) { + if (response == nullptr) { + throw TritonException(Error::Internal, "Input collection failed"); + } + }); + + auto buffer = Buffer(reinterpret_cast(raw_buffer), + reported_bytes / sizeof(T), + reported_mem_type, + reported_device_id, + stream); + + if (memory_type && (reported_mem_type != memory_type || reported_device_id != device_id)) { + throw TritonException(Error::Internal, "data collected in wrong location"); + } + + // Set start time of batch to time latest input tensor was retrieved + compute_start_time_ = std::chrono::steady_clock::now(); + + return Tensor(std::move(shape), std::move(buffer)); + } + + template + auto get_input(std::string const& name, + std::optional const& memory_type, + device_id_t device_id) + { + return get_input(name, memory_type, device_id, stream_); + } + + template + auto get_output(std::string const& name, + std::optional const& memory_type, + device_id_t device_id, + cudaStream_t stream) + { + if (!batch_size_.has_value()) { + throw TritonException(Error::Internal, + "At least one input must be retrieved before any output"); + } + auto shape = get_output_shape_(name, batch_size_.value()); + auto buffer_size = std::reduce(shape.begin(), shape.end(), std::size_t{1}, std::multiplies<>()); + auto final_memory_type = MemoryType{}; + if (memory_type.has_value()) { + final_memory_type = memory_type.value(); + } else { + // If consumer doesn't care, use HostMemory to avoid additional copy on + // non-shared-memory responses. + final_memory_type = HostMemory; + } + auto buffer = Buffer(buffer_size, final_memory_type, device_id, stream); + return OutputTensor(std::move(shape), std::move(buffer), name, responder_); + } + + template + auto get_output(std::string const& name, + std::optional const& memory_type, + device_id_t device_id) + { + return get_output(name, memory_type, device_id, stream_); + } + + auto const& compute_start_time() const { return compute_start_time_; } + + auto stream() const { return stream_; } + + void finalize(TRITONSERVER_Error* err) + { + auto compute_end_time = std::chrono::steady_clock::now(); + if (responder_->Finalize()) { cuda_check(cudaStreamSynchronize(stream_)); } + + send_responses(std::begin(responses_), std::end(responses_), err); + + // Triton resumes ownership of failed requests; only release on success + if (err == nullptr) { + std::for_each( + std::begin(requests_), std::end(requests_), [this, &compute_end_time](auto& request) { + report_statistics_(request, + start_time_, + compute_start_time_, + compute_end_time, + std::chrono::steady_clock::now()); + }); + release_requests(std::begin(requests_), std::end(requests_)); + } + } + + private: + std::vector requests_; + std::vector responses_; + std::function(std::string const&, size_type)> get_output_shape_; + std::function + report_statistics_; + BackendInputCollector collector_; + std::shared_ptr responder_; + cudaStream_t stream_; + std::chrono::time_point start_time_; + std::chrono::time_point compute_start_time_; + std::optional batch_size_; +}; +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/build_control.hpp b/backend/include/rapids_triton/build_control.hpp new file mode 100644 index 0000000..c2a9fec --- /dev/null +++ b/backend/include/rapids_triton/build_control.hpp @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { +namespace backend { +namespace rapids { + +#ifdef TRITON_ENABLE_GPU +auto constexpr IS_GPU_BUILD = true; +#else +auto constexpr IS_GPU_BUILD = false; +#endif + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/cpu_only/cuda_runtime_replacement.hpp b/backend/include/rapids_triton/cpu_only/cuda_runtime_replacement.hpp new file mode 100644 index 0000000..7cb6691 --- /dev/null +++ b/backend/include/rapids_triton/cpu_only/cuda_runtime_replacement.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifdef TRITON_ENABLE_GPU +#include +#else + +namespace triton { +namespace backend { +namespace rapids { + +using cudaStream_t = void*; + +enum struct cudaError_t {cudaSuccess, cudaErrorNonGpuBuild}; +using cudaError = cudaError_t; +auto constexpr cudaSuccess = cudaError_t::cudaSuccess; + +inline void cudaGetLastError() {} + +inline auto const * cudaGetErrorString(cudaError_t err) { + return "CUDA function used in non-GPU build"; +} + +inline auto cudaStreamSynchronize(cudaStream_t stream) { + return cudaError_t::cudaErrorNonGpuBuild; +} + +inline auto cudaGetDevice(int* device_id) { + return cudaError_t::cudaErrorNonGpuBuild; +} + +inline auto cudaGetDeviceCount(int* count) { + return cudaError_t::cudaErrorNonGpuBuild; +} + + +} // namespace rapids +} // namespace backend +} // namespace triton +#endif diff --git a/backend/include/rapids_triton/exceptions.hpp b/backend/include/rapids_triton/exceptions.hpp new file mode 100644 index 0000000..e2d2bc2 --- /dev/null +++ b/backend/include/rapids_triton/exceptions.hpp @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { + +using ErrorCode = TRITONSERVER_Error_Code; + +namespace Error { +auto constexpr Unknown = ErrorCode::TRITONSERVER_ERROR_UNKNOWN; +auto constexpr Internal = ErrorCode::TRITONSERVER_ERROR_INTERNAL; +auto constexpr NotFound = ErrorCode::TRITONSERVER_ERROR_NOT_FOUND; +auto constexpr InvalidArg = ErrorCode::TRITONSERVER_ERROR_INVALID_ARG; +auto constexpr Unavailable = ErrorCode::TRITONSERVER_ERROR_UNAVAILABLE; +auto constexpr Unsupported = ErrorCode::TRITONSERVER_ERROR_UNSUPPORTED; +auto constexpr AlreadyExists = ErrorCode::TRITONSERVER_ERROR_ALREADY_EXISTS; +} // namespace Error + +/** + * @brief Exception thrown if processing cannot continue for a request + * + * This exception should be thrown whenever a condition is encountered that (if + * it is not appropriately handled by some other exception handler) SHOULD + * result in Triton reporting an error for the request being processed. It + * signals that (absent any other fallbacks), this request cannot be fulfilled + * but that the server may still be in a state to continue handling other + * requests, including requests to other models. + */ +struct TritonException : std::exception { + public: + TritonException() : error_(TRITONSERVER_ErrorNew(Error::Unknown, "encountered unknown error")) {} + + TritonException(ErrorCode code, std::string const& msg) + : error_(TRITONSERVER_ErrorNew(code, msg.c_str())) + { + } + + TritonException(ErrorCode code, char const* msg) : error_{TRITONSERVER_ErrorNew(code, msg)} {} + + TritonException(TRITONSERVER_Error* prev_error) : error_(prev_error) {} + + virtual char const* what() const noexcept { return TRITONSERVER_ErrorMessage(error_); } + + auto* error() const { return error_; } + + private: + TRITONSERVER_Error* error_; +}; + +inline void triton_check(TRITONSERVER_Error* err) +{ + if (err != nullptr) { throw TritonException(err); } +} + +inline void cuda_check(cudaError_t const& err) +{ + if constexpr (IS_GPU_BUILD) { + if (err != cudaSuccess) { + cudaGetLastError(); + throw TritonException(Error::Internal, cudaGetErrorString(err)); + } + } else { + throw TritonException(Error::Internal, "cuda_check used in non-GPU build"); + } +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/memory/buffer.hpp b/backend/include/rapids_triton/memory/buffer.hpp new file mode 100644 index 0000000..124437c --- /dev/null +++ b/backend/include/rapids_triton/memory/buffer.hpp @@ -0,0 +1,317 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include + +#ifdef TRITON_ENABLE_GPU +#include +#include +#include +#else +#include +#include +#include +#endif + +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +template +struct Buffer { + using size_type = std::size_t; + using value_type = T; + + using h_buffer = T*; + using d_buffer = T*; + using owned_h_buffer = std::unique_ptr; + using owned_d_buffer = detail::owned_device_buffer; + using data_store = std::variant; + + Buffer() noexcept : device_{}, data_{std::in_place_index<0>, nullptr}, size_{}, stream_{} {} + + /** + * @brief Construct buffer of given size in given memory location (either + * on host or on device) + * A buffer constructed in this way is owning and will release allocated + * resources on deletion + */ + Buffer(size_type size, + MemoryType memory_type = DeviceMemory, + device_id_t device = 0, + cudaStream_t stream = 0) + : device_{device}, + data_{allocate(size, device, memory_type, stream)}, + size_{size}, + stream_{stream} + { + if constexpr (!IS_GPU_BUILD) { + if (memory_type == DeviceMemory) { + throw TritonException( + Error::Internal, + "Cannot use device buffer in non-GPU build" + ); + } + } + } + + /** + * @brief Construct buffer from given source in given memory location (either + * on host or on device) + * A buffer constructed in this way is non-owning; the caller is + * responsible for freeing any resources associated with the input pointer + */ + Buffer(T* input_data, + size_type size, + MemoryType memory_type = DeviceMemory, + device_id_t device = 0, + cudaStream_t stream = 0) + : device_{device}, + data_{[&memory_type, &input_data]() { + auto result = data_store{}; + if (memory_type == HostMemory) { + result = data_store{std::in_place_index<0>, input_data}; + } else { + if constexpr (!IS_GPU_BUILD) { + throw TritonException( + Error::Internal, + "Cannot use device buffer in non-GPU build" + ); + } + result = data_store{std::in_place_index<1>, input_data}; + } + return result; + }()}, + size_{size}, + stream_{stream} + { + } + + /** + * @brief Construct one buffer from another in the given memory location + * (either on host or on device) + * A buffer constructed in this way is owning and will copy the data from + * the original location + */ + Buffer(Buffer const& other, MemoryType memory_type, device_id_t device = 0) + : device_{device}, + data_([&other, &memory_type, &device]() { + auto result = allocate(other.size_, device, memory_type, other.stream_); + copy(result, other.data_, other.size_, other.stream_); + return result; + }()), + size_{other.size_}, + stream_{other.stream_} + { + } + + /** + * @brief Create owning copy of existing buffer + * The memory type of this new buffer will be the same as the original + */ + Buffer(Buffer const& other) : Buffer(other, other.mem_type(), other.device()) {} + + Buffer(Buffer&& other, MemoryType memory_type) + : device_{other.device()}, + data_{[&other, memory_type]() { + data_store result; + if (memory_type == other.mem_type()) { + result = std::move(other.data_); + } else { + result = allocate(other.size_, memory_type, other.device(), other.stream()); + copy(result, other.data_, other.size_, other.stream_); + } + return result; + }()}, + size_{other.size_}, + stream_{other.stream_} + { + } + + Buffer(Buffer&& other) = default; + + Buffer& operator=(Buffer&& other) = default; + + ~Buffer() {} + + /** + * @brief Return where memory for this buffer is located (host or device) + */ + auto mem_type() const noexcept { return data_.index() % 2 == 0 ? HostMemory : DeviceMemory; } + + /** + * @brief Return number of elements in buffer + */ + auto size() const noexcept { return size_; } + + /** + * @brief Return pointer to data stored in buffer + */ + auto* data() const noexcept { return get_raw_ptr(data_); } + + auto device() const noexcept { return device_; } + + /** + * @brief Return CUDA stream associated with this buffer + */ + auto stream() const noexcept { return stream_; } + + void stream_synchronize() const + { + if constexpr (IS_GPU_BUILD) { cuda_check(cudaStreamSynchronize(stream_)); } + } + + /** + * @brief Set CUDA stream for this buffer to new value + * + * @warning This method calls cudaStreamSynchronize on the old stream + * before updating. Be aware of performance implications and try to avoid + * interactions between buffers on different streams where possible. + */ + void set_stream(cudaStream_t new_stream) + { + stream_synchronize(); + stream_ = new_stream; + } + + private: + device_id_t device_; + data_store data_; + size_type size_; + cudaStream_t stream_; + + // Helper function for accessing raw pointer to underlying data of + // data_store + static auto* get_raw_ptr(data_store const& ptr) noexcept + { + /* Switch statement is an optimization relative to std::visit to avoid + * vtable overhead for a small number of alternatives */ + auto* result = static_cast(nullptr); + switch (ptr.index()) { + case 0: result = std::get<0>(ptr); break; + case 1: result = std::get<1>(ptr); break; + case 2: result = std::get<2>(ptr).get(); break; + case 3: result = std::get<3>(ptr).get(); break; + } + return result; + } + + // Helper function for allocating memory in constructors + static auto allocate(size_type size, + device_id_t device = 0, + MemoryType memory_type = DeviceMemory, + cudaStream_t stream = 0) + { + auto result = data_store{}; + if (memory_type == DeviceMemory) { + if constexpr (IS_GPU_BUILD) { + result = data_store{owned_d_buffer{ + device, + size, + stream, + }}; + } else { + throw TritonException(Error::Internal, + "DeviceMemory requested in CPU-only build of FIL backend"); + } + } else { + result = std::make_unique(size); + } + return result; + } + + // Helper function for copying memory in constructors, where there are + // stronger guarantees on conditions that would otherwise need to be + // checked + static void copy(data_store const& dst, data_store const& src, size_type len, cudaStream_t stream) + { + // This function will only be called in constructors, so we allow a + // const_cast here to perform the initial copy of data from a + // Buffer to a newly-created Buffer + auto raw_dst = const_cast*>(get_raw_ptr(dst)); + auto raw_src = get_raw_ptr(src); + + auto dst_mem_type = dst.index() % 2 == 0 ? HostMemory : DeviceMemory; + auto src_mem_type = src.index() % 2 == 0 ? HostMemory : DeviceMemory; + + detail::copy(raw_dst, raw_src, len, stream, dst_mem_type, src_mem_type); + } +}; + +/** + * @brief Copy data from one Buffer to another + * + * @param dst The destination buffer + * @param src The source buffer + * @param dst_begin The offset from the beginning of the destination buffer + * at which to begin copying to. + * @param src_begin The offset from the beginning of the source buffer + * at which to begin copying from. + * @param src_end The offset from the beginning of the source buffer + * before which to end copying from. + */ +template +void copy(Buffer& dst, + Buffer const& src, + typename Buffer::size_type dst_begin, + typename Buffer::size_type src_begin, + typename Buffer::size_type src_end) +{ + if (dst.stream() != src.stream()) { dst.set_stream(src.stream()); } + auto len = src_end - src_begin; + if (len < 0 || src_end > src.size() || len > dst.size() - dst_begin) { + throw TritonException(Error::Internal, "bad copy between buffers"); + } + + auto raw_dst = dst.data() + dst_begin; + auto raw_src = src.data() + src_begin; + + detail::copy(raw_dst, raw_src, len, dst.stream(), dst.mem_type(), src.mem_type()); +} + +template +void copy(Buffer& dst, Buffer const& src) +{ + copy(dst, src, 0, 0, src.size()); +} + +template +void copy(Buffer& dst, Buffer const& src, typename Buffer::size_type dst_begin) +{ + copy(dst, src, dst_begin, 0, src.size()); +} + +template +void copy(Buffer& dst, + Buffer const& src, + typename Buffer::size_type src_begin, + typename Buffer::size_type src_end) +{ + copy(dst, src, 0, src_begin, src_end); +} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/memory/detail/cpu_only/copy.hpp b/backend/include/rapids_triton/memory/detail/cpu_only/copy.hpp new file mode 100644 index 0000000..beb5cb4 --- /dev/null +++ b/backend/include/rapids_triton/memory/detail/cpu_only/copy.hpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include + +#ifndef TRITON_ENABLE_GPU +#include +#endif +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace detail { + +template +void copy(T* dst, + T const* src, + std::size_t len, + cudaStream_t stream, + MemoryType dst_type, + MemoryType src_type) +{ + if (dst_type == DeviceMemory || src_type == DeviceMemory) { + throw TritonException(Error::Internal, "Cannot copy device memory in non-GPU build"); + } else { + std::memcpy(dst, src, len * sizeof(T)); + } +} + +} // namespace detail +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp b/backend/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp new file mode 100644 index 0000000..4006a70 --- /dev/null +++ b/backend/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace detail { + +template +struct owned_device_buffer { + using non_const_T = std::remove_const_t; + owned_device_buffer(device_id_t device_id, std::size_t size, cudaStream_t stream) + { + throw TritonException(Error::Internal, + "Attempted to use device buffer in non-GPU build"); + } + + auto* get() const { return static_cast(nullptr); } +}; + +} // namespace detail +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/memory/detail/cpu_only/resource.hpp b/backend/include/rapids_triton/memory/detail/cpu_only/resource.hpp new file mode 100644 index 0000000..04bb1c5 --- /dev/null +++ b/backend/include/rapids_triton/memory/detail/cpu_only/resource.hpp @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace detail { + +template<> +inline void setup_memory_resource(device_id_t device_id, + TRITONBACKEND_MemoryManager* triton_manager) { } + +} // namespace detail +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/memory/detail/gpu_only/copy.hpp b/backend/include/rapids_triton/memory/detail/gpu_only/copy.hpp new file mode 100644 index 0000000..030b473 --- /dev/null +++ b/backend/include/rapids_triton/memory/detail/gpu_only/copy.hpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifdef TRITON_ENABLE_GPU +#include +#include +#endif + +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace detail { + +template +void copy(T* dst, + T const* src, + std::size_t len, + cudaStream_t stream, + MemoryType dst_type, + MemoryType src_type) +{ + if (dst_type == DeviceMemory || src_type == DeviceMemory) { + try { + raft::copy(dst, src, len, stream); + } catch (raft::cuda_error const& err) { + throw TritonException(Error::Internal, err.what()); + } + } else { + std::memcpy(dst, src, len * sizeof(T)); + } +} + +} // namespace detail +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp b/backend/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp new file mode 100644 index 0000000..34095f7 --- /dev/null +++ b/backend/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace detail { + +template +struct owned_device_buffer { + using non_const_T = std::remove_const_t; + owned_device_buffer(device_id_t device_id, std::size_t size, cudaStream_t stream) + : data_{[&device_id, &size, &stream]() { + auto device_context = device_setter{device_id}; + return rmm::device_buffer{size * sizeof(T), rmm::cuda_stream_view{stream}}; + }()} + { + } + + auto* get() const { return reinterpret_cast(data_.data()); } + + private: + mutable rmm::device_buffer data_; +}; + +} // namespace detail +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/memory/detail/gpu_only/resource.hpp b/backend/include/rapids_triton/memory/detail/gpu_only/resource.hpp new file mode 100644 index 0000000..f2ea115 --- /dev/null +++ b/backend/include/rapids_triton/memory/detail/gpu_only/resource.hpp @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace detail { + +inline auto& resource_lock() +{ + static auto lock = std::mutex{}; + return lock; +} + +/** A struct used solely to keep memory resources in-scope for the lifetime + * of the backend */ +struct resource_data { + resource_data() : base_mr_{}, triton_mrs_{} {} + auto* make_new_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* manager) + { + if (manager == nullptr && triton_mrs_.size() != 0) { + manager = triton_mrs_.back().get_triton_manager(); + } + triton_mrs_.emplace_back(manager, device_id, &base_mr_); + return &(triton_mrs_.back()); + } + + private: + rmm::mr::cuda_memory_resource base_mr_; + std::deque triton_mrs_; +}; + +inline auto& get_device_resources() +{ + static auto device_resources = resource_data{}; + return device_resources; +} + +inline auto is_triton_resource(rmm::cuda_device_id const& device_id) +{ + auto* triton_mr = + dynamic_cast(rmm::mr::get_per_device_resource(device_id)); + return (triton_mr != nullptr && triton_mr->get_triton_manager() != nullptr); +} + +template<> +inline void setup_memory_resource(device_id_t device_id, + TRITONBACKEND_MemoryManager* triton_manager) +{ + auto lock = std::lock_guard{detail::resource_lock()}; + auto rmm_device_id = rmm::cuda_device_id{device_id}; + + if (!detail::is_triton_resource(rmm_device_id)) { + auto& device_resources = detail::get_device_resources(); + rmm::mr::set_per_device_resource(rmm_device_id, + device_resources.make_new_resource(device_id, triton_manager)); + } +} + +/* inline auto* get_memory_resource(device_id_t device_id) +{ + auto rmm_device_id = rmm::cuda_device_id{device_id}; + return rmm::mr::get_per_device_resource(rmm_device_id); +} + +inline auto* get_memory_resource() { return rmm::mr::get_current_device_resource(); } */ + +} // namespace detail +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/memory/detail/owned_device_buffer.hpp b/backend/include/rapids_triton/memory/detail/owned_device_buffer.hpp new file mode 100644 index 0000000..ca25af9 --- /dev/null +++ b/backend/include/rapids_triton/memory/detail/owned_device_buffer.hpp @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +namespace triton { +namespace backend { +namespace rapids { +namespace detail { + +template +struct owned_device_buffer { +}; + +} // namespace detail +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/memory/detail/resource.hpp b/backend/include/rapids_triton/memory/detail/resource.hpp new file mode 100644 index 0000000..cfc4bc4 --- /dev/null +++ b/backend/include/rapids_triton/memory/detail/resource.hpp @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace detail { + +template +inline void setup_memory_resource(device_id_t device_id, + TRITONBACKEND_MemoryManager* triton_manager = nullptr) { +} + +} // namespace detail +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/memory/resource.hpp b/backend/include/rapids_triton/memory/resource.hpp new file mode 100644 index 0000000..afd7f2a --- /dev/null +++ b/backend/include/rapids_triton/memory/resource.hpp @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +#include +#include +#include +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif + +namespace triton { +namespace backend { +namespace rapids { + +inline void setup_memory_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* triton_manager = nullptr) { + detail::setup_memory_resource(device_id, triton_manager); +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/memory/types.hpp b/backend/include/rapids_triton/memory/types.hpp new file mode 100644 index 0000000..884f315 --- /dev/null +++ b/backend/include/rapids_triton/memory/types.hpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { +namespace backend { +namespace rapids { +using MemoryType = TRITONSERVER_MemoryType; +auto constexpr DeviceMemory = TRITONSERVER_MEMORY_GPU; +auto constexpr HostMemory = TRITONSERVER_MEMORY_CPU; +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/model/model.hpp b/backend/include/rapids_triton/model/model.hpp new file mode 100644 index 0000000..3413fca --- /dev/null +++ b/backend/include/rapids_triton/model/model.hpp @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +template +struct Model { + virtual void predict(Batch& batch) const = 0; + + virtual void load() {} + virtual void unload() {} + + /** + * @brief Return the preferred memory type in which to store data for this + * batch or std::nullopt to accept whatever Triton returns + * + * The base implementation of this method will require data on-host if the + * model itself is deployed on the host OR if this backend has not been + * compiled with GPU support. Otherwise, models deployed on device will + * receive memory on device. Overriding this method will allow derived + * model classes to select a preferred memory location based on properties + * of the batch or to simply return std::nullopt if device memory or host + * memory will do equally well. + */ + virtual std::optional preferred_mem_type(Batch& batch) const + { + return (IS_GPU_BUILD && deployment_type_ == GPUDeployment) ? DeviceMemory : HostMemory; + } + virtual std::optional preferred_mem_type_in(Batch& batch) const + { + return preferred_mem_type(batch); + } + virtual std::optional preferred_mem_type_out(Batch& batch) const + { + return preferred_mem_type(batch); + } + + /** + * @brief Retrieve a stream used to set up batches for this model + * + * The base implementation of this method simply returns the default stream + * provided by Triton for use with this model. Child classes may choose to + * override this in order to provide different streams for use with + * successive incoming batches. For instance, one might cycle through + * several streams in order to distribute batches across them, but care + * should be taken to ensure proper synchronization in this case. + */ + virtual cudaStream_t get_stream() const { return default_stream_; } + + /** + * @brief Get input tensor of a particular named input for an entire batch + */ + template + auto get_input(Batch& batch, + std::string const& name, + std::optional const& mem_type, + cudaStream_t stream) const + { + return batch.get_input(name, mem_type, device_id_, stream); + } + template + auto get_input(Batch& batch, + std::string const& name, + std::optional const& mem_type) const + { + return get_input(batch, name, mem_type, default_stream_); + } + template + auto get_input(Batch& batch, std::string const& name) const + { + return get_input(batch, name, preferred_mem_type(batch), default_stream_); + } + + /** + * @brief Get output tensor of a particular named output for an entire batch + */ + template + auto get_output(Batch& batch, + std::string const& name, + std::optional const& mem_type, + device_id_t device_id, + cudaStream_t stream) const + { + return batch.get_output(name, mem_type, device_id, stream); + } + template + auto get_output(Batch& batch, + std::string const& name, + std::optional const& mem_type, + cudaStream_t stream) const + { + return get_output(batch, name, mem_type, device_id_, stream); + } + template + auto get_output(Batch& batch, + std::string const& name, + std::optional const& mem_type) const + { + return get_output(batch, name, mem_type, device_id_, default_stream_); + } + template + auto get_output(Batch& batch, std::string const& name) const + { + return get_output(batch, name, preferred_mem_type(batch), device_id_, default_stream_); + } + + /** + * @brief Retrieve value of configuration parameter + */ + template + auto get_config_param(std::string const& name) const + { + return shared_state_->template get_config_param(name); + } + template + auto get_config_param(std::string const& name, T default_value) const + { + return shared_state_->template get_config_param(name, default_value); + } + template + auto get_config_param(char const* name) const + { + return get_config_param(std::string(name)); + } + template + auto get_config_param(char const* name, T default_value) const + { + return get_config_param(std::string(name), default_value); + } + + Model(std::shared_ptr shared_state, + device_id_t device_id, + cudaStream_t default_stream, + DeploymentType deployment_type, + std::string const& filepath) + : shared_state_{shared_state}, + device_id_{device_id}, + default_stream_{default_stream}, + deployment_type_{deployment_type}, + filepath_{filepath} + { + if constexpr (IS_GPU_BUILD) { setup_memory_resource(device_id_); } + } + + auto get_device_id() const { return device_id_; } + auto get_deployment_type() const { return deployment_type_; } + auto const& get_filepath() const { return filepath_; } + + auto get_output_shape(std::string const& name) const + { + return shared_state_->get_output_shape(name); + } + + protected: + auto get_shared_state() const { return shared_state_; } + + private: + std::shared_ptr shared_state_; + device_id_t device_id_; + cudaStream_t default_stream_; + DeploymentType deployment_type_; + std::string filepath_; +}; +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/model/shared_state.hpp b/backend/include/rapids_triton/model/shared_state.hpp new file mode 100644 index 0000000..aa0d76d --- /dev/null +++ b/backend/include/rapids_triton/model/shared_state.hpp @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +/** + * @brief Stores shared state for multiple instances of the same model + */ +struct SharedModelState { + virtual void load() {} + virtual void unload() {} + + explicit SharedModelState(std::unique_ptr&& config, + bool squeeze_output = false) + : config_{std::move(config)}, + max_batch_size_{get_max_batch_size(*config_)}, + output_shapes_([this, squeeze_output]() { + auto result = std::vector>>{}; + auto output_entries = triton::common::TritonJson::Value{}; + triton_check(config_->MemberAsArray("output", &output_entries)); + + result.reserve(output_entries.ArraySize()); + + // Using a raw loop because TritonJSON::Value access has no iterator interface + for (std::size_t i = 0; i < output_entries.ArraySize(); ++i) { + auto output_entry = triton::common::TritonJson::Value{}; + triton_check(output_entries.IndexAsObject(i, &output_entry)); + auto name = std::string{}; + triton_check(output_entry.MemberAsString("name", &name)); + + auto shape = std::vector{}; + auto reshape_entry = triton::common::TritonJson::Value{}; + if (output_entry.Find("reshape", &reshape_entry)) { + ParseShape(reshape_entry, "shape", &shape); + } else { + ParseShape(output_entry, "dims", &shape); + } + if (shape[0] != -1) { shape.insert(shape.begin(), -1); } + // The squeeze_output option was introduced to handle a bad choice of + // convention in the original FIL backend implementation. For legacy + // compatibility, we introduced this option into RAPIDS-Triton, but + // in general, new backends are advised to avoid using it and defer + // this sort of flattening operation to the consumer. + if (squeeze_output) { + shape.erase(std::remove(shape.begin(), shape.end(), std::int64_t{1}), shape.end()); + } + result.insert( + std::upper_bound(std::begin(output_shapes_), + std::end(output_shapes_), + name, + [](auto& value, auto& entry) { return value < entry.first; }), + {name, shape}); + } + + return result; + }()) + { + } + + template + auto get_config_param(std::string const& name) + { + return get_config_param(name, std::optional{}); + } + + template + auto get_config_param(std::string const& name, T default_value) + { + return get_config_param(name, std::make_optional(default_value)); + } + + auto get_output_shape(std::string const& name) const + { + auto cached_shape = std::lower_bound( + std::begin(output_shapes_), std::end(output_shapes_), name, [](auto& entry, auto& value) { + return entry.first < value; + }); + if (cached_shape == std::end(output_shapes_) || name != cached_shape->first) { + auto log_stream = std::stringstream{}; + log_stream << "No output with name " << name << " in configuration."; + throw TritonException(Error::Internal, log_stream.str()); + } else { + return cached_shape->second; + } + } + + auto get_output_names() const { + auto output_names = std::vector{}; + output_names.reserve(output_shapes_.size()); + std::transform(std::begin(output_shapes_), std::end(output_shapes_), std::back_inserter(output_names), [](auto& output_shape) { + return output_shape.first; + }); + return output_names; + } + + auto check_output_name(std::string const& name) const { + // #TODO: Figure out a way to use std::binary_search here + auto cached_shape = std::lower_bound( + std::begin(output_shapes_), std::end(output_shapes_), name, [](auto& entry, auto& value) { + return entry.first < value; + }); + return cached_shape != std::end(output_shapes_) && name == cached_shape->first; + } + + private: + std::unique_ptr config_; + Batch::size_type max_batch_size_; + std::vector>> mutable output_shapes_; + + template + auto get_config_param(std::string const& name, std::optional const& default_value) + { + auto result = T{}; + if (name == std::string("max_batch_size")) { + result = max_batch_size_; + return result; + } + auto parameters = common::TritonJson::Value{}; + auto json_value = common::TritonJson::Value{}; + if (config_->Find("parameters", ¶meters) && parameters.Find(name.c_str(), &json_value)) { + auto string_repr = std::string{}; + triton_check(json_value.MemberAsString("string_value", &string_repr)); + + auto input_stream = std::istringstream{string_repr}; + + if constexpr (std::is_same_v) { + input_stream >> std::boolalpha >> result; + } else { + input_stream >> result; + } + + if (input_stream.fail()) { + if (default_value) { + result = *default_value; + } else { + throw TritonException(Error::InvalidArg, std::string("Bad input for parameter ") + name); + } + } + } else { + if (default_value) { + result = *default_value; + } else { + throw TritonException( + Error::InvalidArg, + std::string("Required parameter ") + name + std::string(" not found in config")); + } + } + + return result; + } +}; +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/tensor/dtype.hpp b/backend/include/rapids_triton/tensor/dtype.hpp new file mode 100644 index 0000000..325ef4d --- /dev/null +++ b/backend/include/rapids_triton/tensor/dtype.hpp @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { + +using DType = TRITONSERVER_DataType; +auto constexpr DTypeBool = TRITONSERVER_TYPE_BOOL; +auto constexpr DTypeUint8 = TRITONSERVER_TYPE_UINT8; +auto constexpr DTypeChar = DTypeUint8; +auto constexpr DTypeByte = DTypeUint8; +auto constexpr DTypeUint16 = TRITONSERVER_TYPE_UINT16; +auto constexpr DTypeUint32 = TRITONSERVER_TYPE_UINT32; +auto constexpr DTypeUint64 = TRITONSERVER_TYPE_UINT64; +auto constexpr DTypeInt8 = TRITONSERVER_TYPE_INT8; +auto constexpr DTypeInt16 = TRITONSERVER_TYPE_INT16; +auto constexpr DTypeInt32 = TRITONSERVER_TYPE_INT32; +auto constexpr DTypeInt64 = TRITONSERVER_TYPE_INT64; +auto constexpr DTypeFloat32 = TRITONSERVER_TYPE_FP32; +auto constexpr DTypeFloat64 = TRITONSERVER_TYPE_FP64; + +template +struct TritonType { +}; + +template +struct TritonDtype { +}; + +template <> +struct TritonType { + typedef bool type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeBool; +}; + +template <> +struct TritonType { + typedef std::uint8_t type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeUint8; +}; + +template <> +struct TritonType { + typedef std::uint16_t type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeUint16; +}; + +template <> +struct TritonType { + typedef std::uint32_t type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeUint32; +}; + +template <> +struct TritonType { + typedef std::uint64_t type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeUint64; +}; + +template <> +struct TritonType { + typedef std::int8_t type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeInt8; +}; + +template <> +struct TritonType { + typedef std::int16_t type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeInt16; +}; + +template <> +struct TritonType { + typedef std::int32_t type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeInt32; +}; + +template <> +struct TritonType { + typedef std::int64_t type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeInt64; +}; + +template <> +struct TritonType { + typedef float type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeFloat32; +}; + +template <> +struct TritonType { + typedef double type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeFloat64; +}; + +inline std::ostream& operator<<(std::ostream& out, DType const& dtype) +{ + out << TRITONSERVER_DataTypeString(dtype); + return out; +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/tensor/tensor.hpp b/backend/include/rapids_triton/tensor/tensor.hpp new file mode 100644 index 0000000..80f2ba4 --- /dev/null +++ b/backend/include/rapids_triton/tensor/tensor.hpp @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +template +struct BaseTensor { + using size_type = typename Buffer::size_type; + + BaseTensor() : shape_{}, buffer_{} {} + BaseTensor(std::vector const& shape, Buffer&& buffer) + : shape_(shape), buffer_{std::move(buffer)} + { + } + + virtual ~BaseTensor() = 0; + + /** + * @brief Construct a BaseTensor from a collection of buffers + * + * Given a collection of buffers, collate them all into one buffer stored in + * a new BaseTensor + */ + template + BaseTensor(std::vector const& shape, + Iter begin, + Iter end, + MemoryType mem_type, + device_id_t device, + cudaStream_t stream) + : shape_(shape), buffer_([&begin, &end, &mem_type, &device, &stream]() { + auto total_size = std::transform_reduce( + begin, end, size_type{}, std::plus<>{}, [](auto&& buffer) { return buffer.size(); }); + + auto result = Buffer(total_size, mem_type, device, stream); + + std::accumulate(begin, end, size_type{}, [&result](auto offset, auto& buffer) { + copy(result, buffer, offset); + return offset + buffer.size(); + }); + return result; + }()) + { + } + + auto const& shape() const { return shape_; } + auto size() const { return buffer_.size(); } + auto data() const { return buffer_.data(); } + auto& buffer() { return buffer_; } + + auto constexpr dtype() { return TritonDtype::value; } + auto mem_type() const { return buffer_.mem_type(); } + auto stream() const { return buffer_.stream(); } + auto device() const { return buffer_.device(); } + + void stream_synchronize() const + { + if (mem_type() == DeviceMemory) { buffer_.stream_synchronize(); } + } + + void set_stream(cudaStream_t new_stream) { buffer_.set_stream(new_stream); } + + private: + std::vector shape_; + Buffer buffer_; +}; + +template +BaseTensor::~BaseTensor() +{ +} + +template +struct Tensor final : BaseTensor { + Tensor() : BaseTensor{} {} + Tensor(std::vector::size_type> const& shape, Buffer&& buffer) + : BaseTensor(shape, std::move(buffer)) + { + } + + template + Tensor(std::vector::size_type> const& shape, + Iter begin, + Iter end, + MemoryType mem_type, + device_id_t device, + cudaStream_t stream) + : BaseTensor(shape, begin, end, mem_type, device, stream) + { + } +}; + +template +struct OutputTensor final : BaseTensor { + OutputTensor(std::vector::size_type>&& shape, + Buffer&& buffer, + std::string const& name, + std::shared_ptr responder) + : BaseTensor(std::move(shape), std::move(buffer)), name_{name}, responder_{responder} + { + } + /** + * @brief Prepare final output data from this tensor for responding to + * request + * + * This method *must* be called by rapids_triton backends on all of their + * output tensors before returning from their `predict` methods. Because we + * cannot know a priori what names backends might have for their tensors + * and what types will be stored in those tensors, the rapids_triton + * library cannot store references to those tensors that might otherwise be + * used to finalize them. + */ + void finalize() + { + auto& shape = BaseTensor::shape(); + auto triton_shape = std::vector{}; + triton_shape.reserve(shape.size()); + std::transform( + std::begin(shape), std::end(shape), std::back_inserter(triton_shape), [](auto& val) { + return narrow(val); + }); + + // Must call the following because BackendOutputResponder does not expose + // its stream, so we cannot be certain that our data is not being + // processed on another stream. + BaseTensor::stream_synchronize(); + responder_->ProcessTensor(name_.c_str(), + TritonDtype::value, + triton_shape, + reinterpret_cast(BaseTensor::data()), + BaseTensor::mem_type(), + BaseTensor::device()); + } + + private: + std::string name_; + std::shared_ptr responder_; +}; + +template , T>>> +void copy(BaseTensor& dst, BaseTensor& src) +{ + copy(dst.buffer(), src.buffer()); +} + +/** + * @brief Copy data from src Tensor into buffers indicated by iterators + * + * This method is provided to assist with distributing data from a single + * Tensor into many smaller buffers which have been set up to receive a part + * of the data from the src Tensor + */ +template +void copy(Iter begin, Iter end, BaseTensor& src) +{ + std::accumulate(begin, end, typename BaseTensor::size_type{}, [&src](auto offset, auto& dst) { + auto end_offset = offset + dst.size(); + copy(dst.buffer(), src.buffer(), offset, end_offset); + return end_offset; + }); +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/api/execute.hpp b/backend/include/rapids_triton/triton/api/execute.hpp new file mode 100644 index 0000000..f324065 --- /dev/null +++ b/backend/include/rapids_triton/triton/api/execute.hpp @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace triton_api { +template +auto* execute(TRITONBACKEND_ModelInstance* instance, + TRITONBACKEND_Request** raw_requests, + std::size_t request_count) +{ + auto start_time = std::chrono::steady_clock::now(); + + auto* result = static_cast(nullptr); + + try { + auto* model_state = get_model_state(*get_model_from_instance(*instance)); + auto* instance_state = get_instance_state(*instance); + auto& model = instance_state->get_model(); + auto max_batch_size = model.template get_config_param("max_batch_size"); + + /* Note: It is safe to keep a reference to the model in this closure + * and a pointer to the instance in the next because the batch goes + * out of scope at the end of this block and Triton guarantees that + * the liftimes of both the instance and model extend beyond this + * function call. */ + auto output_shape_fetcher = [&model](std::string const& name, Batch::size_type batch_dim) { + auto result = std::vector{}; + auto config_shape = model.get_output_shape(name); + if (config_shape.size() > 0 && config_shape[0] < 0) { config_shape[0] = batch_dim; } + std::transform( + std::begin(config_shape), + std::end(config_shape), + std::back_inserter(result), + [](auto& coord) { + if (coord < 0) { + throw TritonException( + Error::Internal, + "Backends with variable-shape outputs must request desired output shape"); + } else { + return narrow(coord); + } + }); + return result; + }; + auto statistics_reporter = [instance](TRITONBACKEND_Request* request, + time_point req_start, + time_point req_comp_start, + time_point req_comp_end, + time_point req_end) { + report_statistics(*instance, *request, req_start, req_comp_start, req_comp_end, req_end); + }; + + auto batch = Batch(raw_requests, + request_count, + *(model_state->TritonMemoryManager()), + std::move(output_shape_fetcher), + std::move(statistics_reporter), + model_state->EnablePinnedInput(), + model_state->EnablePinnedOutput(), + max_batch_size, + model.get_stream()); + + if constexpr (IS_GPU_BUILD) { + if (model.get_deployment_type() == GPUDeployment) { + cuda_check(cudaSetDevice(model.get_device_id())); + } + } + + auto predict_err = static_cast(nullptr); + try { + model.predict(batch); + } catch (TritonException& err) { + predict_err = err.error(); + } + + auto& compute_start_time = batch.compute_start_time(); + auto compute_end_time = std::chrono::steady_clock::now(); + batch.finalize(predict_err); + auto end_time = std::chrono::steady_clock::now(); + + report_statistics( + *instance, request_count, start_time, compute_start_time, compute_end_time, end_time); + } catch (TritonException& err) { + result = err.error(); + } + + return result; +} +} // namespace triton_api +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/api/initialize.hpp b/backend/include/rapids_triton/triton/api/initialize.hpp new file mode 100644 index 0000000..37824dd --- /dev/null +++ b/backend/include/rapids_triton/triton/api/initialize.hpp @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace triton_api { +inline auto* initialize(TRITONBACKEND_Backend* backend) +{ + auto* result = static_cast(nullptr); + try { + auto name = get_backend_name(*backend); + + log_info(__FILE__, __LINE__) << "TRITONBACKEND_Initialize: " << name; + + if (!check_backend_version(*backend)) { + throw TritonException{Error::Unsupported, + "triton backend API version does not support this backend"}; + } + if constexpr (IS_GPU_BUILD) { + auto device_count = int{}; + auto cuda_err = cudaGetDeviceCount(&device_count); + if (device_count > 0 && cuda_err == cudaSuccess) { + auto device_id = int{}; + cuda_check(cudaGetDevice(&device_id)); + auto* triton_manager = static_cast(nullptr); + triton_check(TRITONBACKEND_BackendMemoryManager(backend, &triton_manager)); + + setup_memory_resource(static_cast(device_id), triton_manager); + } + } + } catch (TritonException& err) { + result = err.error(); + } + return result; +} +} // namespace triton_api +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/api/instance_finalize.hpp b/backend/include/rapids_triton/triton/api/instance_finalize.hpp new file mode 100644 index 0000000..2d1797c --- /dev/null +++ b/backend/include/rapids_triton/triton/api/instance_finalize.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace triton_api { +template +auto* instance_finalize(TRITONBACKEND_ModelInstance* instance) +{ + auto* result = static_cast(nullptr); + try { + auto* instance_state = get_instance_state(*instance); + if (instance_state != nullptr) { + instance_state->unload(); + + log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInstanceFinalize: delete instance state"; + + delete instance_state; + } + } catch (TritonException& err) { + result = err.error(); + } + + return result; +} +} // namespace triton_api +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/api/instance_initialize.hpp b/backend/include/rapids_triton/triton/api/instance_initialize.hpp new file mode 100644 index 0000000..922152f --- /dev/null +++ b/backend/include/rapids_triton/triton/api/instance_initialize.hpp @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace triton_api { +template +auto* instance_initialize(TRITONBACKEND_ModelInstance* instance) +{ + auto* result = static_cast(nullptr); + try { + auto name = get_model_instance_name(*instance); + auto device_id = get_device_id(*instance); + auto deployment_type = get_deployment_type(*instance); + if constexpr (!IS_GPU_BUILD) { + if (deployment_type == GPUDeployment) { + throw TritonException(Error::Unsupported, "KIND_GPU cannot be used in CPU-only build"); + } + } + + log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInstanceInitialize: " << name << " (" + << TRITONSERVER_InstanceGroupKindString(deployment_type) + << " device " << device_id << ")"; + + auto* triton_model = get_model_from_instance(*instance); + auto* model_state = get_model_state(*triton_model); + if constexpr (IS_GPU_BUILD) { + setup_memory_resource(device_id, model_state->TritonMemoryManager()); + } + + auto rapids_model = std::make_unique(*model_state, instance); + if constexpr (IS_GPU_BUILD) { + auto& model = rapids_model->get_model(); + if (model.get_deployment_type() == GPUDeployment) { + cuda_check(cudaSetDevice(model.get_device_id())); + } + } + rapids_model->load(); + + set_instance_state(*instance, std::move(rapids_model)); + } catch (TritonException& err) { + result = err.error(); + } + return result; +} +} // namespace triton_api +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/api/model_finalize.hpp b/backend/include/rapids_triton/triton/api/model_finalize.hpp new file mode 100644 index 0000000..f4d6d6a --- /dev/null +++ b/backend/include/rapids_triton/triton/api/model_finalize.hpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace triton_api { +template +auto* model_finalize(TRITONBACKEND_Model* model) +{ + auto* result = static_cast(nullptr); + try { + auto model_state = get_model_state(*model); + if (model_state != nullptr) { model_state->get_shared_state()->unload(); } + + log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelFinalize: delete model state"; + + delete model_state; + } catch (TritonException& err) { + result = err.error(); + } + + return result; +} +} // namespace triton_api +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/api/model_initialize.hpp b/backend/include/rapids_triton/triton/api/model_initialize.hpp new file mode 100644 index 0000000..aa48013 --- /dev/null +++ b/backend/include/rapids_triton/triton/api/model_initialize.hpp @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +namespace triton_api { +template +auto* model_initialize(TRITONBACKEND_Model* model) +{ + auto* result = static_cast(nullptr); + try { + auto name = get_model_name(*model); + + auto version = get_model_version(*model); + + log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInitialize: " << name << " (version " + << version << ")"; + + auto rapids_model_state = std::make_unique(*model); + rapids_model_state->load(); + + set_model_state(*model, std::move(rapids_model_state)); + } catch (TritonException& err) { + result = err.error(); + } + + return result; +} +} // namespace triton_api +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/backend.hpp b/backend/include/rapids_triton/triton/backend.hpp new file mode 100644 index 0000000..f0f9a5a --- /dev/null +++ b/backend/include/rapids_triton/triton/backend.hpp @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +inline auto get_backend_name(TRITONBACKEND_Backend& backend) +{ + const char* cname; + triton_check(TRITONBACKEND_BackendName(&backend, &cname)); + return std::string(cname); +} + +namespace { +struct backend_version { + std::uint32_t major; + std::uint32_t minor; +}; +} // namespace + +inline auto check_backend_version(TRITONBACKEND_Backend& backend) +{ + auto version = backend_version{}; + triton_check(TRITONBACKEND_ApiVersion(&version.major, &version.minor)); + + log_info(__FILE__, __LINE__) << "Triton TRITONBACKEND API version: " << version.major << "." + << version.minor; + + auto name = get_backend_name(backend); + + log_info(__FILE__, __LINE__) << "'" << name + << "' TRITONBACKEND API version: " << TRITONBACKEND_API_VERSION_MAJOR + << "." << TRITONBACKEND_API_VERSION_MINOR; + + return ((version.major == TRITONBACKEND_API_VERSION_MAJOR) && + (version.minor >= TRITONBACKEND_API_VERSION_MINOR)); +} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/config.hpp b/backend/include/rapids_triton/triton/config.hpp new file mode 100644 index 0000000..4a8300b --- /dev/null +++ b/backend/include/rapids_triton/triton/config.hpp @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +inline auto get_max_batch_size(common::TritonJson::Value& config) +{ + auto reported = int64_t{}; + triton_check(config.MemberAsInt("max_batch_size", &reported)); + return narrow(reported); +} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/deployment.hpp b/backend/include/rapids_triton/triton/deployment.hpp new file mode 100644 index 0000000..e663c47 --- /dev/null +++ b/backend/include/rapids_triton/triton/deployment.hpp @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { +namespace backend { +namespace rapids { +using DeploymentType = TRITONSERVER_InstanceGroupKind; +auto constexpr GPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_GPU; +auto constexpr CPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_CPU; +// Note (wphicks): We currently are not including "Auto" or "Model" because I +// am not sure exactly how those would be used in context. If there is a +// demand, they can be added. +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/device.hpp b/backend/include/rapids_triton/triton/device.hpp new file mode 100644 index 0000000..73b28c4 --- /dev/null +++ b/backend/include/rapids_triton/triton/device.hpp @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { +namespace backend { +namespace rapids { +using device_id_t = std::int32_t; +} +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/input.hpp b/backend/include/rapids_triton/triton/input.hpp new file mode 100644 index 0000000..668875a --- /dev/null +++ b/backend/include/rapids_triton/triton/input.hpp @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) +{ + auto result = static_cast(nullptr); + triton_check(TRITONBACKEND_RequestInput(request, name.c_str(), &result)); + return result; +} + +template +auto get_triton_input_shape(Iter requests_begin, Iter requests_end, std::string const& name) +{ + auto result = std::vector{}; + + auto reported_dtype = DType{}; + auto const* input_shape = static_cast(nullptr); + auto input_dims = uint32_t{}; + + auto batch_dim = std::accumulate( + requests_begin, + requests_end, + int64_t{}, + [&reported_dtype, &input_shape, &input_dims, &name](auto total, auto& request) { + auto* input = get_triton_input(request, name); + triton_check(TRITONBACKEND_InputProperties( + input, nullptr, &reported_dtype, &input_shape, &input_dims, nullptr, nullptr)); + + if (reported_dtype != TritonDtype::value) { + auto log_stream = std::stringstream{}; + log_stream << "incorrect type " << reported_dtype << " for input with required type " + << TritonDtype::value; + throw(TritonException(Error::Internal, log_stream.str())); + } + + if (input_dims != 0) { total += *input_shape; } + return total; + }); + + result.reserve(input_dims); + std::transform(input_shape, input_shape + input_dims, std::back_inserter(result), [](auto& val) { + return narrow(val); + }); + + if (!result.empty()) { result[0] = narrow(batch_dim); } + + return result; +} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/logging.hpp b/backend/include/rapids_triton/triton/logging.hpp new file mode 100644 index 0000000..96455f1 --- /dev/null +++ b/backend/include/rapids_triton/triton/logging.hpp @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { + +namespace { +/** Log message at indicated level */ +inline void log(TRITONSERVER_LogLevel level, char const* filename, int line, char const* message) +{ + triton_check(TRITONSERVER_LogMessage(level, filename, line, message)); +} +} // namespace + +struct log_stream : public std::ostream { + log_stream(TRITONSERVER_LogLevel level, char const* filename, int line) + : std::ostream{}, buffer_{level, filename, line} + { + rdbuf(&buffer_); + } + log_stream(TRITONSERVER_LogLevel level) : std::ostream{}, buffer_{level, __FILE__, __LINE__} + { + rdbuf(&buffer_); + } + + ~log_stream() + { + try { + flush(); + } catch (std::ios_base::failure const& ignored_err) { + // Ignore error if flush fails + } + } + + private: + struct log_buffer : public std::stringbuf { + log_buffer(TRITONSERVER_LogLevel level, char const* filename, int line) + : level_{level}, filename_{filename}, line_{line} + { + } + + virtual int sync() + { + auto msg = str(); + if (!msg.empty()) { + log(level_, filename_, line_, msg.c_str()); + str(""); + } + return 0; + } + + private: + TRITONSERVER_LogLevel level_; + char const* filename_; + int line_; + }; + + log_buffer buffer_; +}; + +/** Log message at INFO level */ +inline void log_info(char const* filename, int line, char const* message) +{ + log(TRITONSERVER_LOG_INFO, filename, line, message); +} +inline void log_info(char const* filename, int line, std::string const& message) +{ + log_info(filename, line, message.c_str()); +} +inline void log_info(char const* message) { log_info(__FILE__, __LINE__, message); } +inline void log_info(std::string const& message) { log_info(__FILE__, __LINE__, message.c_str()); } +inline auto log_info(char const* filename, int line) +{ + return log_stream(TRITONSERVER_LOG_INFO, filename, line); +} +inline auto log_info() { return log_stream(TRITONSERVER_LOG_INFO); } + +/** Log message at WARN level */ +inline void log_warn(char const* filename, int line, char const* message) +{ + log(TRITONSERVER_LOG_WARN, filename, line, message); +} +inline void log_warn(char const* filename, int line, std::string const& message) +{ + log_warn(filename, line, message.c_str()); +} +inline void log_warn(char const* message) { log_warn(__FILE__, __LINE__, message); } +inline void log_warn(std::string const& message) { log_warn(__FILE__, __LINE__, message.c_str()); } +inline auto log_warn(char const* filename, int line) +{ + return log_stream(TRITONSERVER_LOG_WARN, filename, line); +} +inline auto log_warn() { return log_stream(TRITONSERVER_LOG_WARN); } + +/** Log message at ERROR level */ +inline void log_error(char const* filename, int line, char const* message) +{ + log(TRITONSERVER_LOG_ERROR, filename, line, message); +} +inline void log_error(char const* filename, int line, std::string const& message) +{ + log_error(filename, line, message.c_str()); +} +inline void log_error(char const* message) { log_error(__FILE__, __LINE__, message); } +inline void log_error(std::string const& message) +{ + log_error(__FILE__, __LINE__, message.c_str()); +} +inline auto log_error(char const* filename, int line) +{ + return log_stream(TRITONSERVER_LOG_ERROR, filename, line); +} +inline auto log_error() { return log_stream(TRITONSERVER_LOG_ERROR); } + +/** Log message at VERBOSE level */ +inline void log_debug(char const* filename, int line, char const* message) +{ + log(TRITONSERVER_LOG_VERBOSE, filename, line, message); +} +inline void log_debug(char const* filename, int line, std::string const& message) +{ + log_debug(filename, line, message.c_str()); +} +inline void log_debug(char const* message) { log_debug(__FILE__, __LINE__, message); } +inline void log_debug(std::string const& message) +{ + log_debug(__FILE__, __LINE__, message.c_str()); +} +inline auto log_debug(char const* filename, int line) +{ + return log_stream(TRITONSERVER_LOG_VERBOSE, filename, line); +} +inline auto log_debug() { return log_stream(TRITONSERVER_LOG_VERBOSE); } + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/model.hpp b/backend/include/rapids_triton/triton/model.hpp new file mode 100644 index 0000000..1591763 --- /dev/null +++ b/backend/include/rapids_triton/triton/model.hpp @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { + +inline auto get_model_version(TRITONBACKEND_Model& model) +{ + auto version = std::uint64_t{}; + triton_check(TRITONBACKEND_ModelVersion(&model, &version)); + return version; +} + +inline auto get_model_name(TRITONBACKEND_Model& model) +{ + auto* cname = static_cast(nullptr); + triton_check(TRITONBACKEND_ModelName(&model, &cname)); + return std::string(cname); +} + +inline auto get_model_config(TRITONBACKEND_Model& model) +{ + auto* config_message = static_cast(nullptr); + triton_check(TRITONBACKEND_ModelConfig(&model, 1, &config_message)); + + auto* buffer = static_cast(nullptr); + auto byte_size = std::size_t{}; + triton_check(TRITONSERVER_MessageSerializeToJson(config_message, &buffer, &byte_size)); + + auto model_config = std::make_unique(); + auto* err = model_config->Parse(buffer, byte_size); + auto* result = TRITONSERVER_MessageDelete(config_message); + if (err != nullptr) { throw(TritonException(err)); } + if (result != nullptr) { throw(TritonException(result)); } + return model_config; +} + +/** + * @brief Set model state (as used by Triton) to given object + * + * This function accepts a unique_ptr to an object derived from a Triton + * BackendModel object and sets it as the stored state for a model in the + * Triton server. Note that this object is not the same as a RAPIDS-Triton + * "SharedModelState" object. The object that Triton expects must wrap this + * SharedModelState and provide additional interface compatibility. + */ +template +void set_model_state(TRITONBACKEND_Model& model, std::unique_ptr&& model_state) +{ + triton_check(TRITONBACKEND_ModelSetState(&model, reinterpret_cast(model_state.release()))); +} + +/** Given a model, return its associated ModelState object */ +template +auto* get_model_state(TRITONBACKEND_Model& model) +{ + auto* vstate = static_cast(nullptr); + triton_check(TRITONBACKEND_ModelState(&model, &vstate)); + + auto* model_state = reinterpret_cast(vstate); + + return model_state; +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/model_instance.hpp b/backend/include/rapids_triton/triton/model_instance.hpp new file mode 100644 index 0000000..0ba03bd --- /dev/null +++ b/backend/include/rapids_triton/triton/model_instance.hpp @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +/** Get the name of a Triton model instance from the instance itself */ +inline auto get_model_instance_name(TRITONBACKEND_ModelInstance& instance) +{ + auto* cname = static_cast(nullptr); + triton_check(TRITONBACKEND_ModelInstanceName(&instance, &cname)); + return std::string(cname); +} + +/** Get the device on which a Triton model instance is loaded + * + * If this instance is loaded on the host, 0 will be returned. Otherwise the + * GPU device id will be returned.*/ +inline auto get_device_id(TRITONBACKEND_ModelInstance& instance) +{ + auto device_id = device_id_t{}; + triton_check(TRITONBACKEND_ModelInstanceDeviceId(&instance, &device_id)); + return device_id; +} + +/** Determine how a Triton model instance is deployed + * + * Returns enum value indicating whether the instance is deployed on device + * or on the host + */ +inline auto get_deployment_type(TRITONBACKEND_ModelInstance& instance) +{ + auto kind = GPUDeployment; + triton_check(TRITONBACKEND_ModelInstanceKind(&instance, &kind)); + return kind; +} + +/** Return the Triton model from one of its instances + */ +inline auto* get_model_from_instance(TRITONBACKEND_ModelInstance& instance) +{ + auto* model = static_cast(nullptr); + triton_check(TRITONBACKEND_ModelInstanceModel(&instance, &model)); + return model; +} + +/** + * @brief Set Triton model instance state to given object + * + * This function accepts a unique_ptr to an object derived from a Triton + * BackendModelInstance object and sets it as the stored state for a model in the + * Triton server. Note that this object is not the same as a RAPIDS-Triton + * "Model" object. The object that Triton expects must wrap this Model and + * provide additional interface compatibility. + */ +template +void set_instance_state(TRITONBACKEND_ModelInstance& instance, + std::unique_ptr&& model_instance_state) +{ + triton_check(TRITONBACKEND_ModelInstanceSetState( + &instance, reinterpret_cast(model_instance_state.release()))); +} + +/** Get model instance state from instance */ +template +auto* get_instance_state(TRITONBACKEND_ModelInstance& instance) +{ + auto* instance_state = static_cast(nullptr); + triton_check( + TRITONBACKEND_ModelInstanceState(&instance, reinterpret_cast(&instance_state))); + return instance_state; +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/model_instance_state.hpp b/backend/include/rapids_triton/triton/model_instance_state.hpp new file mode 100644 index 0000000..ba05009 --- /dev/null +++ b/backend/include/rapids_triton/triton/model_instance_state.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { + +template +struct ModelInstanceState : public BackendModelInstance { + ModelInstanceState(TritonModelState& model_state, + TRITONBACKEND_ModelInstance* triton_model_instance) + : BackendModelInstance(&model_state, triton_model_instance), + model_(model_state.get_shared_state(), + rapids::get_device_id(*triton_model_instance), + CudaStream(), + Kind(), + JoinPath({model_state.RepositoryPath(), + std::to_string(model_state.Version()), + ArtifactFilename()})) + { + } + + auto& get_model() const { return model_; } + + void load() { model_.load(); } + void unload() { model_.unload(); } + + private: + RapidsModel model_; +}; + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/model_state.hpp b/backend/include/rapids_triton/triton/model_state.hpp new file mode 100644 index 0000000..582b679 --- /dev/null +++ b/backend/include/rapids_triton/triton/model_state.hpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +template +struct TritonModelState : public BackendModel { + TritonModelState(TRITONBACKEND_Model& triton_model) + : BackendModel(&triton_model), + state_{std::make_shared(get_model_config(triton_model))} + { + } + + void load() { state_->load(); } + void unload() { state_->unload(); } + + auto get_shared_state() { return state_; } + + private: + std::shared_ptr state_; +}; + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/output.hpp b/backend/include/rapids_triton/triton/output.hpp new file mode 100644 index 0000000..a48d156 --- /dev/null +++ b/backend/include/rapids_triton/triton/output.hpp @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) +{ + auto* result = static_cast(nullptr); + triton_check(TRITONBACKEND_RequestInput(request, name.c_str(), &result)); + return result; +} + +template +auto get_triton_output_shape(Iter requests_begin, Iter requests_end, std::string const& name) +{ + auto result = std::vector{}; + + auto reported_dtype = DType{}; + auto const* input_shape = static_cast(nullptr); + auto input_dims = uint32_t{}; + + auto batch_dim = + std::reduce(requests_begin, + requests_end, + int64_t{}, + [&reported_dtype, &input_shape, &input_dims, &name](auto& request, auto total) { + auto* input = get_triton_input(request, name); + triton_check(TRITONBACKEND_InputProperties( + input, nullptr, &reported_dtype, &input_shape, &input_dims, nullptr, nullptr)); + + if (reported_dtype != TritonDtype::value) { + auto log_stream = std::stringstream{}; + log_stream << "incorrect type " << reported_dtype + << " for output with required type " << TritonDtype::value; + throw(TritonException(Error::Internal, log_stream.str())); + } + + if (input_dims != 0) { total += *input_shape; } + return total; + }); + + result.reserve(input_dims); + std::transform(input_shape, input_shape + input_dims, std::back_inserter(result), [](auto& val) { + return narrow(val); + }); + + if (!result.empty()) { result[0] = narrow(batch_dim); } + + return result; +} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/requests.hpp b/backend/include/rapids_triton/triton/requests.hpp new file mode 100644 index 0000000..c24c8bb --- /dev/null +++ b/backend/include/rapids_triton/triton/requests.hpp @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +using request_size_t = uint32_t; + +template +void release_requests(Iter begin, Iter end) +{ + std::for_each(begin, end, [](auto& request) { + try { + triton_check(TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL)); + } catch (TritonException& err) { + log_error(__FILE__, __LINE__, err.what()); + } + }); +} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/responses.hpp b/backend/include/rapids_triton/triton/responses.hpp new file mode 100644 index 0000000..b361c5a --- /dev/null +++ b/backend/include/rapids_triton/triton/responses.hpp @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { + +template +auto construct_responses(Iter requests_begin, Iter requests_end) +{ + auto responses = std::vector{}; + + auto requests_size = std::distance(requests_begin, requests_end); + if (!(requests_size > 0)) { + throw TritonException(Error::Internal, + "Invalid iterators for requests when constructing responses"); + } + responses.reserve(requests_size); + + std::transform(requests_begin, requests_end, std::back_inserter(responses), [](auto* request) { + auto* response = static_cast(nullptr); + triton_check(TRITONBACKEND_ResponseNew(&response, request)); + return response; + }); + return responses; +} + +template +void send_responses(Iter begin, Iter end, TRITONSERVER_Error* err) +{ + std::for_each(begin, end, [err](auto& response) { + decltype(err) err_copy; + if (err != nullptr) { + err_copy = TRITONSERVER_ErrorNew(TRITONSERVER_ErrorCode(err), TRITONSERVER_ErrorMessage(err)); + } else { + err_copy = err; + } + + if (response == nullptr) { + log_error(__FILE__, __LINE__) << "Failure in response collation"; + } else { + try { + triton_check( + TRITONBACKEND_ResponseSend(response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, err_copy)); + } catch (TritonException& err) { + log_error(__FILE__, __LINE__, err.what()); + } + } + }); +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/statistics.hpp b/backend/include/rapids_triton/triton/statistics.hpp new file mode 100644 index 0000000..5f4f130 --- /dev/null +++ b/backend/include/rapids_triton/triton/statistics.hpp @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +using time_point = std::chrono::time_point; + +/** + * @brief Report inference statistics for a single request + * + * @param instance The Triton model instance which is processing this request + * @param request The Triton request object itself + * @param start_time The time at which the backend first received the request + * @param compute_start_time The time at which the backend began actual + * inference on the request + * @param compute_end_time The time at which the backend completed inference + * on the request + * @param end_time The time at which the backend finished all processing on + * the request, including copying out results and returning a response + */ +inline void report_statistics(TRITONBACKEND_ModelInstance& instance, + TRITONBACKEND_Request& request, + time_point start_time, + time_point compute_start_time, + time_point compute_end_time, + time_point end_time) +{ + triton_check( + TRITONBACKEND_ModelInstanceReportStatistics(&instance, + &request, + true, + start_time.time_since_epoch().count(), + compute_start_time.time_since_epoch().count(), + compute_end_time.time_since_epoch().count(), + end_time.time_since_epoch().count())); +} + +/** + * @brief Report inference statistics for a batch of requests of given size + * + * @param instance The Triton model instance which is processing this batch + * @param request_count The number of requests in this batch + * @param start_time The time at which the backend first received the batch + * @param compute_start_time The time at which the backend began actual + * inference on the batch + * @param compute_end_time The time at which the backend completed inference + * on the batch + * @param end_time The time at which the backend finished all processing on + * the batch, including copying out results and returning a response + */ +inline void report_statistics(TRITONBACKEND_ModelInstance& instance, + std::size_t request_count, + time_point start_time, + time_point compute_start_time, + time_point compute_end_time, + time_point end_time) +{ + triton_check( + TRITONBACKEND_ModelInstanceReportBatchStatistics(&instance, + request_count, + start_time.time_since_epoch().count(), + compute_start_time.time_since_epoch().count(), + compute_end_time.time_since_epoch().count(), + end_time.time_since_epoch().count())); +} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/triton/triton_memory_resource.hpp b/backend/include/rapids_triton/triton/triton_memory_resource.hpp new file mode 100644 index 0000000..77a879e --- /dev/null +++ b/backend/include/rapids_triton/triton/triton_memory_resource.hpp @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +struct triton_memory_resource final : public rmm::mr::device_memory_resource { + triton_memory_resource(TRITONBACKEND_MemoryManager* manager, + device_id_t device_id, + rmm::mr::device_memory_resource* fallback) + : manager_{manager}, device_id_{device_id}, fallback_{fallback} + { + } + + bool supports_streams() const noexcept override { return false; } + bool supports_get_mem_info() const noexcept override { return false; } + auto* get_triton_manager() const noexcept { return manager_; } + + private: + TRITONBACKEND_MemoryManager* manager_; + std::int64_t device_id_; + rmm::mr::device_memory_resource* fallback_; + + void* do_allocate(std::size_t bytes, rmm::cuda_stream_view stream) override + { + auto* ptr = static_cast(nullptr); + if (manager_ == nullptr) { + ptr = fallback_->allocate(bytes, stream); + } else { + triton_check(TRITONBACKEND_MemoryManagerAllocate( + manager_, &ptr, TRITONSERVER_MEMORY_GPU, device_id_, static_cast(bytes))); + } + return ptr; + } + + void do_deallocate(void* ptr, std::size_t bytes, rmm::cuda_stream_view stream) + { + if (manager_ == nullptr) { + fallback_->deallocate(ptr, bytes, stream); + } else { + triton_check( + TRITONBACKEND_MemoryManagerFree(manager_, ptr, TRITONSERVER_MEMORY_GPU, device_id_)); + } + } + + bool do_is_equal(rmm::mr::device_memory_resource const& other) const noexcept override + { + auto* other_triton_mr = dynamic_cast(&other); + return (other_triton_mr != nullptr && other_triton_mr->get_triton_manager() == manager_); + } + + std::pair do_get_mem_info(rmm::cuda_stream_view stream) const override + { + return {0, 0}; + } +}; + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/utils/const_agnostic.hpp b/backend/include/rapids_triton/utils/const_agnostic.hpp new file mode 100644 index 0000000..cd55aeb --- /dev/null +++ b/backend/include/rapids_triton/utils/const_agnostic.hpp @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { +namespace backend { +namespace rapids { + +template +using const_agnostic_same_t = + std::enable_if_t, std::remove_const_t>>; + +} +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/utils/device_setter.hpp b/backend/include/rapids_triton/utils/device_setter.hpp new file mode 100644 index 0000000..d8aae9b --- /dev/null +++ b/backend/include/rapids_triton/utils/device_setter.hpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifdef TRITON_ENABLE_GPU +#include +#endif +#include +#include + +namespace triton { +namespace backend { +namespace rapids { + +/** Struct for setting cuda device within a code block */ +struct device_setter { + device_setter(device_id_t device) : prev_device_{} { + if constexpr(IS_GPU_BUILD) { + cuda_check(cudaGetDevice(&prev_device_)); + cuda_check(cudaSetDevice(device)); + } else { + throw TritonException(Error::Internal, "Device setter used in non-GPU build"); + } + } + + ~device_setter() { + if constexpr(IS_GPU_BUILD) { + cudaSetDevice(prev_device_); + } + } + private: + device_id_t prev_device_; +}; + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/include/rapids_triton/utils/narrow.hpp b/backend/include/rapids_triton/utils/narrow.hpp new file mode 100644 index 0000000..37465c0 --- /dev/null +++ b/backend/include/rapids_triton/utils/narrow.hpp @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include + +namespace triton { +namespace backend { +namespace rapids { + +template +auto narrow(F from) +{ + auto to = static_cast(from); + + if (static_cast(to) != from || + (std::is_signed::value && !std::is_signed::value && from < F{}) || + (std::is_signed::value && !std::is_signed::value && to < T{}) || + (std::is_signed::value == std::is_signed::value && ((to < T{}) != (from < F{})))) { + throw TritonException(Error::Internal, "invalid narrowing"); + } + return to; +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/scripts/run-clang-format.py b/backend/scripts/run-clang-format.py new file mode 100755 index 0000000..614697b --- /dev/null +++ b/backend/scripts/run-clang-format.py @@ -0,0 +1,148 @@ +# Copyright (c) 2019-2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file was copied from the rapidsai/cuml repo + +from __future__ import print_function +import sys +import re +import os +import subprocess +import argparse +import tempfile +import shutil + + +EXPECTED_VERSION = "11.0.0" +VERSION_REGEX = re.compile(r"clang-format version ([0-9.]+)") +DEFAULT_DIRS = ["cpp/src", + "cpp/include", + "cpp/test"] + + +def parse_args(): + argparser = argparse.ArgumentParser("Runs clang-format on a project") + argparser.add_argument("-dstdir", type=str, default=None, + help="Directory to store the temporary outputs of" + " clang-format. If nothing is passed for this, then" + " a temporary dir will be created using `mkdtemp`") + argparser.add_argument("-exe", type=str, default="clang-format", + help="Path to clang-format exe") + argparser.add_argument("-inplace", default=False, action="store_true", + help="Replace the source files itself.") + argparser.add_argument("-regex", type=str, + default=r"[.](cu|cuh|h|hpp|cpp)$", + help="Regex string to filter in sources") + argparser.add_argument("-ignore", type=str, default=r"cannylab/bh[.]cu$", + help="Regex used to ignore files from matched list") + argparser.add_argument("-v", dest="verbose", action="store_true", + help="Print verbose messages") + argparser.add_argument("dirs", type=str, nargs="*", + help="List of dirs where to find sources") + args = argparser.parse_args() + args.regex_compiled = re.compile(args.regex) + args.ignore_compiled = re.compile(args.ignore) + if args.dstdir is None: + args.dstdir = tempfile.mkdtemp() + ret = subprocess.check_output("%s --version" % args.exe, shell=True) + ret = ret.decode("utf-8") + version = VERSION_REGEX.match(ret) + if version is None: + raise Exception("Failed to figure out clang-format version!") + version = version.group(1) + if version != EXPECTED_VERSION: + raise Exception("clang-format exe must be v%s found '%s'" % \ + (EXPECTED_VERSION, version)) + if len(args.dirs) == 0: + args.dirs = DEFAULT_DIRS + return args + + +def list_all_src_files(file_regex, ignore_regex, srcdirs, dstdir, inplace): + allFiles = [] + for srcdir in srcdirs: + for root, dirs, files in os.walk(srcdir): + for f in files: + if re.search(file_regex, f): + src = os.path.join(root, f) + if re.search(ignore_regex, src): + continue + if inplace: + _dir = root + else: + _dir = os.path.join(dstdir, root) + dst = os.path.join(_dir, f) + allFiles.append((src, dst)) + return allFiles + + +def run_clang_format(src, dst, exe, verbose): + dstdir = os.path.dirname(dst) + if not os.path.exists(dstdir): + os.makedirs(dstdir) + # run the clang format command itself + if src == dst: + cmd = "%s -i %s" % (exe, src) + else: + cmd = "%s %s > %s" % (exe, src, dst) + try: + subprocess.check_call(cmd, shell=True) + except subprocess.CalledProcessError: + print("Failed to run clang-format! Maybe your env is not proper?") + raise + # run the diff to check if there are any formatting issues + cmd = "diff -q %s %s >/dev/null" % (src, dst) + try: + subprocess.check_call(cmd, shell=True) + if verbose: + print("%s passed" % os.path.basename(src)) + except subprocess.CalledProcessError: + print("%s failed! 'diff %s %s' will show formatting violations!" % \ + (os.path.basename(src), src, dst)) + return False + return True + + +def main(): + args = parse_args() + # Attempt to making sure that we run this script from root of repo always + if not os.path.exists(".git"): + print("Error!! This needs to always be run from the root of repo") + sys.exit(-1) + all_files = list_all_src_files(args.regex_compiled, args.ignore_compiled, + args.dirs, args.dstdir, args.inplace) + + # Check whether clang-format exists + if shutil.which("clang-format") is None: + print("clang-format not found. Exiting...") + return + + # actual format checker + status = True + for src, dst in all_files: + if not run_clang_format(src, dst, args.exe, args.verbose): + status = False + if not status: + print("clang-format failed! You have 2 options:") + print(" 1. Look at formatting differences above and fix them manually") + print(" 2. Or run the below command to bulk-fix all these at once") + print("Bulk-fix command: ") + print(" python cpp/scripts/run-clang-format.py %s -inplace" % \ + " ".join(sys.argv[1:])) + sys.exit(-1) + return + + +if __name__ == "__main__": + main() diff --git a/backend/src/CMakeLists.txt b/backend/src/CMakeLists.txt new file mode 100644 index 0000000..1d5904c --- /dev/null +++ b/backend/src/CMakeLists.txt @@ -0,0 +1,68 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +# keep the files in alphabetical order! +add_library( + triton_rapids-identity SHARED + src/api.cc +) + +if(TRITON_ENABLE_GPU) + set_target_properties(triton_rapids-identity + PROPERTIES BUILD_RPATH "\$ORIGIN" + # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) +else() + set_target_properties(triton_rapids-identity + PROPERTIES BUILD_RPATH "\$ORIGIN" + # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) +endif() + +target_compile_options(triton_rapids-identity + PRIVATE "$<$:${RAPIDS_TRITON_CXX_FLAGS}>" + "$<$:${RAPIDS_TRITON_CUDA_FLAGS}>" +) + +target_include_directories(triton_rapids-identity + PRIVATE "$" + "${CMAKE_CURRENT_SOURCE_DIR}/src" +) + +target_link_libraries(triton_rapids-identity +PRIVATE + $<$:rmm::rmm> + $<$:raft::raft> + triton-core-serverstub + triton-backend-utils + "${TRITONSERVER_LIB}" + $ +) + +install( + TARGETS triton_rapids-identity + LIBRARY DESTINATION /opt/tritonserver/backends/rapids-identity +) diff --git a/backend/src/api.cc b/backend/src/api.cc new file mode 100644 index 0000000..7e62aa6 --- /dev/null +++ b/backend/src/api.cc @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace NAMESPACE { + +using ModelState = rapids::TritonModelState; +using ModelInstanceState = + rapids::ModelInstanceState; + +extern "C" { + +/** Confirm that backend is compatible with Triton's backend API version + */ +TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { + return rapids::triton_api::initialize(backend); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { + return rapids::triton_api::model_initialize(model); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { + return rapids::triton_api::model_finalize(model); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize( + TRITONBACKEND_ModelInstance* instance) { + return rapids::triton_api::instance_initialize(instance); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize( + TRITONBACKEND_ModelInstance* instance) { + return rapids::triton_api::instance_finalize(instance); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( + TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** raw_requests, + uint32_t const request_count) { + return rapids::triton_api::execute( + instance, raw_requests, static_cast(request_count)); +} + +} // extern "C" + +} // namespace NAMESPACE +} // namespace backend +} // namespace triton diff --git a/backend/src/model.h b/backend/src/model.h new file mode 100644 index 0000000..5ba95f1 --- /dev/null +++ b/backend/src/model.h @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +#include +#include + +#include +#include +#include // rapids::Batch +#include // rapids::MemoryType +#include // rapids::Model +#include // rapids::copy +#include // rapids::DeploymentType +#include // rapids::device_id_t + +namespace triton { +namespace backend { +namespace NAMESPACE { + +/* Any logic necessary to perform inference with a model and manage its data + * should be implemented in a struct named RapidsModel, as shown here */ + +struct RapidsModel : rapids::Model { + /*************************************************************************** + * BOILERPLATE * + * *********************************************************************** * + * The following constructor can be copied directly into any model + * implementation. + **************************************************************************/ + RapidsModel(std::shared_ptr shared_state, + rapids::device_id_t device_id, + cudaStream_t default_stream, + rapids::DeploymentType deployment_type, + std::string const& filepath) + : rapids::Model( + shared_state, device_id, default_stream, deployment_type, filepath) + { + } + + /*************************************************************************** + * BASIC FEATURES * + * *********************************************************************** * + * The only method that *must* be implemented for a viable model is the + * `predict` method, but the others presented here are often used for basic + * model implementations. Filling out these methods should take care of most + * use cases. + **************************************************************************/ + + /*************************************************************************** + * predict * + * *********************************************************************** * + * This method performs the actual inference step on input data. Implementing + * a predict function requires four steps: + * 1. Call `get_input` on the provided `Batch` object for each of the input + * tensors named in the config file for this backend. This provides a + * `Tensor` object containing the input data. + * 2. Call `get_output` on the provided `Batch` object for each of the output + * tensors named in the config file for this backend. This provides a + * `Tensor` object to which output values can be written. + * 3. Perform inference based on the input Tensors and store the results in + * the output Tensors. `some_tensor.data()` can be used to retrieve a raw + * pointer to the underlying data. + * 4. Call the `finalize` method on all output tensors. + **************************************************************************/ + void predict(rapids::Batch& batch) const + { + // 1. Acquire a tensor representing the input named "input__0" + auto input = get_input(batch, "input__0"); + // 2. Acquire a tensor representing the output named "output__0" + auto output = get_output(batch, "output__0"); + + // 3. Perform inference. In this example, we simply copy the data from the + // input to the output tensor. + rapids::copy(output, input); + + // 4. Call finalize on all output tensors. In this case, we have just one + // output, so we call finalize on it. + output.finalize(); + } + + /*************************************************************************** + * load / unload * + * *********************************************************************** * + * These methods can be used to perform one-time loading/unloading of + * resources when a model is created. For example, data representing the + * model may be loaded onto the GPU in the `load` method and unloaded in the + * `unload` method. This data will then remain loaded while the server is + * running. + * + * While these methods take no arguments, it is typical to read any necessary + * input from the model configuration file by using the `get_config_param` + * method. Any parameters defined in the "parameters" section of the config + * can be accessed by name in this way. The maximum batch size can also be + * retrieved using the name "max_batch_size". + * + * These methods need not be explicitly implemented if no loading/unloading + * logic is required, but we show them here for illustrative purposes. + **************************************************************************/ + void load() {} + void unload() {} + + /*************************************************************************** + * ADVANCED FEATURES * + * *********************************************************************** * + * None of the following methods are required to be implemented in order to + * create a valid model, but they are presented here for those who require + * the additional functionality they provide. + **************************************************************************/ + + /*************************************************************************** + * preferred_mem_type / preferred_mem_type_in / preferred_mem_type_out * + * *********************************************************************** * + * If implemented, `preferred_mem_type` allows for control over when input + * and output data are provided on the host versus on device. In the case + * that a model prefers to receive its input on-host but return output + * on-device (or vice versa), `preferred_mem_type_in` and + * `preferred_mem_type_out` can be used for even more precise control. + * + * In this example, we simply return `std::nullopt` to indicate that the + * model has no preference on its input/output data locations. Note that the + * Batch being processed is taken as input to this function to facilitate + * implementations that may switch their preferred memory location based on + * properties of the batch. + * + * Valid MemoryType options to return are rapids::HostMemory and + * rapids::DeviceMemory. + **************************************************************************/ + std::optional preferred_mem_type(rapids::Batch& batch) const + { + return std::nullopt; + } +}; + +} // namespace NAMESPACE +} // namespace backend +} // namespace triton diff --git a/backend/src/names.h b/backend/src/names.h new file mode 100644 index 0000000..33d6fe0 --- /dev/null +++ b/backend/src/names.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +/* Triton expects certain definitions within its backend libraries to follow + * specific naming conventions. Specifically, for a backend named + * "rapids_identity," most definitions should appear within a namespace called + * triton::backend::rapids_identity. + * + * In order to facilitate this with minimal effort on the part of backend + * developers, we ask that you put the name of your backend here. This macro is + * then used to propagate the correct namespace name wherever it is needed in + * the impl and interface code. */ + +#define NAMESPACE rapids_identity diff --git a/backend/src/shared_state.h b/backend/src/shared_state.h new file mode 100644 index 0000000..b6f8981 --- /dev/null +++ b/backend/src/shared_state.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include + +namespace triton { +namespace backend { +namespace NAMESPACE { + +/* Triton allows multiple instances of a single model to be instantiated at the + * same time (e.g. on different GPUs). All instances of a model share access to + * an object which manages any state that can be shared across all instances. + * Any logic necessary for managing such state should be implemented in a + * struct named RapidsSharedState, as shown here. Models may access this shared + * state object via the `get_shared_state` method, which returns a shared + * pointer to the RapidsSharedState object. + * + * Not all backends require shared state, so leaving this implementation empty + * is entirely valid */ + +struct RapidsSharedState : rapids::SharedModelState { + RapidsSharedState(std::unique_ptr&& config) + : rapids::SharedModelState{std::move(config)} + { + } + void load() {} + void unload() {} +}; + +} // namespace NAMESPACE +} // namespace backend +} // namespace triton diff --git a/backend/test/CMakeLists.txt b/backend/test/CMakeLists.txt new file mode 100644 index 0000000..dbe92b2 --- /dev/null +++ b/backend/test/CMakeLists.txt @@ -0,0 +1,102 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +# keep the files in alphabetical order! +add_executable(test_rapids_triton + test/batch/batch.cpp + test/build_control.cpp + test/exceptions.cpp + test/memory/buffer.cpp + test/memory/detail/copy.cpp + test/memory/detail/owned_device_buffer.cpp + test/memory/resource.cpp + test/memory/types.cpp + test/tensor/dtype.cpp + test/tensor/tensor.cpp + test/test.cpp + test/triton/api/execute.cpp + test/triton/api/initialize.cpp + test/triton/api/instance_finalize.cpp + test/triton/api/instance_initialize.cpp + test/triton/api/model_finalize.cpp + test/triton/api/model_initialize.cpp + test/triton/backend.cpp + test/triton/config.cpp + test/triton/deployment.cpp + test/triton/device.cpp + test/triton/input.cpp + test/triton/logging.cpp + test/triton/model.cpp + test/triton/model_instance.cpp + test/triton/requests.cpp + test/triton/responses.cpp + test/triton/statistics.cpp + test/utils/const_agnostic.cpp + test/utils/narrow.cpp +) + +IF(TRITON_ENABLE_GPU) + set_target_properties(test_rapids_triton + PROPERTIES BUILD_RPATH "\$ORIGIN" + # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) +else() + set_target_properties(test_rapids_triton + PROPERTIES BUILD_RPATH "\$ORIGIN" + # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) +endif() + +target_compile_options(test_rapids_triton + PRIVATE "$<$:${RAPIDS_TRITON_CXX_FLAGS}>" + "$<$:${RAPIDS_TRITON_CUDA_FLAGS}>" +) + +target_include_directories(test_rapids_triton + PUBLIC "$" + "$" +) + + +find_library( + TRITONSERVER_LIB + tritonserver + PATHS /opt/tritonserver/lib +) + +target_link_libraries(test_rapids_triton +PRIVATE + $<$:rmm::rmm> + $<$:raft::raft> + triton-core-serverstub + triton-backend-utils + gmock + gmock_main + GTest::gtest + GTest::gtest_main + "${TRITONSERVER_LIB}" + $ +) diff --git a/backend/test/batch/batch.cpp b/backend/test/batch/batch.cpp new file mode 100644 index 0000000..e522ecb --- /dev/null +++ b/backend/test/batch/batch.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/build_control.cpp b/backend/test/build_control.cpp new file mode 100644 index 0000000..18392b0 --- /dev/null +++ b/backend/test/build_control.cpp @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +namespace triton { +namespace backend { +namespace rapids { + +TEST(RapidsTriton, build_control) +{ +#ifdef TRITON_ENABLE_GPU + ASSERT_EQ(IS_GPU_BUILD, true) << "IS_GPU_BUILD constant has wrong value\n"; +#else + ASSERT_EQ(IS_GPU_BUILD, false) << "IS_GPU_BUILD constant has wrong value\n"; +#endif +} +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/test/exceptions.cpp b/backend/test/exceptions.cpp new file mode 100644 index 0000000..022d184 --- /dev/null +++ b/backend/test/exceptions.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +#include + +#include +#include + +namespace triton { +namespace backend { +namespace rapids { + +TEST(RapidsTriton, default_except) +{ + try { + throw TritonException(); + } catch (TritonException const& err) { + EXPECT_EQ(std::string(err.what()), std::string("encountered unknown error")); + } +} + +TEST(RapidsTriton, msg_except) +{ + auto msg = std::string("TEST ERROR MESSAGE"); + try { + throw TritonException(Error::Internal, msg); + } catch (TritonException const& err) { + EXPECT_EQ(std::string(err.what()), msg); + } + try { + throw TritonException(Error::Internal, msg.c_str()); + } catch (TritonException const& err) { + EXPECT_EQ(std::string(err.what()), msg); + } + try { + throw TritonException(Error::Internal, msg); + } catch (TritonException const& err) { + try { + throw(TritonException(err.error())); + } catch (TritonException const& err2) { + EXPECT_EQ(std::string(err2.what()), msg); + } + } +} + +TEST(RapidsTriton, triton_check) +{ + auto msg = std::string("TEST ERROR MESSAGE"); + EXPECT_THROW(triton_check(TRITONSERVER_ErrorNew(Error::Internal, msg.c_str())), TritonException); + triton_check(nullptr); +} + +TEST(RapidsTriton, cuda_check) +{ +#ifdef TRITON_ENABLE_GPU + EXPECT_THROW(cuda_check(cudaError::cudaErrorMissingConfiguration), TritonException); + cuda_check(cudaError::cudaSuccess); +#else + EXPECT_THROW(cuda_check(cudaError::cudaErrorNonGpuBuild), TritonException); +#endif +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/test/memory/buffer.cpp b/backend/test/memory/buffer.cpp new file mode 100644 index 0000000..1181560 --- /dev/null +++ b/backend/test/memory/buffer.cpp @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef TRITON_ENABLE_GPU +#include +#endif +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +TEST(RapidsTriton, default_buffer) +{ + auto buffer = Buffer(); + EXPECT_EQ(buffer.mem_type(), HostMemory); + EXPECT_EQ(buffer.size(), 0); + EXPECT_EQ(buffer.data(), nullptr); + EXPECT_EQ(buffer.device(), 0); + EXPECT_EQ(buffer.stream(), cudaStream_t{}); +#ifdef TRITON_ENABLE_GPU + auto stream = cudaStream_t{}; + cudaStreamCreate(&stream); + buffer.set_stream(stream); + EXPECT_EQ(buffer.stream(), stream); + cudaStreamDestroy(stream); +#endif +} + +TEST(RapidsTriton, device_buffer) +{ + auto data = std::vector{1, 2, 3}; +#ifdef TRITON_ENABLE_GPU + auto buffer = Buffer(data.size(), DeviceMemory, 0, 0); + + ASSERT_EQ(buffer.mem_type(), DeviceMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_NE(buffer.data(), nullptr); + + auto data_out = std::vector(data.size()); + cudaMemcpy(static_cast(buffer.data()), + static_cast(data.data()), + sizeof(int) * data.size(), + cudaMemcpyHostToDevice); + cudaMemcpy(static_cast(data_out.data()), + static_cast(buffer.data()), + sizeof(int) * data.size(), + cudaMemcpyDeviceToHost); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + +#else + EXPECT_THROW(Buffer(data.size(), DeviceMemory, 0, 0), TritonException); +#endif +} + +TEST(RapidsTriton, non_owning_device_buffer) +{ + auto data = std::vector{1, 2, 3}; +#ifdef TRITON_ENABLE_GPU + auto* ptr_d = static_cast(nullptr); + cudaMalloc(reinterpret_cast(&ptr_d), sizeof(int) * data.size()); + cudaMemcpy(static_cast(ptr_d), + static_cast(data.data()), + sizeof(int) * data.size(), + cudaMemcpyHostToDevice); + auto buffer = Buffer(ptr_d, data.size(), DeviceMemory); + + ASSERT_EQ(buffer.mem_type(), DeviceMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_EQ(buffer.data(), ptr_d); + + auto data_out = std::vector(data.size()); + cudaMemcpy(static_cast(data_out.data()), + static_cast(buffer.data()), + sizeof(int) * data.size(), + cudaMemcpyDeviceToHost); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + + cudaFree(reinterpret_cast(ptr_d)); +#else + ASSERT_THROW(Buffer(data.data(), data.size(), DeviceMemory), TritonException); +#endif +} + +TEST(RapidsTriton, host_buffer) +{ + auto data = std::vector{1, 2, 3}; + auto buffer = Buffer(data.size(), HostMemory, 0, 0); + + ASSERT_EQ(buffer.mem_type(), HostMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_NE(buffer.data(), nullptr); + + std::memcpy( + static_cast(buffer.data()), static_cast(data.data()), data.size() * sizeof(int)); + + auto data_out = std::vector(buffer.data(), buffer.data() + buffer.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +TEST(RapidsTriton, non_owning_host_buffer) +{ + auto data = std::vector{1, 2, 3}; + auto buffer = Buffer(data.data(), data.size(), HostMemory); + + ASSERT_EQ(buffer.mem_type(), HostMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_EQ(buffer.data(), data.data()); + + auto data_out = std::vector(buffer.data(), buffer.data() + buffer.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +TEST(RapidsTriton, copy_buffer) +{ + auto data = std::vector{1, 2, 3}; + auto orig_buffer = Buffer(data.data(), data.size(), HostMemory); + auto buffer = Buffer(orig_buffer); + + ASSERT_EQ(buffer.mem_type(), HostMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_NE(buffer.data(), orig_buffer.data()); + + auto data_out = std::vector(buffer.data(), buffer.data() + buffer.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +TEST(RapidsTriton, move_buffer) +{ + auto data = std::vector{1, 2, 3}; + auto buffer = Buffer(Buffer(data.data(), data.size(), HostMemory)); + + ASSERT_EQ(buffer.mem_type(), HostMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_EQ(buffer.data(), data.data()); + + auto data_out = std::vector(buffer.data(), buffer.data() + buffer.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +TEST(RapidsTriton, move_assignment_buffer) +{ + auto data = std::vector{1, 2, 3}; + +#ifdef TRITON_ENABLE_GPU + auto buffer = Buffer{data.data(), data.size() - 1, DeviceMemory}; +#else + auto buffer = Buffer{data.data(), data.size() - 1, HostMemory}; +#endif + buffer = Buffer{data.size(), HostMemory}; + + ASSERT_EQ(buffer.mem_type(), HostMemory); + ASSERT_EQ(buffer.size(), data.size()); +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/test/memory/detail/copy.cpp b/backend/test/memory/detail/copy.cpp new file mode 100644 index 0000000..d669d6b --- /dev/null +++ b/backend/test/memory/detail/copy.cpp @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef TRITON_ENABLE_GPU +#include +#include +#else +#include +#endif +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +TEST(RapidsTriton, copy) +{ + auto data = std::vector{1, 2, 3}; + auto data_out = std::vector(data.size()); + detail::copy(data_out.data(), data.data(), data.size(), 0, HostMemory, HostMemory); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + + data_out = std::vector(data.size()); +#ifdef TRITON_ENABLE_GPU + auto* ptr_d = static_cast(nullptr); + cudaMalloc(reinterpret_cast(&ptr_d), sizeof(int) * data.size()); + detail::copy(ptr_d, data.data(), data.size(), 0, DeviceMemory, HostMemory); + + cudaMemcpy(static_cast(data_out.data()), + static_cast(ptr_d), + sizeof(int) * data.size(), + cudaMemcpyDeviceToHost); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + cudaFree(reinterpret_cast(ptr_d)); +#else + EXPECT_THROW(detail::copy(data_out.data(), data.data(), data.size(), 0, HostMemory, DeviceMemory), + TritonException); + EXPECT_THROW(detail::copy(data_out.data(), data.data(), data.size(), 0, DeviceMemory, HostMemory), + TritonException); +#endif +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/test/memory/detail/owned_device_buffer.cpp b/backend/test/memory/detail/owned_device_buffer.cpp new file mode 100644 index 0000000..9bd613f --- /dev/null +++ b/backend/test/memory/detail/owned_device_buffer.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef TRITON_ENABLE_GPU +#include +#include +#else +#include +#endif +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +TEST(RapidsTriton, owned_device_buffer) +{ + auto data = std::vector{1, 2, 3}; +#ifdef TRITON_ENABLE_GPU + auto device_id = 0; + cudaGetDevice(&device_id); + auto stream = cudaStream_t{}; + cudaStreamCreate(&stream); + + auto buffer = detail::owned_device_buffer(device_id, data.size(), stream); + auto data_out = std::vector(data.size()); + cudaMemcpy(static_cast(buffer.get()), + static_cast(data.data()), + sizeof(int) * data.size(), + cudaMemcpyHostToDevice); + cudaMemcpy(static_cast(data_out.data()), + static_cast(buffer.get()), + sizeof(int) * data.size(), + cudaMemcpyDeviceToHost); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + cudaStreamDestroy(stream); +#else + // Workaround for ungraceful handling of multiple template parameters in + // EXPECT_THROW + using dev_buffer = detail::owned_device_buffer; + EXPECT_THROW(dev_buffer(0, data.size(), 0), TritonException); +#endif +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/test/memory/resource.cpp b/backend/test/memory/resource.cpp new file mode 100644 index 0000000..5571ef7 --- /dev/null +++ b/backend/test/memory/resource.cpp @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef TRITON_ENABLE_GPU +#include +#include +#include +#include +#endif + +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { + +TEST(RapidsTriton, set_memory_resource) +{ +#ifdef TRITON_ENABLE_GPU + auto device_id = int{}; + cuda_check(cudaGetDevice(&device_id)); + EXPECT_EQ(rmm::mr::get_current_device_resource()->is_equal(rmm::mr::cuda_memory_resource{}), + true); + setup_memory_resource(device_id); + EXPECT_EQ(rmm::mr::get_current_device_resource()->is_equal(rmm::mr::cuda_memory_resource{}), + false); +#else + setup_memory_resource(0); +#endif +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/test/memory/types.cpp b/backend/test/memory/types.cpp new file mode 100644 index 0000000..65eb386 --- /dev/null +++ b/backend/test/memory/types.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/tensor/dtype.cpp b/backend/test/tensor/dtype.cpp new file mode 100644 index 0000000..2e80c60 --- /dev/null +++ b/backend/test/tensor/dtype.cpp @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +namespace triton { +namespace backend { +namespace rapids { + +template +void check_dtype_conversion() +{ + EXPECT_EQ(D, TritonDtype::type>::value); + EXPECT_EQ(D, TritonDtype::type const>::value); +} + +TEST(RapidsTriton, dtype) +{ + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/test/tensor/tensor.cpp b/backend/test/tensor/tensor.cpp new file mode 100644 index 0000000..3a1c5c2 --- /dev/null +++ b/backend/test/tensor/tensor.cpp @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace rapids { + +TEST(RapidsTriton, default_tensor) +{ + auto tensor = Tensor(); + EXPECT_EQ(tensor.buffer().size(), 0); + EXPECT_EQ(tensor.shape().size(), 0); +} + +TEST(RapidsTriton, move_buffer_tensor) +{ + auto shape = std::vector{2, 2}; + auto data = std::vector{1, 2, 3, 4}; + auto tensor = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); + + EXPECT_EQ(data.data(), tensor.data()); + EXPECT_EQ(data.size(), tensor.size()); + EXPECT_THAT(tensor.shape(), ::testing::ElementsAreArray(shape)); + + EXPECT_EQ(tensor.dtype(), DTypeInt32); + EXPECT_EQ(tensor.mem_type(), HostMemory); + EXPECT_EQ(tensor.stream(), cudaStream_t{}); + EXPECT_EQ(tensor.device(), 0); + + auto data_out = std::vector(tensor.data(), tensor.data() + tensor.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +TEST(RapidsTriton, multi_buffer_tensor) +{ + auto shape = std::vector{2, 2}; + auto data = std::vector{1, 2, 3, 4}; + + auto all_buffers = std::vector>{}; + all_buffers.reserve(data.size()); + auto mem_type = HostMemory; + if constexpr (IS_GPU_BUILD) { mem_type = DeviceMemory; } + std::transform(data.begin(), data.end(), std::back_inserter(all_buffers), [mem_type](auto& elem) { + return Buffer{&elem, 1, mem_type}; + }); + auto tensor = + Tensor(shape, all_buffers.begin(), all_buffers.end(), mem_type, 0, cudaStream_t{}); + + auto data_out = std::vector(data.size()); +#ifdef TRITON_ENABLE_GPU + cudaMemcpy(static_cast(data_out.data()), + static_cast(tensor.data()), + sizeof(int) * tensor.size(), + cudaMemcpyDeviceToHost); +#else + std::memcpy(static_cast(data_out.data()), + static_cast(tensor.data()), + sizeof(int) * tensor.size()); +#endif + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +TEST(RapidsTriton, tensor_copy) +{ + auto shape = std::vector{2, 2}; + auto data = std::vector{1, 2, 3, 4}; + + auto data1 = data; + auto tensor1 = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); + auto data2 = std::vector(data1.size()); + auto tensor2 = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); + + rapids::copy(tensor2, tensor1); + + auto data_out = std::vector(tensor2.data(), tensor2.data() + tensor2.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + + auto small_shape = std::vector{2}; + auto small_data = std::vector(2); + auto tensor3 = + Tensor(small_shape, Buffer{small_data.data(), small_data.size(), HostMemory}); + + EXPECT_THROW(rapids::copy(tensor3, tensor1), TritonException); +} + +TEST(RapidsTriton, tensor_multi_copy) +{ + auto shape = std::vector{2, 2}; + auto data = std::vector{1, 2, 3, 4}; + + auto data1 = data; + auto tensor1 = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); + + auto receiver_shape = std::vector{1}; + auto receivers = std::vector>{}; + + receivers.reserve(data.size()); + std::transform( + data.begin(), data.end(), std::back_inserter(receivers), [&receiver_shape](auto& val) { + return Tensor(receiver_shape, Buffer{std::size_t{1}, HostMemory}); + }); + + rapids::copy(receivers.begin(), receivers.end(), tensor1); + + auto data_out = std::vector{}; + data_out.reserve(receivers.size()); + std::transform( + receivers.begin(), receivers.end(), std::back_inserter(data_out), [](auto& tensor) { + return *tensor.data(); + }); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + + // Throw if trying to copy to too many outputs + receivers.emplace_back(receiver_shape, Buffer{std::size_t{1}, HostMemory}); + EXPECT_THROW(rapids::copy(receivers.begin(), receivers.end(), tensor1), TritonException); +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/test/test.cpp b/backend/test/test.cpp new file mode 100644 index 0000000..4a7d7f4 --- /dev/null +++ b/backend/test/test.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +namespace triton { +namespace backend { +namespace rapids { + +TEST(RapidsTriton, installed) { std::cout << test_install() << "\n"; } +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/test/triton/api/execute.cpp b/backend/test/triton/api/execute.cpp new file mode 100644 index 0000000..ba48f8e --- /dev/null +++ b/backend/test/triton/api/execute.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/triton/api/initialize.cpp b/backend/test/triton/api/initialize.cpp new file mode 100644 index 0000000..33b95d4 --- /dev/null +++ b/backend/test/triton/api/initialize.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/triton/api/instance_finalize.cpp b/backend/test/triton/api/instance_finalize.cpp new file mode 100644 index 0000000..f7218a3 --- /dev/null +++ b/backend/test/triton/api/instance_finalize.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/triton/api/instance_initialize.cpp b/backend/test/triton/api/instance_initialize.cpp new file mode 100644 index 0000000..3ed2b91 --- /dev/null +++ b/backend/test/triton/api/instance_initialize.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/triton/api/model_finalize.cpp b/backend/test/triton/api/model_finalize.cpp new file mode 100644 index 0000000..367f16c --- /dev/null +++ b/backend/test/triton/api/model_finalize.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/triton/api/model_initialize.cpp b/backend/test/triton/api/model_initialize.cpp new file mode 100644 index 0000000..071baa3 --- /dev/null +++ b/backend/test/triton/api/model_initialize.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/triton/backend.cpp b/backend/test/triton/backend.cpp new file mode 100644 index 0000000..cacff7d --- /dev/null +++ b/backend/test/triton/backend.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/triton/config.cpp b/backend/test/triton/config.cpp new file mode 100644 index 0000000..7a31891 --- /dev/null +++ b/backend/test/triton/config.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/triton/deployment.cpp b/backend/test/triton/deployment.cpp new file mode 100644 index 0000000..dbd22b7 --- /dev/null +++ b/backend/test/triton/deployment.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/triton/device.cpp b/backend/test/triton/device.cpp new file mode 100644 index 0000000..53e49e0 --- /dev/null +++ b/backend/test/triton/device.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/triton/input.cpp b/backend/test/triton/input.cpp new file mode 100644 index 0000000..1cac65d --- /dev/null +++ b/backend/test/triton/input.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/triton/logging.cpp b/backend/test/triton/logging.cpp new file mode 100644 index 0000000..f1f60cd --- /dev/null +++ b/backend/test/triton/logging.cpp @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +TEST(RapidsTriton, logging) +{ + log_debug("Debug test message"); + log_info("Info test message"); + log_warn("Warn test message"); + log_error("Error test message"); +} + +TEST(RapidsTriton, stream_logging) +{ + log_debug() << "Streamed debug test message"; + log_info() << "Streamed info test message"; + log_warn() << "Streamed warn test message"; + log_error() << "Streamed error test message"; +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/test/triton/model.cpp b/backend/test/triton/model.cpp new file mode 100644 index 0000000..3226e35 --- /dev/null +++ b/backend/test/triton/model.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/triton/model_instance.cpp b/backend/test/triton/model_instance.cpp new file mode 100644 index 0000000..3582f75 --- /dev/null +++ b/backend/test/triton/model_instance.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/triton/requests.cpp b/backend/test/triton/requests.cpp new file mode 100644 index 0000000..cff2af3 --- /dev/null +++ b/backend/test/triton/requests.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/triton/responses.cpp b/backend/test/triton/responses.cpp new file mode 100644 index 0000000..4ca7d52 --- /dev/null +++ b/backend/test/triton/responses.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/triton/statistics.cpp b/backend/test/triton/statistics.cpp new file mode 100644 index 0000000..7c01a6d --- /dev/null +++ b/backend/test/triton/statistics.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/test/utils/const_agnostic.cpp b/backend/test/utils/const_agnostic.cpp new file mode 100644 index 0000000..2a74c69 --- /dev/null +++ b/backend/test/utils/const_agnostic.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +TEST(RapidsTriton, const_agnostic) +{ + static_assert(std::is_same, void>::value); + static_assert(std::is_same, void>::value); +} + +} // namespace rapids +} // namespace backend +} // namespace triton diff --git a/backend/test/utils/narrow.cpp b/backend/test/utils/narrow.cpp new file mode 100644 index 0000000..d9919ca --- /dev/null +++ b/backend/test/utils/narrow.cpp @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +namespace triton { +namespace backend { +namespace rapids { +TEST(RapidsTriton, narrow) +{ + EXPECT_THROW(narrow(-1), TritonException); + narrow(int{5}); + EXPECT_THROW(narrow(std::numeric_limits::max()), TritonException); + narrow(std::size_t{5}); +} + +} // namespace rapids +} // namespace backend +} // namespace triton From 28618731a273e49eba6d27895cbff25f385f8b7d Mon Sep 17 00:00:00 2001 From: GuanLuo Date: Tue, 4 Oct 2022 18:01:05 -0700 Subject: [PATCH 168/199] Replace RAPIDS keyword with developer_tools::backend --- backend/CMakeLists.txt | 90 +++++++++---------- backend/cmake/modules/ConfigureCUDA.cmake | 18 ++-- backend/cmake/thirdparty/get_raft.cmake | 14 +-- backend/cmake/thirdparty/get_rapidjson.cmake | 4 +- .../{rapids_triton.hpp => backend_tools.hpp} | 10 +-- .../developer_tools}/batch/batch.hpp | 48 +++++----- .../developer_tools}/build_control.hpp | 4 +- .../cpu_only/cuda_runtime_replacement.hpp | 4 +- .../developer_tools}/exceptions.hpp | 8 +- .../developer_tools}/memory/buffer.hpp | 26 +++--- .../memory/detail/cpu_only/copy.hpp | 10 +-- .../detail/cpu_only/owned_device_buffer.hpp | 12 +-- .../memory/detail/cpu_only/resource.hpp | 8 +- .../memory/detail/gpu_only/copy.hpp | 8 +- .../detail/gpu_only/owned_device_buffer.hpp | 10 +-- .../memory/detail/gpu_only/resource.hpp | 12 +-- .../memory/detail/owned_device_buffer.hpp | 4 +- .../memory/detail/resource.hpp | 6 +- .../developer_tools}/memory/resource.hpp | 14 +-- .../developer_tools}/memory/types.hpp | 4 +- .../developer_tools}/model/model.hpp | 20 ++--- .../developer_tools}/model/shared_state.hpp | 20 ++--- .../developer_tools}/tensor/dtype.hpp | 6 +- .../developer_tools}/tensor/tensor.hpp | 26 +++--- .../developer_tools}/triton/api/execute.hpp | 16 ++-- .../triton/api/initialize.hpp | 18 ++-- .../triton/api/instance_finalize.hpp | 10 +-- .../triton/api/instance_initialize.hpp | 24 ++--- .../triton/api/model_finalize.hpp | 10 +-- .../triton/api/model_initialize.hpp | 16 ++-- .../developer_tools}/triton/backend.hpp | 8 +- .../developer_tools}/triton/config.hpp | 8 +- .../developer_tools}/triton/deployment.hpp | 4 +- .../developer_tools}/triton/device.hpp | 4 +- .../developer_tools}/triton/input.hpp | 10 +-- .../developer_tools}/triton/logging.hpp | 6 +- .../developer_tools}/triton/model.hpp | 8 +- .../triton/model_instance.hpp | 12 +-- .../triton/model_instance_state.hpp | 22 ++--- .../developer_tools}/triton/model_state.hpp | 16 ++-- .../developer_tools}/triton/output.hpp | 6 +- .../developer_tools}/triton/requests.hpp | 8 +- .../developer_tools}/triton/responses.hpp | 8 +- .../developer_tools}/triton/statistics.hpp | 6 +- .../triton/triton_memory_resource.hpp | 8 +- .../developer_tools}/utils/const_agnostic.hpp | 4 +- .../developer_tools}/utils/device_setter.hpp | 8 +- .../developer_tools}/utils/narrow.hpp | 6 +- backend/src/CMakeLists.txt | 23 +++-- backend/src/api.cc | 36 ++++---- backend/src/model.h | 40 ++++----- backend/src/names.h | 6 +- backend/src/shared_state.h | 16 ++-- backend/test/CMakeLists.txt | 28 +++--- backend/test/batch/batch.cpp | 2 +- backend/test/build_control.cpp | 8 +- backend/test/exceptions.cpp | 16 ++-- backend/test/memory/buffer.cpp | 28 +++--- backend/test/memory/detail/copy.cpp | 14 +-- .../memory/detail/owned_device_buffer.cpp | 14 +-- backend/test/memory/resource.cpp | 12 +-- backend/test/memory/types.cpp | 2 +- backend/test/tensor/dtype.cpp | 8 +- backend/test/tensor/tensor.cpp | 28 +++--- backend/test/test.cpp | 8 +- backend/test/triton/api/execute.cpp | 2 +- backend/test/triton/api/initialize.cpp | 2 +- backend/test/triton/api/instance_finalize.cpp | 2 +- .../test/triton/api/instance_initialize.cpp | 2 +- backend/test/triton/api/model_finalize.cpp | 2 +- backend/test/triton/api/model_initialize.cpp | 2 +- backend/test/triton/backend.cpp | 2 +- backend/test/triton/config.cpp | 2 +- backend/test/triton/deployment.cpp | 2 +- backend/test/triton/device.cpp | 2 +- backend/test/triton/input.cpp | 2 +- backend/test/triton/logging.cpp | 10 +-- backend/test/triton/model.cpp | 2 +- backend/test/triton/model_instance.cpp | 2 +- backend/test/triton/requests.cpp | 2 +- backend/test/triton/responses.cpp | 2 +- backend/test/triton/statistics.cpp | 2 +- backend/test/utils/const_agnostic.cpp | 8 +- backend/test/utils/narrow.cpp | 8 +- 84 files changed, 485 insertions(+), 494 deletions(-) rename backend/include/{rapids_triton.hpp => backend_tools.hpp} (76%) rename backend/include/{rapids_triton => triton/developer_tools}/batch/batch.hpp (86%) rename backend/include/{rapids_triton => triton/developer_tools}/build_control.hpp (93%) rename backend/include/{rapids_triton => triton/developer_tools}/cpu_only/cuda_runtime_replacement.hpp (95%) rename backend/include/{rapids_triton => triton/developer_tools}/exceptions.hpp (94%) rename backend/include/{rapids_triton => triton/developer_tools}/memory/buffer.hpp (92%) rename backend/include/{rapids_triton => triton/developer_tools}/memory/detail/cpu_only/copy.hpp (84%) rename backend/include/{rapids_triton => triton/developer_tools}/memory/detail/cpu_only/owned_device_buffer.hpp (79%) rename backend/include/{rapids_triton => triton/developer_tools}/memory/detail/cpu_only/resource.hpp (84%) rename backend/include/{rapids_triton => triton/developer_tools}/memory/detail/gpu_only/copy.hpp (89%) rename backend/include/{rapids_triton => triton/developer_tools}/memory/detail/gpu_only/owned_device_buffer.hpp (84%) rename backend/include/{rapids_triton => triton/developer_tools}/memory/detail/gpu_only/resource.hpp (90%) rename backend/include/{rapids_triton => triton/developer_tools}/memory/detail/owned_device_buffer.hpp (93%) rename backend/include/{rapids_triton => triton/developer_tools}/memory/detail/resource.hpp (89%) rename backend/include/{rapids_triton => triton/developer_tools}/memory/resource.hpp (72%) rename backend/include/{rapids_triton => triton/developer_tools}/memory/types.hpp (93%) rename backend/include/{rapids_triton => triton/developer_tools}/model/model.hpp (92%) rename backend/include/{rapids_triton => triton/developer_tools}/model/shared_state.hpp (92%) rename backend/include/{rapids_triton => triton/developer_tools}/tensor/dtype.hpp (97%) rename backend/include/{rapids_triton => triton/developer_tools}/tensor/tensor.hpp (88%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/api/execute.hpp (91%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/api/initialize.hpp (81%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/api/instance_finalize.hpp (85%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/api/instance_initialize.hpp (80%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/api/model_finalize.hpp (86%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/api/model_initialize.hpp (78%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/backend.hpp (91%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/config.hpp (86%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/deployment.hpp (94%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/device.hpp (92%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/input.hpp (92%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/logging.hpp (97%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/model.hpp (94%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/model_instance.hpp (91%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/model_instance_state.hpp (66%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/model_state.hpp (71%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/output.hpp (95%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/requests.hpp (87%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/responses.hpp (93%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/statistics.hpp (97%) rename backend/include/{rapids_triton => triton/developer_tools}/triton/triton_memory_resource.hpp (94%) rename backend/include/{rapids_triton => triton/developer_tools}/utils/const_agnostic.hpp (93%) rename backend/include/{rapids_triton => triton/developer_tools}/utils/device_setter.hpp (88%) rename backend/include/{rapids_triton => triton/developer_tools}/utils/narrow.hpp (91%) diff --git a/backend/CMakeLists.txt b/backend/CMakeLists.txt index 136e032..0b3317a 100644 --- a/backend/CMakeLists.txt +++ b/backend/CMakeLists.txt @@ -28,8 +28,8 @@ include(rapids-find) # - User Options ------------------------------------------------------------ option(TRITON_ENABLE_GPU "Enable GPU support in Triton" ON) -option(BUILD_TESTS "Build rapids_triton unit-tests" ON) -option(BUILD_EXAMPLE "Build rapids_identity example backend" OFF) +option(BUILD_TESTS "Build developer_tools_backend unit-tests" ON) +option(BUILD_EXAMPLE "Build developer_tools_backend example backend" OFF) option(CUDA_ENABLE_KERNELINFO "Enable kernel resource usage info" OFF) option(CUDA_ENABLE_LINEINFO "Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler)" OFF) option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF) @@ -41,27 +41,27 @@ set(TRITON_COMMON_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-serve set(TRITON_CORE_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-server/core repo") set(TRITON_BACKEND_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-server/backend repo") -message(VERBOSE "RAPIDS_TRITON: Build RAPIDS_TRITON unit-tests: ${BUILD_TESTS}") -message(VERBOSE "RAPIDS_TRITON: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") -message(VERBOSE "RAPIDS_TRITON: Disable depreaction warnings " ${DISABLE_DEPRECATION_WARNINGS}) -message(VERBOSE "RAPIDS_TRITON: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELINFO}") -message(VERBOSE "RAPIDS_TRITON: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") -message(VERBOSE "RAPIDS_TRITON: Enable nvtx markers: ${NVTX}") -message(VERBOSE "RAPIDS_TRITON: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") -message(VERBOSE "RAPIDS_TRITON: Enable GPU support: ${TRITON_ENABLE_GPU}") -message(VERBOSE "RAPIDS_TRITON: Enable statistics collection in Triton: ${TRITON_ENABLE_STATS}") -message(VERBOSE "RAPIDS_TRITON: Triton common repo tag: ${TRITON_COMMON_REPO_TAG}") -message(VERBOSE "RAPIDS_TRITON: Triton core repo tag: ${TRITON_CORE_REPO_TAG}") -message(VERBOSE "RAPIDS_TRITON: Triton backend repo tag: ${TRITON_BACKEND_REPO_TAG}") +message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Build DEVELOPER_TOOLS_BACKEND unit-tests: ${BUILD_TESTS}") +message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") +message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Disable depreaction warnings " ${DISABLE_DEPRECATION_WARNINGS}) +message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELINFO}") +message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") +message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Enable nvtx markers: ${NVTX}") +message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") +message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Enable GPU support: ${TRITON_ENABLE_GPU}") +message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Enable statistics collection in Triton: ${TRITON_ENABLE_STATS}") +message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Triton common repo tag: ${TRITON_COMMON_REPO_TAG}") +message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Triton core repo tag: ${TRITON_CORE_REPO_TAG}") +message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Triton backend repo tag: ${TRITON_BACKEND_REPO_TAG}") ############################################################################## # - Project Initialization --------------------------------------------------- if(TRITON_ENABLE_GPU) - rapids_cuda_init_architectures(RAPIDS_TRITON) - project(RAPIDS_TRITON VERSION 22.02.00 LANGUAGES CXX CUDA) + rapids_cuda_init_architectures(DEVELOPER_TOOLS_BACKEND) + project(DEVELOPER_TOOLS_BACKEND VERSION 22.02.00 LANGUAGES CXX CUDA) else() - project(RAPIDS_TRITON VERSION 22.02.00 LANGUAGES CXX) + project(DEVELOPER_TOOLS_BACKEND VERSION 22.02.00 LANGUAGES CXX) endif() @@ -77,7 +77,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # Set RMM logging level set(RMM_LOGGING_LEVEL "INFO" CACHE STRING "Choose the logging level.") set_property(CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF") -message(VERBOSE "RAPIDS_TRITON: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") +message(VERBOSE "DEVELOPER_TOOLS_BACKEND: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") ############################################################################## # - Conda environment detection ---------------------------------------------- @@ -85,7 +85,7 @@ message(VERBOSE "RAPIDS_TRITON: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") if(DETECT_CONDA_ENV) rapids_cmake_support_conda_env( conda_env MODIFY_PREFIX_PATH ) if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED ENV{CONDA_PREFIX}) - message(STATUS "RAPIDS_TRITON: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}") + message(STATUS "DEVELOPER_TOOLS_BACKEND: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}") set(CMAKE_INSTALL_PREFIX "$ENV{CONDA_PREFIX}") endif() endif() @@ -98,8 +98,8 @@ if(TRITON_ENABLE_GPU) # * enable the CMake CUDA language # * set other CUDA compilation flags rapids_find_package(CUDAToolkit REQUIRED - BUILD_EXPORT_SET rapids_triton-exports - INSTALL_EXPORT_SET rapids_triton-exports + BUILD_EXPORT_SET developer_tools_backend-exports + INSTALL_EXPORT_SET developer_tools_backend-exports ) include(cmake/modules/ConfigureCUDA.cmake) endif() @@ -125,12 +125,12 @@ endif() ############################################################################## # - install targets----------------------------------------------------------- -add_library(rapids_triton INTERFACE) -add_library(rapids_triton::rapids_triton ALIAS rapids_triton) -target_include_directories(rapids_triton INTERFACE "$" +add_library(developer_tools_backend INTERFACE) +add_library(developer_tools_backend::developer_tools_backend ALIAS developer_tools_backend) +target_include_directories(developer_tools_backend INTERFACE "$" "$") -target_link_libraries(rapids_triton +target_link_libraries(developer_tools_backend INTERFACE $<$:rmm::rmm> $<$:raft::raft> @@ -140,58 +140,58 @@ INTERFACE if (TRITON_ENABLE_GPU) target_compile_features( - rapids_triton INTERFACE cxx_std_17 + developer_tools_backend INTERFACE cxx_std_17 $ ) else() target_compile_features( - rapids_triton INTERFACE cxx_std_17 + developer_tools_backend INTERFACE cxx_std_17 ) endif() rapids_cmake_install_lib_dir(lib_dir) -install(TARGETS rapids_triton +install(TARGETS developer_tools_backend DESTINATION ${lib_dir} - EXPORT rapids_triton-exports + EXPORT developer_tools_backend-exports ) include(GNUInstallDirs) -install(DIRECTORY include/rapids_triton/ - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rapids_triton +install(DIRECTORY include/triton/developer_tools/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/triton/developer_tools ) -# Temporary install of rapids_triton.hpp while the file is removed -install(FILES include/rapids_triton.hpp - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rapids_triton +# Temporary install of backend_tools.hpp while the file is removed +install(FILES include/backend_tools.hpp + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/triton/developer_tools ) ############################################################################## # - install export ----------------------------------------------------------- set(doc_string [=[ -Provide targets for RAPIDS_TRITON. +Provide targets for DEVELOPER_TOOLS_BACKEND. -RAPIDS_TRITON is a header-only library designed to make it easier and faster -to integrate RAPIDS algorithms as Triton backends. +DEVELOPER_TOOLS_BACKEND is a header-only library designed to make it easier and faster +to implement Triton backends. ]=]) - rapids_export(INSTALL rapids_triton - EXPORT_SET rapids_triton-exports - GLOBAL_TARGETS rapids_triton # since we can't hook into EXPORT SETS - NAMESPACE rapids_triton:: + rapids_export(INSTALL developer_tools_backend + EXPORT_SET developer_tools_backend-exports + GLOBAL_TARGETS developer_tools_backend # since we can't hook into EXPORT SETS + NAMESPACE developer_tools_backend:: DOCUMENTATION doc_string ) ############################################################################## # - build export ------------------------------------------------------------- -rapids_export(BUILD rapids_triton - EXPORT_SET rapids_triton-exports - GLOBAL_TARGETS rapids_triton # since we can't hook into EXPORT SETS +rapids_export(BUILD developer_tools_backend + EXPORT_SET developer_tools_backend-exports + GLOBAL_TARGETS developer_tools_backend # since we can't hook into EXPORT SETS LANGUAGES CUDA DOCUMENTATION doc_string - NAMESPACE rapids_triton:: + NAMESPACE developer_tools_backend:: ) ############################################################################## diff --git a/backend/cmake/modules/ConfigureCUDA.cmake b/backend/cmake/modules/ConfigureCUDA.cmake index 20826ab..5643ce6 100644 --- a/backend/cmake/modules/ConfigureCUDA.cmake +++ b/backend/cmake/modules/ConfigureCUDA.cmake @@ -15,29 +15,29 @@ #============================================================================= if(DISABLE_DEPRECATION_WARNINGS) - list(APPEND RAPIDS_TRITON_CXX_FLAGS -Wno-deprecated-declarations) - list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Xcompiler=-Wno-deprecated-declarations) + list(APPEND DEVELOPER_TOOLS_BACKEND_CXX_FLAGS -Wno-deprecated-declarations) + list(APPEND DEVELOPER_TOOLS_BACKEND_CUDA_FLAGS -Xcompiler=-Wno-deprecated-declarations) endif() if(CMAKE_COMPILER_IS_GNUCXX) - list(APPEND RAPIDS_TRITON_CXX_FLAGS -Wall -Werror -Wno-unknown-pragmas -Wno-error=deprecated-declarations) + list(APPEND DEVELOPER_TOOLS_BACKEND_CXX_FLAGS -Wall -Werror -Wno-unknown-pragmas -Wno-error=deprecated-declarations) endif() -list(APPEND RAPIDS_TRITON_CUDA_FLAGS --expt-extended-lambda --expt-relaxed-constexpr) +list(APPEND DEVELOPER_TOOLS_BACKEND_CUDA_FLAGS --expt-extended-lambda --expt-relaxed-constexpr) # set warnings as errors if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 11.2.0) - list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Werror=all-warnings) + list(APPEND DEVELOPER_TOOLS_BACKEND_CUDA_FLAGS -Werror=all-warnings) endif() -list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Xcompiler=-Wall,-Werror,-Wno-error=deprecated-declarations) +list(APPEND DEVELOPER_TOOLS_BACKEND_CUDA_FLAGS -Xcompiler=-Wall,-Werror,-Wno-error=deprecated-declarations) # Option to enable line info in CUDA device compilation to allow introspection when profiling / memchecking if(CUDA_ENABLE_LINEINFO) - list(APPEND RAPIDS_TRITON_CUDA_FLAGS -lineinfo) + list(APPEND DEVELOPER_TOOLS_BACKEND_CUDA_FLAGS -lineinfo) endif() # Debug options if(CMAKE_BUILD_TYPE MATCHES Debug) - message(VERBOSE "RAPIDS_TRITON: Building with debugging flags") - list(APPEND RAPIDS_TRITON_CUDA_FLAGS -G -Xcompiler=-rdynamic) + message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Building with debugging flags") + list(APPEND DEVELOPER_TOOLS_BACKEND_CUDA_FLAGS -G -Xcompiler=-rdynamic) endif() diff --git a/backend/cmake/thirdparty/get_raft.cmake b/backend/cmake/thirdparty/get_raft.cmake index 6a37d7e..78fe8b6 100644 --- a/backend/cmake/thirdparty/get_raft.cmake +++ b/backend/cmake/thirdparty/get_raft.cmake @@ -22,8 +22,8 @@ function(find_and_configure_raft) rapids_cpm_find(raft ${PKG_VERSION} GLOBAL_TARGETS raft::raft - BUILD_EXPORT_SET rapids_triton-exports - INSTALL_EXPORT_SET rapids_triton-exports + BUILD_EXPORT_SET developer_tools_backend-exports + INSTALL_EXPORT_SET developer_tools_backend-exports CPM_ARGS GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git GIT_TAG ${PKG_PINNED_TAG} @@ -33,17 +33,17 @@ function(find_and_configure_raft) "RAFT_COMPILE_LIBRARIES OFF" ) - message(VERBOSE "RAPIDS_TRITON: Using RAFT located in ${raft_SOURCE_DIR}") + message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Using RAFT located in ${raft_SOURCE_DIR}") endfunction() -set(RAPIDS_TRITON_MIN_VERSION_raft "${RAPIDS_TRITON_VERSION_MAJOR}.${RAPIDS_TRITON_VERSION_MINOR}.00") -set(RAPIDS_TRITON_BRANCH_VERSION_raft "${RAPIDS_TRITON_VERSION_MAJOR}.${RAPIDS_TRITON_VERSION_MINOR}") +set(DEVELOPER_TOOLS_BACKEND_MIN_VERSION_raft "${DEVELOPER_TOOLS_BACKEND_VERSION_MAJOR}.${DEVELOPER_TOOLS_BACKEND_VERSION_MINOR}.00") +set(DEVELOPER_TOOLS_BACKEND_BRANCH_VERSION_raft "${DEVELOPER_TOOLS_BACKEND_VERSION_MAJOR}.${DEVELOPER_TOOLS_BACKEND_VERSION_MINOR}") # Change pinned tag here to test a commit in CI # To use a different RAFT locally, set the CMake variable # CPM_raft_SOURCE=/path/to/local/raft -find_and_configure_raft(VERSION ${RAPIDS_TRITON_MIN_VERSION_raft} +find_and_configure_raft(VERSION ${DEVELOPER_TOOLS_BACKEND_MIN_VERSION_raft} FORK rapidsai - PINNED_TAG branch-${RAPIDS_TRITON_BRANCH_VERSION_raft} + PINNED_TAG branch-${DEVELOPER_TOOLS_BACKEND_BRANCH_VERSION_raft} ) diff --git a/backend/cmake/thirdparty/get_rapidjson.cmake b/backend/cmake/thirdparty/get_rapidjson.cmake index e9051c6..3399d91 100644 --- a/backend/cmake/thirdparty/get_rapidjson.cmake +++ b/backend/cmake/thirdparty/get_rapidjson.cmake @@ -19,8 +19,8 @@ function(find_and_configure_rapidjson VERSION) rapids_cpm_find(rapidjson ${VERSION} GLOBAL_TARGETS rapidjson::rapidjson - BUILD_EXPORT_SET rapids_triton-exports - INSTALL_EXPORT_SET rapids_triton-exports + BUILD_EXPORT_SET developer_tools_backend-exports + INSTALL_EXPORT_SET developer_tools_backend-exports CPM_ARGS GIT_REPOSITORY https://github.com/Tencent/rapidjson GIT_TAG "v${VERSION}" diff --git a/backend/include/rapids_triton.hpp b/backend/include/backend_tools.hpp similarity index 76% rename from backend/include/rapids_triton.hpp rename to backend/include/backend_tools.hpp index 16cf368..e11fb2d 100644 --- a/backend/include/rapids_triton.hpp +++ b/backend/include/backend_tools.hpp @@ -18,14 +18,14 @@ #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { -/* Function for testing rapids_triton include +/* Function for testing backend_tools include * - * @return message indicating rapids_triton has been included succesfully*/ -inline auto test_install() { return std::string("rapids_triton set up successfully"); } + * @return message indicating backend_tools has been included succesfully*/ +inline auto test_install() { return std::string("backend_tools set up successfully"); } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/batch/batch.hpp b/backend/include/triton/developer_tools/batch/batch.hpp similarity index 86% rename from backend/include/rapids_triton/batch/batch.hpp rename to backend/include/triton/developer_tools/batch/batch.hpp index db54df8..a372049 100644 --- a/backend/include/rapids_triton/batch/batch.hpp +++ b/backend/include/triton/developer_tools/batch/batch.hpp @@ -19,7 +19,7 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include #include @@ -30,31 +30,31 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { /** * @brief A representation of all data about a single batch of inference * requests * - * Batch objects are the primary interface point between rapids_triton Models - * and the Triton server itself. By calling the `get_input` and `get_output` - * methods of a batch, Model implementations can retrieve the input Tensors - * necessary for prediction and the output Tensors where results can be + * Batch objects are the primary interface point between developer_tools::backend + * Models and the Triton server itself. By calling the `get_input` and + * `get_output` methods of a batch, Model implementations can retrieve the input + * Tensors necessary for prediction and the output Tensors where results can be * stored. * * Batch objects also handle a variety of other tasks necessary for @@ -62,10 +62,10 @@ namespace rapids { * on how long it took to process requests and sending responses to the * client via the Triton server once processing is complete. * - * It is not recommended that developers of rapids_triton backends try to + * It is not recommended that developers of developer_tools backends try to * construct Batch objects directly. Instead, you should make use of the - * rapids::triton_api::execute template, which will construct the Batch for - * you. + * developer_tools::backend::triton_api::execute template, which will construct + * the Batch for you. */ struct Batch { using size_type = std::size_t; @@ -88,7 +88,7 @@ struct Batch { get_output_shape_{std::move(get_output_shape)}, report_statistics_{std::move(report_request_statistics)}, collector_(raw_requests, count, &responses_, &triton_mem_manager, use_pinned_input, stream), - responder_{std::make_shared(raw_requests, + responder_{std::make_shared(raw_requests, count, &responses_, max_batch_size, @@ -261,13 +261,13 @@ struct Batch { time_point const&, time_point const&)> report_statistics_; - BackendInputCollector collector_; - std::shared_ptr responder_; + triton::backend::BackendInputCollector collector_; + std::shared_ptr responder_; cudaStream_t stream_; std::chrono::time_point start_time_; std::chrono::time_point compute_start_time_; std::optional batch_size_; }; -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/build_control.hpp b/backend/include/triton/developer_tools/build_control.hpp similarity index 93% rename from backend/include/rapids_triton/build_control.hpp rename to backend/include/triton/developer_tools/build_control.hpp index c2a9fec..be8693b 100644 --- a/backend/include/rapids_triton/build_control.hpp +++ b/backend/include/triton/developer_tools/build_control.hpp @@ -18,8 +18,8 @@ #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { #ifdef TRITON_ENABLE_GPU auto constexpr IS_GPU_BUILD = true; @@ -27,6 +27,6 @@ auto constexpr IS_GPU_BUILD = true; auto constexpr IS_GPU_BUILD = false; #endif -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/cpu_only/cuda_runtime_replacement.hpp b/backend/include/triton/developer_tools/cpu_only/cuda_runtime_replacement.hpp similarity index 95% rename from backend/include/rapids_triton/cpu_only/cuda_runtime_replacement.hpp rename to backend/include/triton/developer_tools/cpu_only/cuda_runtime_replacement.hpp index 7cb6691..62061e5 100644 --- a/backend/include/rapids_triton/cpu_only/cuda_runtime_replacement.hpp +++ b/backend/include/triton/developer_tools/cpu_only/cuda_runtime_replacement.hpp @@ -20,8 +20,8 @@ #else namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { using cudaStream_t = void*; @@ -48,7 +48,7 @@ inline auto cudaGetDeviceCount(int* count) { } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton #endif diff --git a/backend/include/rapids_triton/exceptions.hpp b/backend/include/triton/developer_tools/exceptions.hpp similarity index 94% rename from backend/include/rapids_triton/exceptions.hpp rename to backend/include/triton/developer_tools/exceptions.hpp index e2d2bc2..ae87702 100644 --- a/backend/include/rapids_triton/exceptions.hpp +++ b/backend/include/triton/developer_tools/exceptions.hpp @@ -18,16 +18,16 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include #include -#include +#include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { using ErrorCode = TRITONSERVER_Error_Code; @@ -89,6 +89,6 @@ inline void cuda_check(cudaError_t const& err) } } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/memory/buffer.hpp b/backend/include/triton/developer_tools/memory/buffer.hpp similarity index 92% rename from backend/include/rapids_triton/memory/buffer.hpp rename to backend/include/triton/developer_tools/memory/buffer.hpp index 124437c..d763b69 100644 --- a/backend/include/rapids_triton/memory/buffer.hpp +++ b/backend/include/triton/developer_tools/memory/buffer.hpp @@ -22,24 +22,24 @@ #ifdef TRITON_ENABLE_GPU #include -#include -#include +#include +#include #else -#include -#include -#include +#include +#include +#include #endif -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { template struct Buffer { using size_type = std::size_t; @@ -312,6 +312,6 @@ void copy(Buffer& dst, { copy(dst, src, 0, src_begin, src_end); } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/memory/detail/cpu_only/copy.hpp b/backend/include/triton/developer_tools/memory/detail/cpu_only/copy.hpp similarity index 84% rename from backend/include/rapids_triton/memory/detail/cpu_only/copy.hpp rename to backend/include/triton/developer_tools/memory/detail/cpu_only/copy.hpp index beb5cb4..5d0db15 100644 --- a/backend/include/rapids_triton/memory/detail/cpu_only/copy.hpp +++ b/backend/include/triton/developer_tools/memory/detail/cpu_only/copy.hpp @@ -19,14 +19,14 @@ #include #ifndef TRITON_ENABLE_GPU -#include +#include #endif -#include -#include +#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { namespace detail { template @@ -45,6 +45,6 @@ void copy(T* dst, } } // namespace detail -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp b/backend/include/triton/developer_tools/memory/detail/cpu_only/owned_device_buffer.hpp similarity index 79% rename from backend/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp rename to backend/include/triton/developer_tools/memory/detail/cpu_only/owned_device_buffer.hpp index 4006a70..6afb257 100644 --- a/backend/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp +++ b/backend/include/triton/developer_tools/memory/detail/cpu_only/owned_device_buffer.hpp @@ -17,14 +17,14 @@ #pragma once #include #include -#include -#include -#include -#include +#include +#include +#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { namespace detail { template @@ -40,6 +40,6 @@ struct owned_device_buffer { }; } // namespace detail -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/memory/detail/cpu_only/resource.hpp b/backend/include/triton/developer_tools/memory/detail/cpu_only/resource.hpp similarity index 84% rename from backend/include/rapids_triton/memory/detail/cpu_only/resource.hpp rename to backend/include/triton/developer_tools/memory/detail/cpu_only/resource.hpp index 04bb1c5..ae28cab 100644 --- a/backend/include/rapids_triton/memory/detail/cpu_only/resource.hpp +++ b/backend/include/triton/developer_tools/memory/detail/cpu_only/resource.hpp @@ -17,12 +17,12 @@ #pragma once #include -#include -#include +#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { namespace detail { template<> @@ -30,6 +30,6 @@ inline void setup_memory_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* triton_manager) { } } // namespace detail -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/memory/detail/gpu_only/copy.hpp b/backend/include/triton/developer_tools/memory/detail/gpu_only/copy.hpp similarity index 89% rename from backend/include/rapids_triton/memory/detail/gpu_only/copy.hpp rename to backend/include/triton/developer_tools/memory/detail/gpu_only/copy.hpp index 030b473..2727aec 100644 --- a/backend/include/rapids_triton/memory/detail/gpu_only/copy.hpp +++ b/backend/include/triton/developer_tools/memory/detail/gpu_only/copy.hpp @@ -22,12 +22,12 @@ #include #include -#include -#include +#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { namespace detail { template @@ -50,6 +50,6 @@ void copy(T* dst, } } // namespace detail -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp b/backend/include/triton/developer_tools/memory/detail/gpu_only/owned_device_buffer.hpp similarity index 84% rename from backend/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp rename to backend/include/triton/developer_tools/memory/detail/gpu_only/owned_device_buffer.hpp index 34095f7..8a275ae 100644 --- a/backend/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp +++ b/backend/include/triton/developer_tools/memory/detail/gpu_only/owned_device_buffer.hpp @@ -16,14 +16,14 @@ #pragma once #include -#include -#include -#include +#include +#include +#include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { namespace detail { template @@ -44,6 +44,6 @@ struct owned_device_buffer { }; } // namespace detail -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/memory/detail/gpu_only/resource.hpp b/backend/include/triton/developer_tools/memory/detail/gpu_only/resource.hpp similarity index 90% rename from backend/include/rapids_triton/memory/detail/gpu_only/resource.hpp rename to backend/include/triton/developer_tools/memory/detail/gpu_only/resource.hpp index f2ea115..28c16dc 100644 --- a/backend/include/rapids_triton/memory/detail/gpu_only/resource.hpp +++ b/backend/include/triton/developer_tools/memory/detail/gpu_only/resource.hpp @@ -19,17 +19,17 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { namespace detail { inline auto& resource_lock() @@ -92,6 +92,6 @@ inline void setup_memory_resource(device_id_t device_id, inline auto* get_memory_resource() { return rmm::mr::get_current_device_resource(); } */ } // namespace detail -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/memory/detail/owned_device_buffer.hpp b/backend/include/triton/developer_tools/memory/detail/owned_device_buffer.hpp similarity index 93% rename from backend/include/rapids_triton/memory/detail/owned_device_buffer.hpp rename to backend/include/triton/developer_tools/memory/detail/owned_device_buffer.hpp index ca25af9..442603b 100644 --- a/backend/include/rapids_triton/memory/detail/owned_device_buffer.hpp +++ b/backend/include/triton/developer_tools/memory/detail/owned_device_buffer.hpp @@ -16,8 +16,8 @@ #pragma once namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { namespace detail { template @@ -25,6 +25,6 @@ struct owned_device_buffer { }; } // namespace detail -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/memory/detail/resource.hpp b/backend/include/triton/developer_tools/memory/detail/resource.hpp similarity index 89% rename from backend/include/rapids_triton/memory/detail/resource.hpp rename to backend/include/triton/developer_tools/memory/detail/resource.hpp index cfc4bc4..38ca530 100644 --- a/backend/include/rapids_triton/memory/detail/resource.hpp +++ b/backend/include/triton/developer_tools/memory/detail/resource.hpp @@ -17,11 +17,11 @@ #pragma once #include -#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { namespace detail { template @@ -30,6 +30,6 @@ inline void setup_memory_resource(device_id_t device_id, } } // namespace detail -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/memory/resource.hpp b/backend/include/triton/developer_tools/memory/resource.hpp similarity index 72% rename from backend/include/rapids_triton/memory/resource.hpp rename to backend/include/triton/developer_tools/memory/resource.hpp index afd7f2a..979a30a 100644 --- a/backend/include/rapids_triton/memory/resource.hpp +++ b/backend/include/triton/developer_tools/memory/resource.hpp @@ -17,23 +17,23 @@ #pragma once #include -#include -#include -#include +#include +#include +#include #ifdef TRITON_ENABLE_GPU -#include +#include #else -#include +#include #endif namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { inline void setup_memory_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* triton_manager = nullptr) { detail::setup_memory_resource(device_id, triton_manager); } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/memory/types.hpp b/backend/include/triton/developer_tools/memory/types.hpp similarity index 93% rename from backend/include/rapids_triton/memory/types.hpp rename to backend/include/triton/developer_tools/memory/types.hpp index 884f315..9de4467 100644 --- a/backend/include/rapids_triton/memory/types.hpp +++ b/backend/include/triton/developer_tools/memory/types.hpp @@ -18,11 +18,11 @@ #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { using MemoryType = TRITONSERVER_MemoryType; auto constexpr DeviceMemory = TRITONSERVER_MEMORY_GPU; auto constexpr HostMemory = TRITONSERVER_MEMORY_CPU; -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/model/model.hpp b/backend/include/triton/developer_tools/model/model.hpp similarity index 92% rename from backend/include/rapids_triton/model/model.hpp rename to backend/include/triton/developer_tools/model/model.hpp index 3413fca..a8684f5 100644 --- a/backend/include/rapids_triton/model/model.hpp +++ b/backend/include/triton/developer_tools/model/model.hpp @@ -18,22 +18,22 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { template struct Model { virtual void predict(Batch& batch) const = 0; @@ -192,6 +192,6 @@ struct Model { DeploymentType deployment_type_; std::string filepath_; }; -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/model/shared_state.hpp b/backend/include/triton/developer_tools/model/shared_state.hpp similarity index 92% rename from backend/include/rapids_triton/model/shared_state.hpp rename to backend/include/triton/developer_tools/model/shared_state.hpp index aa0d76d..1ea2762 100644 --- a/backend/include/rapids_triton/model/shared_state.hpp +++ b/backend/include/triton/developer_tools/model/shared_state.hpp @@ -18,7 +18,7 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include #include @@ -29,15 +29,15 @@ #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { /** * @brief Stores shared state for multiple instances of the same model */ @@ -66,9 +66,9 @@ struct SharedModelState { auto shape = std::vector{}; auto reshape_entry = triton::common::TritonJson::Value{}; if (output_entry.Find("reshape", &reshape_entry)) { - ParseShape(reshape_entry, "shape", &shape); + triton::backend::ParseShape(reshape_entry, "shape", &shape); } else { - ParseShape(output_entry, "dims", &shape); + triton::backend::ParseShape(output_entry, "dims", &shape); } if (shape[0] != -1) { shape.insert(shape.begin(), -1); } // The squeeze_output option was introduced to handle a bad choice of @@ -184,6 +184,6 @@ struct SharedModelState { return result; } }; -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/tensor/dtype.hpp b/backend/include/triton/developer_tools/tensor/dtype.hpp similarity index 97% rename from backend/include/rapids_triton/tensor/dtype.hpp rename to backend/include/triton/developer_tools/tensor/dtype.hpp index 325ef4d..7d150bb 100644 --- a/backend/include/rapids_triton/tensor/dtype.hpp +++ b/backend/include/triton/developer_tools/tensor/dtype.hpp @@ -18,11 +18,11 @@ #include #include #include -#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { using DType = TRITONSERVER_DataType; auto constexpr DTypeBool = TRITONSERVER_TYPE_BOOL; @@ -163,6 +163,6 @@ inline std::ostream& operator<<(std::ostream& out, DType const& dtype) return out; } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/tensor/tensor.hpp b/backend/include/triton/developer_tools/tensor/tensor.hpp similarity index 88% rename from backend/include/rapids_triton/tensor/tensor.hpp rename to backend/include/triton/developer_tools/tensor/tensor.hpp index 80f2ba4..0c74196 100644 --- a/backend/include/rapids_triton/tensor/tensor.hpp +++ b/backend/include/triton/developer_tools/tensor/tensor.hpp @@ -27,20 +27,20 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { template struct BaseTensor { using size_type = typename Buffer::size_type; @@ -133,7 +133,7 @@ struct OutputTensor final : BaseTensor { OutputTensor(std::vector::size_type>&& shape, Buffer&& buffer, std::string const& name, - std::shared_ptr responder) + std::shared_ptr responder) : BaseTensor(std::move(shape), std::move(buffer)), name_{name}, responder_{responder} { } @@ -141,10 +141,10 @@ struct OutputTensor final : BaseTensor { * @brief Prepare final output data from this tensor for responding to * request * - * This method *must* be called by rapids_triton backends on all of their + * This method *must* be called by developer_tools backends on all of their * output tensors before returning from their `predict` methods. Because we * cannot know a priori what names backends might have for their tensors - * and what types will be stored in those tensors, the rapids_triton + * and what types will be stored in those tensors, the developer_tools * library cannot store references to those tensors that might otherwise be * used to finalize them. */ @@ -172,7 +172,7 @@ struct OutputTensor final : BaseTensor { private: std::string name_; - std::shared_ptr responder_; + std::shared_ptr responder_; }; template & src) }); } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/api/execute.hpp b/backend/include/triton/developer_tools/triton/api/execute.hpp similarity index 91% rename from backend/include/rapids_triton/triton/api/execute.hpp rename to backend/include/triton/developer_tools/triton/api/execute.hpp index f324065..45b455b 100644 --- a/backend/include/rapids_triton/triton/api/execute.hpp +++ b/backend/include/triton/developer_tools/triton/api/execute.hpp @@ -20,17 +20,17 @@ #include #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { namespace triton_api { template auto* execute(TRITONBACKEND_ModelInstance* instance, @@ -116,6 +116,6 @@ auto* execute(TRITONBACKEND_ModelInstance* instance, return result; } } // namespace triton_api -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/api/initialize.hpp b/backend/include/triton/developer_tools/triton/api/initialize.hpp similarity index 81% rename from backend/include/rapids_triton/triton/api/initialize.hpp rename to backend/include/triton/developer_tools/triton/api/initialize.hpp index 37824dd..333be7f 100644 --- a/backend/include/rapids_triton/triton/api/initialize.hpp +++ b/backend/include/triton/developer_tools/triton/api/initialize.hpp @@ -18,21 +18,21 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { namespace triton_api { inline auto* initialize(TRITONBACKEND_Backend* backend) { @@ -64,6 +64,6 @@ inline auto* initialize(TRITONBACKEND_Backend* backend) return result; } } // namespace triton_api -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/api/instance_finalize.hpp b/backend/include/triton/developer_tools/triton/api/instance_finalize.hpp similarity index 85% rename from backend/include/rapids_triton/triton/api/instance_finalize.hpp rename to backend/include/triton/developer_tools/triton/api/instance_finalize.hpp index 2d1797c..8984900 100644 --- a/backend/include/rapids_triton/triton/api/instance_finalize.hpp +++ b/backend/include/triton/developer_tools/triton/api/instance_finalize.hpp @@ -16,13 +16,13 @@ #pragma once #include -#include -#include -#include +#include +#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { namespace triton_api { template auto* instance_finalize(TRITONBACKEND_ModelInstance* instance) @@ -44,6 +44,6 @@ auto* instance_finalize(TRITONBACKEND_ModelInstance* instance) return result; } } // namespace triton_api -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/api/instance_initialize.hpp b/backend/include/triton/developer_tools/triton/api/instance_initialize.hpp similarity index 80% rename from backend/include/rapids_triton/triton/api/instance_initialize.hpp rename to backend/include/triton/developer_tools/triton/api/instance_initialize.hpp index 922152f..d32e4c7 100644 --- a/backend/include/rapids_triton/triton/api/instance_initialize.hpp +++ b/backend/include/triton/developer_tools/triton/api/instance_initialize.hpp @@ -18,16 +18,16 @@ #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { namespace triton_api { template auto* instance_initialize(TRITONBACKEND_ModelInstance* instance) @@ -53,22 +53,22 @@ auto* instance_initialize(TRITONBACKEND_ModelInstance* instance) setup_memory_resource(device_id, model_state->TritonMemoryManager()); } - auto rapids_model = std::make_unique(*model_state, instance); + auto tools_model = std::make_unique(*model_state, instance); if constexpr (IS_GPU_BUILD) { - auto& model = rapids_model->get_model(); + auto& model = tools_model->get_model(); if (model.get_deployment_type() == GPUDeployment) { cuda_check(cudaSetDevice(model.get_device_id())); } } - rapids_model->load(); + tools_model->load(); - set_instance_state(*instance, std::move(rapids_model)); + set_instance_state(*instance, std::move(tools_model)); } catch (TritonException& err) { result = err.error(); } return result; } } // namespace triton_api -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/api/model_finalize.hpp b/backend/include/triton/developer_tools/triton/api/model_finalize.hpp similarity index 86% rename from backend/include/rapids_triton/triton/api/model_finalize.hpp rename to backend/include/triton/developer_tools/triton/api/model_finalize.hpp index f4d6d6a..f156725 100644 --- a/backend/include/rapids_triton/triton/api/model_finalize.hpp +++ b/backend/include/triton/developer_tools/triton/api/model_finalize.hpp @@ -17,13 +17,13 @@ #pragma once #include #include -#include -#include -#include +#include +#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { namespace triton_api { template auto* model_finalize(TRITONBACKEND_Model* model) @@ -43,6 +43,6 @@ auto* model_finalize(TRITONBACKEND_Model* model) return result; } } // namespace triton_api -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/api/model_initialize.hpp b/backend/include/triton/developer_tools/triton/api/model_initialize.hpp similarity index 78% rename from backend/include/rapids_triton/triton/api/model_initialize.hpp rename to backend/include/triton/developer_tools/triton/api/model_initialize.hpp index aa48013..1bcffc5 100644 --- a/backend/include/rapids_triton/triton/api/model_initialize.hpp +++ b/backend/include/triton/developer_tools/triton/api/model_initialize.hpp @@ -17,13 +17,13 @@ #pragma once #include #include -#include -#include -#include +#include +#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { namespace triton_api { template auto* model_initialize(TRITONBACKEND_Model* model) @@ -37,10 +37,10 @@ auto* model_initialize(TRITONBACKEND_Model* model) log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInitialize: " << name << " (version " << version << ")"; - auto rapids_model_state = std::make_unique(*model); - rapids_model_state->load(); + auto tools_model_state = std::make_unique(*model); + tools_model_state->load(); - set_model_state(*model, std::move(rapids_model_state)); + set_model_state(*model, std::move(tools_model_state)); } catch (TritonException& err) { result = err.error(); } @@ -48,6 +48,6 @@ auto* model_initialize(TRITONBACKEND_Model* model) return result; } } // namespace triton_api -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/backend.hpp b/backend/include/triton/developer_tools/triton/backend.hpp similarity index 91% rename from backend/include/rapids_triton/triton/backend.hpp rename to backend/include/triton/developer_tools/triton/backend.hpp index f0f9a5a..561327a 100644 --- a/backend/include/rapids_triton/triton/backend.hpp +++ b/backend/include/triton/developer_tools/triton/backend.hpp @@ -18,13 +18,13 @@ #include #include -#include -#include +#include +#include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { inline auto get_backend_name(TRITONBACKEND_Backend& backend) { const char* cname; @@ -56,6 +56,6 @@ inline auto check_backend_version(TRITONBACKEND_Backend& backend) return ((version.major == TRITONBACKEND_API_VERSION_MAJOR) && (version.minor >= TRITONBACKEND_API_VERSION_MINOR)); } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/config.hpp b/backend/include/triton/developer_tools/triton/config.hpp similarity index 86% rename from backend/include/rapids_triton/triton/config.hpp rename to backend/include/triton/developer_tools/triton/config.hpp index 4a8300b..affcf3f 100644 --- a/backend/include/rapids_triton/triton/config.hpp +++ b/backend/include/triton/developer_tools/triton/config.hpp @@ -20,18 +20,18 @@ #include #include -#include -#include +#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { inline auto get_max_batch_size(common::TritonJson::Value& config) { auto reported = int64_t{}; triton_check(config.MemberAsInt("max_batch_size", &reported)); return narrow(reported); } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/deployment.hpp b/backend/include/triton/developer_tools/triton/deployment.hpp similarity index 94% rename from backend/include/rapids_triton/triton/deployment.hpp rename to backend/include/triton/developer_tools/triton/deployment.hpp index e663c47..cc9fd97 100644 --- a/backend/include/rapids_triton/triton/deployment.hpp +++ b/backend/include/triton/developer_tools/triton/deployment.hpp @@ -18,14 +18,14 @@ #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { using DeploymentType = TRITONSERVER_InstanceGroupKind; auto constexpr GPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_GPU; auto constexpr CPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_CPU; // Note (wphicks): We currently are not including "Auto" or "Model" because I // am not sure exactly how those would be used in context. If there is a // demand, they can be added. -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/device.hpp b/backend/include/triton/developer_tools/triton/device.hpp similarity index 92% rename from backend/include/rapids_triton/triton/device.hpp rename to backend/include/triton/developer_tools/triton/device.hpp index 73b28c4..621d493 100644 --- a/backend/include/rapids_triton/triton/device.hpp +++ b/backend/include/triton/developer_tools/triton/device.hpp @@ -18,9 +18,9 @@ #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { using device_id_t = std::int32_t; -} } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/input.hpp b/backend/include/triton/developer_tools/triton/input.hpp similarity index 92% rename from backend/include/rapids_triton/triton/input.hpp rename to backend/include/triton/developer_tools/triton/input.hpp index 668875a..85ef139 100644 --- a/backend/include/rapids_triton/triton/input.hpp +++ b/backend/include/triton/developer_tools/triton/input.hpp @@ -20,16 +20,16 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) { auto result = static_cast(nullptr); @@ -75,6 +75,6 @@ auto get_triton_input_shape(Iter requests_begin, Iter requests_end, std::string return result; } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/logging.hpp b/backend/include/triton/developer_tools/triton/logging.hpp similarity index 97% rename from backend/include/rapids_triton/triton/logging.hpp rename to backend/include/triton/developer_tools/triton/logging.hpp index 96455f1..3f15e83 100644 --- a/backend/include/rapids_triton/triton/logging.hpp +++ b/backend/include/triton/developer_tools/triton/logging.hpp @@ -16,15 +16,15 @@ #pragma once #include -#include +#include #include #include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { namespace { /** Log message at indicated level */ @@ -154,6 +154,6 @@ inline auto log_debug(char const* filename, int line) } inline auto log_debug() { return log_stream(TRITONSERVER_LOG_VERBOSE); } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/model.hpp b/backend/include/triton/developer_tools/triton/model.hpp similarity index 94% rename from backend/include/rapids_triton/triton/model.hpp rename to backend/include/triton/developer_tools/triton/model.hpp index 1591763..d2e31e8 100644 --- a/backend/include/rapids_triton/triton/model.hpp +++ b/backend/include/triton/developer_tools/triton/model.hpp @@ -20,12 +20,12 @@ #include #include #include -#include +#include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { inline auto get_model_version(TRITONBACKEND_Model& model) { @@ -63,7 +63,7 @@ inline auto get_model_config(TRITONBACKEND_Model& model) * * This function accepts a unique_ptr to an object derived from a Triton * BackendModel object and sets it as the stored state for a model in the - * Triton server. Note that this object is not the same as a RAPIDS-Triton + * Triton server. Note that this object is not the same as a Backend-Tools * "SharedModelState" object. The object that Triton expects must wrap this * SharedModelState and provide additional interface compatibility. */ @@ -85,6 +85,6 @@ auto* get_model_state(TRITONBACKEND_Model& model) return model_state; } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/model_instance.hpp b/backend/include/triton/developer_tools/triton/model_instance.hpp similarity index 91% rename from backend/include/rapids_triton/triton/model_instance.hpp rename to backend/include/triton/developer_tools/triton/model_instance.hpp index 0ba03bd..2177f08 100644 --- a/backend/include/rapids_triton/triton/model_instance.hpp +++ b/backend/include/triton/developer_tools/triton/model_instance.hpp @@ -17,14 +17,14 @@ #pragma once #include #include -#include -#include -#include +#include +#include +#include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { /** Get the name of a Triton model instance from the instance itself */ inline auto get_model_instance_name(TRITONBACKEND_ModelInstance& instance) { @@ -70,7 +70,7 @@ inline auto* get_model_from_instance(TRITONBACKEND_ModelInstance& instance) * * This function accepts a unique_ptr to an object derived from a Triton * BackendModelInstance object and sets it as the stored state for a model in the - * Triton server. Note that this object is not the same as a RAPIDS-Triton + * Triton server. Note that this object is not the same as a Backend-Tools * "Model" object. The object that Triton expects must wrap this Model and * provide additional interface compatibility. */ @@ -92,6 +92,6 @@ auto* get_instance_state(TRITONBACKEND_ModelInstance& instance) return instance_state; } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/model_instance_state.hpp b/backend/include/triton/developer_tools/triton/model_instance_state.hpp similarity index 66% rename from backend/include/rapids_triton/triton/model_instance_state.hpp rename to backend/include/triton/developer_tools/triton/model_instance_state.hpp index ba05009..233778c 100644 --- a/backend/include/rapids_triton/triton/model_instance_state.hpp +++ b/backend/include/triton/developer_tools/triton/model_instance_state.hpp @@ -18,23 +18,23 @@ #include #include #include -#include -#include +#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { -template -struct ModelInstanceState : public BackendModelInstance { - ModelInstanceState(TritonModelState& model_state, +template +struct ModelInstanceState : public triton::backend::BackendModelInstance { + ModelInstanceState(TritonModelState& model_state, TRITONBACKEND_ModelInstance* triton_model_instance) - : BackendModelInstance(&model_state, triton_model_instance), + : triton::backend::BackendModelInstance(&model_state, triton_model_instance), model_(model_state.get_shared_state(), - rapids::get_device_id(*triton_model_instance), + backend::get_device_id(*triton_model_instance), CudaStream(), Kind(), - JoinPath({model_state.RepositoryPath(), + triton::backend::JoinPath({model_state.RepositoryPath(), std::to_string(model_state.Version()), ArtifactFilename()})) { @@ -46,9 +46,9 @@ struct ModelInstanceState : public BackendModelInstance { void unload() { model_.unload(); } private: - RapidsModel model_; + ToolsModel model_; }; -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/model_state.hpp b/backend/include/triton/developer_tools/triton/model_state.hpp similarity index 71% rename from backend/include/rapids_triton/triton/model_state.hpp rename to backend/include/triton/developer_tools/triton/model_state.hpp index 582b679..df6118a 100644 --- a/backend/include/rapids_triton/triton/model_state.hpp +++ b/backend/include/triton/developer_tools/triton/model_state.hpp @@ -17,16 +17,16 @@ #pragma once #include #include -#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { -template -struct TritonModelState : public BackendModel { +template +struct TritonModelState : public triton::backend::BackendModel { TritonModelState(TRITONBACKEND_Model& triton_model) - : BackendModel(&triton_model), - state_{std::make_shared(get_model_config(triton_model))} + : triton::backend::BackendModel(&triton_model), + state_{std::make_shared(get_model_config(triton_model))} { } @@ -36,9 +36,9 @@ struct TritonModelState : public BackendModel { auto get_shared_state() { return state_; } private: - std::shared_ptr state_; + std::shared_ptr state_; }; -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/output.hpp b/backend/include/triton/developer_tools/triton/output.hpp similarity index 95% rename from backend/include/rapids_triton/triton/output.hpp rename to backend/include/triton/developer_tools/triton/output.hpp index a48d156..d335f66 100644 --- a/backend/include/rapids_triton/triton/output.hpp +++ b/backend/include/triton/developer_tools/triton/output.hpp @@ -17,11 +17,11 @@ #pragma once #include -#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) { auto* result = static_cast(nullptr); @@ -67,6 +67,6 @@ auto get_triton_output_shape(Iter requests_begin, Iter requests_end, std::string return result; } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/requests.hpp b/backend/include/triton/developer_tools/triton/requests.hpp similarity index 87% rename from backend/include/rapids_triton/triton/requests.hpp rename to backend/include/triton/developer_tools/triton/requests.hpp index c24c8bb..d2a6d06 100644 --- a/backend/include/rapids_triton/triton/requests.hpp +++ b/backend/include/triton/developer_tools/triton/requests.hpp @@ -19,12 +19,12 @@ #include #include -#include -#include +#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { using request_size_t = uint32_t; template @@ -38,6 +38,6 @@ void release_requests(Iter begin, Iter end) } }); } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/responses.hpp b/backend/include/triton/developer_tools/triton/responses.hpp similarity index 93% rename from backend/include/rapids_triton/triton/responses.hpp rename to backend/include/triton/developer_tools/triton/responses.hpp index b361c5a..f276a2e 100644 --- a/backend/include/rapids_triton/triton/responses.hpp +++ b/backend/include/triton/developer_tools/triton/responses.hpp @@ -18,13 +18,13 @@ #include #include #include -#include -#include +#include +#include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { template auto construct_responses(Iter requests_begin, Iter requests_end) @@ -70,6 +70,6 @@ void send_responses(Iter begin, Iter end, TRITONSERVER_Error* err) }); } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/statistics.hpp b/backend/include/triton/developer_tools/triton/statistics.hpp similarity index 97% rename from backend/include/rapids_triton/triton/statistics.hpp rename to backend/include/triton/developer_tools/triton/statistics.hpp index 5f4f130..eacb3ab 100644 --- a/backend/include/rapids_triton/triton/statistics.hpp +++ b/backend/include/triton/developer_tools/triton/statistics.hpp @@ -19,11 +19,11 @@ #include #include #include -#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { using time_point = std::chrono::time_point; /** @@ -84,6 +84,6 @@ inline void report_statistics(TRITONBACKEND_ModelInstance& instance, compute_end_time.time_since_epoch().count(), end_time.time_since_epoch().count())); } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/triton/triton_memory_resource.hpp b/backend/include/triton/developer_tools/triton/triton_memory_resource.hpp similarity index 94% rename from backend/include/rapids_triton/triton/triton_memory_resource.hpp rename to backend/include/triton/developer_tools/triton/triton_memory_resource.hpp index 77a879e..837c8d6 100644 --- a/backend/include/rapids_triton/triton/triton_memory_resource.hpp +++ b/backend/include/triton/developer_tools/triton/triton_memory_resource.hpp @@ -20,16 +20,16 @@ #include #include #include -#include -#include +#include +#include #include #include #include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { struct triton_memory_resource final : public rmm::mr::device_memory_resource { triton_memory_resource(TRITONBACKEND_MemoryManager* manager, device_id_t device_id, @@ -81,6 +81,6 @@ struct triton_memory_resource final : public rmm::mr::device_memory_resource { } }; -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/utils/const_agnostic.hpp b/backend/include/triton/developer_tools/utils/const_agnostic.hpp similarity index 93% rename from backend/include/rapids_triton/utils/const_agnostic.hpp rename to backend/include/triton/developer_tools/utils/const_agnostic.hpp index cd55aeb..4a83eed 100644 --- a/backend/include/rapids_triton/utils/const_agnostic.hpp +++ b/backend/include/triton/developer_tools/utils/const_agnostic.hpp @@ -18,13 +18,13 @@ #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { template using const_agnostic_same_t = std::enable_if_t, std::remove_const_t>>; -} } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/utils/device_setter.hpp b/backend/include/triton/developer_tools/utils/device_setter.hpp similarity index 88% rename from backend/include/rapids_triton/utils/device_setter.hpp rename to backend/include/triton/developer_tools/utils/device_setter.hpp index d8aae9b..ef3fa99 100644 --- a/backend/include/rapids_triton/utils/device_setter.hpp +++ b/backend/include/triton/developer_tools/utils/device_setter.hpp @@ -18,12 +18,12 @@ #ifdef TRITON_ENABLE_GPU #include #endif -#include -#include +#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { /** Struct for setting cuda device within a code block */ struct device_setter { @@ -45,6 +45,6 @@ struct device_setter { device_id_t prev_device_; }; -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/include/rapids_triton/utils/narrow.hpp b/backend/include/triton/developer_tools/utils/narrow.hpp similarity index 91% rename from backend/include/rapids_triton/utils/narrow.hpp rename to backend/include/triton/developer_tools/utils/narrow.hpp index 37465c0..2ed9bb8 100644 --- a/backend/include/rapids_triton/utils/narrow.hpp +++ b/backend/include/triton/developer_tools/utils/narrow.hpp @@ -15,12 +15,12 @@ */ #pragma once -#include +#include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { template auto narrow(F from) @@ -36,6 +36,6 @@ auto narrow(F from) return to; } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/src/CMakeLists.txt b/backend/src/CMakeLists.txt index 1d5904c..1d544bb 100644 --- a/backend/src/CMakeLists.txt +++ b/backend/src/CMakeLists.txt @@ -16,12 +16,12 @@ # keep the files in alphabetical order! add_library( - triton_rapids-identity SHARED + triton_tools-identity SHARED src/api.cc ) if(TRITON_ENABLE_GPU) - set_target_properties(triton_rapids-identity + set_target_properties(triton_tools-identity PROPERTIES BUILD_RPATH "\$ORIGIN" # set target compile options CXX_STANDARD 17 @@ -32,7 +32,7 @@ if(TRITON_ENABLE_GPU) INTERFACE_POSITION_INDEPENDENT_CODE ON ) else() - set_target_properties(triton_rapids-identity + set_target_properties(triton_tools-identity PROPERTIES BUILD_RPATH "\$ORIGIN" # set target compile options CXX_STANDARD 17 @@ -42,27 +42,26 @@ else() ) endif() -target_compile_options(triton_rapids-identity - PRIVATE "$<$:${RAPIDS_TRITON_CXX_FLAGS}>" - "$<$:${RAPIDS_TRITON_CUDA_FLAGS}>" +target_compile_options(triton_tools-identity + PRIVATE "$<$:${DEVELOPER_TOOLS_BACKEND_CXX_FLAGS}>" + "$<$:${DEVELOPER_TOOLS_BACKEND_CUDA_FLAGS}>" ) -target_include_directories(triton_rapids-identity - PRIVATE "$" +target_include_directories(triton_tools-identity + PRIVATE "$" "${CMAKE_CURRENT_SOURCE_DIR}/src" ) -target_link_libraries(triton_rapids-identity +target_link_libraries(triton_tools-identity PRIVATE $<$:rmm::rmm> $<$:raft::raft> triton-core-serverstub triton-backend-utils - "${TRITONSERVER_LIB}" $ ) install( - TARGETS triton_rapids-identity - LIBRARY DESTINATION /opt/tritonserver/backends/rapids-identity + TARGETS triton_tools-identity + LIBRARY DESTINATION /opt/tritonserver/backends/tools-identity ) diff --git a/backend/src/api.cc b/backend/src/api.cc index 7e62aa6..a6117fa 100644 --- a/backend/src/api.cc +++ b/backend/src/api.cc @@ -22,59 +22,59 @@ #include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include namespace triton { -namespace backend { +namespace developer_tools { namespace NAMESPACE { -using ModelState = rapids::TritonModelState; +using ModelState = backend::TritonModelState; using ModelInstanceState = - rapids::ModelInstanceState; + backend::ModelInstanceState; extern "C" { /** Confirm that backend is compatible with Triton's backend API version */ TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { - return rapids::triton_api::initialize(backend); + return backend::triton_api::initialize(backend); } TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { - return rapids::triton_api::model_initialize(model); + return backend::triton_api::model_initialize(model); } TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { - return rapids::triton_api::model_finalize(model); + return backend::triton_api::model_finalize(model); } TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize( TRITONBACKEND_ModelInstance* instance) { - return rapids::triton_api::instance_initialize(instance); } TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize( TRITONBACKEND_ModelInstance* instance) { - return rapids::triton_api::instance_finalize(instance); + return backend::triton_api::instance_finalize(instance); } TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** raw_requests, uint32_t const request_count) { - return rapids::triton_api::execute( + return backend::triton_api::execute( instance, raw_requests, static_cast(request_count)); } } // extern "C" } // namespace NAMESPACE -} // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/src/model.h b/backend/src/model.h index 5ba95f1..7fe0875 100644 --- a/backend/src/model.h +++ b/backend/src/model.h @@ -19,40 +19,40 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include #include #include #include -#include // rapids::Batch -#include // rapids::MemoryType -#include // rapids::Model -#include // rapids::copy -#include // rapids::DeploymentType -#include // rapids::device_id_t +#include // backend::Batch +#include // backend::MemoryType +#include // backend::Model +#include // backend::copy +#include // backend::DeploymentType +#include // backend::device_id_t namespace triton { -namespace backend { +namespace developer_tools { namespace NAMESPACE { /* Any logic necessary to perform inference with a model and manage its data - * should be implemented in a struct named RapidsModel, as shown here */ + * should be implemented in a struct named ToolsModel, as shown here */ -struct RapidsModel : rapids::Model { +struct ToolsModel : backend::Model { /*************************************************************************** * BOILERPLATE * * *********************************************************************** * * The following constructor can be copied directly into any model * implementation. **************************************************************************/ - RapidsModel(std::shared_ptr shared_state, - rapids::device_id_t device_id, + ToolsModel(std::shared_ptr shared_state, + backend::device_id_t device_id, cudaStream_t default_stream, - rapids::DeploymentType deployment_type, + backend::DeploymentType deployment_type, std::string const& filepath) - : rapids::Model( + : backend::Model( shared_state, device_id, default_stream, deployment_type, filepath) { } @@ -82,7 +82,7 @@ struct RapidsModel : rapids::Model { * pointer to the underlying data. * 4. Call the `finalize` method on all output tensors. **************************************************************************/ - void predict(rapids::Batch& batch) const + void predict(backend::Batch& batch) const { // 1. Acquire a tensor representing the input named "input__0" auto input = get_input(batch, "input__0"); @@ -91,7 +91,7 @@ struct RapidsModel : rapids::Model { // 3. Perform inference. In this example, we simply copy the data from the // input to the output tensor. - rapids::copy(output, input); + backend::copy(output, input); // 4. Call finalize on all output tensors. In this case, we have just one // output, so we call finalize on it. @@ -142,15 +142,15 @@ struct RapidsModel : rapids::Model { * implementations that may switch their preferred memory location based on * properties of the batch. * - * Valid MemoryType options to return are rapids::HostMemory and - * rapids::DeviceMemory. + * Valid MemoryType options to return are backend::HostMemory and + * backend::DeviceMemory. **************************************************************************/ - std::optional preferred_mem_type(rapids::Batch& batch) const + std::optional preferred_mem_type(backend::Batch& batch) const { return std::nullopt; } }; } // namespace NAMESPACE -} // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/src/names.h b/backend/src/names.h index 33d6fe0..eba7319 100644 --- a/backend/src/names.h +++ b/backend/src/names.h @@ -18,12 +18,12 @@ /* Triton expects certain definitions within its backend libraries to follow * specific naming conventions. Specifically, for a backend named - * "rapids_identity," most definitions should appear within a namespace called - * triton::backend::rapids_identity. + * "tools_identity," most definitions should appear within a namespace called + * triton::backend::tools_identity. * * In order to facilitate this with minimal effort on the part of backend * developers, we ask that you put the name of your backend here. This macro is * then used to propagate the correct namespace name wherever it is needed in * the impl and interface code. */ -#define NAMESPACE rapids_identity +#define NAMESPACE tools_identity diff --git a/backend/src/shared_state.h b/backend/src/shared_state.h index b6f8981..190020b 100644 --- a/backend/src/shared_state.h +++ b/backend/src/shared_state.h @@ -19,26 +19,26 @@ #include #include -#include +#include namespace triton { -namespace backend { +namespace developer_tools { namespace NAMESPACE { /* Triton allows multiple instances of a single model to be instantiated at the * same time (e.g. on different GPUs). All instances of a model share access to * an object which manages any state that can be shared across all instances. * Any logic necessary for managing such state should be implemented in a - * struct named RapidsSharedState, as shown here. Models may access this shared + * struct named ToolsSharedState, as shown here. Models may access this shared * state object via the `get_shared_state` method, which returns a shared - * pointer to the RapidsSharedState object. + * pointer to the ToolsSharedState object. * * Not all backends require shared state, so leaving this implementation empty * is entirely valid */ -struct RapidsSharedState : rapids::SharedModelState { - RapidsSharedState(std::unique_ptr&& config) - : rapids::SharedModelState{std::move(config)} +struct ToolsSharedState : backend::SharedModelState { + ToolsSharedState(std::unique_ptr&& config) + : backend::SharedModelState{std::move(config)} { } void load() {} @@ -46,5 +46,5 @@ struct RapidsSharedState : rapids::SharedModelState { }; } // namespace NAMESPACE -} // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/test/CMakeLists.txt b/backend/test/CMakeLists.txt index dbe92b2..5a85246 100644 --- a/backend/test/CMakeLists.txt +++ b/backend/test/CMakeLists.txt @@ -15,7 +15,7 @@ #============================================================================= # keep the files in alphabetical order! -add_executable(test_rapids_triton +add_executable(test_developer_tools_backend test/batch/batch.cpp test/build_control.cpp test/exceptions.cpp @@ -49,7 +49,7 @@ add_executable(test_rapids_triton ) IF(TRITON_ENABLE_GPU) - set_target_properties(test_rapids_triton + set_target_properties(test_developer_tools_backend PROPERTIES BUILD_RPATH "\$ORIGIN" # set target compile options CXX_STANDARD 17 @@ -60,7 +60,7 @@ IF(TRITON_ENABLE_GPU) INTERFACE_POSITION_INDEPENDENT_CODE ON ) else() - set_target_properties(test_rapids_triton + set_target_properties(test_developer_tools_backend PROPERTIES BUILD_RPATH "\$ORIGIN" # set target compile options CXX_STANDARD 17 @@ -70,24 +70,17 @@ else() ) endif() -target_compile_options(test_rapids_triton - PRIVATE "$<$:${RAPIDS_TRITON_CXX_FLAGS}>" - "$<$:${RAPIDS_TRITON_CUDA_FLAGS}>" +target_compile_options(test_developer_tools_backend + PRIVATE "$<$:${DEVELOPER_TOOLS_BACKEND_CXX_FLAGS}>" + "$<$:${DEVELOPER_TOOLS_BACKEND_CUDA_FLAGS}>" ) -target_include_directories(test_rapids_triton - PUBLIC "$" - "$" +target_include_directories(test_developer_tools_backend + PUBLIC "$" + "$" ) - -find_library( - TRITONSERVER_LIB - tritonserver - PATHS /opt/tritonserver/lib -) - -target_link_libraries(test_rapids_triton +target_link_libraries(test_developer_tools_backend PRIVATE $<$:rmm::rmm> $<$:raft::raft> @@ -97,6 +90,5 @@ PRIVATE gmock_main GTest::gtest GTest::gtest_main - "${TRITONSERVER_LIB}" $ ) diff --git a/backend/test/batch/batch.cpp b/backend/test/batch/batch.cpp index e522ecb..7c9dd23 100644 --- a/backend/test/batch/batch.cpp +++ b/backend/test/batch/batch.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/build_control.cpp b/backend/test/build_control.cpp index 18392b0..cae9647 100644 --- a/backend/test/build_control.cpp +++ b/backend/test/build_control.cpp @@ -16,13 +16,13 @@ #include -#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { -TEST(RapidsTriton, build_control) +TEST(BackendTools, build_control) { #ifdef TRITON_ENABLE_GPU ASSERT_EQ(IS_GPU_BUILD, true) << "IS_GPU_BUILD constant has wrong value\n"; @@ -30,6 +30,6 @@ TEST(RapidsTriton, build_control) ASSERT_EQ(IS_GPU_BUILD, false) << "IS_GPU_BUILD constant has wrong value\n"; #endif } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/test/exceptions.cpp b/backend/test/exceptions.cpp index 022d184..9544be3 100644 --- a/backend/test/exceptions.cpp +++ b/backend/test/exceptions.cpp @@ -17,18 +17,18 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include -#include +#include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { -TEST(RapidsTriton, default_except) +TEST(BackendTools, default_except) { try { throw TritonException(); @@ -37,7 +37,7 @@ TEST(RapidsTriton, default_except) } } -TEST(RapidsTriton, msg_except) +TEST(BackendTools, msg_except) { auto msg = std::string("TEST ERROR MESSAGE"); try { @@ -61,14 +61,14 @@ TEST(RapidsTriton, msg_except) } } -TEST(RapidsTriton, triton_check) +TEST(BackendTools, triton_check) { auto msg = std::string("TEST ERROR MESSAGE"); EXPECT_THROW(triton_check(TRITONSERVER_ErrorNew(Error::Internal, msg.c_str())), TritonException); triton_check(nullptr); } -TEST(RapidsTriton, cuda_check) +TEST(BackendTools, cuda_check) { #ifdef TRITON_ENABLE_GPU EXPECT_THROW(cuda_check(cudaError::cudaErrorMissingConfiguration), TritonException); @@ -78,6 +78,6 @@ TEST(RapidsTriton, cuda_check) #endif } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/test/memory/buffer.cpp b/backend/test/memory/buffer.cpp index 1181560..798d68f 100644 --- a/backend/test/memory/buffer.cpp +++ b/backend/test/memory/buffer.cpp @@ -21,16 +21,16 @@ #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { -TEST(RapidsTriton, default_buffer) +TEST(BackendTools, default_buffer) { auto buffer = Buffer(); EXPECT_EQ(buffer.mem_type(), HostMemory); @@ -47,7 +47,7 @@ TEST(RapidsTriton, default_buffer) #endif } -TEST(RapidsTriton, device_buffer) +TEST(BackendTools, device_buffer) { auto data = std::vector{1, 2, 3}; #ifdef TRITON_ENABLE_GPU @@ -73,7 +73,7 @@ TEST(RapidsTriton, device_buffer) #endif } -TEST(RapidsTriton, non_owning_device_buffer) +TEST(BackendTools, non_owning_device_buffer) { auto data = std::vector{1, 2, 3}; #ifdef TRITON_ENABLE_GPU @@ -102,7 +102,7 @@ TEST(RapidsTriton, non_owning_device_buffer) #endif } -TEST(RapidsTriton, host_buffer) +TEST(BackendTools, host_buffer) { auto data = std::vector{1, 2, 3}; auto buffer = Buffer(data.size(), HostMemory, 0, 0); @@ -118,7 +118,7 @@ TEST(RapidsTriton, host_buffer) EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, non_owning_host_buffer) +TEST(BackendTools, non_owning_host_buffer) { auto data = std::vector{1, 2, 3}; auto buffer = Buffer(data.data(), data.size(), HostMemory); @@ -131,7 +131,7 @@ TEST(RapidsTriton, non_owning_host_buffer) EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, copy_buffer) +TEST(BackendTools, copy_buffer) { auto data = std::vector{1, 2, 3}; auto orig_buffer = Buffer(data.data(), data.size(), HostMemory); @@ -145,7 +145,7 @@ TEST(RapidsTriton, copy_buffer) EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, move_buffer) +TEST(BackendTools, move_buffer) { auto data = std::vector{1, 2, 3}; auto buffer = Buffer(Buffer(data.data(), data.size(), HostMemory)); @@ -158,7 +158,7 @@ TEST(RapidsTriton, move_buffer) EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, move_assignment_buffer) +TEST(BackendTools, move_assignment_buffer) { auto data = std::vector{1, 2, 3}; @@ -173,6 +173,6 @@ TEST(RapidsTriton, move_assignment_buffer) ASSERT_EQ(buffer.size(), data.size()); } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/test/memory/detail/copy.cpp b/backend/test/memory/detail/copy.cpp index d669d6b..41edbf5 100644 --- a/backend/test/memory/detail/copy.cpp +++ b/backend/test/memory/detail/copy.cpp @@ -16,21 +16,21 @@ #ifdef TRITON_ENABLE_GPU #include -#include +#include #else -#include +#include #endif #include #include -#include -#include +#include +#include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { -TEST(RapidsTriton, copy) +TEST(BackendTools, copy) { auto data = std::vector{1, 2, 3}; auto data_out = std::vector(data.size()); @@ -57,6 +57,6 @@ TEST(RapidsTriton, copy) #endif } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/test/memory/detail/owned_device_buffer.cpp b/backend/test/memory/detail/owned_device_buffer.cpp index 9bd613f..41a2351 100644 --- a/backend/test/memory/detail/owned_device_buffer.cpp +++ b/backend/test/memory/detail/owned_device_buffer.cpp @@ -16,21 +16,21 @@ #ifdef TRITON_ENABLE_GPU #include -#include +#include #else -#include +#include #endif #include #include -#include -#include +#include +#include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { -TEST(RapidsTriton, owned_device_buffer) +TEST(BackendTools, owned_device_buffer) { auto data = std::vector{1, 2, 3}; #ifdef TRITON_ENABLE_GPU @@ -59,6 +59,6 @@ TEST(RapidsTriton, owned_device_buffer) #endif } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/test/memory/resource.cpp b/backend/test/memory/resource.cpp index 5571ef7..487be1d 100644 --- a/backend/test/memory/resource.cpp +++ b/backend/test/memory/resource.cpp @@ -24,15 +24,15 @@ #include #include -#include -#include -#include +#include +#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { -TEST(RapidsTriton, set_memory_resource) +TEST(BackendTools, set_memory_resource) { #ifdef TRITON_ENABLE_GPU auto device_id = int{}; @@ -47,6 +47,6 @@ TEST(RapidsTriton, set_memory_resource) #endif } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/test/memory/types.cpp b/backend/test/memory/types.cpp index 65eb386..f0fdecc 100644 --- a/backend/test/memory/types.cpp +++ b/backend/test/memory/types.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/tensor/dtype.cpp b/backend/test/tensor/dtype.cpp index 2e80c60..21de4f8 100644 --- a/backend/test/tensor/dtype.cpp +++ b/backend/test/tensor/dtype.cpp @@ -16,11 +16,11 @@ #include -#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { template void check_dtype_conversion() @@ -29,7 +29,7 @@ void check_dtype_conversion() EXPECT_EQ(D, TritonDtype::type const>::value); } -TEST(RapidsTriton, dtype) +TEST(BackendTools, dtype) { check_dtype_conversion(); check_dtype_conversion(); @@ -46,6 +46,6 @@ TEST(RapidsTriton, dtype) check_dtype_conversion(); } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/test/tensor/tensor.cpp b/backend/test/tensor/tensor.cpp index 3a1c5c2..4418dbd 100644 --- a/backend/test/tensor/tensor.cpp +++ b/backend/test/tensor/tensor.cpp @@ -17,27 +17,27 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include #include -#include -#include +#include +#include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { -TEST(RapidsTriton, default_tensor) +TEST(BackendTools, default_tensor) { auto tensor = Tensor(); EXPECT_EQ(tensor.buffer().size(), 0); EXPECT_EQ(tensor.shape().size(), 0); } -TEST(RapidsTriton, move_buffer_tensor) +TEST(BackendTools, move_buffer_tensor) { auto shape = std::vector{2, 2}; auto data = std::vector{1, 2, 3, 4}; @@ -56,7 +56,7 @@ TEST(RapidsTriton, move_buffer_tensor) EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, multi_buffer_tensor) +TEST(BackendTools, multi_buffer_tensor) { auto shape = std::vector{2, 2}; auto data = std::vector{1, 2, 3, 4}; @@ -85,7 +85,7 @@ TEST(RapidsTriton, multi_buffer_tensor) EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, tensor_copy) +TEST(BackendTools, tensor_copy) { auto shape = std::vector{2, 2}; auto data = std::vector{1, 2, 3, 4}; @@ -95,7 +95,7 @@ TEST(RapidsTriton, tensor_copy) auto data2 = std::vector(data1.size()); auto tensor2 = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); - rapids::copy(tensor2, tensor1); + backend::copy(tensor2, tensor1); auto data_out = std::vector(tensor2.data(), tensor2.data() + tensor2.size()); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); @@ -105,10 +105,10 @@ TEST(RapidsTriton, tensor_copy) auto tensor3 = Tensor(small_shape, Buffer{small_data.data(), small_data.size(), HostMemory}); - EXPECT_THROW(rapids::copy(tensor3, tensor1), TritonException); + EXPECT_THROW(backend::copy(tensor3, tensor1), TritonException); } -TEST(RapidsTriton, tensor_multi_copy) +TEST(BackendTools, tensor_multi_copy) { auto shape = std::vector{2, 2}; auto data = std::vector{1, 2, 3, 4}; @@ -125,7 +125,7 @@ TEST(RapidsTriton, tensor_multi_copy) return Tensor(receiver_shape, Buffer{std::size_t{1}, HostMemory}); }); - rapids::copy(receivers.begin(), receivers.end(), tensor1); + backend::copy(receivers.begin(), receivers.end(), tensor1); auto data_out = std::vector{}; data_out.reserve(receivers.size()); @@ -137,9 +137,9 @@ TEST(RapidsTriton, tensor_multi_copy) // Throw if trying to copy to too many outputs receivers.emplace_back(receiver_shape, Buffer{std::size_t{1}, HostMemory}); - EXPECT_THROW(rapids::copy(receivers.begin(), receivers.end(), tensor1), TritonException); + EXPECT_THROW(backend::copy(receivers.begin(), receivers.end(), tensor1), TritonException); } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/test/test.cpp b/backend/test/test.cpp index 4a7d7f4..15c9dfc 100644 --- a/backend/test/test.cpp +++ b/backend/test/test.cpp @@ -16,13 +16,13 @@ #include -#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { -TEST(RapidsTriton, installed) { std::cout << test_install() << "\n"; } -} // namespace rapids +TEST(BackendTools, installed) { std::cout << test_install() << "\n"; } } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/test/triton/api/execute.cpp b/backend/test/triton/api/execute.cpp index ba48f8e..41a496f 100644 --- a/backend/test/triton/api/execute.cpp +++ b/backend/test/triton/api/execute.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/triton/api/initialize.cpp b/backend/test/triton/api/initialize.cpp index 33b95d4..ebb4f6d 100644 --- a/backend/test/triton/api/initialize.cpp +++ b/backend/test/triton/api/initialize.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/triton/api/instance_finalize.cpp b/backend/test/triton/api/instance_finalize.cpp index f7218a3..75a5b0d 100644 --- a/backend/test/triton/api/instance_finalize.cpp +++ b/backend/test/triton/api/instance_finalize.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/triton/api/instance_initialize.cpp b/backend/test/triton/api/instance_initialize.cpp index 3ed2b91..356efd1 100644 --- a/backend/test/triton/api/instance_initialize.cpp +++ b/backend/test/triton/api/instance_initialize.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/triton/api/model_finalize.cpp b/backend/test/triton/api/model_finalize.cpp index 367f16c..7e08b57 100644 --- a/backend/test/triton/api/model_finalize.cpp +++ b/backend/test/triton/api/model_finalize.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/triton/api/model_initialize.cpp b/backend/test/triton/api/model_initialize.cpp index 071baa3..a0cc5b1 100644 --- a/backend/test/triton/api/model_initialize.cpp +++ b/backend/test/triton/api/model_initialize.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/triton/backend.cpp b/backend/test/triton/backend.cpp index cacff7d..8bff317 100644 --- a/backend/test/triton/backend.cpp +++ b/backend/test/triton/backend.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/triton/config.cpp b/backend/test/triton/config.cpp index 7a31891..1ec808e 100644 --- a/backend/test/triton/config.cpp +++ b/backend/test/triton/config.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/triton/deployment.cpp b/backend/test/triton/deployment.cpp index dbd22b7..d9f9e66 100644 --- a/backend/test/triton/deployment.cpp +++ b/backend/test/triton/deployment.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/triton/device.cpp b/backend/test/triton/device.cpp index 53e49e0..6fcca89 100644 --- a/backend/test/triton/device.cpp +++ b/backend/test/triton/device.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/triton/input.cpp b/backend/test/triton/input.cpp index 1cac65d..8269e95 100644 --- a/backend/test/triton/input.cpp +++ b/backend/test/triton/input.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/triton/logging.cpp b/backend/test/triton/logging.cpp index f1f60cd..53fbce0 100644 --- a/backend/test/triton/logging.cpp +++ b/backend/test/triton/logging.cpp @@ -17,12 +17,12 @@ #include #include -#include +#include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { -TEST(RapidsTriton, logging) +TEST(BackendTools, logging) { log_debug("Debug test message"); log_info("Info test message"); @@ -30,7 +30,7 @@ TEST(RapidsTriton, logging) log_error("Error test message"); } -TEST(RapidsTriton, stream_logging) +TEST(BackendTools, stream_logging) { log_debug() << "Streamed debug test message"; log_info() << "Streamed info test message"; @@ -38,6 +38,6 @@ TEST(RapidsTriton, stream_logging) log_error() << "Streamed error test message"; } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/test/triton/model.cpp b/backend/test/triton/model.cpp index 3226e35..a8263be 100644 --- a/backend/test/triton/model.cpp +++ b/backend/test/triton/model.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/triton/model_instance.cpp b/backend/test/triton/model_instance.cpp index 3582f75..fcfbbbd 100644 --- a/backend/test/triton/model_instance.cpp +++ b/backend/test/triton/model_instance.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/triton/requests.cpp b/backend/test/triton/requests.cpp index cff2af3..ecef9f8 100644 --- a/backend/test/triton/requests.cpp +++ b/backend/test/triton/requests.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/triton/responses.cpp b/backend/test/triton/responses.cpp index 4ca7d52..34b64ff 100644 --- a/backend/test/triton/responses.cpp +++ b/backend/test/triton/responses.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/triton/statistics.cpp b/backend/test/triton/statistics.cpp index 7c01a6d..1d82994 100644 --- a/backend/test/triton/statistics.cpp +++ b/backend/test/triton/statistics.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/test/utils/const_agnostic.cpp b/backend/test/utils/const_agnostic.cpp index 2a74c69..2d3f6aa 100644 --- a/backend/test/utils/const_agnostic.cpp +++ b/backend/test/utils/const_agnostic.cpp @@ -16,18 +16,18 @@ #include -#include +#include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { -TEST(RapidsTriton, const_agnostic) +TEST(BackendTools, const_agnostic) { static_assert(std::is_same, void>::value); static_assert(std::is_same, void>::value); } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton diff --git a/backend/test/utils/narrow.cpp b/backend/test/utils/narrow.cpp index d9919ca..20caf63 100644 --- a/backend/test/utils/narrow.cpp +++ b/backend/test/utils/narrow.cpp @@ -16,13 +16,13 @@ #include -#include +#include #include namespace triton { +namespace developer_tools { namespace backend { -namespace rapids { -TEST(RapidsTriton, narrow) +TEST(BackendTools, narrow) { EXPECT_THROW(narrow(-1), TritonException); narrow(int{5}); @@ -30,6 +30,6 @@ TEST(RapidsTriton, narrow) narrow(std::size_t{5}); } -} // namespace rapids } // namespace backend +} // namespace developer_tools } // namespace triton From ebbb0730a1034caa075d3e11b51e8a7b3c097cba Mon Sep 17 00:00:00 2001 From: GuanLuo Date: Wed, 5 Oct 2022 16:06:19 -0700 Subject: [PATCH 169/199] Add CI test for backend unit test --- qa/L0_backend_unit_test/test.sh | 75 +++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 qa/L0_backend_unit_test/test.sh diff --git a/qa/L0_backend_unit_test/test.sh b/qa/L0_backend_unit_test/test.sh new file mode 100644 index 0000000..861f5b9 --- /dev/null +++ b/qa/L0_backend_unit_test/test.sh @@ -0,0 +1,75 @@ +# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +export CUDA_VISIBLE_DEVICES=0 + +# fulfill build dependencies +set +e +cmake --version +if [ $? -ne 0 ]; then + echo "cmake not found, installing cmake " + wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \ + gpg --dearmor - | \ + tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \ + apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + cmake-data=3.21.1-0kitware1ubuntu20.04.1 cmake=3.21.1-0kitware1ubuntu20.04.1 +fi +set -e + +apt-get update && \ +apt-get install -y --no-install-recommends \ + rapidjson-dev + +# build and run the unit test for developer_tools::backend +rm -rf ./build +mkdir build && +(cd build && + git clone --single-branch --depth=1 -b ${TRITON_DEVELOPER_TOOLS_REPO_TAG} \ + https://github.com/triton-inference-server/developer_tools.git && \ + cmake developer_tools/backend && make install) + +TEST_LOG=test.log +RET=0 + +set +e +# Must explicitly set LD_LIBRARY_PATH so that the test can find +# libtritonserver.so. +LD_LIBRARY_PATH=/opt/tritonserver/lib:${LD_LIBRARY_PATH} ./build/test_developer_tools_backend >> ${TEST_LOG} 2>&1 +if [ $? -ne 0 ]; then + cat ${TEST_LOG} + RET=1 +fi +set -e + +if [ $RET -eq 0 ]; then + echo -e "\n***\n*** Test Passed\n***" +else + echo -e "\n***\n*** Test FAILED\n***" +fi + +exit $RET From f7ebb6aba6d5670b80608ab5a1671ccf7f40e9c1 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 7 Oct 2022 12:53:01 -0400 Subject: [PATCH 170/199] Add initial group of codeowners --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..76bb196 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +/ @tanmayv25 @GuanLuo +/backend @wphicks @dantegd @divyegala From 5654d51af2ea2c5fde711b5dbc89153a4a106dc7 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 7 Oct 2022 13:51:00 -0400 Subject: [PATCH 171/199] Remove incorrectly-moved files --- backend/.clang-format | 164 --------- backend/CMakeLists.txt | 218 ------------ backend/cmake/doxygen.cmake | 33 -- backend/cmake/modules/ConfigureCUDA.cmake | 43 --- backend/cmake/thirdparty/get_gtest.cmake | 22 -- backend/cmake/thirdparty/get_raft.cmake | 49 --- backend/cmake/thirdparty/get_rapidjson.cmake | 37 -- backend/cmake/thirdparty/get_rmm.cmake | 22 -- backend/cmake/thirdparty/get_triton.cmake | 36 -- backend/include/backend_tools.hpp | 31 -- .../triton/developer_tools/batch/batch.hpp | 273 --------------- .../triton/developer_tools/build_control.hpp | 32 -- .../cpu_only/cuda_runtime_replacement.hpp | 54 --- .../triton/developer_tools/exceptions.hpp | 94 ------ .../triton/developer_tools/memory/buffer.hpp | 317 ------------------ .../memory/detail/cpu_only/copy.hpp | 50 --- .../detail/cpu_only/owned_device_buffer.hpp | 45 --- .../memory/detail/cpu_only/resource.hpp | 35 -- .../memory/detail/gpu_only/copy.hpp | 55 --- .../detail/gpu_only/owned_device_buffer.hpp | 49 --- .../memory/detail/gpu_only/resource.hpp | 97 ------ .../memory/detail/owned_device_buffer.hpp | 30 -- .../memory/detail/resource.hpp | 35 -- .../developer_tools/memory/resource.hpp | 39 --- .../triton/developer_tools/memory/types.hpp | 28 -- .../triton/developer_tools/model/model.hpp | 197 ----------- .../developer_tools/model/shared_state.hpp | 189 ----------- .../triton/developer_tools/tensor/dtype.hpp | 168 ---------- .../triton/developer_tools/tensor/tensor.hpp | 205 ----------- .../developer_tools/triton/api/execute.hpp | 121 ------- .../developer_tools/triton/api/initialize.hpp | 69 ---- .../triton/api/instance_finalize.hpp | 49 --- .../triton/api/instance_initialize.hpp | 74 ---- .../triton/api/model_finalize.hpp | 48 --- .../triton/api/model_initialize.hpp | 53 --- .../triton/developer_tools/triton/backend.hpp | 61 ---- .../triton/developer_tools/triton/config.hpp | 37 -- .../developer_tools/triton/deployment.hpp | 31 -- .../triton/developer_tools/triton/device.hpp | 26 -- .../triton/developer_tools/triton/input.hpp | 80 ----- .../triton/developer_tools/triton/logging.hpp | 159 --------- .../triton/developer_tools/triton/model.hpp | 90 ----- .../developer_tools/triton/model_instance.hpp | 97 ------ .../triton/model_instance_state.hpp | 54 --- .../developer_tools/triton/model_state.hpp | 44 --- .../triton/developer_tools/triton/output.hpp | 72 ---- .../developer_tools/triton/requests.hpp | 43 --- .../developer_tools/triton/responses.hpp | 75 ----- .../developer_tools/triton/statistics.hpp | 89 ----- .../triton/triton_memory_resource.hpp | 86 ----- .../developer_tools/utils/const_agnostic.hpp | 30 -- .../developer_tools/utils/device_setter.hpp | 50 --- .../triton/developer_tools/utils/narrow.hpp | 41 --- backend/scripts/run-clang-format.py | 148 -------- backend/src/CMakeLists.txt | 67 ---- backend/src/api.cc | 80 ----- backend/src/model.h | 156 --------- backend/src/names.h | 29 -- backend/src/shared_state.h | 50 --- backend/test/CMakeLists.txt | 94 ------ backend/test/batch/batch.cpp | 17 - backend/test/build_control.cpp | 35 -- backend/test/exceptions.cpp | 83 ----- backend/test/memory/buffer.cpp | 178 ---------- backend/test/memory/detail/copy.cpp | 62 ---- .../memory/detail/owned_device_buffer.cpp | 64 ---- backend/test/memory/resource.cpp | 52 --- backend/test/memory/types.cpp | 17 - backend/test/tensor/dtype.cpp | 51 --- backend/test/tensor/tensor.cpp | 145 -------- backend/test/test.cpp | 28 -- backend/test/triton/api/execute.cpp | 17 - backend/test/triton/api/initialize.cpp | 17 - backend/test/triton/api/instance_finalize.cpp | 17 - .../test/triton/api/instance_initialize.cpp | 17 - backend/test/triton/api/model_finalize.cpp | 17 - backend/test/triton/api/model_initialize.cpp | 17 - backend/test/triton/backend.cpp | 17 - backend/test/triton/config.cpp | 17 - backend/test/triton/deployment.cpp | 17 - backend/test/triton/device.cpp | 17 - backend/test/triton/input.cpp | 17 - backend/test/triton/logging.cpp | 43 --- backend/test/triton/model.cpp | 17 - backend/test/triton/model_instance.cpp | 17 - backend/test/triton/requests.cpp | 17 - backend/test/triton/responses.cpp | 17 - backend/test/triton/statistics.cpp | 17 - backend/test/utils/const_agnostic.cpp | 33 -- backend/test/utils/narrow.cpp | 35 -- 90 files changed, 5965 deletions(-) delete mode 100644 backend/.clang-format delete mode 100644 backend/CMakeLists.txt delete mode 100644 backend/cmake/doxygen.cmake delete mode 100644 backend/cmake/modules/ConfigureCUDA.cmake delete mode 100644 backend/cmake/thirdparty/get_gtest.cmake delete mode 100644 backend/cmake/thirdparty/get_raft.cmake delete mode 100644 backend/cmake/thirdparty/get_rapidjson.cmake delete mode 100644 backend/cmake/thirdparty/get_rmm.cmake delete mode 100644 backend/cmake/thirdparty/get_triton.cmake delete mode 100644 backend/include/backend_tools.hpp delete mode 100644 backend/include/triton/developer_tools/batch/batch.hpp delete mode 100644 backend/include/triton/developer_tools/build_control.hpp delete mode 100644 backend/include/triton/developer_tools/cpu_only/cuda_runtime_replacement.hpp delete mode 100644 backend/include/triton/developer_tools/exceptions.hpp delete mode 100644 backend/include/triton/developer_tools/memory/buffer.hpp delete mode 100644 backend/include/triton/developer_tools/memory/detail/cpu_only/copy.hpp delete mode 100644 backend/include/triton/developer_tools/memory/detail/cpu_only/owned_device_buffer.hpp delete mode 100644 backend/include/triton/developer_tools/memory/detail/cpu_only/resource.hpp delete mode 100644 backend/include/triton/developer_tools/memory/detail/gpu_only/copy.hpp delete mode 100644 backend/include/triton/developer_tools/memory/detail/gpu_only/owned_device_buffer.hpp delete mode 100644 backend/include/triton/developer_tools/memory/detail/gpu_only/resource.hpp delete mode 100644 backend/include/triton/developer_tools/memory/detail/owned_device_buffer.hpp delete mode 100644 backend/include/triton/developer_tools/memory/detail/resource.hpp delete mode 100644 backend/include/triton/developer_tools/memory/resource.hpp delete mode 100644 backend/include/triton/developer_tools/memory/types.hpp delete mode 100644 backend/include/triton/developer_tools/model/model.hpp delete mode 100644 backend/include/triton/developer_tools/model/shared_state.hpp delete mode 100644 backend/include/triton/developer_tools/tensor/dtype.hpp delete mode 100644 backend/include/triton/developer_tools/tensor/tensor.hpp delete mode 100644 backend/include/triton/developer_tools/triton/api/execute.hpp delete mode 100644 backend/include/triton/developer_tools/triton/api/initialize.hpp delete mode 100644 backend/include/triton/developer_tools/triton/api/instance_finalize.hpp delete mode 100644 backend/include/triton/developer_tools/triton/api/instance_initialize.hpp delete mode 100644 backend/include/triton/developer_tools/triton/api/model_finalize.hpp delete mode 100644 backend/include/triton/developer_tools/triton/api/model_initialize.hpp delete mode 100644 backend/include/triton/developer_tools/triton/backend.hpp delete mode 100644 backend/include/triton/developer_tools/triton/config.hpp delete mode 100644 backend/include/triton/developer_tools/triton/deployment.hpp delete mode 100644 backend/include/triton/developer_tools/triton/device.hpp delete mode 100644 backend/include/triton/developer_tools/triton/input.hpp delete mode 100644 backend/include/triton/developer_tools/triton/logging.hpp delete mode 100644 backend/include/triton/developer_tools/triton/model.hpp delete mode 100644 backend/include/triton/developer_tools/triton/model_instance.hpp delete mode 100644 backend/include/triton/developer_tools/triton/model_instance_state.hpp delete mode 100644 backend/include/triton/developer_tools/triton/model_state.hpp delete mode 100644 backend/include/triton/developer_tools/triton/output.hpp delete mode 100644 backend/include/triton/developer_tools/triton/requests.hpp delete mode 100644 backend/include/triton/developer_tools/triton/responses.hpp delete mode 100644 backend/include/triton/developer_tools/triton/statistics.hpp delete mode 100644 backend/include/triton/developer_tools/triton/triton_memory_resource.hpp delete mode 100644 backend/include/triton/developer_tools/utils/const_agnostic.hpp delete mode 100644 backend/include/triton/developer_tools/utils/device_setter.hpp delete mode 100644 backend/include/triton/developer_tools/utils/narrow.hpp delete mode 100755 backend/scripts/run-clang-format.py delete mode 100644 backend/src/CMakeLists.txt delete mode 100644 backend/src/api.cc delete mode 100644 backend/src/model.h delete mode 100644 backend/src/names.h delete mode 100644 backend/src/shared_state.h delete mode 100644 backend/test/CMakeLists.txt delete mode 100644 backend/test/batch/batch.cpp delete mode 100644 backend/test/build_control.cpp delete mode 100644 backend/test/exceptions.cpp delete mode 100644 backend/test/memory/buffer.cpp delete mode 100644 backend/test/memory/detail/copy.cpp delete mode 100644 backend/test/memory/detail/owned_device_buffer.cpp delete mode 100644 backend/test/memory/resource.cpp delete mode 100644 backend/test/memory/types.cpp delete mode 100644 backend/test/tensor/dtype.cpp delete mode 100644 backend/test/tensor/tensor.cpp delete mode 100644 backend/test/test.cpp delete mode 100644 backend/test/triton/api/execute.cpp delete mode 100644 backend/test/triton/api/initialize.cpp delete mode 100644 backend/test/triton/api/instance_finalize.cpp delete mode 100644 backend/test/triton/api/instance_initialize.cpp delete mode 100644 backend/test/triton/api/model_finalize.cpp delete mode 100644 backend/test/triton/api/model_initialize.cpp delete mode 100644 backend/test/triton/backend.cpp delete mode 100644 backend/test/triton/config.cpp delete mode 100644 backend/test/triton/deployment.cpp delete mode 100644 backend/test/triton/device.cpp delete mode 100644 backend/test/triton/input.cpp delete mode 100644 backend/test/triton/logging.cpp delete mode 100644 backend/test/triton/model.cpp delete mode 100644 backend/test/triton/model_instance.cpp delete mode 100644 backend/test/triton/requests.cpp delete mode 100644 backend/test/triton/responses.cpp delete mode 100644 backend/test/triton/statistics.cpp delete mode 100644 backend/test/utils/const_agnostic.cpp delete mode 100644 backend/test/utils/narrow.cpp diff --git a/backend/.clang-format b/backend/.clang-format deleted file mode 100644 index 0c05436..0000000 --- a/backend/.clang-format +++ /dev/null @@ -1,164 +0,0 @@ ---- -# Refer to the following link for the explanation of each params: -# http://releases.llvm.org/8.0.0/tools/clang/docs/ClangFormatStyleOptions.html -Language: Cpp -# BasedOnStyle: Google -AccessModifierOffset: -1 -AlignAfterOpenBracket: Align -AlignConsecutiveAssignments: true -AlignConsecutiveBitFields: true -AlignConsecutiveDeclarations: false -AlignConsecutiveMacros: true -AlignEscapedNewlines: Left -AlignOperands: true -AlignTrailingComments: true -AllowAllArgumentsOnNextLine: true -AllowAllConstructorInitializersOnNextLine: true -AllowAllParametersOfDeclarationOnNextLine: true -AllowShortBlocksOnASingleLine: true -AllowShortCaseLabelsOnASingleLine: true -AllowShortEnumsOnASingleLine: true -AllowShortFunctionsOnASingleLine: All -AllowShortIfStatementsOnASingleLine: true -AllowShortLambdasOnASingleLine: true -AllowShortLoopsOnASingleLine: false -# This is deprecated -AlwaysBreakAfterDefinitionReturnType: None -AlwaysBreakAfterReturnType: None -AlwaysBreakBeforeMultilineStrings: true -AlwaysBreakTemplateDeclarations: Yes -BinPackArguments: false -BinPackParameters: false -BraceWrapping: - AfterClass: false - AfterControlStatement: false - AfterEnum: false - AfterFunction: false - AfterNamespace: false - AfterObjCDeclaration: false - AfterStruct: false - AfterUnion: false - AfterExternBlock: false - BeforeCatch: false - BeforeElse: false - IndentBraces: false - # disabling the below splits, else, they'll just add to the vertical length of source files! - SplitEmptyFunction: false - SplitEmptyRecord: false - SplitEmptyNamespace: false -BreakAfterJavaFieldAnnotations: false -BreakBeforeBinaryOperators: None -BreakBeforeBraces: WebKit -BreakBeforeInheritanceComma: false -BreakBeforeTernaryOperators: true -BreakConstructorInitializersBeforeComma: false -BreakConstructorInitializers: BeforeColon -BreakInheritanceList: BeforeColon -BreakStringLiterals: true -ColumnLimit: 100 -CommentPragmas: '^ IWYU pragma:' -CompactNamespaces: false -ConstructorInitializerAllOnOneLineOrOnePerLine: true -# Kept the below 2 to be the same as `IndentWidth` to keep everything uniform -ConstructorInitializerIndentWidth: 2 -ContinuationIndentWidth: 2 -Cpp11BracedListStyle: true -DerivePointerAlignment: false -DisableFormat: false -ExperimentalAutoDetectBinPacking: false -FixNamespaceComments: true -ForEachMacros: - - foreach - - Q_FOREACH - - BOOST_FOREACH -IncludeBlocks: Preserve -IncludeCategories: - - Regex: '^' - Priority: 2 - - Regex: '^<.*\.h>' - Priority: 1 - - Regex: '^<.*' - Priority: 2 - - Regex: '.*' - Priority: 3 -IncludeIsMainRegex: '([-_](test|unittest))?$' -IndentCaseLabels: true -IndentPPDirectives: None -IndentWidth: 2 -IndentWrappedFunctionNames: false -JavaScriptQuotes: Leave -JavaScriptWrapImports: true -KeepEmptyLinesAtTheStartOfBlocks: false -MacroBlockBegin: '' -MacroBlockEnd: '' -MaxEmptyLinesToKeep: 1 -NamespaceIndentation: None -ObjCBinPackProtocolList: Never -ObjCBlockIndentWidth: 2 -ObjCSpaceAfterProperty: false -ObjCSpaceBeforeProtocolList: true -PenaltyBreakAssignment: 2 -PenaltyBreakBeforeFirstCallParameter: 1 -PenaltyBreakComment: 300 -PenaltyBreakFirstLessLess: 120 -PenaltyBreakString: 1000 -PenaltyBreakTemplateDeclaration: 10 -PenaltyExcessCharacter: 1000000 -PenaltyReturnTypeOnItsOwnLine: 200 -PointerAlignment: Left -RawStringFormats: - - Language: Cpp - Delimiters: - - cc - - CC - - cpp - - Cpp - - CPP - - 'c++' - - 'C++' - CanonicalDelimiter: '' - - Language: TextProto - Delimiters: - - pb - - PB - - proto - - PROTO - EnclosingFunctions: - - EqualsProto - - EquivToProto - - PARSE_PARTIAL_TEXT_PROTO - - PARSE_TEST_PROTO - - PARSE_TEXT_PROTO - - ParseTextOrDie - - ParseTextProtoOrDie - CanonicalDelimiter: '' - BasedOnStyle: google -# Enabling comment reflow causes doxygen comments to be messed up in their formats! -ReflowComments: true -SortIncludes: true -SortUsingDeclarations: true -SpaceAfterCStyleCast: false -SpaceAfterTemplateKeyword: true -SpaceBeforeAssignmentOperators: true -SpaceBeforeCpp11BracedList: false -SpaceBeforeCtorInitializerColon: true -SpaceBeforeInheritanceColon: true -SpaceBeforeParens: ControlStatements -SpaceBeforeRangeBasedForLoopColon: true -SpaceBeforeSquareBrackets: false -SpaceInEmptyBlock: false -SpaceInEmptyParentheses: false -SpacesBeforeTrailingComments: 2 -SpacesInAngles: false -SpacesInConditionalStatement: false -SpacesInContainerLiterals: true -SpacesInCStyleCastParentheses: false -SpacesInParentheses: false -SpacesInSquareBrackets: false -Standard: c++17 -StatementMacros: - - Q_UNUSED - - QT_REQUIRE_VERSION -# Be consistent with indent-width, even for people who use tab for indentation! -TabWidth: 2 -UseTab: Never diff --git a/backend/CMakeLists.txt b/backend/CMakeLists.txt deleted file mode 100644 index 0b3317a..0000000 --- a/backend/CMakeLists.txt +++ /dev/null @@ -1,218 +0,0 @@ -#============================================================================= -# Copyright (c) 2021-2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -cmake_minimum_required(VERSION 3.21 FATAL_ERROR) -file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.02/RAPIDS.cmake - ${CMAKE_BINARY_DIR}/RAPIDS.cmake) -include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) -include(rapids-cmake) -include(rapids-cpm) -include(rapids-cuda) -include(rapids-export) -include(rapids-find) - -############################################################################## -# - User Options ------------------------------------------------------------ - -option(TRITON_ENABLE_GPU "Enable GPU support in Triton" ON) -option(BUILD_TESTS "Build developer_tools_backend unit-tests" ON) -option(BUILD_EXAMPLE "Build developer_tools_backend example backend" OFF) -option(CUDA_ENABLE_KERNELINFO "Enable kernel resource usage info" OFF) -option(CUDA_ENABLE_LINEINFO "Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler)" OFF) -option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF) -option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" ON) -option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) -option(NVTX "Enable nvtx markers" OFF) -option(TRITON_ENABLE_STATS "Enable statistics collection in Triton" ON) -set(TRITON_COMMON_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-server/common repo") -set(TRITON_CORE_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-server/core repo") -set(TRITON_BACKEND_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-server/backend repo") - -message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Build DEVELOPER_TOOLS_BACKEND unit-tests: ${BUILD_TESTS}") -message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") -message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Disable depreaction warnings " ${DISABLE_DEPRECATION_WARNINGS}) -message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELINFO}") -message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") -message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Enable nvtx markers: ${NVTX}") -message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") -message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Enable GPU support: ${TRITON_ENABLE_GPU}") -message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Enable statistics collection in Triton: ${TRITON_ENABLE_STATS}") -message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Triton common repo tag: ${TRITON_COMMON_REPO_TAG}") -message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Triton core repo tag: ${TRITON_CORE_REPO_TAG}") -message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Triton backend repo tag: ${TRITON_BACKEND_REPO_TAG}") - -############################################################################## -# - Project Initialization --------------------------------------------------- - -if(TRITON_ENABLE_GPU) - rapids_cuda_init_architectures(DEVELOPER_TOOLS_BACKEND) - project(DEVELOPER_TOOLS_BACKEND VERSION 22.02.00 LANGUAGES CXX CUDA) -else() - project(DEVELOPER_TOOLS_BACKEND VERSION 22.02.00 LANGUAGES CXX) -endif() - - -############################################################################## -# - build type --------------------------------------------------------------- - -# Set a default build type if none was specified -rapids_cmake_build_type(Release) - -# this is needed for clang-tidy runs -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - -# Set RMM logging level -set(RMM_LOGGING_LEVEL "INFO" CACHE STRING "Choose the logging level.") -set_property(CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF") -message(VERBOSE "DEVELOPER_TOOLS_BACKEND: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") - -############################################################################## -# - Conda environment detection ---------------------------------------------- - -if(DETECT_CONDA_ENV) - rapids_cmake_support_conda_env( conda_env MODIFY_PREFIX_PATH ) - if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED ENV{CONDA_PREFIX}) - message(STATUS "DEVELOPER_TOOLS_BACKEND: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}") - set(CMAKE_INSTALL_PREFIX "$ENV{CONDA_PREFIX}") - endif() -endif() - -############################################################################## -# - compiler options --------------------------------------------------------- -if(TRITON_ENABLE_GPU) - # * find CUDAToolkit package - # * determine GPU architectures - # * enable the CMake CUDA language - # * set other CUDA compilation flags - rapids_find_package(CUDAToolkit REQUIRED - BUILD_EXPORT_SET developer_tools_backend-exports - INSTALL_EXPORT_SET developer_tools_backend-exports - ) - include(cmake/modules/ConfigureCUDA.cmake) -endif() - -############################################################################## -# - Requirements ------------------------------------------------------------- - -# add third party dependencies using CPM -rapids_cpm_init() - -if(TRITON_ENABLE_GPU) - include(cmake/thirdparty/get_rmm.cmake) - include(cmake/thirdparty/get_raft.cmake) -endif() - -include(cmake/thirdparty/get_rapidjson.cmake) -include(cmake/thirdparty/get_triton.cmake) - -if(BUILD_TESTS) - include(cmake/thirdparty/get_gtest.cmake) -endif() - -############################################################################## -# - install targets----------------------------------------------------------- - -add_library(developer_tools_backend INTERFACE) -add_library(developer_tools_backend::developer_tools_backend ALIAS developer_tools_backend) -target_include_directories(developer_tools_backend INTERFACE "$" - "$") - -target_link_libraries(developer_tools_backend -INTERFACE - $<$:rmm::rmm> - $<$:raft::raft> - triton-core-serverstub - triton-backend-utils -) - -if (TRITON_ENABLE_GPU) - target_compile_features( - developer_tools_backend INTERFACE cxx_std_17 - $ - ) -else() - target_compile_features( - developer_tools_backend INTERFACE cxx_std_17 - ) -endif() - -rapids_cmake_install_lib_dir(lib_dir) -install(TARGETS developer_tools_backend - DESTINATION ${lib_dir} - EXPORT developer_tools_backend-exports - ) - -include(GNUInstallDirs) -install(DIRECTORY include/triton/developer_tools/ - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/triton/developer_tools - ) - -# Temporary install of backend_tools.hpp while the file is removed -install(FILES include/backend_tools.hpp - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/triton/developer_tools - ) - -############################################################################## -# - install export ----------------------------------------------------------- -set(doc_string -[=[ -Provide targets for DEVELOPER_TOOLS_BACKEND. - -DEVELOPER_TOOLS_BACKEND is a header-only library designed to make it easier and faster -to implement Triton backends. - -]=]) - - rapids_export(INSTALL developer_tools_backend - EXPORT_SET developer_tools_backend-exports - GLOBAL_TARGETS developer_tools_backend # since we can't hook into EXPORT SETS - NAMESPACE developer_tools_backend:: - DOCUMENTATION doc_string - ) - -############################################################################## -# - build export ------------------------------------------------------------- - -rapids_export(BUILD developer_tools_backend - EXPORT_SET developer_tools_backend-exports - GLOBAL_TARGETS developer_tools_backend # since we can't hook into EXPORT SETS - LANGUAGES CUDA - DOCUMENTATION doc_string - NAMESPACE developer_tools_backend:: - ) - -############################################################################## -# - build test executable ---------------------------------------------------- - -if(BUILD_TESTS) - include(test/CMakeLists.txt) -endif() - -############################################################################## -# - build example backend ---------------------------------------------------- - -if(BUILD_EXAMPLE) - include(src/CMakeLists.txt) -endif() - -############################################################################## -# - doxygen targets ---------------------------------------------------------- - -# TODO(wphicks) -# include(cmake/doxygen.cmake) -# add_doxygen_target(IN_DOXYFILE Doxyfile.in -# OUT_DOXYFILE ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile -# CWD ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/backend/cmake/doxygen.cmake b/backend/cmake/doxygen.cmake deleted file mode 100644 index 061981f..0000000 --- a/backend/cmake/doxygen.cmake +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -find_package(Doxygen 1.8.11) - -function(add_doxygen_target) - if(Doxygen_FOUND) - set(options "") - set(oneValueArgs IN_DOXYFILE OUT_DOXYFILE CWD) - set(multiValueArgs "") - cmake_parse_arguments(dox "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - configure_file(${dox_IN_DOXYFILE} ${dox_OUT_DOXYFILE} @ONLY) - add_custom_target(doc - ${DOXYGEN_EXECUTABLE} ${dox_OUT_DOXYFILE} - WORKING_DIRECTORY ${dox_CWD} - VERBATIM - COMMENT "Generate doxygen docs") - else() - message("add_doxygen_target: doxygen exe not found") - endif() -endfunction(add_doxygen_target) diff --git a/backend/cmake/modules/ConfigureCUDA.cmake b/backend/cmake/modules/ConfigureCUDA.cmake deleted file mode 100644 index 5643ce6..0000000 --- a/backend/cmake/modules/ConfigureCUDA.cmake +++ /dev/null @@ -1,43 +0,0 @@ -#============================================================================= -# Copyright (c) 2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -if(DISABLE_DEPRECATION_WARNINGS) - list(APPEND DEVELOPER_TOOLS_BACKEND_CXX_FLAGS -Wno-deprecated-declarations) - list(APPEND DEVELOPER_TOOLS_BACKEND_CUDA_FLAGS -Xcompiler=-Wno-deprecated-declarations) -endif() - -if(CMAKE_COMPILER_IS_GNUCXX) - list(APPEND DEVELOPER_TOOLS_BACKEND_CXX_FLAGS -Wall -Werror -Wno-unknown-pragmas -Wno-error=deprecated-declarations) -endif() - -list(APPEND DEVELOPER_TOOLS_BACKEND_CUDA_FLAGS --expt-extended-lambda --expt-relaxed-constexpr) - -# set warnings as errors -if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 11.2.0) - list(APPEND DEVELOPER_TOOLS_BACKEND_CUDA_FLAGS -Werror=all-warnings) -endif() -list(APPEND DEVELOPER_TOOLS_BACKEND_CUDA_FLAGS -Xcompiler=-Wall,-Werror,-Wno-error=deprecated-declarations) - -# Option to enable line info in CUDA device compilation to allow introspection when profiling / memchecking -if(CUDA_ENABLE_LINEINFO) - list(APPEND DEVELOPER_TOOLS_BACKEND_CUDA_FLAGS -lineinfo) -endif() - -# Debug options -if(CMAKE_BUILD_TYPE MATCHES Debug) - message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Building with debugging flags") - list(APPEND DEVELOPER_TOOLS_BACKEND_CUDA_FLAGS -G -Xcompiler=-rdynamic) -endif() diff --git a/backend/cmake/thirdparty/get_gtest.cmake b/backend/cmake/thirdparty/get_gtest.cmake deleted file mode 100644 index a29af07..0000000 --- a/backend/cmake/thirdparty/get_gtest.cmake +++ /dev/null @@ -1,22 +0,0 @@ -#============================================================================= -# Copyright (c) 2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -function(find_and_configure_gtest) - include(${rapids-cmake-dir}/cpm/gtest.cmake) - rapids_cpm_gtest() -endfunction() - -find_and_configure_gtest() diff --git a/backend/cmake/thirdparty/get_raft.cmake b/backend/cmake/thirdparty/get_raft.cmake deleted file mode 100644 index 78fe8b6..0000000 --- a/backend/cmake/thirdparty/get_raft.cmake +++ /dev/null @@ -1,49 +0,0 @@ -#============================================================================= -# Copyright (c) 2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -function(find_and_configure_raft) - - set(oneValueArgs VERSION FORK PINNED_TAG) - cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" - "${multiValueArgs}" ${ARGN} ) - - rapids_cpm_find(raft ${PKG_VERSION} - GLOBAL_TARGETS raft::raft - BUILD_EXPORT_SET developer_tools_backend-exports - INSTALL_EXPORT_SET developer_tools_backend-exports - CPM_ARGS - GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git - GIT_TAG ${PKG_PINNED_TAG} - SOURCE_SUBDIR cpp - OPTIONS - "BUILD_TESTS OFF" - "RAFT_COMPILE_LIBRARIES OFF" - ) - - message(VERBOSE "DEVELOPER_TOOLS_BACKEND: Using RAFT located in ${raft_SOURCE_DIR}") - -endfunction() - -set(DEVELOPER_TOOLS_BACKEND_MIN_VERSION_raft "${DEVELOPER_TOOLS_BACKEND_VERSION_MAJOR}.${DEVELOPER_TOOLS_BACKEND_VERSION_MINOR}.00") -set(DEVELOPER_TOOLS_BACKEND_BRANCH_VERSION_raft "${DEVELOPER_TOOLS_BACKEND_VERSION_MAJOR}.${DEVELOPER_TOOLS_BACKEND_VERSION_MINOR}") - -# Change pinned tag here to test a commit in CI -# To use a different RAFT locally, set the CMake variable -# CPM_raft_SOURCE=/path/to/local/raft -find_and_configure_raft(VERSION ${DEVELOPER_TOOLS_BACKEND_MIN_VERSION_raft} - FORK rapidsai - PINNED_TAG branch-${DEVELOPER_TOOLS_BACKEND_BRANCH_VERSION_raft} - ) diff --git a/backend/cmake/thirdparty/get_rapidjson.cmake b/backend/cmake/thirdparty/get_rapidjson.cmake deleted file mode 100644 index 3399d91..0000000 --- a/backend/cmake/thirdparty/get_rapidjson.cmake +++ /dev/null @@ -1,37 +0,0 @@ -#============================================================================= -# Copyright (c) 2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -# TODO(wphicks): Pass in version -function(find_and_configure_rapidjson VERSION) - - rapids_cpm_find(rapidjson ${VERSION} - GLOBAL_TARGETS rapidjson::rapidjson - BUILD_EXPORT_SET developer_tools_backend-exports - INSTALL_EXPORT_SET developer_tools_backend-exports - CPM_ARGS - GIT_REPOSITORY https://github.com/Tencent/rapidjson - GIT_TAG "v${VERSION}" - GIT_SHALLOW ON - OPTIONS - "RAPIDJSON_BUILD_DOC OFF" - "RAPIDJSON_BUILD_EXAMPLES OFF" - "RAPIDJSON_BUILD_TESTS OFF" - "RAPIDJSON_BUILD_THIRDPARTY_GTEST OFF" - ) - -endfunction() - -find_and_configure_rapidjson("1.1.0") diff --git a/backend/cmake/thirdparty/get_rmm.cmake b/backend/cmake/thirdparty/get_rmm.cmake deleted file mode 100644 index 4a3ca4a..0000000 --- a/backend/cmake/thirdparty/get_rmm.cmake +++ /dev/null @@ -1,22 +0,0 @@ -#============================================================================= -# Copyright (c) 2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -function(find_and_configure_rmm) - include(${rapids-cmake-dir}/cpm/rmm.cmake) - rapids_cpm_rmm() -endfunction() - -find_and_configure_rmm() diff --git a/backend/cmake/thirdparty/get_triton.cmake b/backend/cmake/thirdparty/get_triton.cmake deleted file mode 100644 index e5353ec..0000000 --- a/backend/cmake/thirdparty/get_triton.cmake +++ /dev/null @@ -1,36 +0,0 @@ -#============================================================================= -# Copyright (c) 2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= -include(FetchContent) - -FetchContent_Declare( - repo-common - GIT_REPOSITORY https://github.com/triton-inference-server/common.git - GIT_TAG ${TRITON_COMMON_REPO_TAG} - GIT_SHALLOW ON -) -FetchContent_Declare( - repo-core - GIT_REPOSITORY https://github.com/triton-inference-server/core.git - GIT_TAG ${TRITON_CORE_REPO_TAG} - GIT_SHALLOW ON -) -FetchContent_Declare( - repo-backend - GIT_REPOSITORY https://github.com/triton-inference-server/backend.git - GIT_TAG ${TRITON_BACKEND_REPO_TAG} - GIT_SHALLOW ON -) -FetchContent_MakeAvailable(repo-common repo-core repo-backend) diff --git a/backend/include/backend_tools.hpp b/backend/include/backend_tools.hpp deleted file mode 100644 index e11fb2d..0000000 --- a/backend/include/backend_tools.hpp +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include - -namespace triton { -namespace developer_tools { -namespace backend { - -/* Function for testing backend_tools include - * - * @return message indicating backend_tools has been included succesfully*/ -inline auto test_install() { return std::string("backend_tools set up successfully"); } - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/batch/batch.hpp b/backend/include/triton/developer_tools/batch/batch.hpp deleted file mode 100644 index a372049..0000000 --- a/backend/include/triton/developer_tools/batch/batch.hpp +++ /dev/null @@ -1,273 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#ifdef TRITON_ENABLE_GPU -#include -#else -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -/** - * @brief A representation of all data about a single batch of inference - * requests - * - * Batch objects are the primary interface point between developer_tools::backend - * Models and the Triton server itself. By calling the `get_input` and - * `get_output` methods of a batch, Model implementations can retrieve the input - * Tensors necessary for prediction and the output Tensors where results can be - * stored. - * - * Batch objects also handle a variety of other tasks necessary for - * processing a batch in the Triton model. This includes reporting statistics - * on how long it took to process requests and sending responses to the - * client via the Triton server once processing is complete. - * - * It is not recommended that developers of developer_tools backends try to - * construct Batch objects directly. Instead, you should make use of the - * developer_tools::backend::triton_api::execute template, which will construct - * the Batch for you. - */ -struct Batch { - using size_type = std::size_t; - - Batch(TRITONBACKEND_Request** raw_requests, - request_size_t count, - TRITONBACKEND_MemoryManager& triton_mem_manager, - std::function(std::string const&, size_type)>&& get_output_shape, - std::function&& report_request_statistics, - bool use_pinned_input, - bool use_pinned_output, - size_type max_batch_size, - cudaStream_t stream) - : requests_{raw_requests, raw_requests + count}, - responses_{construct_responses(requests_.begin(), requests_.end())}, - get_output_shape_{std::move(get_output_shape)}, - report_statistics_{std::move(report_request_statistics)}, - collector_(raw_requests, count, &responses_, &triton_mem_manager, use_pinned_input, stream), - responder_{std::make_shared(raw_requests, - count, - &responses_, - max_batch_size, - &triton_mem_manager, - use_pinned_output, - stream)}, - stream_{stream}, - start_time_{std::chrono::steady_clock::now()}, - compute_start_time_{std::chrono::steady_clock::now()}, - batch_size_{} - { - } - - template - auto get_input_shape(std::string const& name) - { - auto result = std::vector{}; - if (!requests_.empty()) { - result = get_triton_input_shape(std::begin(requests_), std::end(requests_), name); - - auto input_batch_dim = size_type{}; - if (result.size() > 0) { input_batch_dim = result[0]; } - - if (batch_size_.has_value()) { - if (batch_size_.value() != input_batch_dim) { - throw TritonException(Error::Internal, - "all input tensors must have same batch dimension"); - } - } else { - batch_size_ = input_batch_dim; - } - } - return result; - } - - template - auto get_input(std::string const& name, - std::optional const& memory_type, - device_id_t device_id, - cudaStream_t stream) - { - auto shape = get_input_shape(name); - auto size_bytes = - sizeof(T) * std::reduce(shape.begin(), shape.end(), std::size_t{1}, std::multiplies<>()); - auto allowed_memory_configs = std::vector>{}; - if (memory_type.has_value()) { - allowed_memory_configs.emplace_back(memory_type.value(), device_id); - } else { - allowed_memory_configs.emplace_back(HostMemory, int64_t{}); - allowed_memory_configs.emplace_back(DeviceMemory, device_id); - } - - auto const* raw_buffer = static_cast(nullptr); - auto reported_bytes = std::size_t{}; - auto reported_mem_type = MemoryType{}; - auto reported_device_id = int64_t{}; - - triton_check( - collector_.ProcessTensor(name.c_str(), - static_cast(nullptr), // Return data without copy if possible - size_bytes, - allowed_memory_configs, - &raw_buffer, - &reported_bytes, - &reported_mem_type, - &reported_device_id)); - - if(collector_.Finalize()){ - if constexpr (IS_GPU_BUILD) { - cuda_check(cudaStreamSynchronize(stream_)); - } else { - throw TritonException(Error::Internal, "stream synchronization required in non-GPU build"); - } - } - - std::for_each(std::begin(responses_), std::end(responses_), [](auto* response) { - if (response == nullptr) { - throw TritonException(Error::Internal, "Input collection failed"); - } - }); - - auto buffer = Buffer(reinterpret_cast(raw_buffer), - reported_bytes / sizeof(T), - reported_mem_type, - reported_device_id, - stream); - - if (memory_type && (reported_mem_type != memory_type || reported_device_id != device_id)) { - throw TritonException(Error::Internal, "data collected in wrong location"); - } - - // Set start time of batch to time latest input tensor was retrieved - compute_start_time_ = std::chrono::steady_clock::now(); - - return Tensor(std::move(shape), std::move(buffer)); - } - - template - auto get_input(std::string const& name, - std::optional const& memory_type, - device_id_t device_id) - { - return get_input(name, memory_type, device_id, stream_); - } - - template - auto get_output(std::string const& name, - std::optional const& memory_type, - device_id_t device_id, - cudaStream_t stream) - { - if (!batch_size_.has_value()) { - throw TritonException(Error::Internal, - "At least one input must be retrieved before any output"); - } - auto shape = get_output_shape_(name, batch_size_.value()); - auto buffer_size = std::reduce(shape.begin(), shape.end(), std::size_t{1}, std::multiplies<>()); - auto final_memory_type = MemoryType{}; - if (memory_type.has_value()) { - final_memory_type = memory_type.value(); - } else { - // If consumer doesn't care, use HostMemory to avoid additional copy on - // non-shared-memory responses. - final_memory_type = HostMemory; - } - auto buffer = Buffer(buffer_size, final_memory_type, device_id, stream); - return OutputTensor(std::move(shape), std::move(buffer), name, responder_); - } - - template - auto get_output(std::string const& name, - std::optional const& memory_type, - device_id_t device_id) - { - return get_output(name, memory_type, device_id, stream_); - } - - auto const& compute_start_time() const { return compute_start_time_; } - - auto stream() const { return stream_; } - - void finalize(TRITONSERVER_Error* err) - { - auto compute_end_time = std::chrono::steady_clock::now(); - if (responder_->Finalize()) { cuda_check(cudaStreamSynchronize(stream_)); } - - send_responses(std::begin(responses_), std::end(responses_), err); - - // Triton resumes ownership of failed requests; only release on success - if (err == nullptr) { - std::for_each( - std::begin(requests_), std::end(requests_), [this, &compute_end_time](auto& request) { - report_statistics_(request, - start_time_, - compute_start_time_, - compute_end_time, - std::chrono::steady_clock::now()); - }); - release_requests(std::begin(requests_), std::end(requests_)); - } - } - - private: - std::vector requests_; - std::vector responses_; - std::function(std::string const&, size_type)> get_output_shape_; - std::function - report_statistics_; - triton::backend::BackendInputCollector collector_; - std::shared_ptr responder_; - cudaStream_t stream_; - std::chrono::time_point start_time_; - std::chrono::time_point compute_start_time_; - std::optional batch_size_; -}; -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/build_control.hpp b/backend/include/triton/developer_tools/build_control.hpp deleted file mode 100644 index be8693b..0000000 --- a/backend/include/triton/developer_tools/build_control.hpp +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include - -namespace triton { -namespace developer_tools { -namespace backend { - -#ifdef TRITON_ENABLE_GPU -auto constexpr IS_GPU_BUILD = true; -#else -auto constexpr IS_GPU_BUILD = false; -#endif - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/cpu_only/cuda_runtime_replacement.hpp b/backend/include/triton/developer_tools/cpu_only/cuda_runtime_replacement.hpp deleted file mode 100644 index 62061e5..0000000 --- a/backend/include/triton/developer_tools/cpu_only/cuda_runtime_replacement.hpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#ifdef TRITON_ENABLE_GPU -#include -#else - -namespace triton { -namespace developer_tools { -namespace backend { - -using cudaStream_t = void*; - -enum struct cudaError_t {cudaSuccess, cudaErrorNonGpuBuild}; -using cudaError = cudaError_t; -auto constexpr cudaSuccess = cudaError_t::cudaSuccess; - -inline void cudaGetLastError() {} - -inline auto const * cudaGetErrorString(cudaError_t err) { - return "CUDA function used in non-GPU build"; -} - -inline auto cudaStreamSynchronize(cudaStream_t stream) { - return cudaError_t::cudaErrorNonGpuBuild; -} - -inline auto cudaGetDevice(int* device_id) { - return cudaError_t::cudaErrorNonGpuBuild; -} - -inline auto cudaGetDeviceCount(int* count) { - return cudaError_t::cudaErrorNonGpuBuild; -} - - -} // namespace backend -} // namespace developer_tools -} // namespace triton -#endif diff --git a/backend/include/triton/developer_tools/exceptions.hpp b/backend/include/triton/developer_tools/exceptions.hpp deleted file mode 100644 index ae87702..0000000 --- a/backend/include/triton/developer_tools/exceptions.hpp +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#ifdef TRITON_ENABLE_GPU -#include -#else -#include -#endif -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { - -using ErrorCode = TRITONSERVER_Error_Code; - -namespace Error { -auto constexpr Unknown = ErrorCode::TRITONSERVER_ERROR_UNKNOWN; -auto constexpr Internal = ErrorCode::TRITONSERVER_ERROR_INTERNAL; -auto constexpr NotFound = ErrorCode::TRITONSERVER_ERROR_NOT_FOUND; -auto constexpr InvalidArg = ErrorCode::TRITONSERVER_ERROR_INVALID_ARG; -auto constexpr Unavailable = ErrorCode::TRITONSERVER_ERROR_UNAVAILABLE; -auto constexpr Unsupported = ErrorCode::TRITONSERVER_ERROR_UNSUPPORTED; -auto constexpr AlreadyExists = ErrorCode::TRITONSERVER_ERROR_ALREADY_EXISTS; -} // namespace Error - -/** - * @brief Exception thrown if processing cannot continue for a request - * - * This exception should be thrown whenever a condition is encountered that (if - * it is not appropriately handled by some other exception handler) SHOULD - * result in Triton reporting an error for the request being processed. It - * signals that (absent any other fallbacks), this request cannot be fulfilled - * but that the server may still be in a state to continue handling other - * requests, including requests to other models. - */ -struct TritonException : std::exception { - public: - TritonException() : error_(TRITONSERVER_ErrorNew(Error::Unknown, "encountered unknown error")) {} - - TritonException(ErrorCode code, std::string const& msg) - : error_(TRITONSERVER_ErrorNew(code, msg.c_str())) - { - } - - TritonException(ErrorCode code, char const* msg) : error_{TRITONSERVER_ErrorNew(code, msg)} {} - - TritonException(TRITONSERVER_Error* prev_error) : error_(prev_error) {} - - virtual char const* what() const noexcept { return TRITONSERVER_ErrorMessage(error_); } - - auto* error() const { return error_; } - - private: - TRITONSERVER_Error* error_; -}; - -inline void triton_check(TRITONSERVER_Error* err) -{ - if (err != nullptr) { throw TritonException(err); } -} - -inline void cuda_check(cudaError_t const& err) -{ - if constexpr (IS_GPU_BUILD) { - if (err != cudaSuccess) { - cudaGetLastError(); - throw TritonException(Error::Internal, cudaGetErrorString(err)); - } - } else { - throw TritonException(Error::Internal, "cuda_check used in non-GPU build"); - } -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/memory/buffer.hpp b/backend/include/triton/developer_tools/memory/buffer.hpp deleted file mode 100644 index d763b69..0000000 --- a/backend/include/triton/developer_tools/memory/buffer.hpp +++ /dev/null @@ -1,317 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include -#include - -#ifdef TRITON_ENABLE_GPU -#include -#include -#include -#else -#include -#include -#include -#endif - -#include -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -template -struct Buffer { - using size_type = std::size_t; - using value_type = T; - - using h_buffer = T*; - using d_buffer = T*; - using owned_h_buffer = std::unique_ptr; - using owned_d_buffer = detail::owned_device_buffer; - using data_store = std::variant; - - Buffer() noexcept : device_{}, data_{std::in_place_index<0>, nullptr}, size_{}, stream_{} {} - - /** - * @brief Construct buffer of given size in given memory location (either - * on host or on device) - * A buffer constructed in this way is owning and will release allocated - * resources on deletion - */ - Buffer(size_type size, - MemoryType memory_type = DeviceMemory, - device_id_t device = 0, - cudaStream_t stream = 0) - : device_{device}, - data_{allocate(size, device, memory_type, stream)}, - size_{size}, - stream_{stream} - { - if constexpr (!IS_GPU_BUILD) { - if (memory_type == DeviceMemory) { - throw TritonException( - Error::Internal, - "Cannot use device buffer in non-GPU build" - ); - } - } - } - - /** - * @brief Construct buffer from given source in given memory location (either - * on host or on device) - * A buffer constructed in this way is non-owning; the caller is - * responsible for freeing any resources associated with the input pointer - */ - Buffer(T* input_data, - size_type size, - MemoryType memory_type = DeviceMemory, - device_id_t device = 0, - cudaStream_t stream = 0) - : device_{device}, - data_{[&memory_type, &input_data]() { - auto result = data_store{}; - if (memory_type == HostMemory) { - result = data_store{std::in_place_index<0>, input_data}; - } else { - if constexpr (!IS_GPU_BUILD) { - throw TritonException( - Error::Internal, - "Cannot use device buffer in non-GPU build" - ); - } - result = data_store{std::in_place_index<1>, input_data}; - } - return result; - }()}, - size_{size}, - stream_{stream} - { - } - - /** - * @brief Construct one buffer from another in the given memory location - * (either on host or on device) - * A buffer constructed in this way is owning and will copy the data from - * the original location - */ - Buffer(Buffer const& other, MemoryType memory_type, device_id_t device = 0) - : device_{device}, - data_([&other, &memory_type, &device]() { - auto result = allocate(other.size_, device, memory_type, other.stream_); - copy(result, other.data_, other.size_, other.stream_); - return result; - }()), - size_{other.size_}, - stream_{other.stream_} - { - } - - /** - * @brief Create owning copy of existing buffer - * The memory type of this new buffer will be the same as the original - */ - Buffer(Buffer const& other) : Buffer(other, other.mem_type(), other.device()) {} - - Buffer(Buffer&& other, MemoryType memory_type) - : device_{other.device()}, - data_{[&other, memory_type]() { - data_store result; - if (memory_type == other.mem_type()) { - result = std::move(other.data_); - } else { - result = allocate(other.size_, memory_type, other.device(), other.stream()); - copy(result, other.data_, other.size_, other.stream_); - } - return result; - }()}, - size_{other.size_}, - stream_{other.stream_} - { - } - - Buffer(Buffer&& other) = default; - - Buffer& operator=(Buffer&& other) = default; - - ~Buffer() {} - - /** - * @brief Return where memory for this buffer is located (host or device) - */ - auto mem_type() const noexcept { return data_.index() % 2 == 0 ? HostMemory : DeviceMemory; } - - /** - * @brief Return number of elements in buffer - */ - auto size() const noexcept { return size_; } - - /** - * @brief Return pointer to data stored in buffer - */ - auto* data() const noexcept { return get_raw_ptr(data_); } - - auto device() const noexcept { return device_; } - - /** - * @brief Return CUDA stream associated with this buffer - */ - auto stream() const noexcept { return stream_; } - - void stream_synchronize() const - { - if constexpr (IS_GPU_BUILD) { cuda_check(cudaStreamSynchronize(stream_)); } - } - - /** - * @brief Set CUDA stream for this buffer to new value - * - * @warning This method calls cudaStreamSynchronize on the old stream - * before updating. Be aware of performance implications and try to avoid - * interactions between buffers on different streams where possible. - */ - void set_stream(cudaStream_t new_stream) - { - stream_synchronize(); - stream_ = new_stream; - } - - private: - device_id_t device_; - data_store data_; - size_type size_; - cudaStream_t stream_; - - // Helper function for accessing raw pointer to underlying data of - // data_store - static auto* get_raw_ptr(data_store const& ptr) noexcept - { - /* Switch statement is an optimization relative to std::visit to avoid - * vtable overhead for a small number of alternatives */ - auto* result = static_cast(nullptr); - switch (ptr.index()) { - case 0: result = std::get<0>(ptr); break; - case 1: result = std::get<1>(ptr); break; - case 2: result = std::get<2>(ptr).get(); break; - case 3: result = std::get<3>(ptr).get(); break; - } - return result; - } - - // Helper function for allocating memory in constructors - static auto allocate(size_type size, - device_id_t device = 0, - MemoryType memory_type = DeviceMemory, - cudaStream_t stream = 0) - { - auto result = data_store{}; - if (memory_type == DeviceMemory) { - if constexpr (IS_GPU_BUILD) { - result = data_store{owned_d_buffer{ - device, - size, - stream, - }}; - } else { - throw TritonException(Error::Internal, - "DeviceMemory requested in CPU-only build of FIL backend"); - } - } else { - result = std::make_unique(size); - } - return result; - } - - // Helper function for copying memory in constructors, where there are - // stronger guarantees on conditions that would otherwise need to be - // checked - static void copy(data_store const& dst, data_store const& src, size_type len, cudaStream_t stream) - { - // This function will only be called in constructors, so we allow a - // const_cast here to perform the initial copy of data from a - // Buffer to a newly-created Buffer - auto raw_dst = const_cast*>(get_raw_ptr(dst)); - auto raw_src = get_raw_ptr(src); - - auto dst_mem_type = dst.index() % 2 == 0 ? HostMemory : DeviceMemory; - auto src_mem_type = src.index() % 2 == 0 ? HostMemory : DeviceMemory; - - detail::copy(raw_dst, raw_src, len, stream, dst_mem_type, src_mem_type); - } -}; - -/** - * @brief Copy data from one Buffer to another - * - * @param dst The destination buffer - * @param src The source buffer - * @param dst_begin The offset from the beginning of the destination buffer - * at which to begin copying to. - * @param src_begin The offset from the beginning of the source buffer - * at which to begin copying from. - * @param src_end The offset from the beginning of the source buffer - * before which to end copying from. - */ -template -void copy(Buffer& dst, - Buffer const& src, - typename Buffer::size_type dst_begin, - typename Buffer::size_type src_begin, - typename Buffer::size_type src_end) -{ - if (dst.stream() != src.stream()) { dst.set_stream(src.stream()); } - auto len = src_end - src_begin; - if (len < 0 || src_end > src.size() || len > dst.size() - dst_begin) { - throw TritonException(Error::Internal, "bad copy between buffers"); - } - - auto raw_dst = dst.data() + dst_begin; - auto raw_src = src.data() + src_begin; - - detail::copy(raw_dst, raw_src, len, dst.stream(), dst.mem_type(), src.mem_type()); -} - -template -void copy(Buffer& dst, Buffer const& src) -{ - copy(dst, src, 0, 0, src.size()); -} - -template -void copy(Buffer& dst, Buffer const& src, typename Buffer::size_type dst_begin) -{ - copy(dst, src, dst_begin, 0, src.size()); -} - -template -void copy(Buffer& dst, - Buffer const& src, - typename Buffer::size_type src_begin, - typename Buffer::size_type src_end) -{ - copy(dst, src, 0, src_begin, src_end); -} -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/memory/detail/cpu_only/copy.hpp b/backend/include/triton/developer_tools/memory/detail/cpu_only/copy.hpp deleted file mode 100644 index 5d0db15..0000000 --- a/backend/include/triton/developer_tools/memory/detail/cpu_only/copy.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include - -#ifndef TRITON_ENABLE_GPU -#include -#endif -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -namespace detail { - -template -void copy(T* dst, - T const* src, - std::size_t len, - cudaStream_t stream, - MemoryType dst_type, - MemoryType src_type) -{ - if (dst_type == DeviceMemory || src_type == DeviceMemory) { - throw TritonException(Error::Internal, "Cannot copy device memory in non-GPU build"); - } else { - std::memcpy(dst, src, len * sizeof(T)); - } -} - -} // namespace detail -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/memory/detail/cpu_only/owned_device_buffer.hpp b/backend/include/triton/developer_tools/memory/detail/cpu_only/owned_device_buffer.hpp deleted file mode 100644 index 6afb257..0000000 --- a/backend/include/triton/developer_tools/memory/detail/cpu_only/owned_device_buffer.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -namespace detail { - -template -struct owned_device_buffer { - using non_const_T = std::remove_const_t; - owned_device_buffer(device_id_t device_id, std::size_t size, cudaStream_t stream) - { - throw TritonException(Error::Internal, - "Attempted to use device buffer in non-GPU build"); - } - - auto* get() const { return static_cast(nullptr); } -}; - -} // namespace detail -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/memory/detail/cpu_only/resource.hpp b/backend/include/triton/developer_tools/memory/detail/cpu_only/resource.hpp deleted file mode 100644 index ae28cab..0000000 --- a/backend/include/triton/developer_tools/memory/detail/cpu_only/resource.hpp +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include - -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -namespace detail { - -template<> -inline void setup_memory_resource(device_id_t device_id, - TRITONBACKEND_MemoryManager* triton_manager) { } - -} // namespace detail -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/memory/detail/gpu_only/copy.hpp b/backend/include/triton/developer_tools/memory/detail/gpu_only/copy.hpp deleted file mode 100644 index 2727aec..0000000 --- a/backend/include/triton/developer_tools/memory/detail/gpu_only/copy.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#ifdef TRITON_ENABLE_GPU -#include -#include -#endif - -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -namespace detail { - -template -void copy(T* dst, - T const* src, - std::size_t len, - cudaStream_t stream, - MemoryType dst_type, - MemoryType src_type) -{ - if (dst_type == DeviceMemory || src_type == DeviceMemory) { - try { - raft::copy(dst, src, len, stream); - } catch (raft::cuda_error const& err) { - throw TritonException(Error::Internal, err.what()); - } - } else { - std::memcpy(dst, src, len * sizeof(T)); - } -} - -} // namespace detail -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/memory/detail/gpu_only/owned_device_buffer.hpp b/backend/include/triton/developer_tools/memory/detail/gpu_only/owned_device_buffer.hpp deleted file mode 100644 index 8a275ae..0000000 --- a/backend/include/triton/developer_tools/memory/detail/gpu_only/owned_device_buffer.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -namespace detail { - -template -struct owned_device_buffer { - using non_const_T = std::remove_const_t; - owned_device_buffer(device_id_t device_id, std::size_t size, cudaStream_t stream) - : data_{[&device_id, &size, &stream]() { - auto device_context = device_setter{device_id}; - return rmm::device_buffer{size * sizeof(T), rmm::cuda_stream_view{stream}}; - }()} - { - } - - auto* get() const { return reinterpret_cast(data_.data()); } - - private: - mutable rmm::device_buffer data_; -}; - -} // namespace detail -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/memory/detail/gpu_only/resource.hpp b/backend/include/triton/developer_tools/memory/detail/gpu_only/resource.hpp deleted file mode 100644 index 28c16dc..0000000 --- a/backend/include/triton/developer_tools/memory/detail/gpu_only/resource.hpp +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -namespace detail { - -inline auto& resource_lock() -{ - static auto lock = std::mutex{}; - return lock; -} - -/** A struct used solely to keep memory resources in-scope for the lifetime - * of the backend */ -struct resource_data { - resource_data() : base_mr_{}, triton_mrs_{} {} - auto* make_new_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* manager) - { - if (manager == nullptr && triton_mrs_.size() != 0) { - manager = triton_mrs_.back().get_triton_manager(); - } - triton_mrs_.emplace_back(manager, device_id, &base_mr_); - return &(triton_mrs_.back()); - } - - private: - rmm::mr::cuda_memory_resource base_mr_; - std::deque triton_mrs_; -}; - -inline auto& get_device_resources() -{ - static auto device_resources = resource_data{}; - return device_resources; -} - -inline auto is_triton_resource(rmm::cuda_device_id const& device_id) -{ - auto* triton_mr = - dynamic_cast(rmm::mr::get_per_device_resource(device_id)); - return (triton_mr != nullptr && triton_mr->get_triton_manager() != nullptr); -} - -template<> -inline void setup_memory_resource(device_id_t device_id, - TRITONBACKEND_MemoryManager* triton_manager) -{ - auto lock = std::lock_guard{detail::resource_lock()}; - auto rmm_device_id = rmm::cuda_device_id{device_id}; - - if (!detail::is_triton_resource(rmm_device_id)) { - auto& device_resources = detail::get_device_resources(); - rmm::mr::set_per_device_resource(rmm_device_id, - device_resources.make_new_resource(device_id, triton_manager)); - } -} - -/* inline auto* get_memory_resource(device_id_t device_id) -{ - auto rmm_device_id = rmm::cuda_device_id{device_id}; - return rmm::mr::get_per_device_resource(rmm_device_id); -} - -inline auto* get_memory_resource() { return rmm::mr::get_current_device_resource(); } */ - -} // namespace detail -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/memory/detail/owned_device_buffer.hpp b/backend/include/triton/developer_tools/memory/detail/owned_device_buffer.hpp deleted file mode 100644 index 442603b..0000000 --- a/backend/include/triton/developer_tools/memory/detail/owned_device_buffer.hpp +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -namespace triton { -namespace developer_tools { -namespace backend { -namespace detail { - -template -struct owned_device_buffer { -}; - -} // namespace detail -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/memory/detail/resource.hpp b/backend/include/triton/developer_tools/memory/detail/resource.hpp deleted file mode 100644 index 38ca530..0000000 --- a/backend/include/triton/developer_tools/memory/detail/resource.hpp +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include - -#include - -namespace triton { -namespace developer_tools { -namespace backend { -namespace detail { - -template -inline void setup_memory_resource(device_id_t device_id, - TRITONBACKEND_MemoryManager* triton_manager = nullptr) { -} - -} // namespace detail -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/memory/resource.hpp b/backend/include/triton/developer_tools/memory/resource.hpp deleted file mode 100644 index 979a30a..0000000 --- a/backend/include/triton/developer_tools/memory/resource.hpp +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include - -#include -#include -#include -#ifdef TRITON_ENABLE_GPU -#include -#else -#include -#endif - -namespace triton { -namespace developer_tools { -namespace backend { - -inline void setup_memory_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* triton_manager = nullptr) { - detail::setup_memory_resource(device_id, triton_manager); -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/memory/types.hpp b/backend/include/triton/developer_tools/memory/types.hpp deleted file mode 100644 index 9de4467..0000000 --- a/backend/include/triton/developer_tools/memory/types.hpp +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include - -namespace triton { -namespace developer_tools { -namespace backend { -using MemoryType = TRITONSERVER_MemoryType; -auto constexpr DeviceMemory = TRITONSERVER_MEMORY_GPU; -auto constexpr HostMemory = TRITONSERVER_MEMORY_CPU; -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/model/model.hpp b/backend/include/triton/developer_tools/model/model.hpp deleted file mode 100644 index a8684f5..0000000 --- a/backend/include/triton/developer_tools/model/model.hpp +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#ifdef TRITON_ENABLE_GPU -#include -#else -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -template -struct Model { - virtual void predict(Batch& batch) const = 0; - - virtual void load() {} - virtual void unload() {} - - /** - * @brief Return the preferred memory type in which to store data for this - * batch or std::nullopt to accept whatever Triton returns - * - * The base implementation of this method will require data on-host if the - * model itself is deployed on the host OR if this backend has not been - * compiled with GPU support. Otherwise, models deployed on device will - * receive memory on device. Overriding this method will allow derived - * model classes to select a preferred memory location based on properties - * of the batch or to simply return std::nullopt if device memory or host - * memory will do equally well. - */ - virtual std::optional preferred_mem_type(Batch& batch) const - { - return (IS_GPU_BUILD && deployment_type_ == GPUDeployment) ? DeviceMemory : HostMemory; - } - virtual std::optional preferred_mem_type_in(Batch& batch) const - { - return preferred_mem_type(batch); - } - virtual std::optional preferred_mem_type_out(Batch& batch) const - { - return preferred_mem_type(batch); - } - - /** - * @brief Retrieve a stream used to set up batches for this model - * - * The base implementation of this method simply returns the default stream - * provided by Triton for use with this model. Child classes may choose to - * override this in order to provide different streams for use with - * successive incoming batches. For instance, one might cycle through - * several streams in order to distribute batches across them, but care - * should be taken to ensure proper synchronization in this case. - */ - virtual cudaStream_t get_stream() const { return default_stream_; } - - /** - * @brief Get input tensor of a particular named input for an entire batch - */ - template - auto get_input(Batch& batch, - std::string const& name, - std::optional const& mem_type, - cudaStream_t stream) const - { - return batch.get_input(name, mem_type, device_id_, stream); - } - template - auto get_input(Batch& batch, - std::string const& name, - std::optional const& mem_type) const - { - return get_input(batch, name, mem_type, default_stream_); - } - template - auto get_input(Batch& batch, std::string const& name) const - { - return get_input(batch, name, preferred_mem_type(batch), default_stream_); - } - - /** - * @brief Get output tensor of a particular named output for an entire batch - */ - template - auto get_output(Batch& batch, - std::string const& name, - std::optional const& mem_type, - device_id_t device_id, - cudaStream_t stream) const - { - return batch.get_output(name, mem_type, device_id, stream); - } - template - auto get_output(Batch& batch, - std::string const& name, - std::optional const& mem_type, - cudaStream_t stream) const - { - return get_output(batch, name, mem_type, device_id_, stream); - } - template - auto get_output(Batch& batch, - std::string const& name, - std::optional const& mem_type) const - { - return get_output(batch, name, mem_type, device_id_, default_stream_); - } - template - auto get_output(Batch& batch, std::string const& name) const - { - return get_output(batch, name, preferred_mem_type(batch), device_id_, default_stream_); - } - - /** - * @brief Retrieve value of configuration parameter - */ - template - auto get_config_param(std::string const& name) const - { - return shared_state_->template get_config_param(name); - } - template - auto get_config_param(std::string const& name, T default_value) const - { - return shared_state_->template get_config_param(name, default_value); - } - template - auto get_config_param(char const* name) const - { - return get_config_param(std::string(name)); - } - template - auto get_config_param(char const* name, T default_value) const - { - return get_config_param(std::string(name), default_value); - } - - Model(std::shared_ptr shared_state, - device_id_t device_id, - cudaStream_t default_stream, - DeploymentType deployment_type, - std::string const& filepath) - : shared_state_{shared_state}, - device_id_{device_id}, - default_stream_{default_stream}, - deployment_type_{deployment_type}, - filepath_{filepath} - { - if constexpr (IS_GPU_BUILD) { setup_memory_resource(device_id_); } - } - - auto get_device_id() const { return device_id_; } - auto get_deployment_type() const { return deployment_type_; } - auto const& get_filepath() const { return filepath_; } - - auto get_output_shape(std::string const& name) const - { - return shared_state_->get_output_shape(name); - } - - protected: - auto get_shared_state() const { return shared_state_; } - - private: - std::shared_ptr shared_state_; - device_id_t device_id_; - cudaStream_t default_stream_; - DeploymentType deployment_type_; - std::string filepath_; -}; -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/model/shared_state.hpp b/backend/include/triton/developer_tools/model/shared_state.hpp deleted file mode 100644 index 1ea2762..0000000 --- a/backend/include/triton/developer_tools/model/shared_state.hpp +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#ifdef TRITON_ENABLE_GPU -#include -#else -#include -#endif -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -/** - * @brief Stores shared state for multiple instances of the same model - */ -struct SharedModelState { - virtual void load() {} - virtual void unload() {} - - explicit SharedModelState(std::unique_ptr&& config, - bool squeeze_output = false) - : config_{std::move(config)}, - max_batch_size_{get_max_batch_size(*config_)}, - output_shapes_([this, squeeze_output]() { - auto result = std::vector>>{}; - auto output_entries = triton::common::TritonJson::Value{}; - triton_check(config_->MemberAsArray("output", &output_entries)); - - result.reserve(output_entries.ArraySize()); - - // Using a raw loop because TritonJSON::Value access has no iterator interface - for (std::size_t i = 0; i < output_entries.ArraySize(); ++i) { - auto output_entry = triton::common::TritonJson::Value{}; - triton_check(output_entries.IndexAsObject(i, &output_entry)); - auto name = std::string{}; - triton_check(output_entry.MemberAsString("name", &name)); - - auto shape = std::vector{}; - auto reshape_entry = triton::common::TritonJson::Value{}; - if (output_entry.Find("reshape", &reshape_entry)) { - triton::backend::ParseShape(reshape_entry, "shape", &shape); - } else { - triton::backend::ParseShape(output_entry, "dims", &shape); - } - if (shape[0] != -1) { shape.insert(shape.begin(), -1); } - // The squeeze_output option was introduced to handle a bad choice of - // convention in the original FIL backend implementation. For legacy - // compatibility, we introduced this option into RAPIDS-Triton, but - // in general, new backends are advised to avoid using it and defer - // this sort of flattening operation to the consumer. - if (squeeze_output) { - shape.erase(std::remove(shape.begin(), shape.end(), std::int64_t{1}), shape.end()); - } - result.insert( - std::upper_bound(std::begin(output_shapes_), - std::end(output_shapes_), - name, - [](auto& value, auto& entry) { return value < entry.first; }), - {name, shape}); - } - - return result; - }()) - { - } - - template - auto get_config_param(std::string const& name) - { - return get_config_param(name, std::optional{}); - } - - template - auto get_config_param(std::string const& name, T default_value) - { - return get_config_param(name, std::make_optional(default_value)); - } - - auto get_output_shape(std::string const& name) const - { - auto cached_shape = std::lower_bound( - std::begin(output_shapes_), std::end(output_shapes_), name, [](auto& entry, auto& value) { - return entry.first < value; - }); - if (cached_shape == std::end(output_shapes_) || name != cached_shape->first) { - auto log_stream = std::stringstream{}; - log_stream << "No output with name " << name << " in configuration."; - throw TritonException(Error::Internal, log_stream.str()); - } else { - return cached_shape->second; - } - } - - auto get_output_names() const { - auto output_names = std::vector{}; - output_names.reserve(output_shapes_.size()); - std::transform(std::begin(output_shapes_), std::end(output_shapes_), std::back_inserter(output_names), [](auto& output_shape) { - return output_shape.first; - }); - return output_names; - } - - auto check_output_name(std::string const& name) const { - // #TODO: Figure out a way to use std::binary_search here - auto cached_shape = std::lower_bound( - std::begin(output_shapes_), std::end(output_shapes_), name, [](auto& entry, auto& value) { - return entry.first < value; - }); - return cached_shape != std::end(output_shapes_) && name == cached_shape->first; - } - - private: - std::unique_ptr config_; - Batch::size_type max_batch_size_; - std::vector>> mutable output_shapes_; - - template - auto get_config_param(std::string const& name, std::optional const& default_value) - { - auto result = T{}; - if (name == std::string("max_batch_size")) { - result = max_batch_size_; - return result; - } - auto parameters = common::TritonJson::Value{}; - auto json_value = common::TritonJson::Value{}; - if (config_->Find("parameters", ¶meters) && parameters.Find(name.c_str(), &json_value)) { - auto string_repr = std::string{}; - triton_check(json_value.MemberAsString("string_value", &string_repr)); - - auto input_stream = std::istringstream{string_repr}; - - if constexpr (std::is_same_v) { - input_stream >> std::boolalpha >> result; - } else { - input_stream >> result; - } - - if (input_stream.fail()) { - if (default_value) { - result = *default_value; - } else { - throw TritonException(Error::InvalidArg, std::string("Bad input for parameter ") + name); - } - } - } else { - if (default_value) { - result = *default_value; - } else { - throw TritonException( - Error::InvalidArg, - std::string("Required parameter ") + name + std::string(" not found in config")); - } - } - - return result; - } -}; -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/tensor/dtype.hpp b/backend/include/triton/developer_tools/tensor/dtype.hpp deleted file mode 100644 index 7d150bb..0000000 --- a/backend/include/triton/developer_tools/tensor/dtype.hpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { - -using DType = TRITONSERVER_DataType; -auto constexpr DTypeBool = TRITONSERVER_TYPE_BOOL; -auto constexpr DTypeUint8 = TRITONSERVER_TYPE_UINT8; -auto constexpr DTypeChar = DTypeUint8; -auto constexpr DTypeByte = DTypeUint8; -auto constexpr DTypeUint16 = TRITONSERVER_TYPE_UINT16; -auto constexpr DTypeUint32 = TRITONSERVER_TYPE_UINT32; -auto constexpr DTypeUint64 = TRITONSERVER_TYPE_UINT64; -auto constexpr DTypeInt8 = TRITONSERVER_TYPE_INT8; -auto constexpr DTypeInt16 = TRITONSERVER_TYPE_INT16; -auto constexpr DTypeInt32 = TRITONSERVER_TYPE_INT32; -auto constexpr DTypeInt64 = TRITONSERVER_TYPE_INT64; -auto constexpr DTypeFloat32 = TRITONSERVER_TYPE_FP32; -auto constexpr DTypeFloat64 = TRITONSERVER_TYPE_FP64; - -template -struct TritonType { -}; - -template -struct TritonDtype { -}; - -template <> -struct TritonType { - typedef bool type; -}; - -template -struct TritonDtype> { - static constexpr DType value = DTypeBool; -}; - -template <> -struct TritonType { - typedef std::uint8_t type; -}; - -template -struct TritonDtype> { - static constexpr DType value = DTypeUint8; -}; - -template <> -struct TritonType { - typedef std::uint16_t type; -}; - -template -struct TritonDtype> { - static constexpr DType value = DTypeUint16; -}; - -template <> -struct TritonType { - typedef std::uint32_t type; -}; - -template -struct TritonDtype> { - static constexpr DType value = DTypeUint32; -}; - -template <> -struct TritonType { - typedef std::uint64_t type; -}; - -template -struct TritonDtype> { - static constexpr DType value = DTypeUint64; -}; - -template <> -struct TritonType { - typedef std::int8_t type; -}; - -template -struct TritonDtype> { - static constexpr DType value = DTypeInt8; -}; - -template <> -struct TritonType { - typedef std::int16_t type; -}; - -template -struct TritonDtype> { - static constexpr DType value = DTypeInt16; -}; - -template <> -struct TritonType { - typedef std::int32_t type; -}; - -template -struct TritonDtype> { - static constexpr DType value = DTypeInt32; -}; - -template <> -struct TritonType { - typedef std::int64_t type; -}; - -template -struct TritonDtype> { - static constexpr DType value = DTypeInt64; -}; - -template <> -struct TritonType { - typedef float type; -}; - -template -struct TritonDtype> { - static constexpr DType value = DTypeFloat32; -}; - -template <> -struct TritonType { - typedef double type; -}; - -template -struct TritonDtype> { - static constexpr DType value = DTypeFloat64; -}; - -inline std::ostream& operator<<(std::ostream& out, DType const& dtype) -{ - out << TRITONSERVER_DataTypeString(dtype); - return out; -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/tensor/tensor.hpp b/backend/include/triton/developer_tools/tensor/tensor.hpp deleted file mode 100644 index 0c74196..0000000 --- a/backend/include/triton/developer_tools/tensor/tensor.hpp +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef TRITON_ENABLE_GPU -#include -#else -#include -#endif - -#include -#include -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -template -struct BaseTensor { - using size_type = typename Buffer::size_type; - - BaseTensor() : shape_{}, buffer_{} {} - BaseTensor(std::vector const& shape, Buffer&& buffer) - : shape_(shape), buffer_{std::move(buffer)} - { - } - - virtual ~BaseTensor() = 0; - - /** - * @brief Construct a BaseTensor from a collection of buffers - * - * Given a collection of buffers, collate them all into one buffer stored in - * a new BaseTensor - */ - template - BaseTensor(std::vector const& shape, - Iter begin, - Iter end, - MemoryType mem_type, - device_id_t device, - cudaStream_t stream) - : shape_(shape), buffer_([&begin, &end, &mem_type, &device, &stream]() { - auto total_size = std::transform_reduce( - begin, end, size_type{}, std::plus<>{}, [](auto&& buffer) { return buffer.size(); }); - - auto result = Buffer(total_size, mem_type, device, stream); - - std::accumulate(begin, end, size_type{}, [&result](auto offset, auto& buffer) { - copy(result, buffer, offset); - return offset + buffer.size(); - }); - return result; - }()) - { - } - - auto const& shape() const { return shape_; } - auto size() const { return buffer_.size(); } - auto data() const { return buffer_.data(); } - auto& buffer() { return buffer_; } - - auto constexpr dtype() { return TritonDtype::value; } - auto mem_type() const { return buffer_.mem_type(); } - auto stream() const { return buffer_.stream(); } - auto device() const { return buffer_.device(); } - - void stream_synchronize() const - { - if (mem_type() == DeviceMemory) { buffer_.stream_synchronize(); } - } - - void set_stream(cudaStream_t new_stream) { buffer_.set_stream(new_stream); } - - private: - std::vector shape_; - Buffer buffer_; -}; - -template -BaseTensor::~BaseTensor() -{ -} - -template -struct Tensor final : BaseTensor { - Tensor() : BaseTensor{} {} - Tensor(std::vector::size_type> const& shape, Buffer&& buffer) - : BaseTensor(shape, std::move(buffer)) - { - } - - template - Tensor(std::vector::size_type> const& shape, - Iter begin, - Iter end, - MemoryType mem_type, - device_id_t device, - cudaStream_t stream) - : BaseTensor(shape, begin, end, mem_type, device, stream) - { - } -}; - -template -struct OutputTensor final : BaseTensor { - OutputTensor(std::vector::size_type>&& shape, - Buffer&& buffer, - std::string const& name, - std::shared_ptr responder) - : BaseTensor(std::move(shape), std::move(buffer)), name_{name}, responder_{responder} - { - } - /** - * @brief Prepare final output data from this tensor for responding to - * request - * - * This method *must* be called by developer_tools backends on all of their - * output tensors before returning from their `predict` methods. Because we - * cannot know a priori what names backends might have for their tensors - * and what types will be stored in those tensors, the developer_tools - * library cannot store references to those tensors that might otherwise be - * used to finalize them. - */ - void finalize() - { - auto& shape = BaseTensor::shape(); - auto triton_shape = std::vector{}; - triton_shape.reserve(shape.size()); - std::transform( - std::begin(shape), std::end(shape), std::back_inserter(triton_shape), [](auto& val) { - return narrow(val); - }); - - // Must call the following because BackendOutputResponder does not expose - // its stream, so we cannot be certain that our data is not being - // processed on another stream. - BaseTensor::stream_synchronize(); - responder_->ProcessTensor(name_.c_str(), - TritonDtype::value, - triton_shape, - reinterpret_cast(BaseTensor::data()), - BaseTensor::mem_type(), - BaseTensor::device()); - } - - private: - std::string name_; - std::shared_ptr responder_; -}; - -template , T>>> -void copy(BaseTensor& dst, BaseTensor& src) -{ - copy(dst.buffer(), src.buffer()); -} - -/** - * @brief Copy data from src Tensor into buffers indicated by iterators - * - * This method is provided to assist with distributing data from a single - * Tensor into many smaller buffers which have been set up to receive a part - * of the data from the src Tensor - */ -template -void copy(Iter begin, Iter end, BaseTensor& src) -{ - std::accumulate(begin, end, typename BaseTensor::size_type{}, [&src](auto offset, auto& dst) { - auto end_offset = offset + dst.size(); - copy(dst.buffer(), src.buffer(), offset, end_offset); - return end_offset; - }); -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/api/execute.hpp b/backend/include/triton/developer_tools/triton/api/execute.hpp deleted file mode 100644 index 45b455b..0000000 --- a/backend/include/triton/developer_tools/triton/api/execute.hpp +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -namespace triton_api { -template -auto* execute(TRITONBACKEND_ModelInstance* instance, - TRITONBACKEND_Request** raw_requests, - std::size_t request_count) -{ - auto start_time = std::chrono::steady_clock::now(); - - auto* result = static_cast(nullptr); - - try { - auto* model_state = get_model_state(*get_model_from_instance(*instance)); - auto* instance_state = get_instance_state(*instance); - auto& model = instance_state->get_model(); - auto max_batch_size = model.template get_config_param("max_batch_size"); - - /* Note: It is safe to keep a reference to the model in this closure - * and a pointer to the instance in the next because the batch goes - * out of scope at the end of this block and Triton guarantees that - * the liftimes of both the instance and model extend beyond this - * function call. */ - auto output_shape_fetcher = [&model](std::string const& name, Batch::size_type batch_dim) { - auto result = std::vector{}; - auto config_shape = model.get_output_shape(name); - if (config_shape.size() > 0 && config_shape[0] < 0) { config_shape[0] = batch_dim; } - std::transform( - std::begin(config_shape), - std::end(config_shape), - std::back_inserter(result), - [](auto& coord) { - if (coord < 0) { - throw TritonException( - Error::Internal, - "Backends with variable-shape outputs must request desired output shape"); - } else { - return narrow(coord); - } - }); - return result; - }; - auto statistics_reporter = [instance](TRITONBACKEND_Request* request, - time_point req_start, - time_point req_comp_start, - time_point req_comp_end, - time_point req_end) { - report_statistics(*instance, *request, req_start, req_comp_start, req_comp_end, req_end); - }; - - auto batch = Batch(raw_requests, - request_count, - *(model_state->TritonMemoryManager()), - std::move(output_shape_fetcher), - std::move(statistics_reporter), - model_state->EnablePinnedInput(), - model_state->EnablePinnedOutput(), - max_batch_size, - model.get_stream()); - - if constexpr (IS_GPU_BUILD) { - if (model.get_deployment_type() == GPUDeployment) { - cuda_check(cudaSetDevice(model.get_device_id())); - } - } - - auto predict_err = static_cast(nullptr); - try { - model.predict(batch); - } catch (TritonException& err) { - predict_err = err.error(); - } - - auto& compute_start_time = batch.compute_start_time(); - auto compute_end_time = std::chrono::steady_clock::now(); - batch.finalize(predict_err); - auto end_time = std::chrono::steady_clock::now(); - - report_statistics( - *instance, request_count, start_time, compute_start_time, compute_end_time, end_time); - } catch (TritonException& err) { - result = err.error(); - } - - return result; -} -} // namespace triton_api -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/api/initialize.hpp b/backend/include/triton/developer_tools/triton/api/initialize.hpp deleted file mode 100644 index 333be7f..0000000 --- a/backend/include/triton/developer_tools/triton/api/initialize.hpp +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#ifdef TRITON_ENABLE_GPU -#include -#else -#include -#endif -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -namespace triton_api { -inline auto* initialize(TRITONBACKEND_Backend* backend) -{ - auto* result = static_cast(nullptr); - try { - auto name = get_backend_name(*backend); - - log_info(__FILE__, __LINE__) << "TRITONBACKEND_Initialize: " << name; - - if (!check_backend_version(*backend)) { - throw TritonException{Error::Unsupported, - "triton backend API version does not support this backend"}; - } - if constexpr (IS_GPU_BUILD) { - auto device_count = int{}; - auto cuda_err = cudaGetDeviceCount(&device_count); - if (device_count > 0 && cuda_err == cudaSuccess) { - auto device_id = int{}; - cuda_check(cudaGetDevice(&device_id)); - auto* triton_manager = static_cast(nullptr); - triton_check(TRITONBACKEND_BackendMemoryManager(backend, &triton_manager)); - - setup_memory_resource(static_cast(device_id), triton_manager); - } - } - } catch (TritonException& err) { - result = err.error(); - } - return result; -} -} // namespace triton_api -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/api/instance_finalize.hpp b/backend/include/triton/developer_tools/triton/api/instance_finalize.hpp deleted file mode 100644 index 8984900..0000000 --- a/backend/include/triton/developer_tools/triton/api/instance_finalize.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -namespace triton_api { -template -auto* instance_finalize(TRITONBACKEND_ModelInstance* instance) -{ - auto* result = static_cast(nullptr); - try { - auto* instance_state = get_instance_state(*instance); - if (instance_state != nullptr) { - instance_state->unload(); - - log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInstanceFinalize: delete instance state"; - - delete instance_state; - } - } catch (TritonException& err) { - result = err.error(); - } - - return result; -} -} // namespace triton_api -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/api/instance_initialize.hpp b/backend/include/triton/developer_tools/triton/api/instance_initialize.hpp deleted file mode 100644 index d32e4c7..0000000 --- a/backend/include/triton/developer_tools/triton/api/instance_initialize.hpp +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -namespace triton_api { -template -auto* instance_initialize(TRITONBACKEND_ModelInstance* instance) -{ - auto* result = static_cast(nullptr); - try { - auto name = get_model_instance_name(*instance); - auto device_id = get_device_id(*instance); - auto deployment_type = get_deployment_type(*instance); - if constexpr (!IS_GPU_BUILD) { - if (deployment_type == GPUDeployment) { - throw TritonException(Error::Unsupported, "KIND_GPU cannot be used in CPU-only build"); - } - } - - log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInstanceInitialize: " << name << " (" - << TRITONSERVER_InstanceGroupKindString(deployment_type) - << " device " << device_id << ")"; - - auto* triton_model = get_model_from_instance(*instance); - auto* model_state = get_model_state(*triton_model); - if constexpr (IS_GPU_BUILD) { - setup_memory_resource(device_id, model_state->TritonMemoryManager()); - } - - auto tools_model = std::make_unique(*model_state, instance); - if constexpr (IS_GPU_BUILD) { - auto& model = tools_model->get_model(); - if (model.get_deployment_type() == GPUDeployment) { - cuda_check(cudaSetDevice(model.get_device_id())); - } - } - tools_model->load(); - - set_instance_state(*instance, std::move(tools_model)); - } catch (TritonException& err) { - result = err.error(); - } - return result; -} -} // namespace triton_api -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/api/model_finalize.hpp b/backend/include/triton/developer_tools/triton/api/model_finalize.hpp deleted file mode 100644 index f156725..0000000 --- a/backend/include/triton/developer_tools/triton/api/model_finalize.hpp +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -namespace triton_api { -template -auto* model_finalize(TRITONBACKEND_Model* model) -{ - auto* result = static_cast(nullptr); - try { - auto model_state = get_model_state(*model); - if (model_state != nullptr) { model_state->get_shared_state()->unload(); } - - log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelFinalize: delete model state"; - - delete model_state; - } catch (TritonException& err) { - result = err.error(); - } - - return result; -} -} // namespace triton_api -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/api/model_initialize.hpp b/backend/include/triton/developer_tools/triton/api/model_initialize.hpp deleted file mode 100644 index 1bcffc5..0000000 --- a/backend/include/triton/developer_tools/triton/api/model_initialize.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -namespace triton_api { -template -auto* model_initialize(TRITONBACKEND_Model* model) -{ - auto* result = static_cast(nullptr); - try { - auto name = get_model_name(*model); - - auto version = get_model_version(*model); - - log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInitialize: " << name << " (version " - << version << ")"; - - auto tools_model_state = std::make_unique(*model); - tools_model_state->load(); - - set_model_state(*model, std::move(tools_model_state)); - } catch (TritonException& err) { - result = err.error(); - } - - return result; -} -} // namespace triton_api -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/backend.hpp b/backend/include/triton/developer_tools/triton/backend.hpp deleted file mode 100644 index 561327a..0000000 --- a/backend/include/triton/developer_tools/triton/backend.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -inline auto get_backend_name(TRITONBACKEND_Backend& backend) -{ - const char* cname; - triton_check(TRITONBACKEND_BackendName(&backend, &cname)); - return std::string(cname); -} - -namespace { -struct backend_version { - std::uint32_t major; - std::uint32_t minor; -}; -} // namespace - -inline auto check_backend_version(TRITONBACKEND_Backend& backend) -{ - auto version = backend_version{}; - triton_check(TRITONBACKEND_ApiVersion(&version.major, &version.minor)); - - log_info(__FILE__, __LINE__) << "Triton TRITONBACKEND API version: " << version.major << "." - << version.minor; - - auto name = get_backend_name(backend); - - log_info(__FILE__, __LINE__) << "'" << name - << "' TRITONBACKEND API version: " << TRITONBACKEND_API_VERSION_MAJOR - << "." << TRITONBACKEND_API_VERSION_MINOR; - - return ((version.major == TRITONBACKEND_API_VERSION_MAJOR) && - (version.minor >= TRITONBACKEND_API_VERSION_MINOR)); -} -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/config.hpp b/backend/include/triton/developer_tools/triton/config.hpp deleted file mode 100644 index affcf3f..0000000 --- a/backend/include/triton/developer_tools/triton/config.hpp +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include - -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -inline auto get_max_batch_size(common::TritonJson::Value& config) -{ - auto reported = int64_t{}; - triton_check(config.MemberAsInt("max_batch_size", &reported)); - return narrow(reported); -} -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/deployment.hpp b/backend/include/triton/developer_tools/triton/deployment.hpp deleted file mode 100644 index cc9fd97..0000000 --- a/backend/include/triton/developer_tools/triton/deployment.hpp +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include - -namespace triton { -namespace developer_tools { -namespace backend { -using DeploymentType = TRITONSERVER_InstanceGroupKind; -auto constexpr GPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_GPU; -auto constexpr CPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_CPU; -// Note (wphicks): We currently are not including "Auto" or "Model" because I -// am not sure exactly how those would be used in context. If there is a -// demand, they can be added. -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/device.hpp b/backend/include/triton/developer_tools/triton/device.hpp deleted file mode 100644 index 621d493..0000000 --- a/backend/include/triton/developer_tools/triton/device.hpp +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include - -namespace triton { -namespace developer_tools { -namespace backend { -using device_id_t = std::int32_t; -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/input.hpp b/backend/include/triton/developer_tools/triton/input.hpp deleted file mode 100644 index 85ef139..0000000 --- a/backend/include/triton/developer_tools/triton/input.hpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) -{ - auto result = static_cast(nullptr); - triton_check(TRITONBACKEND_RequestInput(request, name.c_str(), &result)); - return result; -} - -template -auto get_triton_input_shape(Iter requests_begin, Iter requests_end, std::string const& name) -{ - auto result = std::vector{}; - - auto reported_dtype = DType{}; - auto const* input_shape = static_cast(nullptr); - auto input_dims = uint32_t{}; - - auto batch_dim = std::accumulate( - requests_begin, - requests_end, - int64_t{}, - [&reported_dtype, &input_shape, &input_dims, &name](auto total, auto& request) { - auto* input = get_triton_input(request, name); - triton_check(TRITONBACKEND_InputProperties( - input, nullptr, &reported_dtype, &input_shape, &input_dims, nullptr, nullptr)); - - if (reported_dtype != TritonDtype::value) { - auto log_stream = std::stringstream{}; - log_stream << "incorrect type " << reported_dtype << " for input with required type " - << TritonDtype::value; - throw(TritonException(Error::Internal, log_stream.str())); - } - - if (input_dims != 0) { total += *input_shape; } - return total; - }); - - result.reserve(input_dims); - std::transform(input_shape, input_shape + input_dims, std::back_inserter(result), [](auto& val) { - return narrow(val); - }); - - if (!result.empty()) { result[0] = narrow(batch_dim); } - - return result; -} -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/logging.hpp b/backend/include/triton/developer_tools/triton/logging.hpp deleted file mode 100644 index 3f15e83..0000000 --- a/backend/include/triton/developer_tools/triton/logging.hpp +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include - -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { - -namespace { -/** Log message at indicated level */ -inline void log(TRITONSERVER_LogLevel level, char const* filename, int line, char const* message) -{ - triton_check(TRITONSERVER_LogMessage(level, filename, line, message)); -} -} // namespace - -struct log_stream : public std::ostream { - log_stream(TRITONSERVER_LogLevel level, char const* filename, int line) - : std::ostream{}, buffer_{level, filename, line} - { - rdbuf(&buffer_); - } - log_stream(TRITONSERVER_LogLevel level) : std::ostream{}, buffer_{level, __FILE__, __LINE__} - { - rdbuf(&buffer_); - } - - ~log_stream() - { - try { - flush(); - } catch (std::ios_base::failure const& ignored_err) { - // Ignore error if flush fails - } - } - - private: - struct log_buffer : public std::stringbuf { - log_buffer(TRITONSERVER_LogLevel level, char const* filename, int line) - : level_{level}, filename_{filename}, line_{line} - { - } - - virtual int sync() - { - auto msg = str(); - if (!msg.empty()) { - log(level_, filename_, line_, msg.c_str()); - str(""); - } - return 0; - } - - private: - TRITONSERVER_LogLevel level_; - char const* filename_; - int line_; - }; - - log_buffer buffer_; -}; - -/** Log message at INFO level */ -inline void log_info(char const* filename, int line, char const* message) -{ - log(TRITONSERVER_LOG_INFO, filename, line, message); -} -inline void log_info(char const* filename, int line, std::string const& message) -{ - log_info(filename, line, message.c_str()); -} -inline void log_info(char const* message) { log_info(__FILE__, __LINE__, message); } -inline void log_info(std::string const& message) { log_info(__FILE__, __LINE__, message.c_str()); } -inline auto log_info(char const* filename, int line) -{ - return log_stream(TRITONSERVER_LOG_INFO, filename, line); -} -inline auto log_info() { return log_stream(TRITONSERVER_LOG_INFO); } - -/** Log message at WARN level */ -inline void log_warn(char const* filename, int line, char const* message) -{ - log(TRITONSERVER_LOG_WARN, filename, line, message); -} -inline void log_warn(char const* filename, int line, std::string const& message) -{ - log_warn(filename, line, message.c_str()); -} -inline void log_warn(char const* message) { log_warn(__FILE__, __LINE__, message); } -inline void log_warn(std::string const& message) { log_warn(__FILE__, __LINE__, message.c_str()); } -inline auto log_warn(char const* filename, int line) -{ - return log_stream(TRITONSERVER_LOG_WARN, filename, line); -} -inline auto log_warn() { return log_stream(TRITONSERVER_LOG_WARN); } - -/** Log message at ERROR level */ -inline void log_error(char const* filename, int line, char const* message) -{ - log(TRITONSERVER_LOG_ERROR, filename, line, message); -} -inline void log_error(char const* filename, int line, std::string const& message) -{ - log_error(filename, line, message.c_str()); -} -inline void log_error(char const* message) { log_error(__FILE__, __LINE__, message); } -inline void log_error(std::string const& message) -{ - log_error(__FILE__, __LINE__, message.c_str()); -} -inline auto log_error(char const* filename, int line) -{ - return log_stream(TRITONSERVER_LOG_ERROR, filename, line); -} -inline auto log_error() { return log_stream(TRITONSERVER_LOG_ERROR); } - -/** Log message at VERBOSE level */ -inline void log_debug(char const* filename, int line, char const* message) -{ - log(TRITONSERVER_LOG_VERBOSE, filename, line, message); -} -inline void log_debug(char const* filename, int line, std::string const& message) -{ - log_debug(filename, line, message.c_str()); -} -inline void log_debug(char const* message) { log_debug(__FILE__, __LINE__, message); } -inline void log_debug(std::string const& message) -{ - log_debug(__FILE__, __LINE__, message.c_str()); -} -inline auto log_debug(char const* filename, int line) -{ - return log_stream(TRITONSERVER_LOG_VERBOSE, filename, line); -} -inline auto log_debug() { return log_stream(TRITONSERVER_LOG_VERBOSE); } - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/model.hpp b/backend/include/triton/developer_tools/triton/model.hpp deleted file mode 100644 index d2e31e8..0000000 --- a/backend/include/triton/developer_tools/triton/model.hpp +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { - -inline auto get_model_version(TRITONBACKEND_Model& model) -{ - auto version = std::uint64_t{}; - triton_check(TRITONBACKEND_ModelVersion(&model, &version)); - return version; -} - -inline auto get_model_name(TRITONBACKEND_Model& model) -{ - auto* cname = static_cast(nullptr); - triton_check(TRITONBACKEND_ModelName(&model, &cname)); - return std::string(cname); -} - -inline auto get_model_config(TRITONBACKEND_Model& model) -{ - auto* config_message = static_cast(nullptr); - triton_check(TRITONBACKEND_ModelConfig(&model, 1, &config_message)); - - auto* buffer = static_cast(nullptr); - auto byte_size = std::size_t{}; - triton_check(TRITONSERVER_MessageSerializeToJson(config_message, &buffer, &byte_size)); - - auto model_config = std::make_unique(); - auto* err = model_config->Parse(buffer, byte_size); - auto* result = TRITONSERVER_MessageDelete(config_message); - if (err != nullptr) { throw(TritonException(err)); } - if (result != nullptr) { throw(TritonException(result)); } - return model_config; -} - -/** - * @brief Set model state (as used by Triton) to given object - * - * This function accepts a unique_ptr to an object derived from a Triton - * BackendModel object and sets it as the stored state for a model in the - * Triton server. Note that this object is not the same as a Backend-Tools - * "SharedModelState" object. The object that Triton expects must wrap this - * SharedModelState and provide additional interface compatibility. - */ -template -void set_model_state(TRITONBACKEND_Model& model, std::unique_ptr&& model_state) -{ - triton_check(TRITONBACKEND_ModelSetState(&model, reinterpret_cast(model_state.release()))); -} - -/** Given a model, return its associated ModelState object */ -template -auto* get_model_state(TRITONBACKEND_Model& model) -{ - auto* vstate = static_cast(nullptr); - triton_check(TRITONBACKEND_ModelState(&model, &vstate)); - - auto* model_state = reinterpret_cast(vstate); - - return model_state; -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/model_instance.hpp b/backend/include/triton/developer_tools/triton/model_instance.hpp deleted file mode 100644 index 2177f08..0000000 --- a/backend/include/triton/developer_tools/triton/model_instance.hpp +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -/** Get the name of a Triton model instance from the instance itself */ -inline auto get_model_instance_name(TRITONBACKEND_ModelInstance& instance) -{ - auto* cname = static_cast(nullptr); - triton_check(TRITONBACKEND_ModelInstanceName(&instance, &cname)); - return std::string(cname); -} - -/** Get the device on which a Triton model instance is loaded - * - * If this instance is loaded on the host, 0 will be returned. Otherwise the - * GPU device id will be returned.*/ -inline auto get_device_id(TRITONBACKEND_ModelInstance& instance) -{ - auto device_id = device_id_t{}; - triton_check(TRITONBACKEND_ModelInstanceDeviceId(&instance, &device_id)); - return device_id; -} - -/** Determine how a Triton model instance is deployed - * - * Returns enum value indicating whether the instance is deployed on device - * or on the host - */ -inline auto get_deployment_type(TRITONBACKEND_ModelInstance& instance) -{ - auto kind = GPUDeployment; - triton_check(TRITONBACKEND_ModelInstanceKind(&instance, &kind)); - return kind; -} - -/** Return the Triton model from one of its instances - */ -inline auto* get_model_from_instance(TRITONBACKEND_ModelInstance& instance) -{ - auto* model = static_cast(nullptr); - triton_check(TRITONBACKEND_ModelInstanceModel(&instance, &model)); - return model; -} - -/** - * @brief Set Triton model instance state to given object - * - * This function accepts a unique_ptr to an object derived from a Triton - * BackendModelInstance object and sets it as the stored state for a model in the - * Triton server. Note that this object is not the same as a Backend-Tools - * "Model" object. The object that Triton expects must wrap this Model and - * provide additional interface compatibility. - */ -template -void set_instance_state(TRITONBACKEND_ModelInstance& instance, - std::unique_ptr&& model_instance_state) -{ - triton_check(TRITONBACKEND_ModelInstanceSetState( - &instance, reinterpret_cast(model_instance_state.release()))); -} - -/** Get model instance state from instance */ -template -auto* get_instance_state(TRITONBACKEND_ModelInstance& instance) -{ - auto* instance_state = static_cast(nullptr); - triton_check( - TRITONBACKEND_ModelInstanceState(&instance, reinterpret_cast(&instance_state))); - return instance_state; -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/model_instance_state.hpp b/backend/include/triton/developer_tools/triton/model_instance_state.hpp deleted file mode 100644 index 233778c..0000000 --- a/backend/include/triton/developer_tools/triton/model_instance_state.hpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { - -template -struct ModelInstanceState : public triton::backend::BackendModelInstance { - ModelInstanceState(TritonModelState& model_state, - TRITONBACKEND_ModelInstance* triton_model_instance) - : triton::backend::BackendModelInstance(&model_state, triton_model_instance), - model_(model_state.get_shared_state(), - backend::get_device_id(*triton_model_instance), - CudaStream(), - Kind(), - triton::backend::JoinPath({model_state.RepositoryPath(), - std::to_string(model_state.Version()), - ArtifactFilename()})) - { - } - - auto& get_model() const { return model_; } - - void load() { model_.load(); } - void unload() { model_.unload(); } - - private: - ToolsModel model_; -}; - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/model_state.hpp b/backend/include/triton/developer_tools/triton/model_state.hpp deleted file mode 100644 index df6118a..0000000 --- a/backend/include/triton/developer_tools/triton/model_state.hpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -template -struct TritonModelState : public triton::backend::BackendModel { - TritonModelState(TRITONBACKEND_Model& triton_model) - : triton::backend::BackendModel(&triton_model), - state_{std::make_shared(get_model_config(triton_model))} - { - } - - void load() { state_->load(); } - void unload() { state_->unload(); } - - auto get_shared_state() { return state_; } - - private: - std::shared_ptr state_; -}; - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/output.hpp b/backend/include/triton/developer_tools/triton/output.hpp deleted file mode 100644 index d335f66..0000000 --- a/backend/include/triton/developer_tools/triton/output.hpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) -{ - auto* result = static_cast(nullptr); - triton_check(TRITONBACKEND_RequestInput(request, name.c_str(), &result)); - return result; -} - -template -auto get_triton_output_shape(Iter requests_begin, Iter requests_end, std::string const& name) -{ - auto result = std::vector{}; - - auto reported_dtype = DType{}; - auto const* input_shape = static_cast(nullptr); - auto input_dims = uint32_t{}; - - auto batch_dim = - std::reduce(requests_begin, - requests_end, - int64_t{}, - [&reported_dtype, &input_shape, &input_dims, &name](auto& request, auto total) { - auto* input = get_triton_input(request, name); - triton_check(TRITONBACKEND_InputProperties( - input, nullptr, &reported_dtype, &input_shape, &input_dims, nullptr, nullptr)); - - if (reported_dtype != TritonDtype::value) { - auto log_stream = std::stringstream{}; - log_stream << "incorrect type " << reported_dtype - << " for output with required type " << TritonDtype::value; - throw(TritonException(Error::Internal, log_stream.str())); - } - - if (input_dims != 0) { total += *input_shape; } - return total; - }); - - result.reserve(input_dims); - std::transform(input_shape, input_shape + input_dims, std::back_inserter(result), [](auto& val) { - return narrow(val); - }); - - if (!result.empty()) { result[0] = narrow(batch_dim); } - - return result; -} -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/requests.hpp b/backend/include/triton/developer_tools/triton/requests.hpp deleted file mode 100644 index d2a6d06..0000000 --- a/backend/include/triton/developer_tools/triton/requests.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include - -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -using request_size_t = uint32_t; - -template -void release_requests(Iter begin, Iter end) -{ - std::for_each(begin, end, [](auto& request) { - try { - triton_check(TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL)); - } catch (TritonException& err) { - log_error(__FILE__, __LINE__, err.what()); - } - }); -} -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/responses.hpp b/backend/include/triton/developer_tools/triton/responses.hpp deleted file mode 100644 index f276a2e..0000000 --- a/backend/include/triton/developer_tools/triton/responses.hpp +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { - -template -auto construct_responses(Iter requests_begin, Iter requests_end) -{ - auto responses = std::vector{}; - - auto requests_size = std::distance(requests_begin, requests_end); - if (!(requests_size > 0)) { - throw TritonException(Error::Internal, - "Invalid iterators for requests when constructing responses"); - } - responses.reserve(requests_size); - - std::transform(requests_begin, requests_end, std::back_inserter(responses), [](auto* request) { - auto* response = static_cast(nullptr); - triton_check(TRITONBACKEND_ResponseNew(&response, request)); - return response; - }); - return responses; -} - -template -void send_responses(Iter begin, Iter end, TRITONSERVER_Error* err) -{ - std::for_each(begin, end, [err](auto& response) { - decltype(err) err_copy; - if (err != nullptr) { - err_copy = TRITONSERVER_ErrorNew(TRITONSERVER_ErrorCode(err), TRITONSERVER_ErrorMessage(err)); - } else { - err_copy = err; - } - - if (response == nullptr) { - log_error(__FILE__, __LINE__) << "Failure in response collation"; - } else { - try { - triton_check( - TRITONBACKEND_ResponseSend(response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, err_copy)); - } catch (TritonException& err) { - log_error(__FILE__, __LINE__, err.what()); - } - } - }); -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/statistics.hpp b/backend/include/triton/developer_tools/triton/statistics.hpp deleted file mode 100644 index eacb3ab..0000000 --- a/backend/include/triton/developer_tools/triton/statistics.hpp +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -using time_point = std::chrono::time_point; - -/** - * @brief Report inference statistics for a single request - * - * @param instance The Triton model instance which is processing this request - * @param request The Triton request object itself - * @param start_time The time at which the backend first received the request - * @param compute_start_time The time at which the backend began actual - * inference on the request - * @param compute_end_time The time at which the backend completed inference - * on the request - * @param end_time The time at which the backend finished all processing on - * the request, including copying out results and returning a response - */ -inline void report_statistics(TRITONBACKEND_ModelInstance& instance, - TRITONBACKEND_Request& request, - time_point start_time, - time_point compute_start_time, - time_point compute_end_time, - time_point end_time) -{ - triton_check( - TRITONBACKEND_ModelInstanceReportStatistics(&instance, - &request, - true, - start_time.time_since_epoch().count(), - compute_start_time.time_since_epoch().count(), - compute_end_time.time_since_epoch().count(), - end_time.time_since_epoch().count())); -} - -/** - * @brief Report inference statistics for a batch of requests of given size - * - * @param instance The Triton model instance which is processing this batch - * @param request_count The number of requests in this batch - * @param start_time The time at which the backend first received the batch - * @param compute_start_time The time at which the backend began actual - * inference on the batch - * @param compute_end_time The time at which the backend completed inference - * on the batch - * @param end_time The time at which the backend finished all processing on - * the batch, including copying out results and returning a response - */ -inline void report_statistics(TRITONBACKEND_ModelInstance& instance, - std::size_t request_count, - time_point start_time, - time_point compute_start_time, - time_point compute_end_time, - time_point end_time) -{ - triton_check( - TRITONBACKEND_ModelInstanceReportBatchStatistics(&instance, - request_count, - start_time.time_since_epoch().count(), - compute_start_time.time_since_epoch().count(), - compute_end_time.time_since_epoch().count(), - end_time.time_since_epoch().count())); -} -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/triton/triton_memory_resource.hpp b/backend/include/triton/developer_tools/triton/triton_memory_resource.hpp deleted file mode 100644 index 837c8d6..0000000 --- a/backend/include/triton/developer_tools/triton/triton_memory_resource.hpp +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -struct triton_memory_resource final : public rmm::mr::device_memory_resource { - triton_memory_resource(TRITONBACKEND_MemoryManager* manager, - device_id_t device_id, - rmm::mr::device_memory_resource* fallback) - : manager_{manager}, device_id_{device_id}, fallback_{fallback} - { - } - - bool supports_streams() const noexcept override { return false; } - bool supports_get_mem_info() const noexcept override { return false; } - auto* get_triton_manager() const noexcept { return manager_; } - - private: - TRITONBACKEND_MemoryManager* manager_; - std::int64_t device_id_; - rmm::mr::device_memory_resource* fallback_; - - void* do_allocate(std::size_t bytes, rmm::cuda_stream_view stream) override - { - auto* ptr = static_cast(nullptr); - if (manager_ == nullptr) { - ptr = fallback_->allocate(bytes, stream); - } else { - triton_check(TRITONBACKEND_MemoryManagerAllocate( - manager_, &ptr, TRITONSERVER_MEMORY_GPU, device_id_, static_cast(bytes))); - } - return ptr; - } - - void do_deallocate(void* ptr, std::size_t bytes, rmm::cuda_stream_view stream) - { - if (manager_ == nullptr) { - fallback_->deallocate(ptr, bytes, stream); - } else { - triton_check( - TRITONBACKEND_MemoryManagerFree(manager_, ptr, TRITONSERVER_MEMORY_GPU, device_id_)); - } - } - - bool do_is_equal(rmm::mr::device_memory_resource const& other) const noexcept override - { - auto* other_triton_mr = dynamic_cast(&other); - return (other_triton_mr != nullptr && other_triton_mr->get_triton_manager() == manager_); - } - - std::pair do_get_mem_info(rmm::cuda_stream_view stream) const override - { - return {0, 0}; - } -}; - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/utils/const_agnostic.hpp b/backend/include/triton/developer_tools/utils/const_agnostic.hpp deleted file mode 100644 index 4a83eed..0000000 --- a/backend/include/triton/developer_tools/utils/const_agnostic.hpp +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include - -namespace triton { -namespace developer_tools { -namespace backend { - -template -using const_agnostic_same_t = - std::enable_if_t, std::remove_const_t>>; - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/utils/device_setter.hpp b/backend/include/triton/developer_tools/utils/device_setter.hpp deleted file mode 100644 index ef3fa99..0000000 --- a/backend/include/triton/developer_tools/utils/device_setter.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#ifdef TRITON_ENABLE_GPU -#include -#endif -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { - -/** Struct for setting cuda device within a code block */ -struct device_setter { - device_setter(device_id_t device) : prev_device_{} { - if constexpr(IS_GPU_BUILD) { - cuda_check(cudaGetDevice(&prev_device_)); - cuda_check(cudaSetDevice(device)); - } else { - throw TritonException(Error::Internal, "Device setter used in non-GPU build"); - } - } - - ~device_setter() { - if constexpr(IS_GPU_BUILD) { - cudaSetDevice(prev_device_); - } - } - private: - device_id_t prev_device_; -}; - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/include/triton/developer_tools/utils/narrow.hpp b/backend/include/triton/developer_tools/utils/narrow.hpp deleted file mode 100644 index 2ed9bb8..0000000 --- a/backend/include/triton/developer_tools/utils/narrow.hpp +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { - -template -auto narrow(F from) -{ - auto to = static_cast(from); - - if (static_cast(to) != from || - (std::is_signed::value && !std::is_signed::value && from < F{}) || - (std::is_signed::value && !std::is_signed::value && to < T{}) || - (std::is_signed::value == std::is_signed::value && ((to < T{}) != (from < F{})))) { - throw TritonException(Error::Internal, "invalid narrowing"); - } - return to; -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/scripts/run-clang-format.py b/backend/scripts/run-clang-format.py deleted file mode 100755 index 614697b..0000000 --- a/backend/scripts/run-clang-format.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright (c) 2019-2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This file was copied from the rapidsai/cuml repo - -from __future__ import print_function -import sys -import re -import os -import subprocess -import argparse -import tempfile -import shutil - - -EXPECTED_VERSION = "11.0.0" -VERSION_REGEX = re.compile(r"clang-format version ([0-9.]+)") -DEFAULT_DIRS = ["cpp/src", - "cpp/include", - "cpp/test"] - - -def parse_args(): - argparser = argparse.ArgumentParser("Runs clang-format on a project") - argparser.add_argument("-dstdir", type=str, default=None, - help="Directory to store the temporary outputs of" - " clang-format. If nothing is passed for this, then" - " a temporary dir will be created using `mkdtemp`") - argparser.add_argument("-exe", type=str, default="clang-format", - help="Path to clang-format exe") - argparser.add_argument("-inplace", default=False, action="store_true", - help="Replace the source files itself.") - argparser.add_argument("-regex", type=str, - default=r"[.](cu|cuh|h|hpp|cpp)$", - help="Regex string to filter in sources") - argparser.add_argument("-ignore", type=str, default=r"cannylab/bh[.]cu$", - help="Regex used to ignore files from matched list") - argparser.add_argument("-v", dest="verbose", action="store_true", - help="Print verbose messages") - argparser.add_argument("dirs", type=str, nargs="*", - help="List of dirs where to find sources") - args = argparser.parse_args() - args.regex_compiled = re.compile(args.regex) - args.ignore_compiled = re.compile(args.ignore) - if args.dstdir is None: - args.dstdir = tempfile.mkdtemp() - ret = subprocess.check_output("%s --version" % args.exe, shell=True) - ret = ret.decode("utf-8") - version = VERSION_REGEX.match(ret) - if version is None: - raise Exception("Failed to figure out clang-format version!") - version = version.group(1) - if version != EXPECTED_VERSION: - raise Exception("clang-format exe must be v%s found '%s'" % \ - (EXPECTED_VERSION, version)) - if len(args.dirs) == 0: - args.dirs = DEFAULT_DIRS - return args - - -def list_all_src_files(file_regex, ignore_regex, srcdirs, dstdir, inplace): - allFiles = [] - for srcdir in srcdirs: - for root, dirs, files in os.walk(srcdir): - for f in files: - if re.search(file_regex, f): - src = os.path.join(root, f) - if re.search(ignore_regex, src): - continue - if inplace: - _dir = root - else: - _dir = os.path.join(dstdir, root) - dst = os.path.join(_dir, f) - allFiles.append((src, dst)) - return allFiles - - -def run_clang_format(src, dst, exe, verbose): - dstdir = os.path.dirname(dst) - if not os.path.exists(dstdir): - os.makedirs(dstdir) - # run the clang format command itself - if src == dst: - cmd = "%s -i %s" % (exe, src) - else: - cmd = "%s %s > %s" % (exe, src, dst) - try: - subprocess.check_call(cmd, shell=True) - except subprocess.CalledProcessError: - print("Failed to run clang-format! Maybe your env is not proper?") - raise - # run the diff to check if there are any formatting issues - cmd = "diff -q %s %s >/dev/null" % (src, dst) - try: - subprocess.check_call(cmd, shell=True) - if verbose: - print("%s passed" % os.path.basename(src)) - except subprocess.CalledProcessError: - print("%s failed! 'diff %s %s' will show formatting violations!" % \ - (os.path.basename(src), src, dst)) - return False - return True - - -def main(): - args = parse_args() - # Attempt to making sure that we run this script from root of repo always - if not os.path.exists(".git"): - print("Error!! This needs to always be run from the root of repo") - sys.exit(-1) - all_files = list_all_src_files(args.regex_compiled, args.ignore_compiled, - args.dirs, args.dstdir, args.inplace) - - # Check whether clang-format exists - if shutil.which("clang-format") is None: - print("clang-format not found. Exiting...") - return - - # actual format checker - status = True - for src, dst in all_files: - if not run_clang_format(src, dst, args.exe, args.verbose): - status = False - if not status: - print("clang-format failed! You have 2 options:") - print(" 1. Look at formatting differences above and fix them manually") - print(" 2. Or run the below command to bulk-fix all these at once") - print("Bulk-fix command: ") - print(" python cpp/scripts/run-clang-format.py %s -inplace" % \ - " ".join(sys.argv[1:])) - sys.exit(-1) - return - - -if __name__ == "__main__": - main() diff --git a/backend/src/CMakeLists.txt b/backend/src/CMakeLists.txt deleted file mode 100644 index 1d544bb..0000000 --- a/backend/src/CMakeLists.txt +++ /dev/null @@ -1,67 +0,0 @@ -#============================================================================= -# Copyright (c) 2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -# keep the files in alphabetical order! -add_library( - triton_tools-identity SHARED - src/api.cc -) - -if(TRITON_ENABLE_GPU) - set_target_properties(triton_tools-identity - PROPERTIES BUILD_RPATH "\$ORIGIN" - # set target compile options - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON - ) -else() - set_target_properties(triton_tools-identity - PROPERTIES BUILD_RPATH "\$ORIGIN" - # set target compile options - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON - ) -endif() - -target_compile_options(triton_tools-identity - PRIVATE "$<$:${DEVELOPER_TOOLS_BACKEND_CXX_FLAGS}>" - "$<$:${DEVELOPER_TOOLS_BACKEND_CUDA_FLAGS}>" -) - -target_include_directories(triton_tools-identity - PRIVATE "$" - "${CMAKE_CURRENT_SOURCE_DIR}/src" -) - -target_link_libraries(triton_tools-identity -PRIVATE - $<$:rmm::rmm> - $<$:raft::raft> - triton-core-serverstub - triton-backend-utils - $ -) - -install( - TARGETS triton_tools-identity - LIBRARY DESTINATION /opt/tritonserver/backends/tools-identity -) diff --git a/backend/src/api.cc b/backend/src/api.cc deleted file mode 100644 index a6117fa..0000000 --- a/backend/src/api.cc +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace NAMESPACE { - -using ModelState = backend::TritonModelState; -using ModelInstanceState = - backend::ModelInstanceState; - -extern "C" { - -/** Confirm that backend is compatible with Triton's backend API version - */ -TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { - return backend::triton_api::initialize(backend); -} - -TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { - return backend::triton_api::model_initialize(model); -} - -TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { - return backend::triton_api::model_finalize(model); -} - -TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize( - TRITONBACKEND_ModelInstance* instance) { - return backend::triton_api::instance_initialize(instance); -} - -TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize( - TRITONBACKEND_ModelInstance* instance) { - return backend::triton_api::instance_finalize(instance); -} - -TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( - TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** raw_requests, - uint32_t const request_count) { - return backend::triton_api::execute( - instance, raw_requests, static_cast(request_count)); -} - -} // extern "C" - -} // namespace NAMESPACE -} // namespace developer_tools -} // namespace triton diff --git a/backend/src/model.h b/backend/src/model.h deleted file mode 100644 index 7fe0875..0000000 --- a/backend/src/model.h +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#ifdef TRITON_ENABLE_GPU -#include -#else -#include -#endif -#include -#include - -#include -#include -#include // backend::Batch -#include // backend::MemoryType -#include // backend::Model -#include // backend::copy -#include // backend::DeploymentType -#include // backend::device_id_t - -namespace triton { -namespace developer_tools { -namespace NAMESPACE { - -/* Any logic necessary to perform inference with a model and manage its data - * should be implemented in a struct named ToolsModel, as shown here */ - -struct ToolsModel : backend::Model { - /*************************************************************************** - * BOILERPLATE * - * *********************************************************************** * - * The following constructor can be copied directly into any model - * implementation. - **************************************************************************/ - ToolsModel(std::shared_ptr shared_state, - backend::device_id_t device_id, - cudaStream_t default_stream, - backend::DeploymentType deployment_type, - std::string const& filepath) - : backend::Model( - shared_state, device_id, default_stream, deployment_type, filepath) - { - } - - /*************************************************************************** - * BASIC FEATURES * - * *********************************************************************** * - * The only method that *must* be implemented for a viable model is the - * `predict` method, but the others presented here are often used for basic - * model implementations. Filling out these methods should take care of most - * use cases. - **************************************************************************/ - - /*************************************************************************** - * predict * - * *********************************************************************** * - * This method performs the actual inference step on input data. Implementing - * a predict function requires four steps: - * 1. Call `get_input` on the provided `Batch` object for each of the input - * tensors named in the config file for this backend. This provides a - * `Tensor` object containing the input data. - * 2. Call `get_output` on the provided `Batch` object for each of the output - * tensors named in the config file for this backend. This provides a - * `Tensor` object to which output values can be written. - * 3. Perform inference based on the input Tensors and store the results in - * the output Tensors. `some_tensor.data()` can be used to retrieve a raw - * pointer to the underlying data. - * 4. Call the `finalize` method on all output tensors. - **************************************************************************/ - void predict(backend::Batch& batch) const - { - // 1. Acquire a tensor representing the input named "input__0" - auto input = get_input(batch, "input__0"); - // 2. Acquire a tensor representing the output named "output__0" - auto output = get_output(batch, "output__0"); - - // 3. Perform inference. In this example, we simply copy the data from the - // input to the output tensor. - backend::copy(output, input); - - // 4. Call finalize on all output tensors. In this case, we have just one - // output, so we call finalize on it. - output.finalize(); - } - - /*************************************************************************** - * load / unload * - * *********************************************************************** * - * These methods can be used to perform one-time loading/unloading of - * resources when a model is created. For example, data representing the - * model may be loaded onto the GPU in the `load` method and unloaded in the - * `unload` method. This data will then remain loaded while the server is - * running. - * - * While these methods take no arguments, it is typical to read any necessary - * input from the model configuration file by using the `get_config_param` - * method. Any parameters defined in the "parameters" section of the config - * can be accessed by name in this way. The maximum batch size can also be - * retrieved using the name "max_batch_size". - * - * These methods need not be explicitly implemented if no loading/unloading - * logic is required, but we show them here for illustrative purposes. - **************************************************************************/ - void load() {} - void unload() {} - - /*************************************************************************** - * ADVANCED FEATURES * - * *********************************************************************** * - * None of the following methods are required to be implemented in order to - * create a valid model, but they are presented here for those who require - * the additional functionality they provide. - **************************************************************************/ - - /*************************************************************************** - * preferred_mem_type / preferred_mem_type_in / preferred_mem_type_out * - * *********************************************************************** * - * If implemented, `preferred_mem_type` allows for control over when input - * and output data are provided on the host versus on device. In the case - * that a model prefers to receive its input on-host but return output - * on-device (or vice versa), `preferred_mem_type_in` and - * `preferred_mem_type_out` can be used for even more precise control. - * - * In this example, we simply return `std::nullopt` to indicate that the - * model has no preference on its input/output data locations. Note that the - * Batch being processed is taken as input to this function to facilitate - * implementations that may switch their preferred memory location based on - * properties of the batch. - * - * Valid MemoryType options to return are backend::HostMemory and - * backend::DeviceMemory. - **************************************************************************/ - std::optional preferred_mem_type(backend::Batch& batch) const - { - return std::nullopt; - } -}; - -} // namespace NAMESPACE -} // namespace developer_tools -} // namespace triton diff --git a/backend/src/names.h b/backend/src/names.h deleted file mode 100644 index eba7319..0000000 --- a/backend/src/names.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -/* Triton expects certain definitions within its backend libraries to follow - * specific naming conventions. Specifically, for a backend named - * "tools_identity," most definitions should appear within a namespace called - * triton::backend::tools_identity. - * - * In order to facilitate this with minimal effort on the part of backend - * developers, we ask that you put the name of your backend here. This macro is - * then used to propagate the correct namespace name wherever it is needed in - * the impl and interface code. */ - -#define NAMESPACE tools_identity diff --git a/backend/src/shared_state.h b/backend/src/shared_state.h deleted file mode 100644 index 190020b..0000000 --- a/backend/src/shared_state.h +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include -#include - -namespace triton { -namespace developer_tools { -namespace NAMESPACE { - -/* Triton allows multiple instances of a single model to be instantiated at the - * same time (e.g. on different GPUs). All instances of a model share access to - * an object which manages any state that can be shared across all instances. - * Any logic necessary for managing such state should be implemented in a - * struct named ToolsSharedState, as shown here. Models may access this shared - * state object via the `get_shared_state` method, which returns a shared - * pointer to the ToolsSharedState object. - * - * Not all backends require shared state, so leaving this implementation empty - * is entirely valid */ - -struct ToolsSharedState : backend::SharedModelState { - ToolsSharedState(std::unique_ptr&& config) - : backend::SharedModelState{std::move(config)} - { - } - void load() {} - void unload() {} -}; - -} // namespace NAMESPACE -} // namespace developer_tools -} // namespace triton diff --git a/backend/test/CMakeLists.txt b/backend/test/CMakeLists.txt deleted file mode 100644 index 5a85246..0000000 --- a/backend/test/CMakeLists.txt +++ /dev/null @@ -1,94 +0,0 @@ -#============================================================================= -# Copyright (c) 2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -# keep the files in alphabetical order! -add_executable(test_developer_tools_backend - test/batch/batch.cpp - test/build_control.cpp - test/exceptions.cpp - test/memory/buffer.cpp - test/memory/detail/copy.cpp - test/memory/detail/owned_device_buffer.cpp - test/memory/resource.cpp - test/memory/types.cpp - test/tensor/dtype.cpp - test/tensor/tensor.cpp - test/test.cpp - test/triton/api/execute.cpp - test/triton/api/initialize.cpp - test/triton/api/instance_finalize.cpp - test/triton/api/instance_initialize.cpp - test/triton/api/model_finalize.cpp - test/triton/api/model_initialize.cpp - test/triton/backend.cpp - test/triton/config.cpp - test/triton/deployment.cpp - test/triton/device.cpp - test/triton/input.cpp - test/triton/logging.cpp - test/triton/model.cpp - test/triton/model_instance.cpp - test/triton/requests.cpp - test/triton/responses.cpp - test/triton/statistics.cpp - test/utils/const_agnostic.cpp - test/utils/narrow.cpp -) - -IF(TRITON_ENABLE_GPU) - set_target_properties(test_developer_tools_backend - PROPERTIES BUILD_RPATH "\$ORIGIN" - # set target compile options - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON - ) -else() - set_target_properties(test_developer_tools_backend - PROPERTIES BUILD_RPATH "\$ORIGIN" - # set target compile options - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON - ) -endif() - -target_compile_options(test_developer_tools_backend - PRIVATE "$<$:${DEVELOPER_TOOLS_BACKEND_CXX_FLAGS}>" - "$<$:${DEVELOPER_TOOLS_BACKEND_CUDA_FLAGS}>" -) - -target_include_directories(test_developer_tools_backend - PUBLIC "$" - "$" -) - -target_link_libraries(test_developer_tools_backend -PRIVATE - $<$:rmm::rmm> - $<$:raft::raft> - triton-core-serverstub - triton-backend-utils - gmock - gmock_main - GTest::gtest - GTest::gtest_main - $ -) diff --git a/backend/test/batch/batch.cpp b/backend/test/batch/batch.cpp deleted file mode 100644 index 7c9dd23..0000000 --- a/backend/test/batch/batch.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/build_control.cpp b/backend/test/build_control.cpp deleted file mode 100644 index cae9647..0000000 --- a/backend/test/build_control.cpp +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -namespace triton { -namespace developer_tools { -namespace backend { - -TEST(BackendTools, build_control) -{ -#ifdef TRITON_ENABLE_GPU - ASSERT_EQ(IS_GPU_BUILD, true) << "IS_GPU_BUILD constant has wrong value\n"; -#else - ASSERT_EQ(IS_GPU_BUILD, false) << "IS_GPU_BUILD constant has wrong value\n"; -#endif -} -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/test/exceptions.cpp b/backend/test/exceptions.cpp deleted file mode 100644 index 9544be3..0000000 --- a/backend/test/exceptions.cpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifdef TRITON_ENABLE_GPU -#include -#else -#include -#endif -#include - -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { - -TEST(BackendTools, default_except) -{ - try { - throw TritonException(); - } catch (TritonException const& err) { - EXPECT_EQ(std::string(err.what()), std::string("encountered unknown error")); - } -} - -TEST(BackendTools, msg_except) -{ - auto msg = std::string("TEST ERROR MESSAGE"); - try { - throw TritonException(Error::Internal, msg); - } catch (TritonException const& err) { - EXPECT_EQ(std::string(err.what()), msg); - } - try { - throw TritonException(Error::Internal, msg.c_str()); - } catch (TritonException const& err) { - EXPECT_EQ(std::string(err.what()), msg); - } - try { - throw TritonException(Error::Internal, msg); - } catch (TritonException const& err) { - try { - throw(TritonException(err.error())); - } catch (TritonException const& err2) { - EXPECT_EQ(std::string(err2.what()), msg); - } - } -} - -TEST(BackendTools, triton_check) -{ - auto msg = std::string("TEST ERROR MESSAGE"); - EXPECT_THROW(triton_check(TRITONSERVER_ErrorNew(Error::Internal, msg.c_str())), TritonException); - triton_check(nullptr); -} - -TEST(BackendTools, cuda_check) -{ -#ifdef TRITON_ENABLE_GPU - EXPECT_THROW(cuda_check(cudaError::cudaErrorMissingConfiguration), TritonException); - cuda_check(cudaError::cudaSuccess); -#else - EXPECT_THROW(cuda_check(cudaError::cudaErrorNonGpuBuild), TritonException); -#endif -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/test/memory/buffer.cpp b/backend/test/memory/buffer.cpp deleted file mode 100644 index 798d68f..0000000 --- a/backend/test/memory/buffer.cpp +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifdef TRITON_ENABLE_GPU -#include -#endif -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -TEST(BackendTools, default_buffer) -{ - auto buffer = Buffer(); - EXPECT_EQ(buffer.mem_type(), HostMemory); - EXPECT_EQ(buffer.size(), 0); - EXPECT_EQ(buffer.data(), nullptr); - EXPECT_EQ(buffer.device(), 0); - EXPECT_EQ(buffer.stream(), cudaStream_t{}); -#ifdef TRITON_ENABLE_GPU - auto stream = cudaStream_t{}; - cudaStreamCreate(&stream); - buffer.set_stream(stream); - EXPECT_EQ(buffer.stream(), stream); - cudaStreamDestroy(stream); -#endif -} - -TEST(BackendTools, device_buffer) -{ - auto data = std::vector{1, 2, 3}; -#ifdef TRITON_ENABLE_GPU - auto buffer = Buffer(data.size(), DeviceMemory, 0, 0); - - ASSERT_EQ(buffer.mem_type(), DeviceMemory); - ASSERT_EQ(buffer.size(), data.size()); - ASSERT_NE(buffer.data(), nullptr); - - auto data_out = std::vector(data.size()); - cudaMemcpy(static_cast(buffer.data()), - static_cast(data.data()), - sizeof(int) * data.size(), - cudaMemcpyHostToDevice); - cudaMemcpy(static_cast(data_out.data()), - static_cast(buffer.data()), - sizeof(int) * data.size(), - cudaMemcpyDeviceToHost); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); - -#else - EXPECT_THROW(Buffer(data.size(), DeviceMemory, 0, 0), TritonException); -#endif -} - -TEST(BackendTools, non_owning_device_buffer) -{ - auto data = std::vector{1, 2, 3}; -#ifdef TRITON_ENABLE_GPU - auto* ptr_d = static_cast(nullptr); - cudaMalloc(reinterpret_cast(&ptr_d), sizeof(int) * data.size()); - cudaMemcpy(static_cast(ptr_d), - static_cast(data.data()), - sizeof(int) * data.size(), - cudaMemcpyHostToDevice); - auto buffer = Buffer(ptr_d, data.size(), DeviceMemory); - - ASSERT_EQ(buffer.mem_type(), DeviceMemory); - ASSERT_EQ(buffer.size(), data.size()); - ASSERT_EQ(buffer.data(), ptr_d); - - auto data_out = std::vector(data.size()); - cudaMemcpy(static_cast(data_out.data()), - static_cast(buffer.data()), - sizeof(int) * data.size(), - cudaMemcpyDeviceToHost); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); - - cudaFree(reinterpret_cast(ptr_d)); -#else - ASSERT_THROW(Buffer(data.data(), data.size(), DeviceMemory), TritonException); -#endif -} - -TEST(BackendTools, host_buffer) -{ - auto data = std::vector{1, 2, 3}; - auto buffer = Buffer(data.size(), HostMemory, 0, 0); - - ASSERT_EQ(buffer.mem_type(), HostMemory); - ASSERT_EQ(buffer.size(), data.size()); - ASSERT_NE(buffer.data(), nullptr); - - std::memcpy( - static_cast(buffer.data()), static_cast(data.data()), data.size() * sizeof(int)); - - auto data_out = std::vector(buffer.data(), buffer.data() + buffer.size()); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); -} - -TEST(BackendTools, non_owning_host_buffer) -{ - auto data = std::vector{1, 2, 3}; - auto buffer = Buffer(data.data(), data.size(), HostMemory); - - ASSERT_EQ(buffer.mem_type(), HostMemory); - ASSERT_EQ(buffer.size(), data.size()); - ASSERT_EQ(buffer.data(), data.data()); - - auto data_out = std::vector(buffer.data(), buffer.data() + buffer.size()); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); -} - -TEST(BackendTools, copy_buffer) -{ - auto data = std::vector{1, 2, 3}; - auto orig_buffer = Buffer(data.data(), data.size(), HostMemory); - auto buffer = Buffer(orig_buffer); - - ASSERT_EQ(buffer.mem_type(), HostMemory); - ASSERT_EQ(buffer.size(), data.size()); - ASSERT_NE(buffer.data(), orig_buffer.data()); - - auto data_out = std::vector(buffer.data(), buffer.data() + buffer.size()); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); -} - -TEST(BackendTools, move_buffer) -{ - auto data = std::vector{1, 2, 3}; - auto buffer = Buffer(Buffer(data.data(), data.size(), HostMemory)); - - ASSERT_EQ(buffer.mem_type(), HostMemory); - ASSERT_EQ(buffer.size(), data.size()); - ASSERT_EQ(buffer.data(), data.data()); - - auto data_out = std::vector(buffer.data(), buffer.data() + buffer.size()); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); -} - -TEST(BackendTools, move_assignment_buffer) -{ - auto data = std::vector{1, 2, 3}; - -#ifdef TRITON_ENABLE_GPU - auto buffer = Buffer{data.data(), data.size() - 1, DeviceMemory}; -#else - auto buffer = Buffer{data.data(), data.size() - 1, HostMemory}; -#endif - buffer = Buffer{data.size(), HostMemory}; - - ASSERT_EQ(buffer.mem_type(), HostMemory); - ASSERT_EQ(buffer.size(), data.size()); -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/test/memory/detail/copy.cpp b/backend/test/memory/detail/copy.cpp deleted file mode 100644 index 41edbf5..0000000 --- a/backend/test/memory/detail/copy.cpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifdef TRITON_ENABLE_GPU -#include -#include -#else -#include -#endif -#include -#include - -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -TEST(BackendTools, copy) -{ - auto data = std::vector{1, 2, 3}; - auto data_out = std::vector(data.size()); - detail::copy(data_out.data(), data.data(), data.size(), 0, HostMemory, HostMemory); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); - - data_out = std::vector(data.size()); -#ifdef TRITON_ENABLE_GPU - auto* ptr_d = static_cast(nullptr); - cudaMalloc(reinterpret_cast(&ptr_d), sizeof(int) * data.size()); - detail::copy(ptr_d, data.data(), data.size(), 0, DeviceMemory, HostMemory); - - cudaMemcpy(static_cast(data_out.data()), - static_cast(ptr_d), - sizeof(int) * data.size(), - cudaMemcpyDeviceToHost); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); - cudaFree(reinterpret_cast(ptr_d)); -#else - EXPECT_THROW(detail::copy(data_out.data(), data.data(), data.size(), 0, HostMemory, DeviceMemory), - TritonException); - EXPECT_THROW(detail::copy(data_out.data(), data.data(), data.size(), 0, DeviceMemory, HostMemory), - TritonException); -#endif -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/test/memory/detail/owned_device_buffer.cpp b/backend/test/memory/detail/owned_device_buffer.cpp deleted file mode 100644 index 41a2351..0000000 --- a/backend/test/memory/detail/owned_device_buffer.cpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifdef TRITON_ENABLE_GPU -#include -#include -#else -#include -#endif -#include -#include - -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -TEST(BackendTools, owned_device_buffer) -{ - auto data = std::vector{1, 2, 3}; -#ifdef TRITON_ENABLE_GPU - auto device_id = 0; - cudaGetDevice(&device_id); - auto stream = cudaStream_t{}; - cudaStreamCreate(&stream); - - auto buffer = detail::owned_device_buffer(device_id, data.size(), stream); - auto data_out = std::vector(data.size()); - cudaMemcpy(static_cast(buffer.get()), - static_cast(data.data()), - sizeof(int) * data.size(), - cudaMemcpyHostToDevice); - cudaMemcpy(static_cast(data_out.data()), - static_cast(buffer.get()), - sizeof(int) * data.size(), - cudaMemcpyDeviceToHost); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); - cudaStreamDestroy(stream); -#else - // Workaround for ungraceful handling of multiple template parameters in - // EXPECT_THROW - using dev_buffer = detail::owned_device_buffer; - EXPECT_THROW(dev_buffer(0, data.size(), 0), TritonException); -#endif -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/test/memory/resource.cpp b/backend/test/memory/resource.cpp deleted file mode 100644 index 487be1d..0000000 --- a/backend/test/memory/resource.cpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifdef TRITON_ENABLE_GPU -#include -#include -#include -#include -#endif - -#include -#include - -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { - -TEST(BackendTools, set_memory_resource) -{ -#ifdef TRITON_ENABLE_GPU - auto device_id = int{}; - cuda_check(cudaGetDevice(&device_id)); - EXPECT_EQ(rmm::mr::get_current_device_resource()->is_equal(rmm::mr::cuda_memory_resource{}), - true); - setup_memory_resource(device_id); - EXPECT_EQ(rmm::mr::get_current_device_resource()->is_equal(rmm::mr::cuda_memory_resource{}), - false); -#else - setup_memory_resource(0); -#endif -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/test/memory/types.cpp b/backend/test/memory/types.cpp deleted file mode 100644 index f0fdecc..0000000 --- a/backend/test/memory/types.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/tensor/dtype.cpp b/backend/test/tensor/dtype.cpp deleted file mode 100644 index 21de4f8..0000000 --- a/backend/test/tensor/dtype.cpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -namespace triton { -namespace developer_tools { -namespace backend { - -template -void check_dtype_conversion() -{ - EXPECT_EQ(D, TritonDtype::type>::value); - EXPECT_EQ(D, TritonDtype::type const>::value); -} - -TEST(BackendTools, dtype) -{ - check_dtype_conversion(); - check_dtype_conversion(); - check_dtype_conversion(); - check_dtype_conversion(); - check_dtype_conversion(); - check_dtype_conversion(); - check_dtype_conversion(); - check_dtype_conversion(); - check_dtype_conversion(); - check_dtype_conversion(); - check_dtype_conversion(); - check_dtype_conversion(); - check_dtype_conversion(); -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/test/tensor/tensor.cpp b/backend/test/tensor/tensor.cpp deleted file mode 100644 index 4418dbd..0000000 --- a/backend/test/tensor/tensor.cpp +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifdef TRITON_ENABLE_GPU -#include -#else -#include -#endif -#include -#include - -#include -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { - -TEST(BackendTools, default_tensor) -{ - auto tensor = Tensor(); - EXPECT_EQ(tensor.buffer().size(), 0); - EXPECT_EQ(tensor.shape().size(), 0); -} - -TEST(BackendTools, move_buffer_tensor) -{ - auto shape = std::vector{2, 2}; - auto data = std::vector{1, 2, 3, 4}; - auto tensor = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); - - EXPECT_EQ(data.data(), tensor.data()); - EXPECT_EQ(data.size(), tensor.size()); - EXPECT_THAT(tensor.shape(), ::testing::ElementsAreArray(shape)); - - EXPECT_EQ(tensor.dtype(), DTypeInt32); - EXPECT_EQ(tensor.mem_type(), HostMemory); - EXPECT_EQ(tensor.stream(), cudaStream_t{}); - EXPECT_EQ(tensor.device(), 0); - - auto data_out = std::vector(tensor.data(), tensor.data() + tensor.size()); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); -} - -TEST(BackendTools, multi_buffer_tensor) -{ - auto shape = std::vector{2, 2}; - auto data = std::vector{1, 2, 3, 4}; - - auto all_buffers = std::vector>{}; - all_buffers.reserve(data.size()); - auto mem_type = HostMemory; - if constexpr (IS_GPU_BUILD) { mem_type = DeviceMemory; } - std::transform(data.begin(), data.end(), std::back_inserter(all_buffers), [mem_type](auto& elem) { - return Buffer{&elem, 1, mem_type}; - }); - auto tensor = - Tensor(shape, all_buffers.begin(), all_buffers.end(), mem_type, 0, cudaStream_t{}); - - auto data_out = std::vector(data.size()); -#ifdef TRITON_ENABLE_GPU - cudaMemcpy(static_cast(data_out.data()), - static_cast(tensor.data()), - sizeof(int) * tensor.size(), - cudaMemcpyDeviceToHost); -#else - std::memcpy(static_cast(data_out.data()), - static_cast(tensor.data()), - sizeof(int) * tensor.size()); -#endif - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); -} - -TEST(BackendTools, tensor_copy) -{ - auto shape = std::vector{2, 2}; - auto data = std::vector{1, 2, 3, 4}; - - auto data1 = data; - auto tensor1 = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); - auto data2 = std::vector(data1.size()); - auto tensor2 = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); - - backend::copy(tensor2, tensor1); - - auto data_out = std::vector(tensor2.data(), tensor2.data() + tensor2.size()); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); - - auto small_shape = std::vector{2}; - auto small_data = std::vector(2); - auto tensor3 = - Tensor(small_shape, Buffer{small_data.data(), small_data.size(), HostMemory}); - - EXPECT_THROW(backend::copy(tensor3, tensor1), TritonException); -} - -TEST(BackendTools, tensor_multi_copy) -{ - auto shape = std::vector{2, 2}; - auto data = std::vector{1, 2, 3, 4}; - - auto data1 = data; - auto tensor1 = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); - - auto receiver_shape = std::vector{1}; - auto receivers = std::vector>{}; - - receivers.reserve(data.size()); - std::transform( - data.begin(), data.end(), std::back_inserter(receivers), [&receiver_shape](auto& val) { - return Tensor(receiver_shape, Buffer{std::size_t{1}, HostMemory}); - }); - - backend::copy(receivers.begin(), receivers.end(), tensor1); - - auto data_out = std::vector{}; - data_out.reserve(receivers.size()); - std::transform( - receivers.begin(), receivers.end(), std::back_inserter(data_out), [](auto& tensor) { - return *tensor.data(); - }); - EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); - - // Throw if trying to copy to too many outputs - receivers.emplace_back(receiver_shape, Buffer{std::size_t{1}, HostMemory}); - EXPECT_THROW(backend::copy(receivers.begin(), receivers.end(), tensor1), TritonException); -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/test/test.cpp b/backend/test/test.cpp deleted file mode 100644 index 15c9dfc..0000000 --- a/backend/test/test.cpp +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -namespace triton { -namespace developer_tools { -namespace backend { - -TEST(BackendTools, installed) { std::cout << test_install() << "\n"; } -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/test/triton/api/execute.cpp b/backend/test/triton/api/execute.cpp deleted file mode 100644 index 41a496f..0000000 --- a/backend/test/triton/api/execute.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/triton/api/initialize.cpp b/backend/test/triton/api/initialize.cpp deleted file mode 100644 index ebb4f6d..0000000 --- a/backend/test/triton/api/initialize.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/triton/api/instance_finalize.cpp b/backend/test/triton/api/instance_finalize.cpp deleted file mode 100644 index 75a5b0d..0000000 --- a/backend/test/triton/api/instance_finalize.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/triton/api/instance_initialize.cpp b/backend/test/triton/api/instance_initialize.cpp deleted file mode 100644 index 356efd1..0000000 --- a/backend/test/triton/api/instance_initialize.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/triton/api/model_finalize.cpp b/backend/test/triton/api/model_finalize.cpp deleted file mode 100644 index 7e08b57..0000000 --- a/backend/test/triton/api/model_finalize.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/triton/api/model_initialize.cpp b/backend/test/triton/api/model_initialize.cpp deleted file mode 100644 index a0cc5b1..0000000 --- a/backend/test/triton/api/model_initialize.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/triton/backend.cpp b/backend/test/triton/backend.cpp deleted file mode 100644 index 8bff317..0000000 --- a/backend/test/triton/backend.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/triton/config.cpp b/backend/test/triton/config.cpp deleted file mode 100644 index 1ec808e..0000000 --- a/backend/test/triton/config.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/triton/deployment.cpp b/backend/test/triton/deployment.cpp deleted file mode 100644 index d9f9e66..0000000 --- a/backend/test/triton/deployment.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/triton/device.cpp b/backend/test/triton/device.cpp deleted file mode 100644 index 6fcca89..0000000 --- a/backend/test/triton/device.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/triton/input.cpp b/backend/test/triton/input.cpp deleted file mode 100644 index 8269e95..0000000 --- a/backend/test/triton/input.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/triton/logging.cpp b/backend/test/triton/logging.cpp deleted file mode 100644 index 53fbce0..0000000 --- a/backend/test/triton/logging.cpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -TEST(BackendTools, logging) -{ - log_debug("Debug test message"); - log_info("Info test message"); - log_warn("Warn test message"); - log_error("Error test message"); -} - -TEST(BackendTools, stream_logging) -{ - log_debug() << "Streamed debug test message"; - log_info() << "Streamed info test message"; - log_warn() << "Streamed warn test message"; - log_error() << "Streamed error test message"; -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/test/triton/model.cpp b/backend/test/triton/model.cpp deleted file mode 100644 index a8263be..0000000 --- a/backend/test/triton/model.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/triton/model_instance.cpp b/backend/test/triton/model_instance.cpp deleted file mode 100644 index fcfbbbd..0000000 --- a/backend/test/triton/model_instance.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/triton/requests.cpp b/backend/test/triton/requests.cpp deleted file mode 100644 index ecef9f8..0000000 --- a/backend/test/triton/requests.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/triton/responses.cpp b/backend/test/triton/responses.cpp deleted file mode 100644 index 34b64ff..0000000 --- a/backend/test/triton/responses.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/triton/statistics.cpp b/backend/test/triton/statistics.cpp deleted file mode 100644 index 1d82994..0000000 --- a/backend/test/triton/statistics.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include diff --git a/backend/test/utils/const_agnostic.cpp b/backend/test/utils/const_agnostic.cpp deleted file mode 100644 index 2d3f6aa..0000000 --- a/backend/test/utils/const_agnostic.cpp +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -TEST(BackendTools, const_agnostic) -{ - static_assert(std::is_same, void>::value); - static_assert(std::is_same, void>::value); -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton diff --git a/backend/test/utils/narrow.cpp b/backend/test/utils/narrow.cpp deleted file mode 100644 index 20caf63..0000000 --- a/backend/test/utils/narrow.cpp +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include - -namespace triton { -namespace developer_tools { -namespace backend { -TEST(BackendTools, narrow) -{ - EXPECT_THROW(narrow(-1), TritonException); - narrow(int{5}); - EXPECT_THROW(narrow(std::numeric_limits::max()), TritonException); - narrow(std::size_t{5}); -} - -} // namespace backend -} // namespace developer_tools -} // namespace triton From 5720f3059844e6525bbe57cc9c9ea48b58ee0d33 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 7 Oct 2022 13:52:09 -0400 Subject: [PATCH 172/199] Remove redundant test script --- qa/L0_backend_unit_test/test.sh | 75 --------------------------------- 1 file changed, 75 deletions(-) delete mode 100644 qa/L0_backend_unit_test/test.sh diff --git a/qa/L0_backend_unit_test/test.sh b/qa/L0_backend_unit_test/test.sh deleted file mode 100644 index 861f5b9..0000000 --- a/qa/L0_backend_unit_test/test.sh +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -export CUDA_VISIBLE_DEVICES=0 - -# fulfill build dependencies -set +e -cmake --version -if [ $? -ne 0 ]; then - echo "cmake not found, installing cmake " - wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \ - gpg --dearmor - | \ - tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \ - apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - cmake-data=3.21.1-0kitware1ubuntu20.04.1 cmake=3.21.1-0kitware1ubuntu20.04.1 -fi -set -e - -apt-get update && \ -apt-get install -y --no-install-recommends \ - rapidjson-dev - -# build and run the unit test for developer_tools::backend -rm -rf ./build -mkdir build && -(cd build && - git clone --single-branch --depth=1 -b ${TRITON_DEVELOPER_TOOLS_REPO_TAG} \ - https://github.com/triton-inference-server/developer_tools.git && \ - cmake developer_tools/backend && make install) - -TEST_LOG=test.log -RET=0 - -set +e -# Must explicitly set LD_LIBRARY_PATH so that the test can find -# libtritonserver.so. -LD_LIBRARY_PATH=/opt/tritonserver/lib:${LD_LIBRARY_PATH} ./build/test_developer_tools_backend >> ${TEST_LOG} 2>&1 -if [ $? -ne 0 ]; then - cat ${TEST_LOG} - RET=1 -fi -set -e - -if [ $RET -eq 0 ]; then - echo -e "\n***\n*** Test Passed\n***" -else - echo -e "\n***\n*** Test FAILED\n***" -fi - -exit $RET From 0ee9531a90163c5f3bd974127ac292606ee9ab9e Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 7 Oct 2022 14:37:39 -0400 Subject: [PATCH 173/199] Move linear example to examples directory --- CMakeLists.txt => examples/backend_linear/CMakeLists.txt | 0 Dockerfile => examples/backend_linear/Dockerfile | 0 README.md => examples/backend_linear/README.md | 0 .../backend_linear/cmake}/modules/ConfigureCUDA.cmake | 0 .../backend_linear/cmake}/thirdparty/get_gtest.cmake | 0 .../backend_linear/cmake}/thirdparty/get_rapids-triton.cmake | 0 .../conda}/environments/rapids_triton_dev_cuda11.4.yml | 0 .../backend_linear/conda}/environments/rapids_triton_test.yml | 0 {src => examples/backend_linear/src}/api.cc | 0 {src => examples/backend_linear/src}/gpu_infer.cu | 0 {src => examples/backend_linear/src}/gpu_infer.h | 0 {src => examples/backend_linear/src}/model.h | 0 {src => examples/backend_linear/src}/names.h | 0 {src => examples/backend_linear/src}/shared_state.h | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename CMakeLists.txt => examples/backend_linear/CMakeLists.txt (100%) rename Dockerfile => examples/backend_linear/Dockerfile (100%) rename README.md => examples/backend_linear/README.md (100%) rename {cmake => examples/backend_linear/cmake}/modules/ConfigureCUDA.cmake (100%) rename {cmake => examples/backend_linear/cmake}/thirdparty/get_gtest.cmake (100%) rename {cmake => examples/backend_linear/cmake}/thirdparty/get_rapids-triton.cmake (100%) rename {conda => examples/backend_linear/conda}/environments/rapids_triton_dev_cuda11.4.yml (100%) rename {conda => examples/backend_linear/conda}/environments/rapids_triton_test.yml (100%) rename {src => examples/backend_linear/src}/api.cc (100%) rename {src => examples/backend_linear/src}/gpu_infer.cu (100%) rename {src => examples/backend_linear/src}/gpu_infer.h (100%) rename {src => examples/backend_linear/src}/model.h (100%) rename {src => examples/backend_linear/src}/names.h (100%) rename {src => examples/backend_linear/src}/shared_state.h (100%) diff --git a/CMakeLists.txt b/examples/backend_linear/CMakeLists.txt similarity index 100% rename from CMakeLists.txt rename to examples/backend_linear/CMakeLists.txt diff --git a/Dockerfile b/examples/backend_linear/Dockerfile similarity index 100% rename from Dockerfile rename to examples/backend_linear/Dockerfile diff --git a/README.md b/examples/backend_linear/README.md similarity index 100% rename from README.md rename to examples/backend_linear/README.md diff --git a/cmake/modules/ConfigureCUDA.cmake b/examples/backend_linear/cmake/modules/ConfigureCUDA.cmake similarity index 100% rename from cmake/modules/ConfigureCUDA.cmake rename to examples/backend_linear/cmake/modules/ConfigureCUDA.cmake diff --git a/cmake/thirdparty/get_gtest.cmake b/examples/backend_linear/cmake/thirdparty/get_gtest.cmake similarity index 100% rename from cmake/thirdparty/get_gtest.cmake rename to examples/backend_linear/cmake/thirdparty/get_gtest.cmake diff --git a/cmake/thirdparty/get_rapids-triton.cmake b/examples/backend_linear/cmake/thirdparty/get_rapids-triton.cmake similarity index 100% rename from cmake/thirdparty/get_rapids-triton.cmake rename to examples/backend_linear/cmake/thirdparty/get_rapids-triton.cmake diff --git a/conda/environments/rapids_triton_dev_cuda11.4.yml b/examples/backend_linear/conda/environments/rapids_triton_dev_cuda11.4.yml similarity index 100% rename from conda/environments/rapids_triton_dev_cuda11.4.yml rename to examples/backend_linear/conda/environments/rapids_triton_dev_cuda11.4.yml diff --git a/conda/environments/rapids_triton_test.yml b/examples/backend_linear/conda/environments/rapids_triton_test.yml similarity index 100% rename from conda/environments/rapids_triton_test.yml rename to examples/backend_linear/conda/environments/rapids_triton_test.yml diff --git a/src/api.cc b/examples/backend_linear/src/api.cc similarity index 100% rename from src/api.cc rename to examples/backend_linear/src/api.cc diff --git a/src/gpu_infer.cu b/examples/backend_linear/src/gpu_infer.cu similarity index 100% rename from src/gpu_infer.cu rename to examples/backend_linear/src/gpu_infer.cu diff --git a/src/gpu_infer.h b/examples/backend_linear/src/gpu_infer.h similarity index 100% rename from src/gpu_infer.h rename to examples/backend_linear/src/gpu_infer.h diff --git a/src/model.h b/examples/backend_linear/src/model.h similarity index 100% rename from src/model.h rename to examples/backend_linear/src/model.h diff --git a/src/names.h b/examples/backend_linear/src/names.h similarity index 100% rename from src/names.h rename to examples/backend_linear/src/names.h diff --git a/src/shared_state.h b/examples/backend_linear/src/shared_state.h similarity index 100% rename from src/shared_state.h rename to examples/backend_linear/src/shared_state.h From b497f914faef1f784199972281e81c8e55f658c9 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 7 Oct 2022 14:40:40 -0400 Subject: [PATCH 174/199] Restore README --- README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..6ddaddc --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# triton_developer_tools From fb5a3dbee4d6526f6f71f7f65c3a0c97ab789034 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Fri, 7 Oct 2022 14:41:41 -0400 Subject: [PATCH 175/199] Correct global CODEOWNER configuration --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 76bb196..4bbce44 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,2 @@ -/ @tanmayv25 @GuanLuo +* @tanmayv25 @GuanLuo /backend @wphicks @dantegd @divyegala From cb46f121300c12f347ac430cae94abd2bb38f91b Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 10:31:00 -0400 Subject: [PATCH 176/199] Replace rapids_triton with triton_backend --- .../{rapids_triton.hpp => triton_backend.hpp} | 6 ++-- .../batch/batch.hpp | 28 +++++++++---------- .../build_control.hpp | 0 .../cpu_only/cuda_runtime_replacement.hpp | 0 .../exceptions.hpp | 4 +-- .../memory/buffer.hpp | 22 +++++++-------- .../memory/detail/cpu_only/copy.hpp | 6 ++-- .../detail/cpu_only/owned_device_buffer.hpp | 8 +++--- .../memory/detail/cpu_only/resource.hpp | 4 +-- .../memory/detail/gpu_only/copy.hpp | 4 +-- .../detail/gpu_only/owned_device_buffer.hpp | 6 ++-- .../memory/detail/gpu_only/resource.hpp | 8 +++--- .../memory/detail/owned_device_buffer.hpp | 0 .../memory/detail/resource.hpp | 2 +- .../memory/resource.hpp | 10 +++---- .../memory/types.hpp | 0 .../model/model.hpp | 16 +++++------ .../model/shared_state.hpp | 12 ++++---- .../tensor/dtype.hpp | 2 +- .../tensor/tensor.hpp | 18 ++++++------ .../triton/api/execute.hpp | 12 ++++---- .../triton/api/initialize.hpp | 14 +++++----- .../triton/api/instance_finalize.hpp | 6 ++-- .../triton/api/instance_initialize.hpp | 12 ++++---- .../triton/api/model_finalize.hpp | 6 ++-- .../triton/api/model_initialize.hpp | 6 ++-- .../triton/backend.hpp | 4 +-- .../triton/config.hpp | 4 +-- .../triton/deployment.hpp | 0 .../triton/device.hpp | 0 .../triton/input.hpp | 6 ++-- .../triton/logging.hpp | 2 +- .../triton/model.hpp | 2 +- .../triton/model_instance.hpp | 6 ++-- .../triton/model_instance_state.hpp | 4 +-- .../triton/model_state.hpp | 2 +- .../triton/output.hpp | 2 +- .../triton/requests.hpp | 4 +-- .../triton/responses.hpp | 4 +-- .../triton/statistics.hpp | 2 +- .../triton/triton_memory_resource.hpp | 4 +-- .../utils/const_agnostic.hpp | 0 .../utils/device_setter.hpp | 4 +-- .../utils/narrow.hpp | 2 +- 44 files changed, 132 insertions(+), 132 deletions(-) rename backend/cpp/include/{rapids_triton.hpp => triton_backend.hpp} (78%) rename backend/cpp/include/{rapids_triton => triton_backend}/batch/batch.hpp (93%) rename backend/cpp/include/{rapids_triton => triton_backend}/build_control.hpp (100%) rename backend/cpp/include/{rapids_triton => triton_backend}/cpu_only/cuda_runtime_replacement.hpp (100%) rename backend/cpp/include/{rapids_triton => triton_backend}/exceptions.hpp (96%) rename backend/cpp/include/{rapids_triton => triton_backend}/memory/buffer.hpp (94%) rename backend/cpp/include/{rapids_triton => triton_backend}/memory/detail/cpu_only/copy.hpp (89%) rename backend/cpp/include/{rapids_triton => triton_backend}/memory/detail/cpu_only/owned_device_buffer.hpp (85%) rename backend/cpp/include/{rapids_triton => triton_backend}/memory/detail/cpu_only/resource.hpp (91%) rename backend/cpp/include/{rapids_triton => triton_backend}/memory/detail/gpu_only/copy.hpp (94%) rename backend/cpp/include/{rapids_triton => triton_backend}/memory/detail/gpu_only/owned_device_buffer.hpp (89%) rename backend/cpp/include/{rapids_triton => triton_backend}/memory/detail/gpu_only/resource.hpp (93%) rename backend/cpp/include/{rapids_triton => triton_backend}/memory/detail/owned_device_buffer.hpp (100%) rename backend/cpp/include/{rapids_triton => triton_backend}/memory/detail/resource.hpp (95%) rename backend/cpp/include/{rapids_triton => triton_backend}/memory/resource.hpp (79%) rename backend/cpp/include/{rapids_triton => triton_backend}/memory/types.hpp (100%) rename backend/cpp/include/{rapids_triton => triton_backend}/model/model.hpp (94%) rename backend/cpp/include/{rapids_triton => triton_backend}/model/shared_state.hpp (95%) rename backend/cpp/include/{rapids_triton => triton_backend}/tensor/dtype.hpp (98%) rename backend/cpp/include/{rapids_triton => triton_backend}/tensor/tensor.hpp (92%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/api/execute.hpp (94%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/api/initialize.hpp (85%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/api/instance_finalize.hpp (90%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/api/instance_initialize.hpp (89%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/api/model_finalize.hpp (91%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/api/model_initialize.hpp (92%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/backend.hpp (95%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/config.hpp (92%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/deployment.hpp (100%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/device.hpp (100%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/input.hpp (95%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/logging.hpp (99%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/model.hpp (98%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/model_instance.hpp (96%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/model_instance_state.hpp (94%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/model_state.hpp (96%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/output.hpp (98%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/requests.hpp (93%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/responses.hpp (96%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/statistics.hpp (98%) rename backend/cpp/include/{rapids_triton => triton_backend}/triton/triton_memory_resource.hpp (97%) rename backend/cpp/include/{rapids_triton => triton_backend}/utils/const_agnostic.hpp (100%) rename backend/cpp/include/{rapids_triton => triton_backend}/utils/device_setter.hpp (93%) rename backend/cpp/include/{rapids_triton => triton_backend}/utils/narrow.hpp (96%) diff --git a/backend/cpp/include/rapids_triton.hpp b/backend/cpp/include/triton_backend.hpp similarity index 78% rename from backend/cpp/include/rapids_triton.hpp rename to backend/cpp/include/triton_backend.hpp index 16cf368..7062170 100644 --- a/backend/cpp/include/rapids_triton.hpp +++ b/backend/cpp/include/triton_backend.hpp @@ -21,10 +21,10 @@ namespace triton { namespace backend { namespace rapids { -/* Function for testing rapids_triton include +/* Function for testing triton_backend include * - * @return message indicating rapids_triton has been included succesfully*/ -inline auto test_install() { return std::string("rapids_triton set up successfully"); } + * @return message indicating triton_backend has been included succesfully*/ +inline auto test_install() { return std::string("triton_backend set up successfully"); } } // namespace rapids } // namespace backend diff --git a/backend/cpp/include/rapids_triton/batch/batch.hpp b/backend/cpp/include/triton_backend/batch/batch.hpp similarity index 93% rename from backend/cpp/include/rapids_triton/batch/batch.hpp rename to backend/cpp/include/triton_backend/batch/batch.hpp index db54df8..bbcd524 100644 --- a/backend/cpp/include/rapids_triton/batch/batch.hpp +++ b/backend/cpp/include/triton_backend/batch/batch.hpp @@ -19,7 +19,7 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include #include @@ -30,17 +30,17 @@ #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -51,7 +51,7 @@ namespace rapids { * @brief A representation of all data about a single batch of inference * requests * - * Batch objects are the primary interface point between rapids_triton Models + * Batch objects are the primary interface point between triton_backend Models * and the Triton server itself. By calling the `get_input` and `get_output` * methods of a batch, Model implementations can retrieve the input Tensors * necessary for prediction and the output Tensors where results can be @@ -62,7 +62,7 @@ namespace rapids { * on how long it took to process requests and sending responses to the * client via the Triton server once processing is complete. * - * It is not recommended that developers of rapids_triton backends try to + * It is not recommended that developers of triton_backend backends try to * construct Batch objects directly. Instead, you should make use of the * rapids::triton_api::execute template, which will construct the Batch for * you. diff --git a/backend/cpp/include/rapids_triton/build_control.hpp b/backend/cpp/include/triton_backend/build_control.hpp similarity index 100% rename from backend/cpp/include/rapids_triton/build_control.hpp rename to backend/cpp/include/triton_backend/build_control.hpp diff --git a/backend/cpp/include/rapids_triton/cpu_only/cuda_runtime_replacement.hpp b/backend/cpp/include/triton_backend/cpu_only/cuda_runtime_replacement.hpp similarity index 100% rename from backend/cpp/include/rapids_triton/cpu_only/cuda_runtime_replacement.hpp rename to backend/cpp/include/triton_backend/cpu_only/cuda_runtime_replacement.hpp diff --git a/backend/cpp/include/rapids_triton/exceptions.hpp b/backend/cpp/include/triton_backend/exceptions.hpp similarity index 96% rename from backend/cpp/include/rapids_triton/exceptions.hpp rename to backend/cpp/include/triton_backend/exceptions.hpp index e2d2bc2..acdffb9 100644 --- a/backend/cpp/include/rapids_triton/exceptions.hpp +++ b/backend/cpp/include/triton_backend/exceptions.hpp @@ -18,11 +18,11 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include #include -#include +#include #include namespace triton { diff --git a/backend/cpp/include/rapids_triton/memory/buffer.hpp b/backend/cpp/include/triton_backend/memory/buffer.hpp similarity index 94% rename from backend/cpp/include/rapids_triton/memory/buffer.hpp rename to backend/cpp/include/triton_backend/memory/buffer.hpp index 124437c..92f16bd 100644 --- a/backend/cpp/include/rapids_triton/memory/buffer.hpp +++ b/backend/cpp/include/triton_backend/memory/buffer.hpp @@ -22,20 +22,20 @@ #ifdef TRITON_ENABLE_GPU #include -#include -#include +#include +#include #else -#include -#include -#include +#include +#include +#include #endif -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/memory/detail/cpu_only/copy.hpp b/backend/cpp/include/triton_backend/memory/detail/cpu_only/copy.hpp similarity index 89% rename from backend/cpp/include/rapids_triton/memory/detail/cpu_only/copy.hpp rename to backend/cpp/include/triton_backend/memory/detail/cpu_only/copy.hpp index beb5cb4..dbf47ae 100644 --- a/backend/cpp/include/rapids_triton/memory/detail/cpu_only/copy.hpp +++ b/backend/cpp/include/triton_backend/memory/detail/cpu_only/copy.hpp @@ -19,10 +19,10 @@ #include #ifndef TRITON_ENABLE_GPU -#include +#include #endif -#include -#include +#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp b/backend/cpp/include/triton_backend/memory/detail/cpu_only/owned_device_buffer.hpp similarity index 85% rename from backend/cpp/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp rename to backend/cpp/include/triton_backend/memory/detail/cpu_only/owned_device_buffer.hpp index 4006a70..1458fbe 100644 --- a/backend/cpp/include/rapids_triton/memory/detail/cpu_only/owned_device_buffer.hpp +++ b/backend/cpp/include/triton_backend/memory/detail/cpu_only/owned_device_buffer.hpp @@ -17,10 +17,10 @@ #pragma once #include #include -#include -#include -#include -#include +#include +#include +#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/memory/detail/cpu_only/resource.hpp b/backend/cpp/include/triton_backend/memory/detail/cpu_only/resource.hpp similarity index 91% rename from backend/cpp/include/rapids_triton/memory/detail/cpu_only/resource.hpp rename to backend/cpp/include/triton_backend/memory/detail/cpu_only/resource.hpp index 04bb1c5..89cb81c 100644 --- a/backend/cpp/include/rapids_triton/memory/detail/cpu_only/resource.hpp +++ b/backend/cpp/include/triton_backend/memory/detail/cpu_only/resource.hpp @@ -17,8 +17,8 @@ #pragma once #include -#include -#include +#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/memory/detail/gpu_only/copy.hpp b/backend/cpp/include/triton_backend/memory/detail/gpu_only/copy.hpp similarity index 94% rename from backend/cpp/include/rapids_triton/memory/detail/gpu_only/copy.hpp rename to backend/cpp/include/triton_backend/memory/detail/gpu_only/copy.hpp index 030b473..ff26061 100644 --- a/backend/cpp/include/rapids_triton/memory/detail/gpu_only/copy.hpp +++ b/backend/cpp/include/triton_backend/memory/detail/gpu_only/copy.hpp @@ -22,8 +22,8 @@ #include #include -#include -#include +#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp b/backend/cpp/include/triton_backend/memory/detail/gpu_only/owned_device_buffer.hpp similarity index 89% rename from backend/cpp/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp rename to backend/cpp/include/triton_backend/memory/detail/gpu_only/owned_device_buffer.hpp index 34095f7..ee88a61 100644 --- a/backend/cpp/include/rapids_triton/memory/detail/gpu_only/owned_device_buffer.hpp +++ b/backend/cpp/include/triton_backend/memory/detail/gpu_only/owned_device_buffer.hpp @@ -16,9 +16,9 @@ #pragma once #include -#include -#include -#include +#include +#include +#include #include namespace triton { diff --git a/backend/cpp/include/rapids_triton/memory/detail/gpu_only/resource.hpp b/backend/cpp/include/triton_backend/memory/detail/gpu_only/resource.hpp similarity index 93% rename from backend/cpp/include/rapids_triton/memory/detail/gpu_only/resource.hpp rename to backend/cpp/include/triton_backend/memory/detail/gpu_only/resource.hpp index f2ea115..8d40ae1 100644 --- a/backend/cpp/include/rapids_triton/memory/detail/gpu_only/resource.hpp +++ b/backend/cpp/include/triton_backend/memory/detail/gpu_only/resource.hpp @@ -19,10 +19,10 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include #include diff --git a/backend/cpp/include/rapids_triton/memory/detail/owned_device_buffer.hpp b/backend/cpp/include/triton_backend/memory/detail/owned_device_buffer.hpp similarity index 100% rename from backend/cpp/include/rapids_triton/memory/detail/owned_device_buffer.hpp rename to backend/cpp/include/triton_backend/memory/detail/owned_device_buffer.hpp diff --git a/backend/cpp/include/rapids_triton/memory/detail/resource.hpp b/backend/cpp/include/triton_backend/memory/detail/resource.hpp similarity index 95% rename from backend/cpp/include/rapids_triton/memory/detail/resource.hpp rename to backend/cpp/include/triton_backend/memory/detail/resource.hpp index cfc4bc4..5a15d90 100644 --- a/backend/cpp/include/rapids_triton/memory/detail/resource.hpp +++ b/backend/cpp/include/triton_backend/memory/detail/resource.hpp @@ -17,7 +17,7 @@ #pragma once #include -#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/memory/resource.hpp b/backend/cpp/include/triton_backend/memory/resource.hpp similarity index 79% rename from backend/cpp/include/rapids_triton/memory/resource.hpp rename to backend/cpp/include/triton_backend/memory/resource.hpp index afd7f2a..9dc33f5 100644 --- a/backend/cpp/include/rapids_triton/memory/resource.hpp +++ b/backend/cpp/include/triton_backend/memory/resource.hpp @@ -17,13 +17,13 @@ #pragma once #include -#include -#include -#include +#include +#include +#include #ifdef TRITON_ENABLE_GPU -#include +#include #else -#include +#include #endif namespace triton { diff --git a/backend/cpp/include/rapids_triton/memory/types.hpp b/backend/cpp/include/triton_backend/memory/types.hpp similarity index 100% rename from backend/cpp/include/rapids_triton/memory/types.hpp rename to backend/cpp/include/triton_backend/memory/types.hpp diff --git a/backend/cpp/include/rapids_triton/model/model.hpp b/backend/cpp/include/triton_backend/model/model.hpp similarity index 94% rename from backend/cpp/include/rapids_triton/model/model.hpp rename to backend/cpp/include/triton_backend/model/model.hpp index 3413fca..a1ec5be 100644 --- a/backend/cpp/include/rapids_triton/model/model.hpp +++ b/backend/cpp/include/triton_backend/model/model.hpp @@ -18,16 +18,16 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include #include diff --git a/backend/cpp/include/rapids_triton/model/shared_state.hpp b/backend/cpp/include/triton_backend/model/shared_state.hpp similarity index 95% rename from backend/cpp/include/rapids_triton/model/shared_state.hpp rename to backend/cpp/include/triton_backend/model/shared_state.hpp index aa0d76d..8a8b65f 100644 --- a/backend/cpp/include/rapids_triton/model/shared_state.hpp +++ b/backend/cpp/include/triton_backend/model/shared_state.hpp @@ -18,7 +18,7 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include #include @@ -29,11 +29,11 @@ #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/tensor/dtype.hpp b/backend/cpp/include/triton_backend/tensor/dtype.hpp similarity index 98% rename from backend/cpp/include/rapids_triton/tensor/dtype.hpp rename to backend/cpp/include/triton_backend/tensor/dtype.hpp index 325ef4d..1389586 100644 --- a/backend/cpp/include/rapids_triton/tensor/dtype.hpp +++ b/backend/cpp/include/triton_backend/tensor/dtype.hpp @@ -18,7 +18,7 @@ #include #include #include -#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/tensor/tensor.hpp b/backend/cpp/include/triton_backend/tensor/tensor.hpp similarity index 92% rename from backend/cpp/include/rapids_triton/tensor/tensor.hpp rename to backend/cpp/include/triton_backend/tensor/tensor.hpp index 80f2ba4..e4fab28 100644 --- a/backend/cpp/include/rapids_triton/tensor/tensor.hpp +++ b/backend/cpp/include/triton_backend/tensor/tensor.hpp @@ -27,16 +27,16 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include namespace triton { namespace backend { @@ -141,10 +141,10 @@ struct OutputTensor final : BaseTensor { * @brief Prepare final output data from this tensor for responding to * request * - * This method *must* be called by rapids_triton backends on all of their + * This method *must* be called by triton_backend backends on all of their * output tensors before returning from their `predict` methods. Because we * cannot know a priori what names backends might have for their tensors - * and what types will be stored in those tensors, the rapids_triton + * and what types will be stored in those tensors, the triton_backend * library cannot store references to those tensors that might otherwise be * used to finalize them. */ diff --git a/backend/cpp/include/rapids_triton/triton/api/execute.hpp b/backend/cpp/include/triton_backend/triton/api/execute.hpp similarity index 94% rename from backend/cpp/include/rapids_triton/triton/api/execute.hpp rename to backend/cpp/include/triton_backend/triton/api/execute.hpp index f324065..d9ce7c9 100644 --- a/backend/cpp/include/rapids_triton/triton/api/execute.hpp +++ b/backend/cpp/include/triton_backend/triton/api/execute.hpp @@ -20,12 +20,12 @@ #include #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include namespace triton { diff --git a/backend/cpp/include/rapids_triton/triton/api/initialize.hpp b/backend/cpp/include/triton_backend/triton/api/initialize.hpp similarity index 85% rename from backend/cpp/include/rapids_triton/triton/api/initialize.hpp rename to backend/cpp/include/triton_backend/triton/api/initialize.hpp index 37824dd..2b1ebb2 100644 --- a/backend/cpp/include/rapids_triton/triton/api/initialize.hpp +++ b/backend/cpp/include/triton_backend/triton/api/initialize.hpp @@ -18,16 +18,16 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include namespace triton { diff --git a/backend/cpp/include/rapids_triton/triton/api/instance_finalize.hpp b/backend/cpp/include/triton_backend/triton/api/instance_finalize.hpp similarity index 90% rename from backend/cpp/include/rapids_triton/triton/api/instance_finalize.hpp rename to backend/cpp/include/triton_backend/triton/api/instance_finalize.hpp index 2d1797c..48f64cb 100644 --- a/backend/cpp/include/rapids_triton/triton/api/instance_finalize.hpp +++ b/backend/cpp/include/triton_backend/triton/api/instance_finalize.hpp @@ -16,9 +16,9 @@ #pragma once #include -#include -#include -#include +#include +#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/triton/api/instance_initialize.hpp b/backend/cpp/include/triton_backend/triton/api/instance_initialize.hpp similarity index 89% rename from backend/cpp/include/rapids_triton/triton/api/instance_initialize.hpp rename to backend/cpp/include/triton_backend/triton/api/instance_initialize.hpp index 922152f..cfd5e02 100644 --- a/backend/cpp/include/rapids_triton/triton/api/instance_initialize.hpp +++ b/backend/cpp/include/triton_backend/triton/api/instance_initialize.hpp @@ -18,12 +18,12 @@ #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/triton/api/model_finalize.hpp b/backend/cpp/include/triton_backend/triton/api/model_finalize.hpp similarity index 91% rename from backend/cpp/include/rapids_triton/triton/api/model_finalize.hpp rename to backend/cpp/include/triton_backend/triton/api/model_finalize.hpp index f4d6d6a..6d22833 100644 --- a/backend/cpp/include/rapids_triton/triton/api/model_finalize.hpp +++ b/backend/cpp/include/triton_backend/triton/api/model_finalize.hpp @@ -17,9 +17,9 @@ #pragma once #include #include -#include -#include -#include +#include +#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/triton/api/model_initialize.hpp b/backend/cpp/include/triton_backend/triton/api/model_initialize.hpp similarity index 92% rename from backend/cpp/include/rapids_triton/triton/api/model_initialize.hpp rename to backend/cpp/include/triton_backend/triton/api/model_initialize.hpp index aa48013..6ab863a 100644 --- a/backend/cpp/include/rapids_triton/triton/api/model_initialize.hpp +++ b/backend/cpp/include/triton_backend/triton/api/model_initialize.hpp @@ -17,9 +17,9 @@ #pragma once #include #include -#include -#include -#include +#include +#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/triton/backend.hpp b/backend/cpp/include/triton_backend/triton/backend.hpp similarity index 95% rename from backend/cpp/include/rapids_triton/triton/backend.hpp rename to backend/cpp/include/triton_backend/triton/backend.hpp index f0f9a5a..d1f7f59 100644 --- a/backend/cpp/include/rapids_triton/triton/backend.hpp +++ b/backend/cpp/include/triton_backend/triton/backend.hpp @@ -18,8 +18,8 @@ #include #include -#include -#include +#include +#include #include namespace triton { diff --git a/backend/cpp/include/rapids_triton/triton/config.hpp b/backend/cpp/include/triton_backend/triton/config.hpp similarity index 92% rename from backend/cpp/include/rapids_triton/triton/config.hpp rename to backend/cpp/include/triton_backend/triton/config.hpp index 4a8300b..9253dd1 100644 --- a/backend/cpp/include/rapids_triton/triton/config.hpp +++ b/backend/cpp/include/triton_backend/triton/config.hpp @@ -20,8 +20,8 @@ #include #include -#include -#include +#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/triton/deployment.hpp b/backend/cpp/include/triton_backend/triton/deployment.hpp similarity index 100% rename from backend/cpp/include/rapids_triton/triton/deployment.hpp rename to backend/cpp/include/triton_backend/triton/deployment.hpp diff --git a/backend/cpp/include/rapids_triton/triton/device.hpp b/backend/cpp/include/triton_backend/triton/device.hpp similarity index 100% rename from backend/cpp/include/rapids_triton/triton/device.hpp rename to backend/cpp/include/triton_backend/triton/device.hpp diff --git a/backend/cpp/include/rapids_triton/triton/input.hpp b/backend/cpp/include/triton_backend/triton/input.hpp similarity index 95% rename from backend/cpp/include/rapids_triton/triton/input.hpp rename to backend/cpp/include/triton_backend/triton/input.hpp index 668875a..ba3d55e 100644 --- a/backend/cpp/include/rapids_triton/triton/input.hpp +++ b/backend/cpp/include/triton_backend/triton/input.hpp @@ -20,9 +20,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include #include diff --git a/backend/cpp/include/rapids_triton/triton/logging.hpp b/backend/cpp/include/triton_backend/triton/logging.hpp similarity index 99% rename from backend/cpp/include/rapids_triton/triton/logging.hpp rename to backend/cpp/include/triton_backend/triton/logging.hpp index 96455f1..7149bc6 100644 --- a/backend/cpp/include/rapids_triton/triton/logging.hpp +++ b/backend/cpp/include/triton_backend/triton/logging.hpp @@ -16,7 +16,7 @@ #pragma once #include -#include +#include #include #include diff --git a/backend/cpp/include/rapids_triton/triton/model.hpp b/backend/cpp/include/triton_backend/triton/model.hpp similarity index 98% rename from backend/cpp/include/rapids_triton/triton/model.hpp rename to backend/cpp/include/triton_backend/triton/model.hpp index 1591763..798c838 100644 --- a/backend/cpp/include/rapids_triton/triton/model.hpp +++ b/backend/cpp/include/triton_backend/triton/model.hpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include namespace triton { diff --git a/backend/cpp/include/rapids_triton/triton/model_instance.hpp b/backend/cpp/include/triton_backend/triton/model_instance.hpp similarity index 96% rename from backend/cpp/include/rapids_triton/triton/model_instance.hpp rename to backend/cpp/include/triton_backend/triton/model_instance.hpp index 0ba03bd..174891c 100644 --- a/backend/cpp/include/rapids_triton/triton/model_instance.hpp +++ b/backend/cpp/include/triton_backend/triton/model_instance.hpp @@ -17,9 +17,9 @@ #pragma once #include #include -#include -#include -#include +#include +#include +#include #include namespace triton { diff --git a/backend/cpp/include/rapids_triton/triton/model_instance_state.hpp b/backend/cpp/include/triton_backend/triton/model_instance_state.hpp similarity index 94% rename from backend/cpp/include/rapids_triton/triton/model_instance_state.hpp rename to backend/cpp/include/triton_backend/triton/model_instance_state.hpp index ba05009..c780abe 100644 --- a/backend/cpp/include/rapids_triton/triton/model_instance_state.hpp +++ b/backend/cpp/include/triton_backend/triton/model_instance_state.hpp @@ -18,8 +18,8 @@ #include #include #include -#include -#include +#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/triton/model_state.hpp b/backend/cpp/include/triton_backend/triton/model_state.hpp similarity index 96% rename from backend/cpp/include/rapids_triton/triton/model_state.hpp rename to backend/cpp/include/triton_backend/triton/model_state.hpp index 582b679..f9b0b82 100644 --- a/backend/cpp/include/rapids_triton/triton/model_state.hpp +++ b/backend/cpp/include/triton_backend/triton/model_state.hpp @@ -17,7 +17,7 @@ #pragma once #include #include -#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/triton/output.hpp b/backend/cpp/include/triton_backend/triton/output.hpp similarity index 98% rename from backend/cpp/include/rapids_triton/triton/output.hpp rename to backend/cpp/include/triton_backend/triton/output.hpp index a48d156..f1f74ec 100644 --- a/backend/cpp/include/rapids_triton/triton/output.hpp +++ b/backend/cpp/include/triton_backend/triton/output.hpp @@ -17,7 +17,7 @@ #pragma once #include -#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/triton/requests.hpp b/backend/cpp/include/triton_backend/triton/requests.hpp similarity index 93% rename from backend/cpp/include/rapids_triton/triton/requests.hpp rename to backend/cpp/include/triton_backend/triton/requests.hpp index c24c8bb..560d2dd 100644 --- a/backend/cpp/include/rapids_triton/triton/requests.hpp +++ b/backend/cpp/include/triton_backend/triton/requests.hpp @@ -19,8 +19,8 @@ #include #include -#include -#include +#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/triton/responses.hpp b/backend/cpp/include/triton_backend/triton/responses.hpp similarity index 96% rename from backend/cpp/include/rapids_triton/triton/responses.hpp rename to backend/cpp/include/triton_backend/triton/responses.hpp index b361c5a..9cb7bfe 100644 --- a/backend/cpp/include/rapids_triton/triton/responses.hpp +++ b/backend/cpp/include/triton_backend/triton/responses.hpp @@ -18,8 +18,8 @@ #include #include #include -#include -#include +#include +#include #include namespace triton { diff --git a/backend/cpp/include/rapids_triton/triton/statistics.hpp b/backend/cpp/include/triton_backend/triton/statistics.hpp similarity index 98% rename from backend/cpp/include/rapids_triton/triton/statistics.hpp rename to backend/cpp/include/triton_backend/triton/statistics.hpp index 5f4f130..fb18a8b 100644 --- a/backend/cpp/include/rapids_triton/triton/statistics.hpp +++ b/backend/cpp/include/triton_backend/triton/statistics.hpp @@ -19,7 +19,7 @@ #include #include #include -#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/triton/triton_memory_resource.hpp b/backend/cpp/include/triton_backend/triton/triton_memory_resource.hpp similarity index 97% rename from backend/cpp/include/rapids_triton/triton/triton_memory_resource.hpp rename to backend/cpp/include/triton_backend/triton/triton_memory_resource.hpp index 77a879e..bdf13d3 100644 --- a/backend/cpp/include/rapids_triton/triton/triton_memory_resource.hpp +++ b/backend/cpp/include/triton_backend/triton/triton_memory_resource.hpp @@ -20,8 +20,8 @@ #include #include #include -#include -#include +#include +#include #include #include #include diff --git a/backend/cpp/include/rapids_triton/utils/const_agnostic.hpp b/backend/cpp/include/triton_backend/utils/const_agnostic.hpp similarity index 100% rename from backend/cpp/include/rapids_triton/utils/const_agnostic.hpp rename to backend/cpp/include/triton_backend/utils/const_agnostic.hpp diff --git a/backend/cpp/include/rapids_triton/utils/device_setter.hpp b/backend/cpp/include/triton_backend/utils/device_setter.hpp similarity index 93% rename from backend/cpp/include/rapids_triton/utils/device_setter.hpp rename to backend/cpp/include/triton_backend/utils/device_setter.hpp index d8aae9b..f2f8b6c 100644 --- a/backend/cpp/include/rapids_triton/utils/device_setter.hpp +++ b/backend/cpp/include/triton_backend/utils/device_setter.hpp @@ -18,8 +18,8 @@ #ifdef TRITON_ENABLE_GPU #include #endif -#include -#include +#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/include/rapids_triton/utils/narrow.hpp b/backend/cpp/include/triton_backend/utils/narrow.hpp similarity index 96% rename from backend/cpp/include/rapids_triton/utils/narrow.hpp rename to backend/cpp/include/triton_backend/utils/narrow.hpp index 37465c0..80f7600 100644 --- a/backend/cpp/include/rapids_triton/utils/narrow.hpp +++ b/backend/cpp/include/triton_backend/utils/narrow.hpp @@ -15,7 +15,7 @@ */ #pragma once -#include +#include #include namespace triton { From 15a690d7ecdb39844b8245377f31e88b9adc9932 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 10:31:54 -0400 Subject: [PATCH 177/199] Replace rapids namespace with dev_tools --- backend/cpp/include/triton_backend.hpp | 4 ++-- backend/cpp/include/triton_backend/batch/batch.hpp | 6 +++--- backend/cpp/include/triton_backend/build_control.hpp | 4 ++-- .../cpu_only/cuda_runtime_replacement.hpp | 4 ++-- backend/cpp/include/triton_backend/exceptions.hpp | 4 ++-- backend/cpp/include/triton_backend/memory/buffer.hpp | 4 ++-- .../triton_backend/memory/detail/cpu_only/copy.hpp | 4 ++-- .../memory/detail/cpu_only/owned_device_buffer.hpp | 4 ++-- .../memory/detail/cpu_only/resource.hpp | 4 ++-- .../triton_backend/memory/detail/gpu_only/copy.hpp | 4 ++-- .../memory/detail/gpu_only/owned_device_buffer.hpp | 4 ++-- .../memory/detail/gpu_only/resource.hpp | 4 ++-- .../memory/detail/owned_device_buffer.hpp | 4 ++-- .../triton_backend/memory/detail/resource.hpp | 4 ++-- .../cpp/include/triton_backend/memory/resource.hpp | 4 ++-- backend/cpp/include/triton_backend/memory/types.hpp | 4 ++-- backend/cpp/include/triton_backend/model/model.hpp | 4 ++-- .../include/triton_backend/model/shared_state.hpp | 4 ++-- backend/cpp/include/triton_backend/tensor/dtype.hpp | 4 ++-- backend/cpp/include/triton_backend/tensor/tensor.hpp | 4 ++-- .../include/triton_backend/triton/api/execute.hpp | 4 ++-- .../include/triton_backend/triton/api/initialize.hpp | 4 ++-- .../triton_backend/triton/api/instance_finalize.hpp | 4 ++-- .../triton/api/instance_initialize.hpp | 12 ++++++------ .../triton_backend/triton/api/model_finalize.hpp | 4 ++-- .../triton_backend/triton/api/model_initialize.hpp | 10 +++++----- .../cpp/include/triton_backend/triton/backend.hpp | 4 ++-- backend/cpp/include/triton_backend/triton/config.hpp | 4 ++-- .../cpp/include/triton_backend/triton/deployment.hpp | 4 ++-- backend/cpp/include/triton_backend/triton/device.hpp | 2 +- backend/cpp/include/triton_backend/triton/input.hpp | 4 ++-- .../cpp/include/triton_backend/triton/logging.hpp | 4 ++-- backend/cpp/include/triton_backend/triton/model.hpp | 4 ++-- .../include/triton_backend/triton/model_instance.hpp | 4 ++-- .../triton_backend/triton/model_instance_state.hpp | 6 +++--- .../include/triton_backend/triton/model_state.hpp | 4 ++-- backend/cpp/include/triton_backend/triton/output.hpp | 4 ++-- .../cpp/include/triton_backend/triton/requests.hpp | 4 ++-- .../cpp/include/triton_backend/triton/responses.hpp | 4 ++-- .../cpp/include/triton_backend/triton/statistics.hpp | 4 ++-- .../triton_backend/triton/triton_memory_resource.hpp | 4 ++-- .../include/triton_backend/utils/const_agnostic.hpp | 2 +- .../include/triton_backend/utils/device_setter.hpp | 4 ++-- backend/cpp/include/triton_backend/utils/narrow.hpp | 4 ++-- 44 files changed, 95 insertions(+), 95 deletions(-) diff --git a/backend/cpp/include/triton_backend.hpp b/backend/cpp/include/triton_backend.hpp index 7062170..fbc7c25 100644 --- a/backend/cpp/include/triton_backend.hpp +++ b/backend/cpp/include/triton_backend.hpp @@ -19,13 +19,13 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { /* Function for testing triton_backend include * * @return message indicating triton_backend has been included succesfully*/ inline auto test_install() { return std::string("triton_backend set up successfully"); } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/batch/batch.hpp b/backend/cpp/include/triton_backend/batch/batch.hpp index bbcd524..25894a4 100644 --- a/backend/cpp/include/triton_backend/batch/batch.hpp +++ b/backend/cpp/include/triton_backend/batch/batch.hpp @@ -46,7 +46,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { /** * @brief A representation of all data about a single batch of inference * requests @@ -64,7 +64,7 @@ namespace rapids { * * It is not recommended that developers of triton_backend backends try to * construct Batch objects directly. Instead, you should make use of the - * rapids::triton_api::execute template, which will construct the Batch for + * dev_tools::triton_api::execute template, which will construct the Batch for * you. */ struct Batch { @@ -268,6 +268,6 @@ struct Batch { std::chrono::time_point compute_start_time_; std::optional batch_size_; }; -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/build_control.hpp b/backend/cpp/include/triton_backend/build_control.hpp index c2a9fec..f2d497a 100644 --- a/backend/cpp/include/triton_backend/build_control.hpp +++ b/backend/cpp/include/triton_backend/build_control.hpp @@ -19,7 +19,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { #ifdef TRITON_ENABLE_GPU auto constexpr IS_GPU_BUILD = true; @@ -27,6 +27,6 @@ auto constexpr IS_GPU_BUILD = true; auto constexpr IS_GPU_BUILD = false; #endif -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/cpu_only/cuda_runtime_replacement.hpp b/backend/cpp/include/triton_backend/cpu_only/cuda_runtime_replacement.hpp index 7cb6691..06acf28 100644 --- a/backend/cpp/include/triton_backend/cpu_only/cuda_runtime_replacement.hpp +++ b/backend/cpp/include/triton_backend/cpu_only/cuda_runtime_replacement.hpp @@ -21,7 +21,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { using cudaStream_t = void*; @@ -48,7 +48,7 @@ inline auto cudaGetDeviceCount(int* count) { } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton #endif diff --git a/backend/cpp/include/triton_backend/exceptions.hpp b/backend/cpp/include/triton_backend/exceptions.hpp index acdffb9..f6fe24c 100644 --- a/backend/cpp/include/triton_backend/exceptions.hpp +++ b/backend/cpp/include/triton_backend/exceptions.hpp @@ -27,7 +27,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { using ErrorCode = TRITONSERVER_Error_Code; @@ -89,6 +89,6 @@ inline void cuda_check(cudaError_t const& err) } } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/buffer.hpp b/backend/cpp/include/triton_backend/memory/buffer.hpp index 92f16bd..1c9857c 100644 --- a/backend/cpp/include/triton_backend/memory/buffer.hpp +++ b/backend/cpp/include/triton_backend/memory/buffer.hpp @@ -39,7 +39,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { template struct Buffer { using size_type = std::size_t; @@ -312,6 +312,6 @@ void copy(Buffer& dst, { copy(dst, src, 0, src_begin, src_end); } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/detail/cpu_only/copy.hpp b/backend/cpp/include/triton_backend/memory/detail/cpu_only/copy.hpp index dbf47ae..1d4778e 100644 --- a/backend/cpp/include/triton_backend/memory/detail/cpu_only/copy.hpp +++ b/backend/cpp/include/triton_backend/memory/detail/cpu_only/copy.hpp @@ -26,7 +26,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { namespace detail { template @@ -45,6 +45,6 @@ void copy(T* dst, } } // namespace detail -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/detail/cpu_only/owned_device_buffer.hpp b/backend/cpp/include/triton_backend/memory/detail/cpu_only/owned_device_buffer.hpp index 1458fbe..ba7fece 100644 --- a/backend/cpp/include/triton_backend/memory/detail/cpu_only/owned_device_buffer.hpp +++ b/backend/cpp/include/triton_backend/memory/detail/cpu_only/owned_device_buffer.hpp @@ -24,7 +24,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { namespace detail { template @@ -40,6 +40,6 @@ struct owned_device_buffer { }; } // namespace detail -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/detail/cpu_only/resource.hpp b/backend/cpp/include/triton_backend/memory/detail/cpu_only/resource.hpp index 89cb81c..2a96a58 100644 --- a/backend/cpp/include/triton_backend/memory/detail/cpu_only/resource.hpp +++ b/backend/cpp/include/triton_backend/memory/detail/cpu_only/resource.hpp @@ -22,7 +22,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { namespace detail { template<> @@ -30,6 +30,6 @@ inline void setup_memory_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* triton_manager) { } } // namespace detail -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/detail/gpu_only/copy.hpp b/backend/cpp/include/triton_backend/memory/detail/gpu_only/copy.hpp index ff26061..b1c021e 100644 --- a/backend/cpp/include/triton_backend/memory/detail/gpu_only/copy.hpp +++ b/backend/cpp/include/triton_backend/memory/detail/gpu_only/copy.hpp @@ -27,7 +27,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { namespace detail { template @@ -50,6 +50,6 @@ void copy(T* dst, } } // namespace detail -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/detail/gpu_only/owned_device_buffer.hpp b/backend/cpp/include/triton_backend/memory/detail/gpu_only/owned_device_buffer.hpp index ee88a61..f897c58 100644 --- a/backend/cpp/include/triton_backend/memory/detail/gpu_only/owned_device_buffer.hpp +++ b/backend/cpp/include/triton_backend/memory/detail/gpu_only/owned_device_buffer.hpp @@ -23,7 +23,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { namespace detail { template @@ -44,6 +44,6 @@ struct owned_device_buffer { }; } // namespace detail -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/detail/gpu_only/resource.hpp b/backend/cpp/include/triton_backend/memory/detail/gpu_only/resource.hpp index 8d40ae1..639572a 100644 --- a/backend/cpp/include/triton_backend/memory/detail/gpu_only/resource.hpp +++ b/backend/cpp/include/triton_backend/memory/detail/gpu_only/resource.hpp @@ -29,7 +29,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { namespace detail { inline auto& resource_lock() @@ -92,6 +92,6 @@ inline void setup_memory_resource(device_id_t device_id, inline auto* get_memory_resource() { return rmm::mr::get_current_device_resource(); } */ } // namespace detail -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/detail/owned_device_buffer.hpp b/backend/cpp/include/triton_backend/memory/detail/owned_device_buffer.hpp index ca25af9..61a470c 100644 --- a/backend/cpp/include/triton_backend/memory/detail/owned_device_buffer.hpp +++ b/backend/cpp/include/triton_backend/memory/detail/owned_device_buffer.hpp @@ -17,7 +17,7 @@ #pragma once namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { namespace detail { template @@ -25,6 +25,6 @@ struct owned_device_buffer { }; } // namespace detail -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/detail/resource.hpp b/backend/cpp/include/triton_backend/memory/detail/resource.hpp index 5a15d90..cf922b1 100644 --- a/backend/cpp/include/triton_backend/memory/detail/resource.hpp +++ b/backend/cpp/include/triton_backend/memory/detail/resource.hpp @@ -21,7 +21,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { namespace detail { template @@ -30,6 +30,6 @@ inline void setup_memory_resource(device_id_t device_id, } } // namespace detail -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/resource.hpp b/backend/cpp/include/triton_backend/memory/resource.hpp index 9dc33f5..e95378f 100644 --- a/backend/cpp/include/triton_backend/memory/resource.hpp +++ b/backend/cpp/include/triton_backend/memory/resource.hpp @@ -28,12 +28,12 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { inline void setup_memory_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* triton_manager = nullptr) { detail::setup_memory_resource(device_id, triton_manager); } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/types.hpp b/backend/cpp/include/triton_backend/memory/types.hpp index 884f315..57386f1 100644 --- a/backend/cpp/include/triton_backend/memory/types.hpp +++ b/backend/cpp/include/triton_backend/memory/types.hpp @@ -19,10 +19,10 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { using MemoryType = TRITONSERVER_MemoryType; auto constexpr DeviceMemory = TRITONSERVER_MEMORY_GPU; auto constexpr HostMemory = TRITONSERVER_MEMORY_CPU; -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/model/model.hpp b/backend/cpp/include/triton_backend/model/model.hpp index a1ec5be..a46151f 100644 --- a/backend/cpp/include/triton_backend/model/model.hpp +++ b/backend/cpp/include/triton_backend/model/model.hpp @@ -33,7 +33,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { template struct Model { virtual void predict(Batch& batch) const = 0; @@ -192,6 +192,6 @@ struct Model { DeploymentType deployment_type_; std::string filepath_; }; -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/model/shared_state.hpp b/backend/cpp/include/triton_backend/model/shared_state.hpp index 8a8b65f..dada7a3 100644 --- a/backend/cpp/include/triton_backend/model/shared_state.hpp +++ b/backend/cpp/include/triton_backend/model/shared_state.hpp @@ -37,7 +37,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { /** * @brief Stores shared state for multiple instances of the same model */ @@ -184,6 +184,6 @@ struct SharedModelState { return result; } }; -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/tensor/dtype.hpp b/backend/cpp/include/triton_backend/tensor/dtype.hpp index 1389586..9545541 100644 --- a/backend/cpp/include/triton_backend/tensor/dtype.hpp +++ b/backend/cpp/include/triton_backend/tensor/dtype.hpp @@ -22,7 +22,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { using DType = TRITONSERVER_DataType; auto constexpr DTypeBool = TRITONSERVER_TYPE_BOOL; @@ -163,6 +163,6 @@ inline std::ostream& operator<<(std::ostream& out, DType const& dtype) return out; } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/tensor/tensor.hpp b/backend/cpp/include/triton_backend/tensor/tensor.hpp index e4fab28..c52bbe3 100644 --- a/backend/cpp/include/triton_backend/tensor/tensor.hpp +++ b/backend/cpp/include/triton_backend/tensor/tensor.hpp @@ -40,7 +40,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { template struct BaseTensor { using size_type = typename Buffer::size_type; @@ -200,6 +200,6 @@ void copy(Iter begin, Iter end, BaseTensor& src) }); } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/api/execute.hpp b/backend/cpp/include/triton_backend/triton/api/execute.hpp index d9ce7c9..3233e89 100644 --- a/backend/cpp/include/triton_backend/triton/api/execute.hpp +++ b/backend/cpp/include/triton_backend/triton/api/execute.hpp @@ -30,7 +30,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { namespace triton_api { template auto* execute(TRITONBACKEND_ModelInstance* instance, @@ -116,6 +116,6 @@ auto* execute(TRITONBACKEND_ModelInstance* instance, return result; } } // namespace triton_api -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/api/initialize.hpp b/backend/cpp/include/triton_backend/triton/api/initialize.hpp index 2b1ebb2..25dd376 100644 --- a/backend/cpp/include/triton_backend/triton/api/initialize.hpp +++ b/backend/cpp/include/triton_backend/triton/api/initialize.hpp @@ -32,7 +32,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { namespace triton_api { inline auto* initialize(TRITONBACKEND_Backend* backend) { @@ -64,6 +64,6 @@ inline auto* initialize(TRITONBACKEND_Backend* backend) return result; } } // namespace triton_api -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/api/instance_finalize.hpp b/backend/cpp/include/triton_backend/triton/api/instance_finalize.hpp index 48f64cb..10a249b 100644 --- a/backend/cpp/include/triton_backend/triton/api/instance_finalize.hpp +++ b/backend/cpp/include/triton_backend/triton/api/instance_finalize.hpp @@ -22,7 +22,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { namespace triton_api { template auto* instance_finalize(TRITONBACKEND_ModelInstance* instance) @@ -44,6 +44,6 @@ auto* instance_finalize(TRITONBACKEND_ModelInstance* instance) return result; } } // namespace triton_api -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/api/instance_initialize.hpp b/backend/cpp/include/triton_backend/triton/api/instance_initialize.hpp index cfd5e02..5c71084 100644 --- a/backend/cpp/include/triton_backend/triton/api/instance_initialize.hpp +++ b/backend/cpp/include/triton_backend/triton/api/instance_initialize.hpp @@ -27,7 +27,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { namespace triton_api { template auto* instance_initialize(TRITONBACKEND_ModelInstance* instance) @@ -53,22 +53,22 @@ auto* instance_initialize(TRITONBACKEND_ModelInstance* instance) setup_memory_resource(device_id, model_state->TritonMemoryManager()); } - auto rapids_model = std::make_unique(*model_state, instance); + auto dev_tools_model = std::make_unique(*model_state, instance); if constexpr (IS_GPU_BUILD) { - auto& model = rapids_model->get_model(); + auto& model = dev_tools_model->get_model(); if (model.get_deployment_type() == GPUDeployment) { cuda_check(cudaSetDevice(model.get_device_id())); } } - rapids_model->load(); + dev_tools_model->load(); - set_instance_state(*instance, std::move(rapids_model)); + set_instance_state(*instance, std::move(dev_tools_model)); } catch (TritonException& err) { result = err.error(); } return result; } } // namespace triton_api -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/api/model_finalize.hpp b/backend/cpp/include/triton_backend/triton/api/model_finalize.hpp index 6d22833..f385cc1 100644 --- a/backend/cpp/include/triton_backend/triton/api/model_finalize.hpp +++ b/backend/cpp/include/triton_backend/triton/api/model_finalize.hpp @@ -23,7 +23,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { namespace triton_api { template auto* model_finalize(TRITONBACKEND_Model* model) @@ -43,6 +43,6 @@ auto* model_finalize(TRITONBACKEND_Model* model) return result; } } // namespace triton_api -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/api/model_initialize.hpp b/backend/cpp/include/triton_backend/triton/api/model_initialize.hpp index 6ab863a..8877b90 100644 --- a/backend/cpp/include/triton_backend/triton/api/model_initialize.hpp +++ b/backend/cpp/include/triton_backend/triton/api/model_initialize.hpp @@ -23,7 +23,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { namespace triton_api { template auto* model_initialize(TRITONBACKEND_Model* model) @@ -37,10 +37,10 @@ auto* model_initialize(TRITONBACKEND_Model* model) log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInitialize: " << name << " (version " << version << ")"; - auto rapids_model_state = std::make_unique(*model); - rapids_model_state->load(); + auto dev_tools_model_state = std::make_unique(*model); + dev_tools_model_state->load(); - set_model_state(*model, std::move(rapids_model_state)); + set_model_state(*model, std::move(dev_tools_model_state)); } catch (TritonException& err) { result = err.error(); } @@ -48,6 +48,6 @@ auto* model_initialize(TRITONBACKEND_Model* model) return result; } } // namespace triton_api -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/backend.hpp b/backend/cpp/include/triton_backend/triton/backend.hpp index d1f7f59..76b848f 100644 --- a/backend/cpp/include/triton_backend/triton/backend.hpp +++ b/backend/cpp/include/triton_backend/triton/backend.hpp @@ -24,7 +24,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { inline auto get_backend_name(TRITONBACKEND_Backend& backend) { const char* cname; @@ -56,6 +56,6 @@ inline auto check_backend_version(TRITONBACKEND_Backend& backend) return ((version.major == TRITONBACKEND_API_VERSION_MAJOR) && (version.minor >= TRITONBACKEND_API_VERSION_MINOR)); } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/config.hpp b/backend/cpp/include/triton_backend/triton/config.hpp index 9253dd1..80974a7 100644 --- a/backend/cpp/include/triton_backend/triton/config.hpp +++ b/backend/cpp/include/triton_backend/triton/config.hpp @@ -25,13 +25,13 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { inline auto get_max_batch_size(common::TritonJson::Value& config) { auto reported = int64_t{}; triton_check(config.MemberAsInt("max_batch_size", &reported)); return narrow(reported); } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/deployment.hpp b/backend/cpp/include/triton_backend/triton/deployment.hpp index e663c47..c985d66 100644 --- a/backend/cpp/include/triton_backend/triton/deployment.hpp +++ b/backend/cpp/include/triton_backend/triton/deployment.hpp @@ -19,13 +19,13 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { using DeploymentType = TRITONSERVER_InstanceGroupKind; auto constexpr GPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_GPU; auto constexpr CPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_CPU; // Note (wphicks): We currently are not including "Auto" or "Model" because I // am not sure exactly how those would be used in context. If there is a // demand, they can be added. -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/device.hpp b/backend/cpp/include/triton_backend/triton/device.hpp index 73b28c4..480fbda 100644 --- a/backend/cpp/include/triton_backend/triton/device.hpp +++ b/backend/cpp/include/triton_backend/triton/device.hpp @@ -19,7 +19,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { using device_id_t = std::int32_t; } } // namespace backend diff --git a/backend/cpp/include/triton_backend/triton/input.hpp b/backend/cpp/include/triton_backend/triton/input.hpp index ba3d55e..0e724a7 100644 --- a/backend/cpp/include/triton_backend/triton/input.hpp +++ b/backend/cpp/include/triton_backend/triton/input.hpp @@ -29,7 +29,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) { auto result = static_cast(nullptr); @@ -75,6 +75,6 @@ auto get_triton_input_shape(Iter requests_begin, Iter requests_end, std::string return result; } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/logging.hpp b/backend/cpp/include/triton_backend/triton/logging.hpp index 7149bc6..c7a4072 100644 --- a/backend/cpp/include/triton_backend/triton/logging.hpp +++ b/backend/cpp/include/triton_backend/triton/logging.hpp @@ -24,7 +24,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { namespace { /** Log message at indicated level */ @@ -154,6 +154,6 @@ inline auto log_debug(char const* filename, int line) } inline auto log_debug() { return log_stream(TRITONSERVER_LOG_VERBOSE); } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/model.hpp b/backend/cpp/include/triton_backend/triton/model.hpp index 798c838..0070a0d 100644 --- a/backend/cpp/include/triton_backend/triton/model.hpp +++ b/backend/cpp/include/triton_backend/triton/model.hpp @@ -25,7 +25,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { inline auto get_model_version(TRITONBACKEND_Model& model) { @@ -85,6 +85,6 @@ auto* get_model_state(TRITONBACKEND_Model& model) return model_state; } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/model_instance.hpp b/backend/cpp/include/triton_backend/triton/model_instance.hpp index 174891c..aacb8da 100644 --- a/backend/cpp/include/triton_backend/triton/model_instance.hpp +++ b/backend/cpp/include/triton_backend/triton/model_instance.hpp @@ -24,7 +24,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { /** Get the name of a Triton model instance from the instance itself */ inline auto get_model_instance_name(TRITONBACKEND_ModelInstance& instance) { @@ -92,6 +92,6 @@ auto* get_instance_state(TRITONBACKEND_ModelInstance& instance) return instance_state; } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/model_instance_state.hpp b/backend/cpp/include/triton_backend/triton/model_instance_state.hpp index c780abe..a78e54c 100644 --- a/backend/cpp/include/triton_backend/triton/model_instance_state.hpp +++ b/backend/cpp/include/triton_backend/triton/model_instance_state.hpp @@ -23,7 +23,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { template struct ModelInstanceState : public BackendModelInstance { @@ -31,7 +31,7 @@ struct ModelInstanceState : public BackendModelInstance { TRITONBACKEND_ModelInstance* triton_model_instance) : BackendModelInstance(&model_state, triton_model_instance), model_(model_state.get_shared_state(), - rapids::get_device_id(*triton_model_instance), + dev_tools::get_device_id(*triton_model_instance), CudaStream(), Kind(), JoinPath({model_state.RepositoryPath(), @@ -49,6 +49,6 @@ struct ModelInstanceState : public BackendModelInstance { RapidsModel model_; }; -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/model_state.hpp b/backend/cpp/include/triton_backend/triton/model_state.hpp index f9b0b82..f050dee 100644 --- a/backend/cpp/include/triton_backend/triton/model_state.hpp +++ b/backend/cpp/include/triton_backend/triton/model_state.hpp @@ -21,7 +21,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { template struct TritonModelState : public BackendModel { TritonModelState(TRITONBACKEND_Model& triton_model) @@ -39,6 +39,6 @@ struct TritonModelState : public BackendModel { std::shared_ptr state_; }; -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/output.hpp b/backend/cpp/include/triton_backend/triton/output.hpp index f1f74ec..6e54c5e 100644 --- a/backend/cpp/include/triton_backend/triton/output.hpp +++ b/backend/cpp/include/triton_backend/triton/output.hpp @@ -21,7 +21,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) { auto* result = static_cast(nullptr); @@ -67,6 +67,6 @@ auto get_triton_output_shape(Iter requests_begin, Iter requests_end, std::string return result; } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/requests.hpp b/backend/cpp/include/triton_backend/triton/requests.hpp index 560d2dd..ad1a91b 100644 --- a/backend/cpp/include/triton_backend/triton/requests.hpp +++ b/backend/cpp/include/triton_backend/triton/requests.hpp @@ -24,7 +24,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { using request_size_t = uint32_t; template @@ -38,6 +38,6 @@ void release_requests(Iter begin, Iter end) } }); } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/responses.hpp b/backend/cpp/include/triton_backend/triton/responses.hpp index 9cb7bfe..721ce3a 100644 --- a/backend/cpp/include/triton_backend/triton/responses.hpp +++ b/backend/cpp/include/triton_backend/triton/responses.hpp @@ -24,7 +24,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { template auto construct_responses(Iter requests_begin, Iter requests_end) @@ -70,6 +70,6 @@ void send_responses(Iter begin, Iter end, TRITONSERVER_Error* err) }); } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/statistics.hpp b/backend/cpp/include/triton_backend/triton/statistics.hpp index fb18a8b..4d25361 100644 --- a/backend/cpp/include/triton_backend/triton/statistics.hpp +++ b/backend/cpp/include/triton_backend/triton/statistics.hpp @@ -23,7 +23,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { using time_point = std::chrono::time_point; /** @@ -84,6 +84,6 @@ inline void report_statistics(TRITONBACKEND_ModelInstance& instance, compute_end_time.time_since_epoch().count(), end_time.time_since_epoch().count())); } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/triton_memory_resource.hpp b/backend/cpp/include/triton_backend/triton/triton_memory_resource.hpp index bdf13d3..4c4b197 100644 --- a/backend/cpp/include/triton_backend/triton/triton_memory_resource.hpp +++ b/backend/cpp/include/triton_backend/triton/triton_memory_resource.hpp @@ -29,7 +29,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { struct triton_memory_resource final : public rmm::mr::device_memory_resource { triton_memory_resource(TRITONBACKEND_MemoryManager* manager, device_id_t device_id, @@ -81,6 +81,6 @@ struct triton_memory_resource final : public rmm::mr::device_memory_resource { } }; -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/utils/const_agnostic.hpp b/backend/cpp/include/triton_backend/utils/const_agnostic.hpp index cd55aeb..4ca9dcc 100644 --- a/backend/cpp/include/triton_backend/utils/const_agnostic.hpp +++ b/backend/cpp/include/triton_backend/utils/const_agnostic.hpp @@ -19,7 +19,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { template using const_agnostic_same_t = diff --git a/backend/cpp/include/triton_backend/utils/device_setter.hpp b/backend/cpp/include/triton_backend/utils/device_setter.hpp index f2f8b6c..8ea21a3 100644 --- a/backend/cpp/include/triton_backend/utils/device_setter.hpp +++ b/backend/cpp/include/triton_backend/utils/device_setter.hpp @@ -23,7 +23,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { /** Struct for setting cuda device within a code block */ struct device_setter { @@ -45,6 +45,6 @@ struct device_setter { device_id_t prev_device_; }; -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/include/triton_backend/utils/narrow.hpp b/backend/cpp/include/triton_backend/utils/narrow.hpp index 80f7600..da9e524 100644 --- a/backend/cpp/include/triton_backend/utils/narrow.hpp +++ b/backend/cpp/include/triton_backend/utils/narrow.hpp @@ -20,7 +20,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { template auto narrow(F from) @@ -36,6 +36,6 @@ auto narrow(F from) return to; } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton From 0abaca19e0cd22b3db100f51094dd94002b1b04e Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 10:35:16 -0400 Subject: [PATCH 178/199] Replace Rapids template names with Backend --- .../include/triton_backend/triton/model_instance_state.hpp | 6 +++--- backend/cpp/include/triton_backend/triton/model_state.hpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/cpp/include/triton_backend/triton/model_instance_state.hpp b/backend/cpp/include/triton_backend/triton/model_instance_state.hpp index a78e54c..efac340 100644 --- a/backend/cpp/include/triton_backend/triton/model_instance_state.hpp +++ b/backend/cpp/include/triton_backend/triton/model_instance_state.hpp @@ -25,9 +25,9 @@ namespace triton { namespace backend { namespace dev_tools { -template +template struct ModelInstanceState : public BackendModelInstance { - ModelInstanceState(TritonModelState& model_state, + ModelInstanceState(TritonModelState& model_state, TRITONBACKEND_ModelInstance* triton_model_instance) : BackendModelInstance(&model_state, triton_model_instance), model_(model_state.get_shared_state(), @@ -46,7 +46,7 @@ struct ModelInstanceState : public BackendModelInstance { void unload() { model_.unload(); } private: - RapidsModel model_; + BackendModel model_; }; } // namespace dev_tools diff --git a/backend/cpp/include/triton_backend/triton/model_state.hpp b/backend/cpp/include/triton_backend/triton/model_state.hpp index f050dee..c9ac1bc 100644 --- a/backend/cpp/include/triton_backend/triton/model_state.hpp +++ b/backend/cpp/include/triton_backend/triton/model_state.hpp @@ -22,11 +22,11 @@ namespace triton { namespace backend { namespace dev_tools { -template +template struct TritonModelState : public BackendModel { TritonModelState(TRITONBACKEND_Model& triton_model) : BackendModel(&triton_model), - state_{std::make_shared(get_model_config(triton_model))} + state_{std::make_shared(get_model_config(triton_model))} { } @@ -36,7 +36,7 @@ struct TritonModelState : public BackendModel { auto get_shared_state() { return state_; } private: - std::shared_ptr state_; + std::shared_ptr state_; }; } // namespace dev_tools From da604ca593fa8d864a1a090baa9fa50ed39f09b5 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 10:38:33 -0400 Subject: [PATCH 179/199] Rewrite comments referencing RAPIDS --- backend/cpp/include/triton_backend/model/shared_state.hpp | 6 +++--- backend/cpp/include/triton_backend/triton/model.hpp | 2 +- .../cpp/include/triton_backend/triton/model_instance.hpp | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/cpp/include/triton_backend/model/shared_state.hpp b/backend/cpp/include/triton_backend/model/shared_state.hpp index dada7a3..f09b3fa 100644 --- a/backend/cpp/include/triton_backend/model/shared_state.hpp +++ b/backend/cpp/include/triton_backend/model/shared_state.hpp @@ -73,9 +73,9 @@ struct SharedModelState { if (shape[0] != -1) { shape.insert(shape.begin(), -1); } // The squeeze_output option was introduced to handle a bad choice of // convention in the original FIL backend implementation. For legacy - // compatibility, we introduced this option into RAPIDS-Triton, but - // in general, new backends are advised to avoid using it and defer - // this sort of flattening operation to the consumer. + // compatibility, we introduced this option into Triton's dev tools, + // but in general, new backends are advised to avoid using it and + // defer this sort of flattening operation to the consumer. if (squeeze_output) { shape.erase(std::remove(shape.begin(), shape.end(), std::int64_t{1}), shape.end()); } diff --git a/backend/cpp/include/triton_backend/triton/model.hpp b/backend/cpp/include/triton_backend/triton/model.hpp index 0070a0d..6f05f77 100644 --- a/backend/cpp/include/triton_backend/triton/model.hpp +++ b/backend/cpp/include/triton_backend/triton/model.hpp @@ -63,7 +63,7 @@ inline auto get_model_config(TRITONBACKEND_Model& model) * * This function accepts a unique_ptr to an object derived from a Triton * BackendModel object and sets it as the stored state for a model in the - * Triton server. Note that this object is not the same as a RAPIDS-Triton + * Triton server. Note that this object is not the same as a dev tools * "SharedModelState" object. The object that Triton expects must wrap this * SharedModelState and provide additional interface compatibility. */ diff --git a/backend/cpp/include/triton_backend/triton/model_instance.hpp b/backend/cpp/include/triton_backend/triton/model_instance.hpp index aacb8da..220e007 100644 --- a/backend/cpp/include/triton_backend/triton/model_instance.hpp +++ b/backend/cpp/include/triton_backend/triton/model_instance.hpp @@ -70,7 +70,7 @@ inline auto* get_model_from_instance(TRITONBACKEND_ModelInstance& instance) * * This function accepts a unique_ptr to an object derived from a Triton * BackendModelInstance object and sets it as the stored state for a model in the - * Triton server. Note that this object is not the same as a RAPIDS-Triton + * Triton server. Note that this object is not the same as a dev tools * "Model" object. The object that Triton expects must wrap this Model and * provide additional interface compatibility. */ From d476da923f96a16188668526235f5d509ad21b4c Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 10:39:49 -0400 Subject: [PATCH 180/199] Revert "Replace Rapids template names with Backend" This reverts commit 0abaca19e0cd22b3db100f51094dd94002b1b04e. --- .../include/triton_backend/triton/model_instance_state.hpp | 6 +++--- backend/cpp/include/triton_backend/triton/model_state.hpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/cpp/include/triton_backend/triton/model_instance_state.hpp b/backend/cpp/include/triton_backend/triton/model_instance_state.hpp index efac340..a78e54c 100644 --- a/backend/cpp/include/triton_backend/triton/model_instance_state.hpp +++ b/backend/cpp/include/triton_backend/triton/model_instance_state.hpp @@ -25,9 +25,9 @@ namespace triton { namespace backend { namespace dev_tools { -template +template struct ModelInstanceState : public BackendModelInstance { - ModelInstanceState(TritonModelState& model_state, + ModelInstanceState(TritonModelState& model_state, TRITONBACKEND_ModelInstance* triton_model_instance) : BackendModelInstance(&model_state, triton_model_instance), model_(model_state.get_shared_state(), @@ -46,7 +46,7 @@ struct ModelInstanceState : public BackendModelInstance { void unload() { model_.unload(); } private: - BackendModel model_; + RapidsModel model_; }; } // namespace dev_tools diff --git a/backend/cpp/include/triton_backend/triton/model_state.hpp b/backend/cpp/include/triton_backend/triton/model_state.hpp index c9ac1bc..f050dee 100644 --- a/backend/cpp/include/triton_backend/triton/model_state.hpp +++ b/backend/cpp/include/triton_backend/triton/model_state.hpp @@ -22,11 +22,11 @@ namespace triton { namespace backend { namespace dev_tools { -template +template struct TritonModelState : public BackendModel { TritonModelState(TRITONBACKEND_Model& triton_model) : BackendModel(&triton_model), - state_{std::make_shared(get_model_config(triton_model))} + state_{std::make_shared(get_model_config(triton_model))} { } @@ -36,7 +36,7 @@ struct TritonModelState : public BackendModel { auto get_shared_state() { return state_; } private: - std::shared_ptr state_; + std::shared_ptr state_; }; } // namespace dev_tools From 17bebde9e93690d117a76f90f29d7eed4cc412e2 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 10:40:15 -0400 Subject: [PATCH 181/199] Replace Rapids with DevTools in template names --- .../include/triton_backend/triton/model_instance_state.hpp | 6 +++--- backend/cpp/include/triton_backend/triton/model_state.hpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/cpp/include/triton_backend/triton/model_instance_state.hpp b/backend/cpp/include/triton_backend/triton/model_instance_state.hpp index a78e54c..83d515a 100644 --- a/backend/cpp/include/triton_backend/triton/model_instance_state.hpp +++ b/backend/cpp/include/triton_backend/triton/model_instance_state.hpp @@ -25,9 +25,9 @@ namespace triton { namespace backend { namespace dev_tools { -template +template struct ModelInstanceState : public BackendModelInstance { - ModelInstanceState(TritonModelState& model_state, + ModelInstanceState(TritonModelState& model_state, TRITONBACKEND_ModelInstance* triton_model_instance) : BackendModelInstance(&model_state, triton_model_instance), model_(model_state.get_shared_state(), @@ -46,7 +46,7 @@ struct ModelInstanceState : public BackendModelInstance { void unload() { model_.unload(); } private: - RapidsModel model_; + DevToolsModel model_; }; } // namespace dev_tools diff --git a/backend/cpp/include/triton_backend/triton/model_state.hpp b/backend/cpp/include/triton_backend/triton/model_state.hpp index f050dee..1af6065 100644 --- a/backend/cpp/include/triton_backend/triton/model_state.hpp +++ b/backend/cpp/include/triton_backend/triton/model_state.hpp @@ -22,11 +22,11 @@ namespace triton { namespace backend { namespace dev_tools { -template +template struct TritonModelState : public BackendModel { TritonModelState(TRITONBACKEND_Model& triton_model) : BackendModel(&triton_model), - state_{std::make_shared(get_model_config(triton_model))} + state_{std::make_shared(get_model_config(triton_model))} { } @@ -36,7 +36,7 @@ struct TritonModelState : public BackendModel { auto get_shared_state() { return state_; } private: - std::shared_ptr state_; + std::shared_ptr state_; }; } // namespace dev_tools From e69a803ffb0bba0487f4c96dbb8db2c4cc9efc39 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 10:45:00 -0400 Subject: [PATCH 182/199] Update include path for dev_tools --- backend/cpp/src/api.cc | 16 ++++++++-------- backend/cpp/src/model.h | 14 +++++++------- backend/cpp/src/shared_state.h | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/backend/cpp/src/api.cc b/backend/cpp/src/api.cc index 7e62aa6..8007ee0 100644 --- a/backend/cpp/src/api.cc +++ b/backend/cpp/src/api.cc @@ -22,14 +22,14 @@ #include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/src/model.h b/backend/cpp/src/model.h index 5ba95f1..7653817 100644 --- a/backend/cpp/src/model.h +++ b/backend/cpp/src/model.h @@ -19,19 +19,19 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include #include #include #include -#include // rapids::Batch -#include // rapids::MemoryType -#include // rapids::Model -#include // rapids::copy -#include // rapids::DeploymentType -#include // rapids::device_id_t +#include // rapids::Batch +#include // rapids::MemoryType +#include // rapids::Model +#include // rapids::copy +#include // rapids::DeploymentType +#include // rapids::device_id_t namespace triton { namespace backend { diff --git a/backend/cpp/src/shared_state.h b/backend/cpp/src/shared_state.h index b6f8981..1a9b524 100644 --- a/backend/cpp/src/shared_state.h +++ b/backend/cpp/src/shared_state.h @@ -19,7 +19,7 @@ #include #include -#include +#include namespace triton { namespace backend { From 399fe747a82be74acc392e85568f15341c76ba37 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 10:45:34 -0400 Subject: [PATCH 183/199] Update template names to DevTools in identity --- backend/cpp/src/api.cc | 4 ++-- backend/cpp/src/model.h | 8 ++++---- backend/cpp/src/shared_state.h | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/backend/cpp/src/api.cc b/backend/cpp/src/api.cc index 8007ee0..cd8dc9e 100644 --- a/backend/cpp/src/api.cc +++ b/backend/cpp/src/api.cc @@ -35,9 +35,9 @@ namespace triton { namespace backend { namespace NAMESPACE { -using ModelState = rapids::TritonModelState; +using ModelState = rapids::TritonModelState; using ModelInstanceState = - rapids::ModelInstanceState; + rapids::ModelInstanceState; extern "C" { diff --git a/backend/cpp/src/model.h b/backend/cpp/src/model.h index 7653817..dca829b 100644 --- a/backend/cpp/src/model.h +++ b/backend/cpp/src/model.h @@ -38,21 +38,21 @@ namespace backend { namespace NAMESPACE { /* Any logic necessary to perform inference with a model and manage its data - * should be implemented in a struct named RapidsModel, as shown here */ + * should be implemented in a struct named DevToolsModel, as shown here */ -struct RapidsModel : rapids::Model { +struct DevToolsModel : rapids::Model { /*************************************************************************** * BOILERPLATE * * *********************************************************************** * * The following constructor can be copied directly into any model * implementation. **************************************************************************/ - RapidsModel(std::shared_ptr shared_state, + DevToolsModel(std::shared_ptr shared_state, rapids::device_id_t device_id, cudaStream_t default_stream, rapids::DeploymentType deployment_type, std::string const& filepath) - : rapids::Model( + : rapids::Model( shared_state, device_id, default_stream, deployment_type, filepath) { } diff --git a/backend/cpp/src/shared_state.h b/backend/cpp/src/shared_state.h index 1a9b524..ed3e5ef 100644 --- a/backend/cpp/src/shared_state.h +++ b/backend/cpp/src/shared_state.h @@ -29,15 +29,15 @@ namespace NAMESPACE { * same time (e.g. on different GPUs). All instances of a model share access to * an object which manages any state that can be shared across all instances. * Any logic necessary for managing such state should be implemented in a - * struct named RapidsSharedState, as shown here. Models may access this shared + * struct named DevToolsSharedState, as shown here. Models may access this shared * state object via the `get_shared_state` method, which returns a shared - * pointer to the RapidsSharedState object. + * pointer to the DevToolsSharedState object. * * Not all backends require shared state, so leaving this implementation empty * is entirely valid */ -struct RapidsSharedState : rapids::SharedModelState { - RapidsSharedState(std::unique_ptr&& config) +struct DevToolsSharedState : rapids::SharedModelState { + DevToolsSharedState(std::unique_ptr&& config) : rapids::SharedModelState{std::move(config)} { } From fa82f5b242e64a578074dfd2e371e02ede49d379 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 11:08:23 -0400 Subject: [PATCH 184/199] Update name of identity backend --- backend/cpp/src/CMakeLists.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/backend/cpp/src/CMakeLists.txt b/backend/cpp/src/CMakeLists.txt index 1d5904c..bfd3f19 100644 --- a/backend/cpp/src/CMakeLists.txt +++ b/backend/cpp/src/CMakeLists.txt @@ -16,12 +16,12 @@ # keep the files in alphabetical order! add_library( - triton_rapids-identity SHARED + triton_dev_tools-identity SHARED src/api.cc ) if(TRITON_ENABLE_GPU) - set_target_properties(triton_rapids-identity + set_target_properties(triton_dev_tools-identity PROPERTIES BUILD_RPATH "\$ORIGIN" # set target compile options CXX_STANDARD 17 @@ -32,7 +32,7 @@ if(TRITON_ENABLE_GPU) INTERFACE_POSITION_INDEPENDENT_CODE ON ) else() - set_target_properties(triton_rapids-identity + set_target_properties(triton_dev_tools-identity PROPERTIES BUILD_RPATH "\$ORIGIN" # set target compile options CXX_STANDARD 17 @@ -42,17 +42,17 @@ else() ) endif() -target_compile_options(triton_rapids-identity +target_compile_options(triton_dev_tools-identity PRIVATE "$<$:${RAPIDS_TRITON_CXX_FLAGS}>" "$<$:${RAPIDS_TRITON_CUDA_FLAGS}>" ) -target_include_directories(triton_rapids-identity +target_include_directories(triton_dev_tools-identity PRIVATE "$" "${CMAKE_CURRENT_SOURCE_DIR}/src" ) -target_link_libraries(triton_rapids-identity +target_link_libraries(triton_dev_tools-identity PRIVATE $<$:rmm::rmm> $<$:raft::raft> @@ -63,6 +63,6 @@ PRIVATE ) install( - TARGETS triton_rapids-identity - LIBRARY DESTINATION /opt/tritonserver/backends/rapids-identity + TARGETS triton_dev_tools-identity + LIBRARY DESTINATION /opt/tritonserver/backends/dev_tools-identity ) From aff27a0187ab4ea067ad22fb916257faf752cfd8 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 11:09:01 -0400 Subject: [PATCH 185/199] Update rapids namespace to dev_tools in identity --- backend/cpp/src/api.cc | 16 ++++++++-------- backend/cpp/src/model.h | 30 +++++++++++++++--------------- backend/cpp/src/names.h | 6 +++--- backend/cpp/src/shared_state.h | 4 ++-- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/backend/cpp/src/api.cc b/backend/cpp/src/api.cc index cd8dc9e..dbeb77b 100644 --- a/backend/cpp/src/api.cc +++ b/backend/cpp/src/api.cc @@ -35,41 +35,41 @@ namespace triton { namespace backend { namespace NAMESPACE { -using ModelState = rapids::TritonModelState; +using ModelState = dev_tools::TritonModelState; using ModelInstanceState = - rapids::ModelInstanceState; + dev_tools::ModelInstanceState; extern "C" { /** Confirm that backend is compatible with Triton's backend API version */ TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { - return rapids::triton_api::initialize(backend); + return dev_tools::triton_api::initialize(backend); } TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { - return rapids::triton_api::model_initialize(model); + return dev_tools::triton_api::model_initialize(model); } TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { - return rapids::triton_api::model_finalize(model); + return dev_tools::triton_api::model_finalize(model); } TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize( TRITONBACKEND_ModelInstance* instance) { - return rapids::triton_api::instance_initialize(instance); } TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize( TRITONBACKEND_ModelInstance* instance) { - return rapids::triton_api::instance_finalize(instance); + return dev_tools::triton_api::instance_finalize(instance); } TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** raw_requests, uint32_t const request_count) { - return rapids::triton_api::execute( + return dev_tools::triton_api::execute( instance, raw_requests, static_cast(request_count)); } diff --git a/backend/cpp/src/model.h b/backend/cpp/src/model.h index dca829b..a292d20 100644 --- a/backend/cpp/src/model.h +++ b/backend/cpp/src/model.h @@ -26,12 +26,12 @@ #include #include -#include // rapids::Batch -#include // rapids::MemoryType -#include // rapids::Model -#include // rapids::copy -#include // rapids::DeploymentType -#include // rapids::device_id_t +#include // dev_tools::Batch +#include // dev_tools::MemoryType +#include // dev_tools::Model +#include // dev_tools::copy +#include // dev_tools::DeploymentType +#include // dev_tools::device_id_t namespace triton { namespace backend { @@ -40,7 +40,7 @@ namespace NAMESPACE { /* Any logic necessary to perform inference with a model and manage its data * should be implemented in a struct named DevToolsModel, as shown here */ -struct DevToolsModel : rapids::Model { +struct DevToolsModel : dev_tools::Model { /*************************************************************************** * BOILERPLATE * * *********************************************************************** * @@ -48,11 +48,11 @@ struct DevToolsModel : rapids::Model { * implementation. **************************************************************************/ DevToolsModel(std::shared_ptr shared_state, - rapids::device_id_t device_id, + dev_tools::device_id_t device_id, cudaStream_t default_stream, - rapids::DeploymentType deployment_type, + dev_tools::DeploymentType deployment_type, std::string const& filepath) - : rapids::Model( + : dev_tools::Model( shared_state, device_id, default_stream, deployment_type, filepath) { } @@ -82,7 +82,7 @@ struct DevToolsModel : rapids::Model { * pointer to the underlying data. * 4. Call the `finalize` method on all output tensors. **************************************************************************/ - void predict(rapids::Batch& batch) const + void predict(dev_tools::Batch& batch) const { // 1. Acquire a tensor representing the input named "input__0" auto input = get_input(batch, "input__0"); @@ -91,7 +91,7 @@ struct DevToolsModel : rapids::Model { // 3. Perform inference. In this example, we simply copy the data from the // input to the output tensor. - rapids::copy(output, input); + dev_tools::copy(output, input); // 4. Call finalize on all output tensors. In this case, we have just one // output, so we call finalize on it. @@ -142,10 +142,10 @@ struct DevToolsModel : rapids::Model { * implementations that may switch their preferred memory location based on * properties of the batch. * - * Valid MemoryType options to return are rapids::HostMemory and - * rapids::DeviceMemory. + * Valid MemoryType options to return are dev_tools::HostMemory and + * dev_tools::DeviceMemory. **************************************************************************/ - std::optional preferred_mem_type(rapids::Batch& batch) const + std::optional preferred_mem_type(dev_tools::Batch& batch) const { return std::nullopt; } diff --git a/backend/cpp/src/names.h b/backend/cpp/src/names.h index 33d6fe0..3477995 100644 --- a/backend/cpp/src/names.h +++ b/backend/cpp/src/names.h @@ -18,12 +18,12 @@ /* Triton expects certain definitions within its backend libraries to follow * specific naming conventions. Specifically, for a backend named - * "rapids_identity," most definitions should appear within a namespace called - * triton::backend::rapids_identity. + * "dev_tools_identity," most definitions should appear within a namespace called + * triton::backend::dev_tools_identity. * * In order to facilitate this with minimal effort on the part of backend * developers, we ask that you put the name of your backend here. This macro is * then used to propagate the correct namespace name wherever it is needed in * the impl and interface code. */ -#define NAMESPACE rapids_identity +#define NAMESPACE dev_tools_identity diff --git a/backend/cpp/src/shared_state.h b/backend/cpp/src/shared_state.h index ed3e5ef..5daccf4 100644 --- a/backend/cpp/src/shared_state.h +++ b/backend/cpp/src/shared_state.h @@ -36,9 +36,9 @@ namespace NAMESPACE { * Not all backends require shared state, so leaving this implementation empty * is entirely valid */ -struct DevToolsSharedState : rapids::SharedModelState { +struct DevToolsSharedState : dev_tools::SharedModelState { DevToolsSharedState(std::unique_ptr&& config) - : rapids::SharedModelState{std::move(config)} + : dev_tools::SharedModelState{std::move(config)} { } void load() {} From f86e69b932caea7013977064615e46cf2654fb27 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 11:13:09 -0400 Subject: [PATCH 186/199] Update CMake variable names --- backend/cpp/src/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/cpp/src/CMakeLists.txt b/backend/cpp/src/CMakeLists.txt index bfd3f19..665b284 100644 --- a/backend/cpp/src/CMakeLists.txt +++ b/backend/cpp/src/CMakeLists.txt @@ -43,12 +43,12 @@ else() endif() target_compile_options(triton_dev_tools-identity - PRIVATE "$<$:${RAPIDS_TRITON_CXX_FLAGS}>" - "$<$:${RAPIDS_TRITON_CUDA_FLAGS}>" + PRIVATE "$<$:${DEV_TOOLS_CXX_FLAGS}>" + "$<$:${DEV_TOOLS_CUDA_FLAGS}>" ) target_include_directories(triton_dev_tools-identity - PRIVATE "$" + PRIVATE "$" "${CMAKE_CURRENT_SOURCE_DIR}/src" ) From 386a8eb33ff098bb644b2922e68cfc0f71eeff61 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 11:15:22 -0400 Subject: [PATCH 187/199] Update include path in tests --- backend/cpp/test/CMakeLists.txt | 12 ++++++------ backend/cpp/test/batch/batch.cpp | 2 +- backend/cpp/test/build_control.cpp | 2 +- backend/cpp/test/exceptions.cpp | 4 ++-- backend/cpp/test/memory/buffer.cpp | 8 ++++---- backend/cpp/test/memory/detail/copy.cpp | 8 ++++---- .../cpp/test/memory/detail/owned_device_buffer.cpp | 8 ++++---- backend/cpp/test/memory/resource.cpp | 6 +++--- backend/cpp/test/memory/types.cpp | 2 +- backend/cpp/test/tensor/dtype.cpp | 2 +- backend/cpp/test/tensor/tensor.cpp | 6 +++--- backend/cpp/test/test.cpp | 2 +- backend/cpp/test/triton/api/execute.cpp | 2 +- backend/cpp/test/triton/api/initialize.cpp | 2 +- backend/cpp/test/triton/api/instance_finalize.cpp | 2 +- backend/cpp/test/triton/api/instance_initialize.cpp | 2 +- backend/cpp/test/triton/api/model_finalize.cpp | 2 +- backend/cpp/test/triton/api/model_initialize.cpp | 2 +- backend/cpp/test/triton/backend.cpp | 2 +- backend/cpp/test/triton/config.cpp | 2 +- backend/cpp/test/triton/deployment.cpp | 2 +- backend/cpp/test/triton/device.cpp | 2 +- backend/cpp/test/triton/input.cpp | 2 +- backend/cpp/test/triton/logging.cpp | 2 +- backend/cpp/test/triton/model.cpp | 2 +- backend/cpp/test/triton/model_instance.cpp | 2 +- backend/cpp/test/triton/requests.cpp | 2 +- backend/cpp/test/triton/responses.cpp | 2 +- backend/cpp/test/triton/statistics.cpp | 2 +- backend/cpp/test/utils/const_agnostic.cpp | 2 +- backend/cpp/test/utils/narrow.cpp | 2 +- 31 files changed, 50 insertions(+), 50 deletions(-) diff --git a/backend/cpp/test/CMakeLists.txt b/backend/cpp/test/CMakeLists.txt index dbe92b2..386c6ca 100644 --- a/backend/cpp/test/CMakeLists.txt +++ b/backend/cpp/test/CMakeLists.txt @@ -15,7 +15,7 @@ #============================================================================= # keep the files in alphabetical order! -add_executable(test_rapids_triton +add_executable(test_triton_backend test/batch/batch.cpp test/build_control.cpp test/exceptions.cpp @@ -49,7 +49,7 @@ add_executable(test_rapids_triton ) IF(TRITON_ENABLE_GPU) - set_target_properties(test_rapids_triton + set_target_properties(test_triton_backend PROPERTIES BUILD_RPATH "\$ORIGIN" # set target compile options CXX_STANDARD 17 @@ -60,7 +60,7 @@ IF(TRITON_ENABLE_GPU) INTERFACE_POSITION_INDEPENDENT_CODE ON ) else() - set_target_properties(test_rapids_triton + set_target_properties(test_triton_backend PROPERTIES BUILD_RPATH "\$ORIGIN" # set target compile options CXX_STANDARD 17 @@ -70,12 +70,12 @@ else() ) endif() -target_compile_options(test_rapids_triton +target_compile_options(test_triton_backend PRIVATE "$<$:${RAPIDS_TRITON_CXX_FLAGS}>" "$<$:${RAPIDS_TRITON_CUDA_FLAGS}>" ) -target_include_directories(test_rapids_triton +target_include_directories(test_triton_backend PUBLIC "$" "$" ) @@ -87,7 +87,7 @@ find_library( PATHS /opt/tritonserver/lib ) -target_link_libraries(test_rapids_triton +target_link_libraries(test_triton_backend PRIVATE $<$:rmm::rmm> $<$:raft::raft> diff --git a/backend/cpp/test/batch/batch.cpp b/backend/cpp/test/batch/batch.cpp index e522ecb..61e7768 100644 --- a/backend/cpp/test/batch/batch.cpp +++ b/backend/cpp/test/batch/batch.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/build_control.cpp b/backend/cpp/test/build_control.cpp index 18392b0..4f5a9d3 100644 --- a/backend/cpp/test/build_control.cpp +++ b/backend/cpp/test/build_control.cpp @@ -16,7 +16,7 @@ #include -#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/test/exceptions.cpp b/backend/cpp/test/exceptions.cpp index 022d184..0e9ee67 100644 --- a/backend/cpp/test/exceptions.cpp +++ b/backend/cpp/test/exceptions.cpp @@ -17,11 +17,11 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include -#include +#include #include namespace triton { diff --git a/backend/cpp/test/memory/buffer.cpp b/backend/cpp/test/memory/buffer.cpp index 1181560..e1f7563 100644 --- a/backend/cpp/test/memory/buffer.cpp +++ b/backend/cpp/test/memory/buffer.cpp @@ -21,10 +21,10 @@ #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include namespace triton { diff --git a/backend/cpp/test/memory/detail/copy.cpp b/backend/cpp/test/memory/detail/copy.cpp index d669d6b..28ee3e8 100644 --- a/backend/cpp/test/memory/detail/copy.cpp +++ b/backend/cpp/test/memory/detail/copy.cpp @@ -16,15 +16,15 @@ #ifdef TRITON_ENABLE_GPU #include -#include +#include #else -#include +#include #endif #include #include -#include -#include +#include +#include #include namespace triton { diff --git a/backend/cpp/test/memory/detail/owned_device_buffer.cpp b/backend/cpp/test/memory/detail/owned_device_buffer.cpp index 9bd613f..1c9aeba 100644 --- a/backend/cpp/test/memory/detail/owned_device_buffer.cpp +++ b/backend/cpp/test/memory/detail/owned_device_buffer.cpp @@ -16,15 +16,15 @@ #ifdef TRITON_ENABLE_GPU #include -#include +#include #else -#include +#include #endif #include #include -#include -#include +#include +#include #include namespace triton { diff --git a/backend/cpp/test/memory/resource.cpp b/backend/cpp/test/memory/resource.cpp index 5571ef7..55ec5f3 100644 --- a/backend/cpp/test/memory/resource.cpp +++ b/backend/cpp/test/memory/resource.cpp @@ -24,9 +24,9 @@ #include #include -#include -#include -#include +#include +#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/test/memory/types.cpp b/backend/cpp/test/memory/types.cpp index 65eb386..cc86dc2 100644 --- a/backend/cpp/test/memory/types.cpp +++ b/backend/cpp/test/memory/types.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/tensor/dtype.cpp b/backend/cpp/test/tensor/dtype.cpp index 2e80c60..5f0a329 100644 --- a/backend/cpp/test/tensor/dtype.cpp +++ b/backend/cpp/test/tensor/dtype.cpp @@ -16,7 +16,7 @@ #include -#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/test/tensor/tensor.cpp b/backend/cpp/test/tensor/tensor.cpp index 3a1c5c2..e28ad83 100644 --- a/backend/cpp/test/tensor/tensor.cpp +++ b/backend/cpp/test/tensor/tensor.cpp @@ -17,13 +17,13 @@ #ifdef TRITON_ENABLE_GPU #include #else -#include +#include #endif #include #include -#include -#include +#include +#include #include namespace triton { diff --git a/backend/cpp/test/test.cpp b/backend/cpp/test/test.cpp index 4a7d7f4..0e3d136 100644 --- a/backend/cpp/test/test.cpp +++ b/backend/cpp/test/test.cpp @@ -16,7 +16,7 @@ #include -#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/test/triton/api/execute.cpp b/backend/cpp/test/triton/api/execute.cpp index ba48f8e..f4582b8 100644 --- a/backend/cpp/test/triton/api/execute.cpp +++ b/backend/cpp/test/triton/api/execute.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/triton/api/initialize.cpp b/backend/cpp/test/triton/api/initialize.cpp index 33b95d4..9862e5b 100644 --- a/backend/cpp/test/triton/api/initialize.cpp +++ b/backend/cpp/test/triton/api/initialize.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/triton/api/instance_finalize.cpp b/backend/cpp/test/triton/api/instance_finalize.cpp index f7218a3..4f71439 100644 --- a/backend/cpp/test/triton/api/instance_finalize.cpp +++ b/backend/cpp/test/triton/api/instance_finalize.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/triton/api/instance_initialize.cpp b/backend/cpp/test/triton/api/instance_initialize.cpp index 3ed2b91..a0349d4 100644 --- a/backend/cpp/test/triton/api/instance_initialize.cpp +++ b/backend/cpp/test/triton/api/instance_initialize.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/triton/api/model_finalize.cpp b/backend/cpp/test/triton/api/model_finalize.cpp index 367f16c..a7b85e1 100644 --- a/backend/cpp/test/triton/api/model_finalize.cpp +++ b/backend/cpp/test/triton/api/model_finalize.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/triton/api/model_initialize.cpp b/backend/cpp/test/triton/api/model_initialize.cpp index 071baa3..138efbf 100644 --- a/backend/cpp/test/triton/api/model_initialize.cpp +++ b/backend/cpp/test/triton/api/model_initialize.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/triton/backend.cpp b/backend/cpp/test/triton/backend.cpp index cacff7d..aa5020e 100644 --- a/backend/cpp/test/triton/backend.cpp +++ b/backend/cpp/test/triton/backend.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/triton/config.cpp b/backend/cpp/test/triton/config.cpp index 7a31891..1d4291f 100644 --- a/backend/cpp/test/triton/config.cpp +++ b/backend/cpp/test/triton/config.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/triton/deployment.cpp b/backend/cpp/test/triton/deployment.cpp index dbd22b7..cb0347b 100644 --- a/backend/cpp/test/triton/deployment.cpp +++ b/backend/cpp/test/triton/deployment.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/triton/device.cpp b/backend/cpp/test/triton/device.cpp index 53e49e0..cefe845 100644 --- a/backend/cpp/test/triton/device.cpp +++ b/backend/cpp/test/triton/device.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/triton/input.cpp b/backend/cpp/test/triton/input.cpp index 1cac65d..c7b31c2 100644 --- a/backend/cpp/test/triton/input.cpp +++ b/backend/cpp/test/triton/input.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/triton/logging.cpp b/backend/cpp/test/triton/logging.cpp index f1f60cd..f1dba6c 100644 --- a/backend/cpp/test/triton/logging.cpp +++ b/backend/cpp/test/triton/logging.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include namespace triton { namespace backend { diff --git a/backend/cpp/test/triton/model.cpp b/backend/cpp/test/triton/model.cpp index 3226e35..c46eac1 100644 --- a/backend/cpp/test/triton/model.cpp +++ b/backend/cpp/test/triton/model.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/triton/model_instance.cpp b/backend/cpp/test/triton/model_instance.cpp index 3582f75..6655785 100644 --- a/backend/cpp/test/triton/model_instance.cpp +++ b/backend/cpp/test/triton/model_instance.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/triton/requests.cpp b/backend/cpp/test/triton/requests.cpp index cff2af3..bc8a7b0 100644 --- a/backend/cpp/test/triton/requests.cpp +++ b/backend/cpp/test/triton/requests.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/triton/responses.cpp b/backend/cpp/test/triton/responses.cpp index 4ca7d52..8129f90 100644 --- a/backend/cpp/test/triton/responses.cpp +++ b/backend/cpp/test/triton/responses.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/triton/statistics.cpp b/backend/cpp/test/triton/statistics.cpp index 7c01a6d..4c0b189 100644 --- a/backend/cpp/test/triton/statistics.cpp +++ b/backend/cpp/test/triton/statistics.cpp @@ -14,4 +14,4 @@ * limitations under the License. */ -#include +#include diff --git a/backend/cpp/test/utils/const_agnostic.cpp b/backend/cpp/test/utils/const_agnostic.cpp index 2a74c69..fe185b8 100644 --- a/backend/cpp/test/utils/const_agnostic.cpp +++ b/backend/cpp/test/utils/const_agnostic.cpp @@ -16,7 +16,7 @@ #include -#include +#include #include namespace triton { diff --git a/backend/cpp/test/utils/narrow.cpp b/backend/cpp/test/utils/narrow.cpp index d9919ca..2b08b7b 100644 --- a/backend/cpp/test/utils/narrow.cpp +++ b/backend/cpp/test/utils/narrow.cpp @@ -16,7 +16,7 @@ #include -#include +#include #include namespace triton { From c6da4f2016c55e918f0683c38f41d2cdf05ab966 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 11:17:03 -0400 Subject: [PATCH 188/199] Update remaining names in tests --- backend/cpp/test/CMakeLists.txt | 8 +++---- backend/cpp/test/build_control.cpp | 6 ++--- backend/cpp/test/exceptions.cpp | 12 +++++----- backend/cpp/test/memory/buffer.cpp | 20 ++++++++--------- backend/cpp/test/memory/detail/copy.cpp | 6 ++--- .../memory/detail/owned_device_buffer.cpp | 6 ++--- backend/cpp/test/memory/resource.cpp | 6 ++--- backend/cpp/test/tensor/dtype.cpp | 6 ++--- backend/cpp/test/tensor/tensor.cpp | 22 +++++++++---------- backend/cpp/test/test.cpp | 6 ++--- backend/cpp/test/triton/logging.cpp | 8 +++---- backend/cpp/test/utils/const_agnostic.cpp | 6 ++--- backend/cpp/test/utils/narrow.cpp | 6 ++--- 13 files changed, 59 insertions(+), 59 deletions(-) diff --git a/backend/cpp/test/CMakeLists.txt b/backend/cpp/test/CMakeLists.txt index 386c6ca..9d8230e 100644 --- a/backend/cpp/test/CMakeLists.txt +++ b/backend/cpp/test/CMakeLists.txt @@ -71,13 +71,13 @@ else() endif() target_compile_options(test_triton_backend - PRIVATE "$<$:${RAPIDS_TRITON_CXX_FLAGS}>" - "$<$:${RAPIDS_TRITON_CUDA_FLAGS}>" + PRIVATE "$<$:${DEV_TOOLS_CXX_FLAGS}>" + "$<$:${DEV_TOOLS_CUDA_FLAGS}>" ) target_include_directories(test_triton_backend - PUBLIC "$" - "$" + PUBLIC "$" + "$" ) diff --git a/backend/cpp/test/build_control.cpp b/backend/cpp/test/build_control.cpp index 4f5a9d3..4e45cbb 100644 --- a/backend/cpp/test/build_control.cpp +++ b/backend/cpp/test/build_control.cpp @@ -20,9 +20,9 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { -TEST(RapidsTriton, build_control) +TEST(DevToolsTriton, build_control) { #ifdef TRITON_ENABLE_GPU ASSERT_EQ(IS_GPU_BUILD, true) << "IS_GPU_BUILD constant has wrong value\n"; @@ -30,6 +30,6 @@ TEST(RapidsTriton, build_control) ASSERT_EQ(IS_GPU_BUILD, false) << "IS_GPU_BUILD constant has wrong value\n"; #endif } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/test/exceptions.cpp b/backend/cpp/test/exceptions.cpp index 0e9ee67..7bdabfc 100644 --- a/backend/cpp/test/exceptions.cpp +++ b/backend/cpp/test/exceptions.cpp @@ -26,9 +26,9 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { -TEST(RapidsTriton, default_except) +TEST(DevToolsTriton, default_except) { try { throw TritonException(); @@ -37,7 +37,7 @@ TEST(RapidsTriton, default_except) } } -TEST(RapidsTriton, msg_except) +TEST(DevToolsTriton, msg_except) { auto msg = std::string("TEST ERROR MESSAGE"); try { @@ -61,14 +61,14 @@ TEST(RapidsTriton, msg_except) } } -TEST(RapidsTriton, triton_check) +TEST(DevToolsTriton, triton_check) { auto msg = std::string("TEST ERROR MESSAGE"); EXPECT_THROW(triton_check(TRITONSERVER_ErrorNew(Error::Internal, msg.c_str())), TritonException); triton_check(nullptr); } -TEST(RapidsTriton, cuda_check) +TEST(DevToolsTriton, cuda_check) { #ifdef TRITON_ENABLE_GPU EXPECT_THROW(cuda_check(cudaError::cudaErrorMissingConfiguration), TritonException); @@ -78,6 +78,6 @@ TEST(RapidsTriton, cuda_check) #endif } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/test/memory/buffer.cpp b/backend/cpp/test/memory/buffer.cpp index e1f7563..6edc3b7 100644 --- a/backend/cpp/test/memory/buffer.cpp +++ b/backend/cpp/test/memory/buffer.cpp @@ -29,8 +29,8 @@ namespace triton { namespace backend { -namespace rapids { -TEST(RapidsTriton, default_buffer) +namespace dev_tools { +TEST(DevToolsTriton, default_buffer) { auto buffer = Buffer(); EXPECT_EQ(buffer.mem_type(), HostMemory); @@ -47,7 +47,7 @@ TEST(RapidsTriton, default_buffer) #endif } -TEST(RapidsTriton, device_buffer) +TEST(DevToolsTriton, device_buffer) { auto data = std::vector{1, 2, 3}; #ifdef TRITON_ENABLE_GPU @@ -73,7 +73,7 @@ TEST(RapidsTriton, device_buffer) #endif } -TEST(RapidsTriton, non_owning_device_buffer) +TEST(DevToolsTriton, non_owning_device_buffer) { auto data = std::vector{1, 2, 3}; #ifdef TRITON_ENABLE_GPU @@ -102,7 +102,7 @@ TEST(RapidsTriton, non_owning_device_buffer) #endif } -TEST(RapidsTriton, host_buffer) +TEST(DevToolsTriton, host_buffer) { auto data = std::vector{1, 2, 3}; auto buffer = Buffer(data.size(), HostMemory, 0, 0); @@ -118,7 +118,7 @@ TEST(RapidsTriton, host_buffer) EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, non_owning_host_buffer) +TEST(DevToolsTriton, non_owning_host_buffer) { auto data = std::vector{1, 2, 3}; auto buffer = Buffer(data.data(), data.size(), HostMemory); @@ -131,7 +131,7 @@ TEST(RapidsTriton, non_owning_host_buffer) EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, copy_buffer) +TEST(DevToolsTriton, copy_buffer) { auto data = std::vector{1, 2, 3}; auto orig_buffer = Buffer(data.data(), data.size(), HostMemory); @@ -145,7 +145,7 @@ TEST(RapidsTriton, copy_buffer) EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, move_buffer) +TEST(DevToolsTriton, move_buffer) { auto data = std::vector{1, 2, 3}; auto buffer = Buffer(Buffer(data.data(), data.size(), HostMemory)); @@ -158,7 +158,7 @@ TEST(RapidsTriton, move_buffer) EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, move_assignment_buffer) +TEST(DevToolsTriton, move_assignment_buffer) { auto data = std::vector{1, 2, 3}; @@ -173,6 +173,6 @@ TEST(RapidsTriton, move_assignment_buffer) ASSERT_EQ(buffer.size(), data.size()); } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/test/memory/detail/copy.cpp b/backend/cpp/test/memory/detail/copy.cpp index 28ee3e8..da59fa7 100644 --- a/backend/cpp/test/memory/detail/copy.cpp +++ b/backend/cpp/test/memory/detail/copy.cpp @@ -29,8 +29,8 @@ namespace triton { namespace backend { -namespace rapids { -TEST(RapidsTriton, copy) +namespace dev_tools { +TEST(DevToolsTriton, copy) { auto data = std::vector{1, 2, 3}; auto data_out = std::vector(data.size()); @@ -57,6 +57,6 @@ TEST(RapidsTriton, copy) #endif } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/test/memory/detail/owned_device_buffer.cpp b/backend/cpp/test/memory/detail/owned_device_buffer.cpp index 1c9aeba..f924a07 100644 --- a/backend/cpp/test/memory/detail/owned_device_buffer.cpp +++ b/backend/cpp/test/memory/detail/owned_device_buffer.cpp @@ -29,8 +29,8 @@ namespace triton { namespace backend { -namespace rapids { -TEST(RapidsTriton, owned_device_buffer) +namespace dev_tools { +TEST(DevToolsTriton, owned_device_buffer) { auto data = std::vector{1, 2, 3}; #ifdef TRITON_ENABLE_GPU @@ -59,6 +59,6 @@ TEST(RapidsTriton, owned_device_buffer) #endif } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/test/memory/resource.cpp b/backend/cpp/test/memory/resource.cpp index 55ec5f3..5f413bd 100644 --- a/backend/cpp/test/memory/resource.cpp +++ b/backend/cpp/test/memory/resource.cpp @@ -30,9 +30,9 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { -TEST(RapidsTriton, set_memory_resource) +TEST(DevToolsTriton, set_memory_resource) { #ifdef TRITON_ENABLE_GPU auto device_id = int{}; @@ -47,6 +47,6 @@ TEST(RapidsTriton, set_memory_resource) #endif } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/test/tensor/dtype.cpp b/backend/cpp/test/tensor/dtype.cpp index 5f0a329..53cf17f 100644 --- a/backend/cpp/test/tensor/dtype.cpp +++ b/backend/cpp/test/tensor/dtype.cpp @@ -20,7 +20,7 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { template void check_dtype_conversion() @@ -29,7 +29,7 @@ void check_dtype_conversion() EXPECT_EQ(D, TritonDtype::type const>::value); } -TEST(RapidsTriton, dtype) +TEST(DevToolsTriton, dtype) { check_dtype_conversion(); check_dtype_conversion(); @@ -46,6 +46,6 @@ TEST(RapidsTriton, dtype) check_dtype_conversion(); } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/test/tensor/tensor.cpp b/backend/cpp/test/tensor/tensor.cpp index e28ad83..6e63450 100644 --- a/backend/cpp/test/tensor/tensor.cpp +++ b/backend/cpp/test/tensor/tensor.cpp @@ -28,16 +28,16 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { -TEST(RapidsTriton, default_tensor) +TEST(DevToolsTriton, default_tensor) { auto tensor = Tensor(); EXPECT_EQ(tensor.buffer().size(), 0); EXPECT_EQ(tensor.shape().size(), 0); } -TEST(RapidsTriton, move_buffer_tensor) +TEST(DevToolsTriton, move_buffer_tensor) { auto shape = std::vector{2, 2}; auto data = std::vector{1, 2, 3, 4}; @@ -56,7 +56,7 @@ TEST(RapidsTriton, move_buffer_tensor) EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, multi_buffer_tensor) +TEST(DevToolsTriton, multi_buffer_tensor) { auto shape = std::vector{2, 2}; auto data = std::vector{1, 2, 3, 4}; @@ -85,7 +85,7 @@ TEST(RapidsTriton, multi_buffer_tensor) EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); } -TEST(RapidsTriton, tensor_copy) +TEST(DevToolsTriton, tensor_copy) { auto shape = std::vector{2, 2}; auto data = std::vector{1, 2, 3, 4}; @@ -95,7 +95,7 @@ TEST(RapidsTriton, tensor_copy) auto data2 = std::vector(data1.size()); auto tensor2 = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); - rapids::copy(tensor2, tensor1); + dev_tools::copy(tensor2, tensor1); auto data_out = std::vector(tensor2.data(), tensor2.data() + tensor2.size()); EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); @@ -105,10 +105,10 @@ TEST(RapidsTriton, tensor_copy) auto tensor3 = Tensor(small_shape, Buffer{small_data.data(), small_data.size(), HostMemory}); - EXPECT_THROW(rapids::copy(tensor3, tensor1), TritonException); + EXPECT_THROW(dev_tools::copy(tensor3, tensor1), TritonException); } -TEST(RapidsTriton, tensor_multi_copy) +TEST(DevToolsTriton, tensor_multi_copy) { auto shape = std::vector{2, 2}; auto data = std::vector{1, 2, 3, 4}; @@ -125,7 +125,7 @@ TEST(RapidsTriton, tensor_multi_copy) return Tensor(receiver_shape, Buffer{std::size_t{1}, HostMemory}); }); - rapids::copy(receivers.begin(), receivers.end(), tensor1); + dev_tools::copy(receivers.begin(), receivers.end(), tensor1); auto data_out = std::vector{}; data_out.reserve(receivers.size()); @@ -137,9 +137,9 @@ TEST(RapidsTriton, tensor_multi_copy) // Throw if trying to copy to too many outputs receivers.emplace_back(receiver_shape, Buffer{std::size_t{1}, HostMemory}); - EXPECT_THROW(rapids::copy(receivers.begin(), receivers.end(), tensor1), TritonException); + EXPECT_THROW(dev_tools::copy(receivers.begin(), receivers.end(), tensor1), TritonException); } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/test/test.cpp b/backend/cpp/test/test.cpp index 0e3d136..e6bba9c 100644 --- a/backend/cpp/test/test.cpp +++ b/backend/cpp/test/test.cpp @@ -20,9 +20,9 @@ namespace triton { namespace backend { -namespace rapids { +namespace dev_tools { -TEST(RapidsTriton, installed) { std::cout << test_install() << "\n"; } -} // namespace rapids +TEST(DevToolsTriton, installed) { std::cout << test_install() << "\n"; } +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/test/triton/logging.cpp b/backend/cpp/test/triton/logging.cpp index f1dba6c..ca55c5f 100644 --- a/backend/cpp/test/triton/logging.cpp +++ b/backend/cpp/test/triton/logging.cpp @@ -21,8 +21,8 @@ namespace triton { namespace backend { -namespace rapids { -TEST(RapidsTriton, logging) +namespace dev_tools { +TEST(DevToolsTriton, logging) { log_debug("Debug test message"); log_info("Info test message"); @@ -30,7 +30,7 @@ TEST(RapidsTriton, logging) log_error("Error test message"); } -TEST(RapidsTriton, stream_logging) +TEST(DevToolsTriton, stream_logging) { log_debug() << "Streamed debug test message"; log_info() << "Streamed info test message"; @@ -38,6 +38,6 @@ TEST(RapidsTriton, stream_logging) log_error() << "Streamed error test message"; } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/test/utils/const_agnostic.cpp b/backend/cpp/test/utils/const_agnostic.cpp index fe185b8..2ab28c2 100644 --- a/backend/cpp/test/utils/const_agnostic.cpp +++ b/backend/cpp/test/utils/const_agnostic.cpp @@ -21,13 +21,13 @@ namespace triton { namespace backend { -namespace rapids { -TEST(RapidsTriton, const_agnostic) +namespace dev_tools { +TEST(DevToolsTriton, const_agnostic) { static_assert(std::is_same, void>::value); static_assert(std::is_same, void>::value); } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton diff --git a/backend/cpp/test/utils/narrow.cpp b/backend/cpp/test/utils/narrow.cpp index 2b08b7b..9741912 100644 --- a/backend/cpp/test/utils/narrow.cpp +++ b/backend/cpp/test/utils/narrow.cpp @@ -21,8 +21,8 @@ namespace triton { namespace backend { -namespace rapids { -TEST(RapidsTriton, narrow) +namespace dev_tools { +TEST(DevToolsTriton, narrow) { EXPECT_THROW(narrow(-1), TritonException); narrow(int{5}); @@ -30,6 +30,6 @@ TEST(RapidsTriton, narrow) narrow(std::size_t{5}); } -} // namespace rapids +} // namespace dev_tools } // namespace backend } // namespace triton From 3143d9a52c8ad7bac154ad4cb27a428f96264532 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 11:19:36 -0400 Subject: [PATCH 189/199] Update names in cmake --- backend/cpp/cmake/modules/ConfigureCUDA.cmake | 18 +++++++++--------- backend/cpp/cmake/thirdparty/get_raft.cmake | 10 +++++----- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/backend/cpp/cmake/modules/ConfigureCUDA.cmake b/backend/cpp/cmake/modules/ConfigureCUDA.cmake index 20826ab..3b34212 100644 --- a/backend/cpp/cmake/modules/ConfigureCUDA.cmake +++ b/backend/cpp/cmake/modules/ConfigureCUDA.cmake @@ -15,29 +15,29 @@ #============================================================================= if(DISABLE_DEPRECATION_WARNINGS) - list(APPEND RAPIDS_TRITON_CXX_FLAGS -Wno-deprecated-declarations) - list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Xcompiler=-Wno-deprecated-declarations) + list(APPEND DEV_TOOLS_CXX_FLAGS -Wno-deprecated-declarations) + list(APPEND DEV_TOOLS_CUDA_FLAGS -Xcompiler=-Wno-deprecated-declarations) endif() if(CMAKE_COMPILER_IS_GNUCXX) - list(APPEND RAPIDS_TRITON_CXX_FLAGS -Wall -Werror -Wno-unknown-pragmas -Wno-error=deprecated-declarations) + list(APPEND DEV_TOOLS_CXX_FLAGS -Wall -Werror -Wno-unknown-pragmas -Wno-error=deprecated-declarations) endif() -list(APPEND RAPIDS_TRITON_CUDA_FLAGS --expt-extended-lambda --expt-relaxed-constexpr) +list(APPEND DEV_TOOLS_CUDA_FLAGS --expt-extended-lambda --expt-relaxed-constexpr) # set warnings as errors if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 11.2.0) - list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Werror=all-warnings) + list(APPEND DEV_TOOLS_CUDA_FLAGS -Werror=all-warnings) endif() -list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Xcompiler=-Wall,-Werror,-Wno-error=deprecated-declarations) +list(APPEND DEV_TOOLS_CUDA_FLAGS -Xcompiler=-Wall,-Werror,-Wno-error=deprecated-declarations) # Option to enable line info in CUDA device compilation to allow introspection when profiling / memchecking if(CUDA_ENABLE_LINEINFO) - list(APPEND RAPIDS_TRITON_CUDA_FLAGS -lineinfo) + list(APPEND DEV_TOOLS_CUDA_FLAGS -lineinfo) endif() # Debug options if(CMAKE_BUILD_TYPE MATCHES Debug) - message(VERBOSE "RAPIDS_TRITON: Building with debugging flags") - list(APPEND RAPIDS_TRITON_CUDA_FLAGS -G -Xcompiler=-rdynamic) + message(VERBOSE "DEV_TOOLS: Building with debugging flags") + list(APPEND DEV_TOOLS_CUDA_FLAGS -G -Xcompiler=-rdynamic) endif() diff --git a/backend/cpp/cmake/thirdparty/get_raft.cmake b/backend/cpp/cmake/thirdparty/get_raft.cmake index 6a37d7e..28c4460 100644 --- a/backend/cpp/cmake/thirdparty/get_raft.cmake +++ b/backend/cpp/cmake/thirdparty/get_raft.cmake @@ -33,17 +33,17 @@ function(find_and_configure_raft) "RAFT_COMPILE_LIBRARIES OFF" ) - message(VERBOSE "RAPIDS_TRITON: Using RAFT located in ${raft_SOURCE_DIR}") + message(VERBOSE "DEV_TOOLS: Using RAFT located in ${raft_SOURCE_DIR}") endfunction() -set(RAPIDS_TRITON_MIN_VERSION_raft "${RAPIDS_TRITON_VERSION_MAJOR}.${RAPIDS_TRITON_VERSION_MINOR}.00") -set(RAPIDS_TRITON_BRANCH_VERSION_raft "${RAPIDS_TRITON_VERSION_MAJOR}.${RAPIDS_TRITON_VERSION_MINOR}") +set(DEV_TOOLS_MIN_VERSION_raft "${DEV_TOOLS_VERSION_MAJOR}.${DEV_TOOLS_VERSION_MINOR}.00") +set(DEV_TOOLS_BRANCH_VERSION_raft "${DEV_TOOLS_VERSION_MAJOR}.${DEV_TOOLS_VERSION_MINOR}") # Change pinned tag here to test a commit in CI # To use a different RAFT locally, set the CMake variable # CPM_raft_SOURCE=/path/to/local/raft -find_and_configure_raft(VERSION ${RAPIDS_TRITON_MIN_VERSION_raft} +find_and_configure_raft(VERSION ${DEV_TOOLS_MIN_VERSION_raft} FORK rapidsai - PINNED_TAG branch-${RAPIDS_TRITON_BRANCH_VERSION_raft} + PINNED_TAG branch-${DEV_TOOLS_BRANCH_VERSION_raft} ) From 793b57d2bcc71891dc7b68603ba0565c021bf038 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 11:25:54 -0400 Subject: [PATCH 190/199] Update names in Dockerfile and CMakeLists --- backend/Dockerfile | 30 ++++++------- backend/cpp/CMakeLists.txt | 90 +++++++++++++++++++------------------- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 748dc6d..a668bf2 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -42,7 +42,7 @@ RUN wget \ && bash Miniconda3-latest-Linux-x86_64.sh -b \ && rm -f Miniconda3-latest-Linux-x86_64.sh -COPY ./conda/environments/rapids_triton_dev.yml /environment.yml +COPY ./conda/environments/triton_backend.yml /environment.yml RUN conda env update -f /environment.yml \ && rm /environment.yml \ @@ -52,11 +52,11 @@ RUN conda env update -f /environment.yml \ ENV PYTHONDONTWRITEBYTECODE=false -SHELL ["conda", "run", "--no-capture-output", "-n", "rapids_triton_dev", "/bin/bash", "-c"] +SHELL ["conda", "run", "--no-capture-output", "-n", "triton_backend", "/bin/bash", "-c"] FROM base as build-stage -COPY ./cpp /rapids_triton +COPY ./cpp /triton_backend ARG TRITON_VERSION ENV TRITON_VERSION=$TRITON_VERSION @@ -70,9 +70,9 @@ ENV BUILD_EXAMPLE=$BUILD_EXAMPLE ARG TRITON_ENABLE_GPU ENV TRITON_ENABLE_GPU=$TRITON_ENABLE_GPU -RUN mkdir /rapids_triton/build +RUN mkdir /triton_backend/build -WORKDIR /rapids_triton/build +WORKDIR /triton_backend/build RUN cmake \ -GNinja \ @@ -88,7 +88,7 @@ RUN --mount=type=cache,target=/ccache/ ninja install FROM base as test-install -COPY ./conda/environments/rapids_triton_test.yml /environment.yml +COPY ./conda/environments/triton_backend_test.yml /environment.yml RUN conda env update -f /environment.yml \ && rm /environment.yml \ @@ -96,33 +96,33 @@ RUN conda env update -f /environment.yml \ && find /root/miniconda3/ -follow -type f -name '*.pyc' -delete \ && find /root/miniconda3/ -follow -type f -name '*.js.map' -delete -COPY ./python /rapids_triton +COPY ./python /triton_backend -RUN conda run -n rapids_triton_test pip install /rapids_triton \ - && rm -rf /rapids_triton +RUN conda run -n triton_backend_test pip install /triton_backend \ + && rm -rf /triton_backend FROM build-stage as test-stage COPY --from=test-install /root/miniconda3 /root/miniconda3 -ENV TEST_EXE=/rapids_triton/build/test_rapids_triton +ENV TEST_EXE=/triton_backend/build/test_triton_backend COPY qa /qa -ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "rapids_triton_test", "/bin/bash", "/qa/entrypoint.sh"] +ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "triton_backend_test", "/bin/bash", "/qa/entrypoint.sh"] FROM ${BASE_IMAGE} RUN mkdir /models # Remove existing backend install -RUN if [ -d /opt/tritonserver/backends/rapids-identity ]; \ +RUN if [ -d /opt/tritonserver/backends/dev_tools-identity ]; \ then \ - rm -rf /opt/tritonserver/backends/rapids-identity/*; \ + rm -rf /opt/tritonserver/backends/dev_tools-identity/*; \ fi COPY --from=build-stage \ - /opt/tritonserver/backends/rapids-identity \ - /opt/tritonserver/backends/rapids-identity + /opt/tritonserver/backends/dev_tools-identity \ + /opt/tritonserver/backends/dev_tools-identity ENTRYPOINT ["tritonserver", "--model-repository=/models"] diff --git a/backend/cpp/CMakeLists.txt b/backend/cpp/CMakeLists.txt index 280997f..4540a2b 100644 --- a/backend/cpp/CMakeLists.txt +++ b/backend/cpp/CMakeLists.txt @@ -28,8 +28,8 @@ include(rapids-find) # - User Options ------------------------------------------------------------ option(TRITON_ENABLE_GPU "Enable GPU support in Triton" ON) -option(BUILD_TESTS "Build rapids_triton unit-tests" ON) -option(BUILD_EXAMPLE "Build rapids_identity example backend" OFF) +option(BUILD_TESTS "Build backend dev tools unit-tests" ON) +option(BUILD_EXAMPLE "Build dev tools identity example backend" OFF) option(CUDA_ENABLE_KERNELINFO "Enable kernel resource usage info" OFF) option(CUDA_ENABLE_LINEINFO "Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler)" OFF) option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF) @@ -41,27 +41,27 @@ set(TRITON_COMMON_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-serve set(TRITON_CORE_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-server/core repo") set(TRITON_BACKEND_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-server/backend repo") -message(VERBOSE "RAPIDS_TRITON: Build RAPIDS_TRITON unit-tests: ${BUILD_TESTS}") -message(VERBOSE "RAPIDS_TRITON: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") -message(VERBOSE "RAPIDS_TRITON: Disable depreaction warnings " ${DISABLE_DEPRECATION_WARNINGS}) -message(VERBOSE "RAPIDS_TRITON: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELINFO}") -message(VERBOSE "RAPIDS_TRITON: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") -message(VERBOSE "RAPIDS_TRITON: Enable nvtx markers: ${NVTX}") -message(VERBOSE "RAPIDS_TRITON: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") -message(VERBOSE "RAPIDS_TRITON: Enable GPU support: ${TRITON_ENABLE_GPU}") -message(VERBOSE "RAPIDS_TRITON: Enable statistics collection in Triton: ${TRITON_ENABLE_STATS}") -message(VERBOSE "RAPIDS_TRITON: Triton common repo tag: ${TRITON_COMMON_REPO_TAG}") -message(VERBOSE "RAPIDS_TRITON: Triton core repo tag: ${TRITON_CORE_REPO_TAG}") -message(VERBOSE "RAPIDS_TRITON: Triton backend repo tag: ${TRITON_BACKEND_REPO_TAG}") +message(VERBOSE "DEV_TOOLS: Build DEV_TOOLS unit-tests: ${BUILD_TESTS}") +message(VERBOSE "DEV_TOOLS: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") +message(VERBOSE "DEV_TOOLS: Disable depreaction warnings " ${DISABLE_DEPRECATION_WARNINGS}) +message(VERBOSE "DEV_TOOLS: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELINFO}") +message(VERBOSE "DEV_TOOLS: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") +message(VERBOSE "DEV_TOOLS: Enable nvtx markers: ${NVTX}") +message(VERBOSE "DEV_TOOLS: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") +message(VERBOSE "DEV_TOOLS: Enable GPU support: ${TRITON_ENABLE_GPU}") +message(VERBOSE "DEV_TOOLS: Enable statistics collection in Triton: ${TRITON_ENABLE_STATS}") +message(VERBOSE "DEV_TOOLS: Triton common repo tag: ${TRITON_COMMON_REPO_TAG}") +message(VERBOSE "DEV_TOOLS: Triton core repo tag: ${TRITON_CORE_REPO_TAG}") +message(VERBOSE "DEV_TOOLS: Triton backend repo tag: ${TRITON_BACKEND_REPO_TAG}") ############################################################################## # - Project Initialization --------------------------------------------------- if(TRITON_ENABLE_GPU) - rapids_cuda_init_architectures(RAPIDS_TRITON) - project(RAPIDS_TRITON VERSION 22.02.00 LANGUAGES CXX CUDA) + rapids_cuda_init_architectures(DEV_TOOLS) + project(DEV_TOOLS VERSION 22.02.00 LANGUAGES CXX CUDA) else() - project(RAPIDS_TRITON VERSION 22.02.00 LANGUAGES CXX) + project(DEV_TOOLS VERSION 22.02.00 LANGUAGES CXX) endif() @@ -77,7 +77,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # Set RMM logging level set(RMM_LOGGING_LEVEL "INFO" CACHE STRING "Choose the logging level.") set_property(CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF") -message(VERBOSE "RAPIDS_TRITON: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") +message(VERBOSE "DEV_TOOLS: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") ############################################################################## # - Conda environment detection ---------------------------------------------- @@ -85,7 +85,7 @@ message(VERBOSE "RAPIDS_TRITON: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") if(DETECT_CONDA_ENV) rapids_cmake_support_conda_env( conda_env MODIFY_PREFIX_PATH ) if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED ENV{CONDA_PREFIX}) - message(STATUS "RAPIDS_TRITON: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}") + message(STATUS "DEV_TOOLS: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}") set(CMAKE_INSTALL_PREFIX "$ENV{CONDA_PREFIX}") endif() endif() @@ -102,8 +102,8 @@ if(TRITON_ENABLE_GPU) # * enable the CMake CUDA language # * set other CUDA compilation flags rapids_find_package(CUDAToolkit REQUIRED - BUILD_EXPORT_SET rapids_triton-exports - INSTALL_EXPORT_SET rapids_triton-exports + BUILD_EXPORT_SET triton_backend-exports + INSTALL_EXPORT_SET triton_backend-exports ) include(cmake/modules/ConfigureCUDA.cmake) endif() @@ -129,12 +129,12 @@ endif() ############################################################################## # - install targets----------------------------------------------------------- -add_library(rapids_triton INTERFACE) -add_library(rapids_triton::rapids_triton ALIAS rapids_triton) -target_include_directories(rapids_triton INTERFACE "$" +add_library(triton_backend INTERFACE) +add_library(triton_backend::triton_backend ALIAS triton_backend) +target_include_directories(triton_backend INTERFACE "$" "$") -target_link_libraries(rapids_triton +target_link_libraries(triton_backend INTERFACE $<$:rmm::rmm> $<$:raft::raft> @@ -144,58 +144,58 @@ INTERFACE if (TRITON_ENABLE_GPU) target_compile_features( - rapids_triton INTERFACE cxx_std_17 + triton_backend INTERFACE cxx_std_17 $ ) else() target_compile_features( - rapids_triton INTERFACE cxx_std_17 + triton_backend INTERFACE cxx_std_17 ) endif() rapids_cmake_install_lib_dir(lib_dir) -install(TARGETS rapids_triton +install(TARGETS triton_backend DESTINATION ${lib_dir} - EXPORT rapids_triton-exports + EXPORT triton_backend-exports ) include(GNUInstallDirs) -install(DIRECTORY include/rapids_triton/ - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rapids_triton +install(DIRECTORY include/triton_backend/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/triton_backend ) -# Temporary install of rapids_triton.hpp while the file is removed -install(FILES include/rapids_triton.hpp - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rapids_triton +# Temporary install of triton_backend.hpp while the file is removed +install(FILES include/triton_backend.hpp + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/triton_backend ) ############################################################################## # - install export ----------------------------------------------------------- set(doc_string [=[ -Provide targets for RAPIDS_TRITON. +Provide targets for Triton Backend Developer Tools. -RAPIDS_TRITON is a header-only library designed to make it easier and faster -to integrate RAPIDS algorithms as Triton backends. +Triton Backend Developer Tools is a header-only library designed to make it +easier and faster to create custom C++ Triton backends. ]=]) - rapids_export(INSTALL rapids_triton - EXPORT_SET rapids_triton-exports - GLOBAL_TARGETS rapids_triton # since we can't hook into EXPORT SETS - NAMESPACE rapids_triton:: + rapids_export(INSTALL triton_backend + EXPORT_SET triton_backend-exports + GLOBAL_TARGETS triton_backend # since we can't hook into EXPORT SETS + NAMESPACE triton_backend:: DOCUMENTATION doc_string ) ############################################################################## # - build export ------------------------------------------------------------- -rapids_export(BUILD rapids_triton - EXPORT_SET rapids_triton-exports - GLOBAL_TARGETS rapids_triton # since we can't hook into EXPORT SETS +rapids_export(BUILD triton_backend + EXPORT_SET triton_backend-exports + GLOBAL_TARGETS triton_backend # since we can't hook into EXPORT SETS LANGUAGES CUDA DOCUMENTATION doc_string - NAMESPACE rapids_triton:: + NAMESPACE triton_backend:: ) ############################################################################## From fad694ac79d495197b8e0862335379e5eca1ff4b Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 11:28:10 -0400 Subject: [PATCH 191/199] Update naming in build scripts --- backend/build.sh | 8 ++++---- backend/ci/local/build.sh | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/backend/build.sh b/backend/build.sh index ab15d97..f07628a 100755 --- a/backend/build.sh +++ b/backend/build.sh @@ -99,10 +99,10 @@ do ;; --tag-commit ) [ -z $EXAMPLE_TAG ] \ - && EXAMPLE_TAG="rapids_triton_identity:$(cd $REPODIR; git rev-parse --short HEAD)" \ + && EXAMPLE_TAG="triton_dt_identity:$(cd $REPODIR; git rev-parse --short HEAD)" \ || true [ -z $TEST_TAG ] \ - && TEST_TAG="rapids_triton_identity_test:$(cd $REPODIR; git rev-parse --short HEAD)" \ + && TEST_TAG="triton_dt_identity_test:$(cd $REPODIR; git rev-parse --short HEAD)" \ || true ;; --) @@ -115,11 +115,11 @@ done if [ -z $EXAMPLE_TAG ] then - EXAMPLE_TAG='rapids_triton_identity' + EXAMPLE_TAG='triton_dt_identity' fi if [ -z $TEST_TAG ] then - TEST_TAG='rapids_triton_identity_test' + TEST_TAG='triton_dt_identity_test' fi DOCKER_ARGS="$DOCKER_ARGS --build-arg BUILD_TYPE=${BUILD_TYPE}" diff --git a/backend/ci/local/build.sh b/backend/ci/local/build.sh index b13f9cb..757242d 100755 --- a/backend/ci/local/build.sh +++ b/backend/ci/local/build.sh @@ -4,16 +4,16 @@ set -e REPODIR=$(cd $(dirname $0)/../../; pwd) -EXAMPLE_TAG=rapids_triton_identity \ - TEST_TAG=rapids_triton_identity_test \ +EXAMPLE_TAG=triton_dt_identity \ + TEST_TAG=triton_dt_identity_test \ $REPODIR/build.sh if [ -z $CUDA_VISIBLE_DEVICES ] then - docker run --gpus all --rm rapids_triton_identity_test + docker run --gpus all --rm triton_dt_identity_test else - docker run --gpus $CUDA_VISIBLE_DEVICES --rm rapids_triton_identity_test + docker run --gpus $CUDA_VISIBLE_DEVICES --rm triton_dt_identity_test fi -EXAMPLE_TAG=rapids_triton_identity:cpu \ - TEST_TAG=rapids_triton_identity_test:cpu \ +EXAMPLE_TAG=triton_dt_identity:cpu \ + TEST_TAG=triton_dt_identity_test:cpu \ $REPODIR/build.sh --cpu-only -docker run -v "${REPODIR}/qa/logs:/qa/logs" --gpus all --rm rapids_triton_identity_test:cpu +docker run -v "${REPODIR}/qa/logs:/qa/logs" --gpus all --rm triton_dt_identity_test:cpu From 47a83f67e3b871688e940532245e8261d29ed4c3 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 11:31:46 -0400 Subject: [PATCH 192/199] Update conda names --- .../environments/{rapids_triton_dev.yml => triton_backend.yml} | 2 +- .../{rapids_triton_test.yml => triton_backend_test.yml} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename backend/conda/environments/{rapids_triton_dev.yml => triton_backend.yml} (79%) rename backend/conda/environments/{rapids_triton_test.yml => triton_backend_test.yml} (83%) diff --git a/backend/conda/environments/rapids_triton_dev.yml b/backend/conda/environments/triton_backend.yml similarity index 79% rename from backend/conda/environments/rapids_triton_dev.yml rename to backend/conda/environments/triton_backend.yml index 1edf8a9..e131efb 100644 --- a/backend/conda/environments/rapids_triton_dev.yml +++ b/backend/conda/environments/triton_backend.yml @@ -1,5 +1,5 @@ --- -name: rapids_triton_dev +name: triton_backend_dev channels: - conda-forge dependencies: diff --git a/backend/conda/environments/rapids_triton_test.yml b/backend/conda/environments/triton_backend_test.yml similarity index 83% rename from backend/conda/environments/rapids_triton_test.yml rename to backend/conda/environments/triton_backend_test.yml index 4a86579..b34e710 100644 --- a/backend/conda/environments/rapids_triton_test.yml +++ b/backend/conda/environments/triton_backend_test.yml @@ -1,5 +1,5 @@ --- -name: rapids_triton_test +name: triton_backend_test channels: - conda-forge dependencies: From 3fa14a7c4f0e2878694ca6ddf8214e5f9523dbe7 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 13:35:47 -0400 Subject: [PATCH 193/199] Bring in updates from post 22.08 RAPIDS-Triton --- backend/Dockerfile | 2 +- backend/ci/local/build.sh | 4 ++-- backend/conda/environments/triton_backend.yml | 4 +++- backend/qa/run_tests.sh | 1 + 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index a668bf2..3d0b2ff 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -16,7 +16,7 @@ # Arguments for controlling build details ########################################################################################### # Version of Triton to use -ARG TRITON_VERSION=21.12 +ARG TRITON_VERSION=22.08 # Base container image ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 # Whether or not to build indicated components diff --git a/backend/ci/local/build.sh b/backend/ci/local/build.sh index 757242d..4e86976 100755 --- a/backend/ci/local/build.sh +++ b/backend/ci/local/build.sh @@ -9,9 +9,9 @@ EXAMPLE_TAG=triton_dt_identity \ $REPODIR/build.sh if [ -z $CUDA_VISIBLE_DEVICES ] then - docker run --gpus all --rm triton_dt_identity_test + docker run -v "${REPODIR}/qa/logs:/qa/logs" --gpus all --rm triton_dt_identity_test else - docker run --gpus $CUDA_VISIBLE_DEVICES --rm triton_dt_identity_test + docker run -v "${REPODIR}/qa/logs:/qa/logs" --gpus $CUDA_VISIBLE_DEVICES --rm triton_dt_identity_test fi EXAMPLE_TAG=triton_dt_identity:cpu \ TEST_TAG=triton_dt_identity_test:cpu \ diff --git a/backend/conda/environments/triton_backend.yml b/backend/conda/environments/triton_backend.yml index e131efb..75a4cb9 100644 --- a/backend/conda/environments/triton_backend.yml +++ b/backend/conda/environments/triton_backend.yml @@ -4,6 +4,8 @@ channels: - conda-forge dependencies: - ccache - - cmake>=3.21 + - cmake>=3.23.1 + - libstdcxx-ng<=11.2.0 + - libgcc-ng<=11.2.0 - ninja - rapidjson diff --git a/backend/qa/run_tests.sh b/backend/qa/run_tests.sh index 592b413..69a3d0d 100755 --- a/backend/qa/run_tests.sh +++ b/backend/qa/run_tests.sh @@ -97,6 +97,7 @@ finally() { docker rm -f $CONTAINER_NAME > /dev/null 2>&1 else kill -15 $TRITON_PID + wait fi } From 2dbff7e945ab036f61e7333057e5bc55a379f064 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Mon, 10 Oct 2022 16:47:49 -0400 Subject: [PATCH 194/199] Begin filling out top-level README --- README.md | 47 ++++++++++++++++++- backend/Dockerfile | 2 +- .../identity/config.pbtxt | 6 +-- .../model_repository/identity/config.pbtxt | 6 +-- 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 6ddaddc..7009fdf 100644 --- a/README.md +++ b/README.md @@ -1 +1,46 @@ -# triton_developer_tools +# Triton Developer Tools +This repository contains tools to make it easier to develop custom C++ backends +for NVIDIA's Triton Inference Server and to use Triton as a library within an +existing C++ project. It consists of two sub-projects: +- [Triton Backend Developer Tools](https://github.com/triton-inference-server/developer_tools/tree/main/backend): A header-only library to make it quick and easy to add high-performance custom functionality to Triton at the C++ level +- Triton Library Developer Tools: Coming soon, this library will allow easy + integration of Triton as a library within other C++ projects + +## Backend Developer Tools + +### Why might I want to use Triton's Backend Developer Tools? +- You have an inference model which you wish to serve in Triton, but Triton + does not currently support your model type +- You want to add specialized data manipulation logic to Triton and require + performance beyond what the Python backend can provide +- You want to ensure that your custom C++ backend stays up-to-date with the + latest Triton performance features + +### Why might I _not_ want to use Triton's Backend Developer Tools? +- The functionality you're looking for is already provided by an existing + Triton backend +- You want to write your additional logic in Python rather than C++ + +### Example +To create a custom backend with Triton's Backend Developer Tools, you can +make use of the Backend Template (TODO(wphicks): link) and have a working +Triton backend in minutes. For example, consider a simple example of a backend +that simply returns any array of floats provided as input. In order to create +such a backend, the only code we would have to write would be the +following `predict` method: +``` + void predict(dev_tools::Batch& batch) const { + dev_tools::Tensor input = get_input(batch, "input__0"); + dev_tools::Tensor output = get_output(batch, "output__0"); + + dev_tools::copy(output, input); + + output.finalize(); + } +``` +For a complete walkthrough of this example and much more detail on how to add +your own functionality to Triton, check out the Backend Developer Tools docs +TODO(wphicks): Link. + +## Library Developer Tools +TODO(wphicks) diff --git a/backend/Dockerfile b/backend/Dockerfile index 3d0b2ff..86d5016 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -52,7 +52,7 @@ RUN conda env update -f /environment.yml \ ENV PYTHONDONTWRITEBYTECODE=false -SHELL ["conda", "run", "--no-capture-output", "-n", "triton_backend", "/bin/bash", "-c"] +SHELL ["conda", "run", "--no-capture-output", "-n", "triton_backend_dev", "/bin/bash", "-c"] FROM base as build-stage diff --git a/backend/qa/L0_e2e/cpu_model_repository/identity/config.pbtxt b/backend/qa/L0_e2e/cpu_model_repository/identity/config.pbtxt index 33f4eed..86193e9 100644 --- a/backend/qa/L0_e2e/cpu_model_repository/identity/config.pbtxt +++ b/backend/qa/L0_e2e/cpu_model_repository/identity/config.pbtxt @@ -1,4 +1,4 @@ -backend: "rapids-identity" +backend: "dev_tools-identity" max_batch_size: 32768 input [ { @@ -17,6 +17,4 @@ output [ version_policy: { all { }} instance_group [{ kind: KIND_CPU }] parameters [ ] -dynamic_batching { - max_queue_delay_microseconds: 50 -} +dynamic_batching { } diff --git a/backend/qa/L0_e2e/model_repository/identity/config.pbtxt b/backend/qa/L0_e2e/model_repository/identity/config.pbtxt index 572434a..40b4974 100644 --- a/backend/qa/L0_e2e/model_repository/identity/config.pbtxt +++ b/backend/qa/L0_e2e/model_repository/identity/config.pbtxt @@ -1,4 +1,4 @@ -backend: "rapids-identity" +backend: "dev_tools-identity" max_batch_size: 32768 input [ { @@ -17,6 +17,4 @@ output [ version_policy: { all { }} instance_group [{ kind: KIND_GPU }] parameters [ ] -dynamic_batching { - max_queue_delay_microseconds: 50 -} +dynamic_batching { } From a039d7a4c03bb493938be075d223ae473c82c496 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 15 Dec 2022 14:12:44 -0500 Subject: [PATCH 195/199] Update conda dependencies --- examples/backend_linear/CMakeLists.txt | 24 +++++++++---------- examples/backend_linear/Dockerfile | 2 +- .../rapids_triton_dev_cuda11.4.yml | 6 +++-- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/examples/backend_linear/CMakeLists.txt b/examples/backend_linear/CMakeLists.txt index 354990a..b59135e 100644 --- a/examples/backend_linear/CMakeLists.txt +++ b/examples/backend_linear/CMakeLists.txt @@ -24,7 +24,7 @@ set(BACKEND_TARGET "triton_${BACKEND_NAME}") ############################################################################## # - Prepare rapids-cmake ----------------------------------------------------- -file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-21.10/RAPIDS.cmake +file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.12/RAPIDS.cmake ${CMAKE_BINARY_DIR}/RAPIDS.cmake) include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) include(rapids-cmake) @@ -33,9 +33,9 @@ include(rapids-cuda) include(rapids-export) include(rapids-find) -rapids_cuda_init_architectures(RAPIDS_TRITON_BACKEND) +rapids_cuda_init_architectures(TRITON_LINEAR_EXAMPLE) -project(RAPIDS_TRITON_BACKEND VERSION 21.10.00 LANGUAGES CXX CUDA) +project(TRITON_LINEAR_EXAMPLE VERSION 21.10.00 LANGUAGES CXX CUDA) ############################################################################## # - build type --------------------------------------------------------------- @@ -56,16 +56,16 @@ option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(NVTX "Enable nvtx markers" OFF) -message(VERBOSE "RAPIDS_TRITON_BACKEND: Enabling detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") -message(VERBOSE "RAPIDS_TRITON_BACKEND: Enabling kernelinfo in nvcc: ${CUDA_ENABLE_KERNEL_INFO}") -message(VERBOSE "RAPIDS_TRITON_BACKEND: Enabling lineinfo in nvcc: ${CUDA_ENABLE_LINE_INFO}") -message(VERBOSE "RAPIDS_TRITON_BACKEND: Enabling nvtx markers: ${NVTX}") -message(VERBOSE "RAPIDS_TRITON_BACKEND: Build RAPIDS_TRITON_BACKEND unit-tests: ${BUILD_TESTS}") +message(VERBOSE "TRITON_LINEAR_EXAMPLE: Enabling detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") +message(VERBOSE "TRITON_LINEAR_EXAMPLE: Enabling kernelinfo in nvcc: ${CUDA_ENABLE_KERNEL_INFO}") +message(VERBOSE "TRITON_LINEAR_EXAMPLE: Enabling lineinfo in nvcc: ${CUDA_ENABLE_LINE_INFO}") +message(VERBOSE "TRITON_LINEAR_EXAMPLE: Enabling nvtx markers: ${NVTX}") +message(VERBOSE "TRITON_LINEAR_EXAMPLE: Build TRITON_LINEAR_EXAMPLE unit-tests: ${BUILD_TESTS}") # Set RMM logging level set(RMM_LOGGING_LEVEL "INFO" CACHE STRING "Choose the logging level.") set_property(CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF") -message(VERBOSE "RAPIDS_TRITON_BACKEND: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") +message(VERBOSE "TRITON_LINEAR_EXAMPLE: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") ############################################################################## # - Conda environment detection ---------------------------------------------- @@ -86,9 +86,9 @@ endif() # * enable the CMake CUDA language # * set other CUDA compilation flags rapids_find_package(CUDAToolkit REQUIRED - BUILD_EXPORT_SET cuml-exports - INSTALL_EXPORT_SET cuml-exports - ) + BUILD_EXPORT_SET triton-linear-example-exports + INSTALL_EXPORT_SET triton-linear-example-exports +) include(cmake/modules/ConfigureCUDA.cmake) ############################################################################## diff --git a/examples/backend_linear/Dockerfile b/examples/backend_linear/Dockerfile index fb8f478..c92ed94 100644 --- a/examples/backend_linear/Dockerfile +++ b/examples/backend_linear/Dockerfile @@ -2,7 +2,7 @@ # Arguments for controlling build details ########################################################################################### # Version of Triton to use -ARG TRITON_VERSION=21.08 +ARG TRITON_VERSION=22.11 # Base container image ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 # Whether or not to build indicated components diff --git a/examples/backend_linear/conda/environments/rapids_triton_dev_cuda11.4.yml b/examples/backend_linear/conda/environments/rapids_triton_dev_cuda11.4.yml index 409c7bc..2801cca 100644 --- a/examples/backend_linear/conda/environments/rapids_triton_dev_cuda11.4.yml +++ b/examples/backend_linear/conda/environments/rapids_triton_dev_cuda11.4.yml @@ -4,7 +4,9 @@ channels: - nvidia - conda-forge dependencies: - - cmake>=3.21 - - cudatoolkit=11.4 + - cmake>=3.21,!=3.25.0 + - cudatoolkit=11.5 + - libstdcxx-ng<=11.2.0 + - libgcc-ng<=11.2.0 - ninja - rapidjson From b6186215d54c5a04a65c8899ccabb7ea4749865f Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 15 Dec 2022 16:49:26 -0500 Subject: [PATCH 196/199] Correctly fetch backend dependencies --- backend/cpp/CMakeLists.txt | 4 ++-- examples/backend_linear/CMakeLists.txt | 2 +- .../cmake/thirdparty/get_rapids-triton.cmake | 20 +++++++++---------- .../rapids_triton_dev_cuda11.4.yml | 1 + 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/backend/cpp/CMakeLists.txt b/backend/cpp/CMakeLists.txt index 4540a2b..a9e0aed 100644 --- a/backend/cpp/CMakeLists.txt +++ b/backend/cpp/CMakeLists.txt @@ -59,9 +59,9 @@ message(VERBOSE "DEV_TOOLS: Triton backend repo tag: ${TRITON_BACKEND_REPO_TAG}" if(TRITON_ENABLE_GPU) rapids_cuda_init_architectures(DEV_TOOLS) - project(DEV_TOOLS VERSION 22.02.00 LANGUAGES CXX CUDA) + project(DEV_TOOLS VERSION 23.01.00 LANGUAGES CXX CUDA) else() - project(DEV_TOOLS VERSION 22.02.00 LANGUAGES CXX) + project(DEV_TOOLS VERSION 23.01.00 LANGUAGES CXX) endif() diff --git a/examples/backend_linear/CMakeLists.txt b/examples/backend_linear/CMakeLists.txt index b59135e..25c0558 100644 --- a/examples/backend_linear/CMakeLists.txt +++ b/examples/backend_linear/CMakeLists.txt @@ -137,7 +137,7 @@ target_include_directories(${BACKEND_TARGET} target_link_libraries(${BACKEND_TARGET} PRIVATE - rapids_triton::rapids_triton + triton_backend::triton_backend triton-core-serverstub triton-backend-utils "${TRITONSERVER_LIB}" diff --git a/examples/backend_linear/cmake/thirdparty/get_rapids-triton.cmake b/examples/backend_linear/cmake/thirdparty/get_rapids-triton.cmake index c63b906..54ad51b 100644 --- a/examples/backend_linear/cmake/thirdparty/get_rapids-triton.cmake +++ b/examples/backend_linear/cmake/thirdparty/get_rapids-triton.cmake @@ -14,20 +14,20 @@ # limitations under the License. #============================================================================= -function(find_and_configure_rapids_triton) +function(find_and_configure_backend_dev_tools) set(oneValueArgs VERSION FORK PINNED_TAG) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) - rapids_cpm_find(rapids_triton ${PKG_VERSION} - GLOBAL_TARGETS rapids_triton::rapids_triton - BUILD_EXPORT_SET rapids_triton_linear-exports - INSTALL_EXPORT_SET rapids_triton_linear-exports + rapids_cpm_find(triton_backend ${PKG_VERSION} + GLOBAL_TARGETS triton_backend::triton_backend + BUILD_EXPORT_SET triton_backend-exports + INSTALL_EXPORT_SET triton_backend-exports CPM_ARGS - GIT_REPOSITORY https://github.com/${PKG_FORK}/rapids-triton.git + GIT_REPOSITORY https://github.com/${PKG_FORK}/triton_developer_tools.git GIT_TAG ${PKG_PINNED_TAG} - SOURCE_SUBDIR cpp + SOURCE_SUBDIR backend/cpp OPTIONS "BUILD_TESTS OFF" "BUILD_EXAMPLE OFF" @@ -40,7 +40,7 @@ endfunction() # Change pinned tag here to test a commit in CI # To use a different RAFT locally, set the CMake variable # CPM_raft_SOURCE=/path/to/local/raft -find_and_configure_rapids_triton(VERSION 21.10 - FORK rapidsai - PINNED_TAG branch-21.10 +find_and_configure_backend_dev_tools(VERSION 23.01 + FORK wphicks + PINNED_TAG fea-backend ) diff --git a/examples/backend_linear/conda/environments/rapids_triton_dev_cuda11.4.yml b/examples/backend_linear/conda/environments/rapids_triton_dev_cuda11.4.yml index 2801cca..fc39c82 100644 --- a/examples/backend_linear/conda/environments/rapids_triton_dev_cuda11.4.yml +++ b/examples/backend_linear/conda/environments/rapids_triton_dev_cuda11.4.yml @@ -4,6 +4,7 @@ channels: - nvidia - conda-forge dependencies: + - ccache - cmake>=3.21,!=3.25.0 - cudatoolkit=11.5 - libstdcxx-ng<=11.2.0 From 3d2cf748e4edfd8090e594eabc185bc58e45be2c Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 15 Dec 2022 16:51:24 -0500 Subject: [PATCH 197/199] Update include paths --- examples/backend_linear/src/api.cc | 16 ++++++++-------- examples/backend_linear/src/gpu_infer.cu | 4 ++-- examples/backend_linear/src/gpu_infer.h | 4 ++-- examples/backend_linear/src/model.h | 16 ++++++++-------- examples/backend_linear/src/shared_state.h | 4 ++-- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/examples/backend_linear/src/api.cc b/examples/backend_linear/src/api.cc index 7e62aa6..8007ee0 100644 --- a/examples/backend_linear/src/api.cc +++ b/examples/backend_linear/src/api.cc @@ -22,14 +22,14 @@ #include #include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include namespace triton { namespace backend { diff --git a/examples/backend_linear/src/gpu_infer.cu b/examples/backend_linear/src/gpu_infer.cu index 059341e..e2feb6f 100644 --- a/examples/backend_linear/src/gpu_infer.cu +++ b/examples/backend_linear/src/gpu_infer.cu @@ -19,8 +19,8 @@ #include #include #include -#include -#include +#include +#include namespace triton { namespace backend { namespace NAMESPACE { diff --git a/examples/backend_linear/src/gpu_infer.h b/examples/backend_linear/src/gpu_infer.h index b188282..3400b97 100644 --- a/examples/backend_linear/src/gpu_infer.h +++ b/examples/backend_linear/src/gpu_infer.h @@ -20,8 +20,8 @@ #include #include -#include -#include +#include +#include namespace triton { namespace backend { diff --git a/examples/backend_linear/src/model.h b/examples/backend_linear/src/model.h index 4a9f9ee..72d5e48 100644 --- a/examples/backend_linear/src/model.h +++ b/examples/backend_linear/src/model.h @@ -25,14 +25,14 @@ #include #include #include -#include // rapids::Batch -#include // rapids::Buffer, rapids::copy -#include // rapids::MemoryType -#include // rapids::Model -#include // rapids::copy -#include // rapids::DeploymentType -#include // rapids::device_id_t -#include // rapids::log_info +#include // rapids::Batch +#include // rapids::Buffer, rapids::copy +#include // rapids::MemoryType +#include // rapids::Model +#include // rapids::copy +#include // rapids::DeploymentType +#include // rapids::device_id_t +#include // rapids::log_info #include namespace triton { diff --git a/examples/backend_linear/src/shared_state.h b/examples/backend_linear/src/shared_state.h index 790751a..bd6873f 100644 --- a/examples/backend_linear/src/shared_state.h +++ b/examples/backend_linear/src/shared_state.h @@ -19,8 +19,8 @@ #include #include -#include -#include +#include +#include #include namespace triton { From 507e44eb50ac75f4ed964d56727c63f0ee845678 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Thu, 15 Dec 2022 17:01:36 -0500 Subject: [PATCH 198/199] Update namespace for developer_tools --- examples/backend_linear/README.md | 30 ++++++------- examples/backend_linear/src/api.cc | 16 +++---- examples/backend_linear/src/model.h | 52 +++++++++++----------- examples/backend_linear/src/shared_state.h | 6 +-- 4 files changed, 52 insertions(+), 52 deletions(-) diff --git a/examples/backend_linear/README.md b/examples/backend_linear/README.md index ba666a9..78bd193 100644 --- a/examples/backend_linear/README.md +++ b/examples/backend_linear/README.md @@ -220,13 +220,13 @@ it from the configuration each time (which may involve additional parsing). All RAPIDS-Triton backends store their shared state in a class called `RapidsSharedState` in their own namespace, which inherits from -`rapids::SharedModelState`. A basic implementation of this class for the +`dev_tools::SharedModelState`. A basic implementation of this class for the current backend might look something like the following: ```cpp -struct RapidsSharedState : rapids::SharedModelState { +struct RapidsSharedState : dev_tools::SharedModelState { RapidsSharedState(std::unique_ptr&& config) - : rapids::SharedModelState{std::move(config)} {} + : dev_tools::SharedModelState{std::move(config)} {} void load() {} void unload() {} @@ -266,7 +266,7 @@ We will instead use it here simply to illustrate the use of RAPIDS-Triton logging functions. Logging functions are defined in `rapids_triton/triton/logging.hpp` and may be invoked as follows: ```cpp -void unload() { rapids::log_info(__FILE__, __LINE__) << "Unloading shared state..."; } +void unload() { dev_tools::log_info(__FILE__, __LINE__) << "Unloading shared state..."; } ``` The arguments to `log_info` and related functions may be omitted, but if included may provide some use in debugging. @@ -302,11 +302,11 @@ on host or device **or** a non-owning view on host/device memory that was allocated elsewhere. We'll introduce a `Buffer` member to RapidsModel to store **c**: ```cpp -rapids::Buffer c{}; +dev_tools::Buffer c{}; ``` Now, we are ready to actually load **c** from its file representation. -`rapids::Model`, the abstract parent class of `RapidsModel` implementations, +`dev_tools::Model`, the abstract parent class of `RapidsModel` implementations, defines several methods to help with model deserialization, including: - `get_filepath`, which returns the path to the model file if it is specified in the config or the model directory if it is not @@ -341,20 +341,20 @@ text file representation: We then query the model to figure out exactly what sort of Buffer will be needed to store **c**: ```cpp -auto memory_type = rapids::MemoryType{}; -if (get_deployment_type() == rapids::GPUDeployment) { - memory_type = rapids::DeviceMemory; +auto memory_type = dev_tools::MemoryType{}; +if (get_deployment_type() == dev_tools::GPUDeployment) { + memory_type = dev_tools::DeviceMemory; } else { - memory_type = rapids::HostMemory; + memory_type = dev_tools::HostMemory; } -c = rapids::Buffer(model_vec.size(), memory_type, get_device_id()); +c = dev_tools::Buffer(model_vec.size(), memory_type, get_device_id()); ``` -Finally, we use the helper function `rapids::copy` to copy the values of **c** +Finally, we use the helper function `dev_tools::copy` to copy the values of **c** from a Buffer-based view of `model_vec` to the `c` Buffer itself: ```cpp -rapids::copy(c, rapids::Buffer(model_vec.data(), model_vec.size(), - rapids::HostMemory)); +dev_tools::copy(c, dev_tools::Buffer(model_vec.data(), model_vec.size(), + dev_tools::HostMemory)); ``` By taking advantage of Buffer's RAII semantics, we eliminate the need to @@ -385,7 +385,7 @@ returned tensor to determine how inference will proceed. For tensors on the host, our inference logic might look something like: ```cpp -if (u.mem_type() == rapids::HostMemory) { +if (u.mem_type() == dev_tools::HostMemory) { auto alpha = get_shared_state()->alpha; for (std::size_t i{}; i < u.size(); ++i) { r.data()[i] = diff --git a/examples/backend_linear/src/api.cc b/examples/backend_linear/src/api.cc index 8007ee0..00f5780 100644 --- a/examples/backend_linear/src/api.cc +++ b/examples/backend_linear/src/api.cc @@ -35,41 +35,41 @@ namespace triton { namespace backend { namespace NAMESPACE { -using ModelState = rapids::TritonModelState; +using ModelState = dev_tools::TritonModelState; using ModelInstanceState = - rapids::ModelInstanceState; + dev_tools::ModelInstanceState; extern "C" { /** Confirm that backend is compatible with Triton's backend API version */ TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { - return rapids::triton_api::initialize(backend); + return dev_tools::triton_api::initialize(backend); } TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { - return rapids::triton_api::model_initialize(model); + return dev_tools::triton_api::model_initialize(model); } TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { - return rapids::triton_api::model_finalize(model); + return dev_tools::triton_api::model_finalize(model); } TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize( TRITONBACKEND_ModelInstance* instance) { - return rapids::triton_api::instance_initialize(instance); } TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize( TRITONBACKEND_ModelInstance* instance) { - return rapids::triton_api::instance_finalize(instance); + return dev_tools::triton_api::instance_finalize(instance); } TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** raw_requests, uint32_t const request_count) { - return rapids::triton_api::execute( + return dev_tools::triton_api::execute( instance, raw_requests, static_cast(request_count)); } diff --git a/examples/backend_linear/src/model.h b/examples/backend_linear/src/model.h index 72d5e48..173918d 100644 --- a/examples/backend_linear/src/model.h +++ b/examples/backend_linear/src/model.h @@ -25,37 +25,37 @@ #include #include #include -#include // rapids::Batch -#include // rapids::Buffer, rapids::copy -#include // rapids::MemoryType -#include // rapids::Model -#include // rapids::copy -#include // rapids::DeploymentType -#include // rapids::device_id_t -#include // rapids::log_info +#include // dev_tools::Batch +#include // dev_tools::Buffer, dev_tools::copy +#include // dev_tools::MemoryType +#include // dev_tools::Model +#include // dev_tools::copy +#include // dev_tools::DeploymentType +#include // dev_tools::device_id_t +#include // dev_tools::log_info #include namespace triton { namespace backend { namespace NAMESPACE { -struct RapidsModel : rapids::Model { +struct RapidsModel : dev_tools::Model { RapidsModel(std::shared_ptr shared_state, - rapids::device_id_t device_id, cudaStream_t default_stream, - rapids::DeploymentType deployment_type, + dev_tools::device_id_t device_id, cudaStream_t default_stream, + dev_tools::DeploymentType deployment_type, std::string const& filepath) - : rapids::Model(shared_state, device_id, + : dev_tools::Model(shared_state, device_id, default_stream, deployment_type, filepath) {} - void predict(rapids::Batch& batch) const { + void predict(dev_tools::Batch& batch) const { auto u = get_input(batch, "u"); auto v = get_input(batch, "v"); auto r = get_output(batch, "r"); auto alpha = get_shared_state()->alpha; - if (u.mem_type() == rapids::HostMemory) { + if (u.mem_type() == dev_tools::HostMemory) { for (auto i = std::size_t{}; i < u.size(); ++i) { r.data()[i] = alpha * u.data()[i] + v.data()[i] + c.data()[i % c.size()]; @@ -91,30 +91,30 @@ struct RapidsModel : rapids::Model { } // Construct buffer to hold c based on details of this model deployment - auto memory_type = rapids::MemoryType{}; - if constexpr (rapids::IS_GPU_BUILD) { - if (get_deployment_type() == rapids::GPUDeployment) { - memory_type = rapids::DeviceMemory; + auto memory_type = dev_tools::MemoryType{}; + if constexpr (dev_tools::IS_GPU_BUILD) { + if (get_deployment_type() == dev_tools::GPUDeployment) { + memory_type = dev_tools::DeviceMemory; } else { - memory_type = rapids::HostMemory; + memory_type = dev_tools::HostMemory; } } else { - memory_type = rapids::HostMemory; + memory_type = dev_tools::HostMemory; } - c = rapids::Buffer(model_vec.size(), memory_type, get_device_id(), + c = dev_tools::Buffer(model_vec.size(), memory_type, get_device_id(), get_stream()); /* Use a Buffer view on model_vec to safely copy data to its final - * location. Making use of rapids::copy here provides additional safety + * location. Making use of dev_tools::copy here provides additional safety * checks to avoid buffer overruns. Note that the destination buffer comes - * first in rapids::copy calls, so we are copying *into* c */ - rapids::copy(c, rapids::Buffer(model_vec.data(), model_vec.size(), - rapids::HostMemory)); + * first in dev_tools::copy calls, so we are copying *into* c */ + dev_tools::copy(c, dev_tools::Buffer(model_vec.data(), model_vec.size(), + dev_tools::HostMemory)); } private: - rapids::Buffer c{}; + dev_tools::Buffer c{}; }; } // namespace NAMESPACE diff --git a/examples/backend_linear/src/shared_state.h b/examples/backend_linear/src/shared_state.h index bd6873f..d81e8c6 100644 --- a/examples/backend_linear/src/shared_state.h +++ b/examples/backend_linear/src/shared_state.h @@ -27,12 +27,12 @@ namespace triton { namespace backend { namespace NAMESPACE { -struct RapidsSharedState : rapids::SharedModelState { +struct RapidsSharedState : dev_tools::SharedModelState { RapidsSharedState(std::unique_ptr&& config) - : rapids::SharedModelState{std::move(config)} {} + : dev_tools::SharedModelState{std::move(config)} {} void load() { alpha = get_config_param("alpha"); } void unload() { - rapids::log_info(__FILE__, __LINE__) << "Unloading shared state..."; + dev_tools::log_info(__FILE__, __LINE__) << "Unloading shared state..."; } float alpha = 1.0f; From 1d95ae2365374e63e3816a2a5cd70c5313e0d927 Mon Sep 17 00:00:00 2001 From: William Hicks Date: Tue, 17 Jan 2023 11:45:20 -0500 Subject: [PATCH 199/199] Bump version numbers --- backend/Dockerfile | 2 +- backend/cpp/CMakeLists.txt | 6 +++--- backend/cpp/cmake/thirdparty/get_raft.cmake | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 86d5016..14846f0 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -16,7 +16,7 @@ # Arguments for controlling build details ########################################################################################### # Version of Triton to use -ARG TRITON_VERSION=22.08 +ARG TRITON_VERSION=22.12 # Base container image ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 # Whether or not to build indicated components diff --git a/backend/cpp/CMakeLists.txt b/backend/cpp/CMakeLists.txt index a9e0aed..0fb5aac 100644 --- a/backend/cpp/CMakeLists.txt +++ b/backend/cpp/CMakeLists.txt @@ -37,9 +37,9 @@ option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(NVTX "Enable nvtx markers" OFF) option(TRITON_ENABLE_STATS "Enable statistics collection in Triton" ON) -set(TRITON_COMMON_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-server/common repo") -set(TRITON_CORE_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-server/core repo") -set(TRITON_BACKEND_REPO_TAG "r21.12" CACHE STRING "Tag for triton-inference-server/backend repo") +set(TRITON_COMMON_REPO_TAG "r22.12" CACHE STRING "Tag for triton-inference-server/common repo") +set(TRITON_CORE_REPO_TAG "r22.12" CACHE STRING "Tag for triton-inference-server/core repo") +set(TRITON_BACKEND_REPO_TAG "r22.12" CACHE STRING "Tag for triton-inference-server/backend repo") message(VERBOSE "DEV_TOOLS: Build DEV_TOOLS unit-tests: ${BUILD_TESTS}") message(VERBOSE "DEV_TOOLS: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") diff --git a/backend/cpp/cmake/thirdparty/get_raft.cmake b/backend/cpp/cmake/thirdparty/get_raft.cmake index 28c4460..d0afb92 100644 --- a/backend/cpp/cmake/thirdparty/get_raft.cmake +++ b/backend/cpp/cmake/thirdparty/get_raft.cmake @@ -37,8 +37,8 @@ function(find_and_configure_raft) endfunction() -set(DEV_TOOLS_MIN_VERSION_raft "${DEV_TOOLS_VERSION_MAJOR}.${DEV_TOOLS_VERSION_MINOR}.00") -set(DEV_TOOLS_BRANCH_VERSION_raft "${DEV_TOOLS_VERSION_MAJOR}.${DEV_TOOLS_VERSION_MINOR}") +set(DEV_TOOLS_MIN_VERSION_raft "22.12.00") +set(DEV_TOOLS_BRANCH_VERSION_raft "22.12") # Change pinned tag here to test a commit in CI # To use a different RAFT locally, set the CMake variable