diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..4bbce44 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +* @tanmayv25 @GuanLuo +/backend @wphicks @dantegd @divyegala diff --git a/.gitignore b/.gitignore index 0e9f099..df9e3e2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ -/build -/.vscode -*.so +backend/cpp/build +backend/qa/logs diff --git a/README.md b/README.md index adb360f..7009fdf 100644 --- a/README.md +++ b/README.md @@ -1 +1,46 @@ -# triton_developer_tools \ No newline at end of file +# Triton Developer Tools +This repository contains tools to make it easier to develop custom C++ backends +for NVIDIA's Triton Inference Server and to use Triton as a library within an +existing C++ project. It consists of two sub-projects: +- [Triton Backend Developer Tools](https://github.com/triton-inference-server/developer_tools/tree/main/backend): A header-only library to make it quick and easy to add high-performance custom functionality to Triton at the C++ level +- Triton Library Developer Tools: Coming soon, this library will allow easy + integration of Triton as a library within other C++ projects + +## Backend Developer Tools + +### Why might I want to use Triton's Backend Developer Tools? +- You have an inference model which you wish to serve in Triton, but Triton + does not currently support your model type +- You want to add specialized data manipulation logic to Triton and require + performance beyond what the Python backend can provide +- You want to ensure that your custom C++ backend stays up-to-date with the + latest Triton performance features + +### Why might I _not_ want to use Triton's Backend Developer Tools? +- The functionality you're looking for is already provided by an existing + Triton backend +- You want to write your additional logic in Python rather than C++ + +### Example +To create a custom backend with Triton's Backend Developer Tools, you can +make use of the Backend Template (TODO(wphicks): link) and have a working +Triton backend in minutes. For example, consider a simple example of a backend +that simply returns any array of floats provided as input. In order to create +such a backend, the only code we would have to write would be the +following `predict` method: +``` + void predict(dev_tools::Batch& batch) const { + dev_tools::Tensor input = get_input(batch, "input__0"); + dev_tools::Tensor output = get_output(batch, "output__0"); + + dev_tools::copy(output, input); + + output.finalize(); + } +``` +For a complete walkthrough of this example and much more detail on how to add +your own functionality to Triton, check out the Backend Developer Tools docs +TODO(wphicks): Link. + +## Library Developer Tools +TODO(wphicks) diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 0000000..a2d3658 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1 @@ +cpp/build diff --git a/backend/CONTRIBUTING.md b/backend/CONTRIBUTING.md new file mode 100644 index 0000000..dbb1fa4 --- /dev/null +++ b/backend/CONTRIBUTING.md @@ -0,0 +1,93 @@ + + +# Contributing to RAPIDS-Triton + +You can help improve RAPIDS-Triton in any of the following ways: +- Submitting a bug report, feature request or documentation issue +- Proposing and implementing a new feature +- Implementing a feature or bug-fix for an outstanding issue + +## Bug reports +When submitting a bug report, please include a *minimum* *reproducible* +example. Ideally, this should be a snippet of code that other developers can +copy, paste, and immediately run to try to reproduce the error. Please: +- Do include import statements and any other code necessary to immediately run + your example +- Avoid examples that require other developers to download models or data + unless you cannot reproduce the problem with synthetically-generated data + +## Code Contributions +To contribute code to this project, please follow these steps: +1. Find an issue to work on or submit an issue documenting the problem you + would like to work on. +2. Comment on the issue saying that you plan to work on it. +3. Review the conventions below for information to help you make your changes + in a way that is consistent with the rest of the codebase. +4. Code! +5. Create your pull request. +6. Wait for other developers to review your code and update your PR as needed. +7. Once a PR is approved, it will be merged into the main branch. + +### Coding Conventions +* RAPIDS-Triton follows [Almost Always Auto + (AAA)](https://herbsutter.com/2013/08/12/gotw-94-solution-aaa-style-almost-always-auto/) + style. Please maintain this style in any contributions, with the possible + exception of some docs, where type information may be helpful for new users + trying to understand a snippet in isolation. +* Avoid raw loops where possible. +* C++ versions of types should be used instead of C versions except when + interfacing with C code (e.g. use `std::size_t` instead of `size_t`). +* Avoid using output pointers in function signatures. Prefer instead to + actually return the value computed by the function and take advantage of + return value optimization and move semantics. + +### Signing Your Work +* We require that all contributors "sign-off" on their commits. This certifies that the contribution is your original work, or you have rights to submit it under the same license, or a compatible license. + * Any contribution which contains commits that are not Signed-Off will not be accepted. +* To sign off on a commit you simply use the `--signoff` (or `-s`) option when committing your changes: + ```bash + $ git commit -s -m "Add cool feature." + ``` + This will append the following to your commit message: + ``` + Signed-off-by: Your Name + ``` +* Full text of the DCO: + ``` + Developer Certificate of Origin + Version 1.1 + + Copyright (C) 2004, 2006 The Linux Foundation and its contributors. + 1 Letterman Drive + Suite D4700 + San Francisco, CA, 94129 + + Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. + + + Developer's Certificate of Origin 1.1 + + By making a contribution to this project, I certify that: + + (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or + + (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or + + (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. + + (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. + ``` diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..14846f0 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,128 @@ +# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +########################################################################################### +# Arguments for controlling build details +########################################################################################### +# Version of Triton to use +ARG TRITON_VERSION=22.12 +# Base container image +ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 +# Whether or not to build indicated components +ARG BUILD_TESTS=OFF +ARG BUILD_EXAMPLE=ON +# Whether or not to enable GPU build +ARG TRITON_ENABLE_GPU=ON + +FROM ${BASE_IMAGE} as base + +ENV PATH="/root/miniconda3/bin:${PATH}" + +RUN apt-get update \ + && apt-get install --no-install-recommends -y wget patchelf \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +ENV PYTHONDONTWRITEBYTECODE=true + +RUN wget \ + https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && mkdir /root/.conda \ + && bash Miniconda3-latest-Linux-x86_64.sh -b \ + && rm -f Miniconda3-latest-Linux-x86_64.sh + +COPY ./conda/environments/triton_backend.yml /environment.yml + +RUN conda env update -f /environment.yml \ + && rm /environment.yml \ + && conda clean -afy \ + && find /root/miniconda3/ -follow -type f -name '*.pyc' -delete \ + && find /root/miniconda3/ -follow -type f -name '*.js.map' -delete + +ENV PYTHONDONTWRITEBYTECODE=false + +SHELL ["conda", "run", "--no-capture-output", "-n", "triton_backend_dev", "/bin/bash", "-c"] + +FROM base as build-stage + +COPY ./cpp /triton_backend + +ARG TRITON_VERSION +ENV TRITON_VERSION=$TRITON_VERSION + +ARG BUILD_TYPE=Release +ENV BUILD_TYPE=$BUILD_TYPE +ARG BUILD_TESTS +ENV BUILD_TESTS=$BUILD_TESTS +ARG BUILD_EXAMPLE +ENV BUILD_EXAMPLE=$BUILD_EXAMPLE +ARG TRITON_ENABLE_GPU +ENV TRITON_ENABLE_GPU=$TRITON_ENABLE_GPU + +RUN mkdir /triton_backend/build + +WORKDIR /triton_backend/build + +RUN cmake \ + -GNinja \ + -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ + -DBUILD_TESTS="${BUILD_TESTS}" \ + -DBUILD_EXAMPLE="${BUILD_EXAMPLE}" \ + -DTRITON_ENABLE_GPU="${TRITON_ENABLE_GPU}" \ + .. + +ENV CCACHE_DIR=/ccache + +RUN --mount=type=cache,target=/ccache/ ninja install + +FROM base as test-install + +COPY ./conda/environments/triton_backend_test.yml /environment.yml + +RUN conda env update -f /environment.yml \ + && rm /environment.yml \ + && conda clean -afy \ + && find /root/miniconda3/ -follow -type f -name '*.pyc' -delete \ + && find /root/miniconda3/ -follow -type f -name '*.js.map' -delete + +COPY ./python /triton_backend + +RUN conda run -n triton_backend_test pip install /triton_backend \ + && rm -rf /triton_backend + +FROM build-stage as test-stage + +COPY --from=test-install /root/miniconda3 /root/miniconda3 + +ENV TEST_EXE=/triton_backend/build/test_triton_backend + +COPY qa /qa + +ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "triton_backend_test", "/bin/bash", "/qa/entrypoint.sh"] + +FROM ${BASE_IMAGE} + +RUN mkdir /models + +# Remove existing backend install +RUN if [ -d /opt/tritonserver/backends/dev_tools-identity ]; \ + then \ + rm -rf /opt/tritonserver/backends/dev_tools-identity/*; \ + fi + +COPY --from=build-stage \ + /opt/tritonserver/backends/dev_tools-identity \ + /opt/tritonserver/backends/dev_tools-identity + +ENTRYPOINT ["tritonserver", "--model-repository=/models"] diff --git a/backend/LICENSE b/backend/LICENSE new file mode 100644 index 0000000..b360c42 --- /dev/null +++ b/backend/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2021 NVIDIA CORPORATION + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 0000000..59a4973 --- /dev/null +++ b/backend/README.md @@ -0,0 +1,96 @@ + + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +# The RAPIDS-Triton Library + +This project is designed to make it easy to integrate any C++-based algorithm +into the NVIDIA Triton Inference Server. Originally developed to assist with +the integration of RAPIDS algorithms, this library can be used by anyone to +quickly get up and running with a custom backend for Triton. + +## Background + +### Triton + +The [NVIDIA Triton Inference +Server](https://developer.nvidia.com/nvidia-triton-inference-server) offers a +complete open-source solution for deployment of machine learning models from a +wide variety of ML frameworks (PyTorch, Tensorflow, ONNX, XGBoost, etc.) on +both CPU and GPU hardware. It allows you to maximize inference performance in +production (whether that means maximizing throughput, minimizing latency, or +optimizing some other metric) regardless of how you may have trained your ML +model. Through smart batching, efficient pipeline handling, and tools to +simplify deployments almost anywhere, Triton helps make production inference +serving simpler and more cost-effective. + +### Custom Backends + +While Triton natively supports many common ML frameworks, you may wish to take +advantage of Triton's features for something a little more specialized. Triton +provides support for different kinds of models via "backends:" modular +libraries which provide the specialized logic for those models. Triton allows +you to create custom backends in +[Python](https://github.com/triton-inference-server/python_backend), but for +those who wish to use C++ directly, RAPIDS-Triton can help simplify the process +of developing your backend. + +The goal of RAPIDS-Triton is not to facilitate every possible use case of the +Triton backend API but to make the most common uses of this API easier by +providing a simpler interface to them. That being said, if there is a feature +of the Triton backend API which RAPIDS-Triton does not expose and which you +wish to use in a custom backend, please [submit a feature +request](https://github.com/rapidsai/rapids-triton/issues), and we will see if +it can be added. + +## Simple Example + +In the `cpp/src` directory of this repository, you can see a complete, +annotated example of a backend built with RAPIDS-Triton. The core of any +backend is defining the `predict` function for your model as shown below: + +``` + void predict(rapids::Batch& batch) const { + rapids::Tensor input = get_input(batch, "input__0"); + rapids::Tensor output = get_output(batch, "output__0"); + + rapids::copy(output, input); + + output.finalize(); + } +``` + +In this example, we ask Triton to provide a tensor named `"input__0"` and copy +it to an output tensor named `"output__0"`. Thus, our "inference" function in +this simple example is just a passthrough from one input tensor to one output +tensor. + +To do something more sophisticated in this `predict` function, we might take +advantage of the `data()` method of Tensor objects, which provides a raw +pointer (on host or device) to the underlying data along with `size()`, and +`mem_type()` to determine the number of elements in the Tensor and whether they +are stored on host or device respectively. Note that `finalize()` must be +called on all output tensors before returning from the predict function. + +For a much more detailed look at developing backends with RAPIDS-Triton, +check out our complete [usage guide](https://github.com/rapidsai/rapids-triton/blob/main/docs/usage.md). + +## Contributing + +If you wish to contribute to RAPIDS-Triton, please see our [contributors' +guide](https://github.com/rapidsai/rapids-triton/blob/main/CONTRIBUTING.md) for +tips and full details on how to get started. diff --git a/backend/build.sh b/backend/build.sh new file mode 100755 index 0000000..f07628a --- /dev/null +++ b/backend/build.sh @@ -0,0 +1,165 @@ +#!/bin/bash +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +REPODIR=$(cd $(dirname $0); pwd) + +NUMARGS=$# +ARGS=$* +VALIDTARGETS="example tests" +VALIDFLAGS="--cpu-only -g -h --help" +VALIDARGS="${VALIDTARGETS} ${VALIDFLAGS}" +HELP="$0 [ ...] [ ...] + where is: + example - build the identity backend example + tests - build container(s) with unit tests + and is: + -g - build for debug + -h - print this text + --cpu-only - build CPU-only versions of targets + --tag-commit - tag docker images based on current git commit + + default action (no args) is to build all targets + The following environment variables are also accepted to allow further customization: + BASE_IMAGE - Base image for Docker images + TRITON_VERSION - Triton version to use for build + EXAMPLE_TAG - The tag to use for the server image + TEST_TAG - The tag to use for the test image +" + +BUILD_TYPE=Release +TRITON_ENABLE_GPU=ON +DOCKER_ARGS="" + +export DOCKER_BUILDKIT=1 + +function hasArg { + (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") +} + +function completeBuild { + (( ${NUMARGS} == 0 )) && return + for a in ${ARGS}; do + if (echo " ${VALIDTARGETS} " | grep -q " ${a} "); then + false; return + fi + done + true +} + +if hasArg -h || hasArg --help; then + echo "${HELP}" + exit 0 +fi + +# Long arguments +LONG_ARGUMENT_LIST=( + "cpu-only" + "tag-commit" +) + +# Short arguments +ARGUMENT_LIST=( + "g" +) + +# read arguments +opts=$(getopt \ + --longoptions "$(printf "%s," "${LONG_ARGUMENT_LIST[@]}")" \ + --name "$(basename "$0")" \ + --options "$(printf "%s" "${ARGUMENT_LIST[@]}")" \ + -- "$@" +) + +if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi + +eval set -- "$opts" + +while true +do + case "$1" in + -g | --debug ) + BUILD_TYPE=Debug + ;; + --cpu-only ) + TRITON_ENABLE_GPU=OFF + ;; + --tag-commit ) + [ -z $EXAMPLE_TAG ] \ + && EXAMPLE_TAG="triton_dt_identity:$(cd $REPODIR; git rev-parse --short HEAD)" \ + || true + [ -z $TEST_TAG ] \ + && TEST_TAG="triton_dt_identity_test:$(cd $REPODIR; git rev-parse --short HEAD)" \ + || true + ;; + --) + shift + break + ;; + esac + shift +done + +if [ -z $EXAMPLE_TAG ] +then + EXAMPLE_TAG='triton_dt_identity' +fi +if [ -z $TEST_TAG ] +then + TEST_TAG='triton_dt_identity_test' +fi + +DOCKER_ARGS="$DOCKER_ARGS --build-arg BUILD_TYPE=${BUILD_TYPE}" +DOCKER_ARGS="$DOCKER_ARGS --build-arg TRITON_ENABLE_GPU=${TRITON_ENABLE_GPU}" + +if [ ! -z $BASE_IMAGE ] +then + DOCKER_ARGS="$DOCKER_ARGS --build-arg BASE_IMAGE=${BASE_IMAGE}" +fi +if [ ! -z $TRITON_VERSION ] +then + DOCKER_ARGS="$DOCKER_ARGS --build-arg TRITON_VERSION=${TRITON_VERSION}" +fi + +if completeBuild || hasArg example +then + BACKEND=1 + DOCKER_ARGS="$DOCKER_ARGS --build-arg BUILD_EXAMPLE=ON" +fi + +if completeBuild || hasArg tests +then + TESTS=1 + DOCKER_ARGS="$DOCKER_ARGS --build-arg BUILD_TESTS=ON" +fi + +if [ $BACKEND -eq 1 ] +then + docker build \ + $DOCKER_ARGS \ + -t "$EXAMPLE_TAG" \ + $REPODIR +fi + +if [ $TESTS -eq 1 ] +then + docker build \ + $DOCKER_ARGS \ + -t "$EXAMPLE_TAG" \ + --target test-stage \ + -t "$TEST_TAG" \ + $REPODIR +fi diff --git a/backend/ci/local/build.sh b/backend/ci/local/build.sh new file mode 100755 index 0000000..4e86976 --- /dev/null +++ b/backend/ci/local/build.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +REPODIR=$(cd $(dirname $0)/../../; pwd) + +EXAMPLE_TAG=triton_dt_identity \ + TEST_TAG=triton_dt_identity_test \ + $REPODIR/build.sh +if [ -z $CUDA_VISIBLE_DEVICES ] +then + docker run -v "${REPODIR}/qa/logs:/qa/logs" --gpus all --rm triton_dt_identity_test +else + docker run -v "${REPODIR}/qa/logs:/qa/logs" --gpus $CUDA_VISIBLE_DEVICES --rm triton_dt_identity_test +fi +EXAMPLE_TAG=triton_dt_identity:cpu \ + TEST_TAG=triton_dt_identity_test:cpu \ + $REPODIR/build.sh --cpu-only +docker run -v "${REPODIR}/qa/logs:/qa/logs" --gpus all --rm triton_dt_identity_test:cpu diff --git a/backend/conda/environments/triton_backend.yml b/backend/conda/environments/triton_backend.yml new file mode 100644 index 0000000..75a4cb9 --- /dev/null +++ b/backend/conda/environments/triton_backend.yml @@ -0,0 +1,11 @@ +--- +name: triton_backend_dev +channels: + - conda-forge +dependencies: + - ccache + - cmake>=3.23.1 + - libstdcxx-ng<=11.2.0 + - libgcc-ng<=11.2.0 + - ninja + - rapidjson diff --git a/backend/conda/environments/triton_backend_test.yml b/backend/conda/environments/triton_backend_test.yml new file mode 100644 index 0000000..b34e710 --- /dev/null +++ b/backend/conda/environments/triton_backend_test.yml @@ -0,0 +1,12 @@ +--- +name: triton_backend_test +channels: + - conda-forge +dependencies: + - flake8 + - pip + - python + - pytest + - numpy + - pip: + - tritonclient[all] diff --git a/backend/cpp/.clang-format b/backend/cpp/.clang-format new file mode 100644 index 0000000..0c05436 --- /dev/null +++ b/backend/cpp/.clang-format @@ -0,0 +1,164 @@ +--- +# Refer to the following link for the explanation of each params: +# http://releases.llvm.org/8.0.0/tools/clang/docs/ClangFormatStyleOptions.html +Language: Cpp +# BasedOnStyle: Google +AccessModifierOffset: -1 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: true +AlignConsecutiveBitFields: true +AlignConsecutiveDeclarations: false +AlignConsecutiveMacros: true +AlignEscapedNewlines: Left +AlignOperands: true +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllConstructorInitializersOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: true +AllowShortCaseLabelsOnASingleLine: true +AllowShortEnumsOnASingleLine: true +AllowShortFunctionsOnASingleLine: All +AllowShortIfStatementsOnASingleLine: true +AllowShortLambdasOnASingleLine: true +AllowShortLoopsOnASingleLine: false +# This is deprecated +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: Yes +BinPackArguments: false +BinPackParameters: false +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false + # disabling the below splits, else, they'll just add to the vertical length of source files! + SplitEmptyFunction: false + SplitEmptyRecord: false + SplitEmptyNamespace: false +BreakAfterJavaFieldAnnotations: false +BreakBeforeBinaryOperators: None +BreakBeforeBraces: WebKit +BreakBeforeInheritanceComma: false +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakInheritanceList: BeforeColon +BreakStringLiterals: true +ColumnLimit: 100 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +# Kept the below 2 to be the same as `IndentWidth` to keep everything uniform +ConstructorInitializerIndentWidth: 2 +ContinuationIndentWidth: 2 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^' + Priority: 2 + - Regex: '^<.*\.h>' + Priority: 1 + - Regex: '^<.*' + Priority: 2 + - Regex: '.*' + Priority: 3 +IncludeIsMainRegex: '([-_](test|unittest))?$' +IndentCaseLabels: true +IndentPPDirectives: None +IndentWidth: 2 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Never +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +RawStringFormats: + - Language: Cpp + Delimiters: + - cc + - CC + - cpp + - Cpp + - CPP + - 'c++' + - 'C++' + CanonicalDelimiter: '' + - Language: TextProto + Delimiters: + - pb + - PB + - proto + - PROTO + EnclosingFunctions: + - EqualsProto + - EquivToProto + - PARSE_PARTIAL_TEXT_PROTO + - PARSE_TEST_PROTO + - PARSE_TEXT_PROTO + - ParseTextOrDie + - ParseTextProtoOrDie + CanonicalDelimiter: '' + BasedOnStyle: google +# Enabling comment reflow causes doxygen comments to be messed up in their formats! +ReflowComments: true +SortIncludes: true +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: true +SpaceBeforeSquareBrackets: false +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: c++17 +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +# Be consistent with indent-width, even for people who use tab for indentation! +TabWidth: 2 +UseTab: Never diff --git a/backend/cpp/CMakeLists.txt b/backend/cpp/CMakeLists.txt new file mode 100644 index 0000000..0fb5aac --- /dev/null +++ b/backend/cpp/CMakeLists.txt @@ -0,0 +1,222 @@ +#============================================================================= +# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +cmake_minimum_required(VERSION 3.21 FATAL_ERROR) +file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.02/RAPIDS.cmake + ${CMAKE_BINARY_DIR}/RAPIDS.cmake) +include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) +include(rapids-cmake) +include(rapids-cpm) +include(rapids-cuda) +include(rapids-export) +include(rapids-find) + +############################################################################## +# - User Options ------------------------------------------------------------ + +option(TRITON_ENABLE_GPU "Enable GPU support in Triton" ON) +option(BUILD_TESTS "Build backend dev tools unit-tests" ON) +option(BUILD_EXAMPLE "Build dev tools identity example backend" OFF) +option(CUDA_ENABLE_KERNELINFO "Enable kernel resource usage info" OFF) +option(CUDA_ENABLE_LINEINFO "Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler)" OFF) +option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF) +option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" ON) +option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) +option(NVTX "Enable nvtx markers" OFF) +option(TRITON_ENABLE_STATS "Enable statistics collection in Triton" ON) +set(TRITON_COMMON_REPO_TAG "r22.12" CACHE STRING "Tag for triton-inference-server/common repo") +set(TRITON_CORE_REPO_TAG "r22.12" CACHE STRING "Tag for triton-inference-server/core repo") +set(TRITON_BACKEND_REPO_TAG "r22.12" CACHE STRING "Tag for triton-inference-server/backend repo") + +message(VERBOSE "DEV_TOOLS: Build DEV_TOOLS unit-tests: ${BUILD_TESTS}") +message(VERBOSE "DEV_TOOLS: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") +message(VERBOSE "DEV_TOOLS: Disable depreaction warnings " ${DISABLE_DEPRECATION_WARNINGS}) +message(VERBOSE "DEV_TOOLS: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELINFO}") +message(VERBOSE "DEV_TOOLS: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") +message(VERBOSE "DEV_TOOLS: Enable nvtx markers: ${NVTX}") +message(VERBOSE "DEV_TOOLS: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") +message(VERBOSE "DEV_TOOLS: Enable GPU support: ${TRITON_ENABLE_GPU}") +message(VERBOSE "DEV_TOOLS: Enable statistics collection in Triton: ${TRITON_ENABLE_STATS}") +message(VERBOSE "DEV_TOOLS: Triton common repo tag: ${TRITON_COMMON_REPO_TAG}") +message(VERBOSE "DEV_TOOLS: Triton core repo tag: ${TRITON_CORE_REPO_TAG}") +message(VERBOSE "DEV_TOOLS: Triton backend repo tag: ${TRITON_BACKEND_REPO_TAG}") + +############################################################################## +# - Project Initialization --------------------------------------------------- + +if(TRITON_ENABLE_GPU) + rapids_cuda_init_architectures(DEV_TOOLS) + project(DEV_TOOLS VERSION 23.01.00 LANGUAGES CXX CUDA) +else() + project(DEV_TOOLS VERSION 23.01.00 LANGUAGES CXX) +endif() + + +############################################################################## +# - build type --------------------------------------------------------------- + +# Set a default build type if none was specified +rapids_cmake_build_type(Release) + +# this is needed for clang-tidy runs +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +# Set RMM logging level +set(RMM_LOGGING_LEVEL "INFO" CACHE STRING "Choose the logging level.") +set_property(CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF") +message(VERBOSE "DEV_TOOLS: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") + +############################################################################## +# - Conda environment detection ---------------------------------------------- + +if(DETECT_CONDA_ENV) + rapids_cmake_support_conda_env( conda_env MODIFY_PREFIX_PATH ) + if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED ENV{CONDA_PREFIX}) + message(STATUS "DEV_TOOLS: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}") + set(CMAKE_INSTALL_PREFIX "$ENV{CONDA_PREFIX}") + endif() +endif() + +############################################################################## +# - compiler options --------------------------------------------------------- +set(CMAKE_C_COMPILER_LAUNCHER ccache) +set(CMAKE_CXX_COMPILER_LAUNCHER ccache) +if(TRITON_ENABLE_GPU) + set(CMAKE_CUDA_COMPILER_LAUNCHER ccache) + + # * find CUDAToolkit package + # * determine GPU architectures + # * enable the CMake CUDA language + # * set other CUDA compilation flags + rapids_find_package(CUDAToolkit REQUIRED + BUILD_EXPORT_SET triton_backend-exports + INSTALL_EXPORT_SET triton_backend-exports + ) + include(cmake/modules/ConfigureCUDA.cmake) +endif() + +############################################################################## +# - Requirements ------------------------------------------------------------- + +# add third party dependencies using CPM +rapids_cpm_init() + +if(TRITON_ENABLE_GPU) + include(cmake/thirdparty/get_rmm.cmake) + include(cmake/thirdparty/get_raft.cmake) +endif() + +include(cmake/thirdparty/get_rapidjson.cmake) +include(cmake/thirdparty/get_triton.cmake) + +if(BUILD_TESTS) + include(cmake/thirdparty/get_gtest.cmake) +endif() + +############################################################################## +# - install targets----------------------------------------------------------- + +add_library(triton_backend INTERFACE) +add_library(triton_backend::triton_backend ALIAS triton_backend) +target_include_directories(triton_backend INTERFACE "$" + "$") + +target_link_libraries(triton_backend +INTERFACE + $<$:rmm::rmm> + $<$:raft::raft> + triton-core-serverstub + triton-backend-utils +) + +if (TRITON_ENABLE_GPU) + target_compile_features( + triton_backend INTERFACE cxx_std_17 + $ + ) +else() + target_compile_features( + triton_backend INTERFACE cxx_std_17 + ) +endif() + +rapids_cmake_install_lib_dir(lib_dir) +install(TARGETS triton_backend + DESTINATION ${lib_dir} + EXPORT triton_backend-exports + ) + +include(GNUInstallDirs) +install(DIRECTORY include/triton_backend/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/triton_backend + ) + +# Temporary install of triton_backend.hpp while the file is removed +install(FILES include/triton_backend.hpp + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/triton_backend + ) + +############################################################################## +# - install export ----------------------------------------------------------- +set(doc_string +[=[ +Provide targets for Triton Backend Developer Tools. + +Triton Backend Developer Tools is a header-only library designed to make it +easier and faster to create custom C++ Triton backends. + +]=]) + + rapids_export(INSTALL triton_backend + EXPORT_SET triton_backend-exports + GLOBAL_TARGETS triton_backend # since we can't hook into EXPORT SETS + NAMESPACE triton_backend:: + DOCUMENTATION doc_string + ) + +############################################################################## +# - build export ------------------------------------------------------------- + +rapids_export(BUILD triton_backend + EXPORT_SET triton_backend-exports + GLOBAL_TARGETS triton_backend # since we can't hook into EXPORT SETS + LANGUAGES CUDA + DOCUMENTATION doc_string + NAMESPACE triton_backend:: + ) + +############################################################################## +# - build test executable ---------------------------------------------------- + +if(BUILD_TESTS) + include(test/CMakeLists.txt) +endif() + +############################################################################## +# - build example backend ---------------------------------------------------- + +if(BUILD_EXAMPLE) + include(src/CMakeLists.txt) +endif() + +############################################################################## +# - doxygen targets ---------------------------------------------------------- + +# TODO(wphicks) +# include(cmake/doxygen.cmake) +# add_doxygen_target(IN_DOXYFILE Doxyfile.in +# OUT_DOXYFILE ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile +# CWD ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/backend/cpp/cmake/doxygen.cmake b/backend/cpp/cmake/doxygen.cmake new file mode 100644 index 0000000..061981f --- /dev/null +++ b/backend/cpp/cmake/doxygen.cmake @@ -0,0 +1,33 @@ +# Copyright (c) 2020, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +find_package(Doxygen 1.8.11) + +function(add_doxygen_target) + if(Doxygen_FOUND) + set(options "") + set(oneValueArgs IN_DOXYFILE OUT_DOXYFILE CWD) + set(multiValueArgs "") + cmake_parse_arguments(dox "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + configure_file(${dox_IN_DOXYFILE} ${dox_OUT_DOXYFILE} @ONLY) + add_custom_target(doc + ${DOXYGEN_EXECUTABLE} ${dox_OUT_DOXYFILE} + WORKING_DIRECTORY ${dox_CWD} + VERBATIM + COMMENT "Generate doxygen docs") + else() + message("add_doxygen_target: doxygen exe not found") + endif() +endfunction(add_doxygen_target) diff --git a/backend/cpp/cmake/modules/ConfigureCUDA.cmake b/backend/cpp/cmake/modules/ConfigureCUDA.cmake new file mode 100644 index 0000000..3b34212 --- /dev/null +++ b/backend/cpp/cmake/modules/ConfigureCUDA.cmake @@ -0,0 +1,43 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +if(DISABLE_DEPRECATION_WARNINGS) + list(APPEND DEV_TOOLS_CXX_FLAGS -Wno-deprecated-declarations) + list(APPEND DEV_TOOLS_CUDA_FLAGS -Xcompiler=-Wno-deprecated-declarations) +endif() + +if(CMAKE_COMPILER_IS_GNUCXX) + list(APPEND DEV_TOOLS_CXX_FLAGS -Wall -Werror -Wno-unknown-pragmas -Wno-error=deprecated-declarations) +endif() + +list(APPEND DEV_TOOLS_CUDA_FLAGS --expt-extended-lambda --expt-relaxed-constexpr) + +# set warnings as errors +if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 11.2.0) + list(APPEND DEV_TOOLS_CUDA_FLAGS -Werror=all-warnings) +endif() +list(APPEND DEV_TOOLS_CUDA_FLAGS -Xcompiler=-Wall,-Werror,-Wno-error=deprecated-declarations) + +# Option to enable line info in CUDA device compilation to allow introspection when profiling / memchecking +if(CUDA_ENABLE_LINEINFO) + list(APPEND DEV_TOOLS_CUDA_FLAGS -lineinfo) +endif() + +# Debug options +if(CMAKE_BUILD_TYPE MATCHES Debug) + message(VERBOSE "DEV_TOOLS: Building with debugging flags") + list(APPEND DEV_TOOLS_CUDA_FLAGS -G -Xcompiler=-rdynamic) +endif() diff --git a/backend/cpp/cmake/thirdparty/get_gtest.cmake b/backend/cpp/cmake/thirdparty/get_gtest.cmake new file mode 100644 index 0000000..a29af07 --- /dev/null +++ b/backend/cpp/cmake/thirdparty/get_gtest.cmake @@ -0,0 +1,22 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_gtest) + include(${rapids-cmake-dir}/cpm/gtest.cmake) + rapids_cpm_gtest() +endfunction() + +find_and_configure_gtest() diff --git a/backend/cpp/cmake/thirdparty/get_raft.cmake b/backend/cpp/cmake/thirdparty/get_raft.cmake new file mode 100644 index 0000000..d0afb92 --- /dev/null +++ b/backend/cpp/cmake/thirdparty/get_raft.cmake @@ -0,0 +1,49 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_raft) + + set(oneValueArgs VERSION FORK PINNED_TAG) + cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN} ) + + rapids_cpm_find(raft ${PKG_VERSION} + GLOBAL_TARGETS raft::raft + BUILD_EXPORT_SET rapids_triton-exports + INSTALL_EXPORT_SET rapids_triton-exports + CPM_ARGS + GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git + GIT_TAG ${PKG_PINNED_TAG} + SOURCE_SUBDIR cpp + OPTIONS + "BUILD_TESTS OFF" + "RAFT_COMPILE_LIBRARIES OFF" + ) + + message(VERBOSE "DEV_TOOLS: Using RAFT located in ${raft_SOURCE_DIR}") + +endfunction() + +set(DEV_TOOLS_MIN_VERSION_raft "22.12.00") +set(DEV_TOOLS_BRANCH_VERSION_raft "22.12") + +# Change pinned tag here to test a commit in CI +# To use a different RAFT locally, set the CMake variable +# CPM_raft_SOURCE=/path/to/local/raft +find_and_configure_raft(VERSION ${DEV_TOOLS_MIN_VERSION_raft} + FORK rapidsai + PINNED_TAG branch-${DEV_TOOLS_BRANCH_VERSION_raft} + ) diff --git a/backend/cpp/cmake/thirdparty/get_rapidjson.cmake b/backend/cpp/cmake/thirdparty/get_rapidjson.cmake new file mode 100644 index 0000000..e9051c6 --- /dev/null +++ b/backend/cpp/cmake/thirdparty/get_rapidjson.cmake @@ -0,0 +1,37 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +# TODO(wphicks): Pass in version +function(find_and_configure_rapidjson VERSION) + + rapids_cpm_find(rapidjson ${VERSION} + GLOBAL_TARGETS rapidjson::rapidjson + BUILD_EXPORT_SET rapids_triton-exports + INSTALL_EXPORT_SET rapids_triton-exports + CPM_ARGS + GIT_REPOSITORY https://github.com/Tencent/rapidjson + GIT_TAG "v${VERSION}" + GIT_SHALLOW ON + OPTIONS + "RAPIDJSON_BUILD_DOC OFF" + "RAPIDJSON_BUILD_EXAMPLES OFF" + "RAPIDJSON_BUILD_TESTS OFF" + "RAPIDJSON_BUILD_THIRDPARTY_GTEST OFF" + ) + +endfunction() + +find_and_configure_rapidjson("1.1.0") diff --git a/backend/cpp/cmake/thirdparty/get_rmm.cmake b/backend/cpp/cmake/thirdparty/get_rmm.cmake new file mode 100644 index 0000000..4a3ca4a --- /dev/null +++ b/backend/cpp/cmake/thirdparty/get_rmm.cmake @@ -0,0 +1,22 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_rmm) + include(${rapids-cmake-dir}/cpm/rmm.cmake) + rapids_cpm_rmm() +endfunction() + +find_and_configure_rmm() diff --git a/backend/cpp/cmake/thirdparty/get_triton.cmake b/backend/cpp/cmake/thirdparty/get_triton.cmake new file mode 100644 index 0000000..e5353ec --- /dev/null +++ b/backend/cpp/cmake/thirdparty/get_triton.cmake @@ -0,0 +1,36 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= +include(FetchContent) + +FetchContent_Declare( + repo-common + GIT_REPOSITORY https://github.com/triton-inference-server/common.git + GIT_TAG ${TRITON_COMMON_REPO_TAG} + GIT_SHALLOW ON +) +FetchContent_Declare( + repo-core + GIT_REPOSITORY https://github.com/triton-inference-server/core.git + GIT_TAG ${TRITON_CORE_REPO_TAG} + GIT_SHALLOW ON +) +FetchContent_Declare( + repo-backend + GIT_REPOSITORY https://github.com/triton-inference-server/backend.git + GIT_TAG ${TRITON_BACKEND_REPO_TAG} + GIT_SHALLOW ON +) +FetchContent_MakeAvailable(repo-common repo-core repo-backend) diff --git a/backend/cpp/include/triton_backend.hpp b/backend/cpp/include/triton_backend.hpp new file mode 100644 index 0000000..fbc7c25 --- /dev/null +++ b/backend/cpp/include/triton_backend.hpp @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { +namespace backend { +namespace dev_tools { + +/* Function for testing triton_backend include + * + * @return message indicating triton_backend has been included succesfully*/ +inline auto test_install() { return std::string("triton_backend set up successfully"); } + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/batch/batch.hpp b/backend/cpp/include/triton_backend/batch/batch.hpp new file mode 100644 index 0000000..25894a4 --- /dev/null +++ b/backend/cpp/include/triton_backend/batch/batch.hpp @@ -0,0 +1,273 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +/** + * @brief A representation of all data about a single batch of inference + * requests + * + * Batch objects are the primary interface point between triton_backend Models + * and the Triton server itself. By calling the `get_input` and `get_output` + * methods of a batch, Model implementations can retrieve the input Tensors + * necessary for prediction and the output Tensors where results can be + * stored. + * + * Batch objects also handle a variety of other tasks necessary for + * processing a batch in the Triton model. This includes reporting statistics + * on how long it took to process requests and sending responses to the + * client via the Triton server once processing is complete. + * + * It is not recommended that developers of triton_backend backends try to + * construct Batch objects directly. Instead, you should make use of the + * dev_tools::triton_api::execute template, which will construct the Batch for + * you. + */ +struct Batch { + using size_type = std::size_t; + + Batch(TRITONBACKEND_Request** raw_requests, + request_size_t count, + TRITONBACKEND_MemoryManager& triton_mem_manager, + std::function(std::string const&, size_type)>&& get_output_shape, + std::function&& report_request_statistics, + bool use_pinned_input, + bool use_pinned_output, + size_type max_batch_size, + cudaStream_t stream) + : requests_{raw_requests, raw_requests + count}, + responses_{construct_responses(requests_.begin(), requests_.end())}, + get_output_shape_{std::move(get_output_shape)}, + report_statistics_{std::move(report_request_statistics)}, + collector_(raw_requests, count, &responses_, &triton_mem_manager, use_pinned_input, stream), + responder_{std::make_shared(raw_requests, + count, + &responses_, + max_batch_size, + &triton_mem_manager, + use_pinned_output, + stream)}, + stream_{stream}, + start_time_{std::chrono::steady_clock::now()}, + compute_start_time_{std::chrono::steady_clock::now()}, + batch_size_{} + { + } + + template + auto get_input_shape(std::string const& name) + { + auto result = std::vector{}; + if (!requests_.empty()) { + result = get_triton_input_shape(std::begin(requests_), std::end(requests_), name); + + auto input_batch_dim = size_type{}; + if (result.size() > 0) { input_batch_dim = result[0]; } + + if (batch_size_.has_value()) { + if (batch_size_.value() != input_batch_dim) { + throw TritonException(Error::Internal, + "all input tensors must have same batch dimension"); + } + } else { + batch_size_ = input_batch_dim; + } + } + return result; + } + + template + auto get_input(std::string const& name, + std::optional const& memory_type, + device_id_t device_id, + cudaStream_t stream) + { + auto shape = get_input_shape(name); + auto size_bytes = + sizeof(T) * std::reduce(shape.begin(), shape.end(), std::size_t{1}, std::multiplies<>()); + auto allowed_memory_configs = std::vector>{}; + if (memory_type.has_value()) { + allowed_memory_configs.emplace_back(memory_type.value(), device_id); + } else { + allowed_memory_configs.emplace_back(HostMemory, int64_t{}); + allowed_memory_configs.emplace_back(DeviceMemory, device_id); + } + + auto const* raw_buffer = static_cast(nullptr); + auto reported_bytes = std::size_t{}; + auto reported_mem_type = MemoryType{}; + auto reported_device_id = int64_t{}; + + triton_check( + collector_.ProcessTensor(name.c_str(), + static_cast(nullptr), // Return data without copy if possible + size_bytes, + allowed_memory_configs, + &raw_buffer, + &reported_bytes, + &reported_mem_type, + &reported_device_id)); + + if(collector_.Finalize()){ + if constexpr (IS_GPU_BUILD) { + cuda_check(cudaStreamSynchronize(stream_)); + } else { + throw TritonException(Error::Internal, "stream synchronization required in non-GPU build"); + } + } + + std::for_each(std::begin(responses_), std::end(responses_), [](auto* response) { + if (response == nullptr) { + throw TritonException(Error::Internal, "Input collection failed"); + } + }); + + auto buffer = Buffer(reinterpret_cast(raw_buffer), + reported_bytes / sizeof(T), + reported_mem_type, + reported_device_id, + stream); + + if (memory_type && (reported_mem_type != memory_type || reported_device_id != device_id)) { + throw TritonException(Error::Internal, "data collected in wrong location"); + } + + // Set start time of batch to time latest input tensor was retrieved + compute_start_time_ = std::chrono::steady_clock::now(); + + return Tensor(std::move(shape), std::move(buffer)); + } + + template + auto get_input(std::string const& name, + std::optional const& memory_type, + device_id_t device_id) + { + return get_input(name, memory_type, device_id, stream_); + } + + template + auto get_output(std::string const& name, + std::optional const& memory_type, + device_id_t device_id, + cudaStream_t stream) + { + if (!batch_size_.has_value()) { + throw TritonException(Error::Internal, + "At least one input must be retrieved before any output"); + } + auto shape = get_output_shape_(name, batch_size_.value()); + auto buffer_size = std::reduce(shape.begin(), shape.end(), std::size_t{1}, std::multiplies<>()); + auto final_memory_type = MemoryType{}; + if (memory_type.has_value()) { + final_memory_type = memory_type.value(); + } else { + // If consumer doesn't care, use HostMemory to avoid additional copy on + // non-shared-memory responses. + final_memory_type = HostMemory; + } + auto buffer = Buffer(buffer_size, final_memory_type, device_id, stream); + return OutputTensor(std::move(shape), std::move(buffer), name, responder_); + } + + template + auto get_output(std::string const& name, + std::optional const& memory_type, + device_id_t device_id) + { + return get_output(name, memory_type, device_id, stream_); + } + + auto const& compute_start_time() const { return compute_start_time_; } + + auto stream() const { return stream_; } + + void finalize(TRITONSERVER_Error* err) + { + auto compute_end_time = std::chrono::steady_clock::now(); + if (responder_->Finalize()) { cuda_check(cudaStreamSynchronize(stream_)); } + + send_responses(std::begin(responses_), std::end(responses_), err); + + // Triton resumes ownership of failed requests; only release on success + if (err == nullptr) { + std::for_each( + std::begin(requests_), std::end(requests_), [this, &compute_end_time](auto& request) { + report_statistics_(request, + start_time_, + compute_start_time_, + compute_end_time, + std::chrono::steady_clock::now()); + }); + release_requests(std::begin(requests_), std::end(requests_)); + } + } + + private: + std::vector requests_; + std::vector responses_; + std::function(std::string const&, size_type)> get_output_shape_; + std::function + report_statistics_; + BackendInputCollector collector_; + std::shared_ptr responder_; + cudaStream_t stream_; + std::chrono::time_point start_time_; + std::chrono::time_point compute_start_time_; + std::optional batch_size_; +}; +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/build_control.hpp b/backend/cpp/include/triton_backend/build_control.hpp new file mode 100644 index 0000000..f2d497a --- /dev/null +++ b/backend/cpp/include/triton_backend/build_control.hpp @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { +namespace backend { +namespace dev_tools { + +#ifdef TRITON_ENABLE_GPU +auto constexpr IS_GPU_BUILD = true; +#else +auto constexpr IS_GPU_BUILD = false; +#endif + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/cpu_only/cuda_runtime_replacement.hpp b/backend/cpp/include/triton_backend/cpu_only/cuda_runtime_replacement.hpp new file mode 100644 index 0000000..06acf28 --- /dev/null +++ b/backend/cpp/include/triton_backend/cpu_only/cuda_runtime_replacement.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifdef TRITON_ENABLE_GPU +#include +#else + +namespace triton { +namespace backend { +namespace dev_tools { + +using cudaStream_t = void*; + +enum struct cudaError_t {cudaSuccess, cudaErrorNonGpuBuild}; +using cudaError = cudaError_t; +auto constexpr cudaSuccess = cudaError_t::cudaSuccess; + +inline void cudaGetLastError() {} + +inline auto const * cudaGetErrorString(cudaError_t err) { + return "CUDA function used in non-GPU build"; +} + +inline auto cudaStreamSynchronize(cudaStream_t stream) { + return cudaError_t::cudaErrorNonGpuBuild; +} + +inline auto cudaGetDevice(int* device_id) { + return cudaError_t::cudaErrorNonGpuBuild; +} + +inline auto cudaGetDeviceCount(int* count) { + return cudaError_t::cudaErrorNonGpuBuild; +} + + +} // namespace dev_tools +} // namespace backend +} // namespace triton +#endif diff --git a/backend/cpp/include/triton_backend/exceptions.hpp b/backend/cpp/include/triton_backend/exceptions.hpp new file mode 100644 index 0000000..f6fe24c --- /dev/null +++ b/backend/cpp/include/triton_backend/exceptions.hpp @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { + +using ErrorCode = TRITONSERVER_Error_Code; + +namespace Error { +auto constexpr Unknown = ErrorCode::TRITONSERVER_ERROR_UNKNOWN; +auto constexpr Internal = ErrorCode::TRITONSERVER_ERROR_INTERNAL; +auto constexpr NotFound = ErrorCode::TRITONSERVER_ERROR_NOT_FOUND; +auto constexpr InvalidArg = ErrorCode::TRITONSERVER_ERROR_INVALID_ARG; +auto constexpr Unavailable = ErrorCode::TRITONSERVER_ERROR_UNAVAILABLE; +auto constexpr Unsupported = ErrorCode::TRITONSERVER_ERROR_UNSUPPORTED; +auto constexpr AlreadyExists = ErrorCode::TRITONSERVER_ERROR_ALREADY_EXISTS; +} // namespace Error + +/** + * @brief Exception thrown if processing cannot continue for a request + * + * This exception should be thrown whenever a condition is encountered that (if + * it is not appropriately handled by some other exception handler) SHOULD + * result in Triton reporting an error for the request being processed. It + * signals that (absent any other fallbacks), this request cannot be fulfilled + * but that the server may still be in a state to continue handling other + * requests, including requests to other models. + */ +struct TritonException : std::exception { + public: + TritonException() : error_(TRITONSERVER_ErrorNew(Error::Unknown, "encountered unknown error")) {} + + TritonException(ErrorCode code, std::string const& msg) + : error_(TRITONSERVER_ErrorNew(code, msg.c_str())) + { + } + + TritonException(ErrorCode code, char const* msg) : error_{TRITONSERVER_ErrorNew(code, msg)} {} + + TritonException(TRITONSERVER_Error* prev_error) : error_(prev_error) {} + + virtual char const* what() const noexcept { return TRITONSERVER_ErrorMessage(error_); } + + auto* error() const { return error_; } + + private: + TRITONSERVER_Error* error_; +}; + +inline void triton_check(TRITONSERVER_Error* err) +{ + if (err != nullptr) { throw TritonException(err); } +} + +inline void cuda_check(cudaError_t const& err) +{ + if constexpr (IS_GPU_BUILD) { + if (err != cudaSuccess) { + cudaGetLastError(); + throw TritonException(Error::Internal, cudaGetErrorString(err)); + } + } else { + throw TritonException(Error::Internal, "cuda_check used in non-GPU build"); + } +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/buffer.hpp b/backend/cpp/include/triton_backend/memory/buffer.hpp new file mode 100644 index 0000000..1c9857c --- /dev/null +++ b/backend/cpp/include/triton_backend/memory/buffer.hpp @@ -0,0 +1,317 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include + +#ifdef TRITON_ENABLE_GPU +#include +#include +#include +#else +#include +#include +#include +#endif + +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +template +struct Buffer { + using size_type = std::size_t; + using value_type = T; + + using h_buffer = T*; + using d_buffer = T*; + using owned_h_buffer = std::unique_ptr; + using owned_d_buffer = detail::owned_device_buffer; + using data_store = std::variant; + + Buffer() noexcept : device_{}, data_{std::in_place_index<0>, nullptr}, size_{}, stream_{} {} + + /** + * @brief Construct buffer of given size in given memory location (either + * on host or on device) + * A buffer constructed in this way is owning and will release allocated + * resources on deletion + */ + Buffer(size_type size, + MemoryType memory_type = DeviceMemory, + device_id_t device = 0, + cudaStream_t stream = 0) + : device_{device}, + data_{allocate(size, device, memory_type, stream)}, + size_{size}, + stream_{stream} + { + if constexpr (!IS_GPU_BUILD) { + if (memory_type == DeviceMemory) { + throw TritonException( + Error::Internal, + "Cannot use device buffer in non-GPU build" + ); + } + } + } + + /** + * @brief Construct buffer from given source in given memory location (either + * on host or on device) + * A buffer constructed in this way is non-owning; the caller is + * responsible for freeing any resources associated with the input pointer + */ + Buffer(T* input_data, + size_type size, + MemoryType memory_type = DeviceMemory, + device_id_t device = 0, + cudaStream_t stream = 0) + : device_{device}, + data_{[&memory_type, &input_data]() { + auto result = data_store{}; + if (memory_type == HostMemory) { + result = data_store{std::in_place_index<0>, input_data}; + } else { + if constexpr (!IS_GPU_BUILD) { + throw TritonException( + Error::Internal, + "Cannot use device buffer in non-GPU build" + ); + } + result = data_store{std::in_place_index<1>, input_data}; + } + return result; + }()}, + size_{size}, + stream_{stream} + { + } + + /** + * @brief Construct one buffer from another in the given memory location + * (either on host or on device) + * A buffer constructed in this way is owning and will copy the data from + * the original location + */ + Buffer(Buffer const& other, MemoryType memory_type, device_id_t device = 0) + : device_{device}, + data_([&other, &memory_type, &device]() { + auto result = allocate(other.size_, device, memory_type, other.stream_); + copy(result, other.data_, other.size_, other.stream_); + return result; + }()), + size_{other.size_}, + stream_{other.stream_} + { + } + + /** + * @brief Create owning copy of existing buffer + * The memory type of this new buffer will be the same as the original + */ + Buffer(Buffer const& other) : Buffer(other, other.mem_type(), other.device()) {} + + Buffer(Buffer&& other, MemoryType memory_type) + : device_{other.device()}, + data_{[&other, memory_type]() { + data_store result; + if (memory_type == other.mem_type()) { + result = std::move(other.data_); + } else { + result = allocate(other.size_, memory_type, other.device(), other.stream()); + copy(result, other.data_, other.size_, other.stream_); + } + return result; + }()}, + size_{other.size_}, + stream_{other.stream_} + { + } + + Buffer(Buffer&& other) = default; + + Buffer& operator=(Buffer&& other) = default; + + ~Buffer() {} + + /** + * @brief Return where memory for this buffer is located (host or device) + */ + auto mem_type() const noexcept { return data_.index() % 2 == 0 ? HostMemory : DeviceMemory; } + + /** + * @brief Return number of elements in buffer + */ + auto size() const noexcept { return size_; } + + /** + * @brief Return pointer to data stored in buffer + */ + auto* data() const noexcept { return get_raw_ptr(data_); } + + auto device() const noexcept { return device_; } + + /** + * @brief Return CUDA stream associated with this buffer + */ + auto stream() const noexcept { return stream_; } + + void stream_synchronize() const + { + if constexpr (IS_GPU_BUILD) { cuda_check(cudaStreamSynchronize(stream_)); } + } + + /** + * @brief Set CUDA stream for this buffer to new value + * + * @warning This method calls cudaStreamSynchronize on the old stream + * before updating. Be aware of performance implications and try to avoid + * interactions between buffers on different streams where possible. + */ + void set_stream(cudaStream_t new_stream) + { + stream_synchronize(); + stream_ = new_stream; + } + + private: + device_id_t device_; + data_store data_; + size_type size_; + cudaStream_t stream_; + + // Helper function for accessing raw pointer to underlying data of + // data_store + static auto* get_raw_ptr(data_store const& ptr) noexcept + { + /* Switch statement is an optimization relative to std::visit to avoid + * vtable overhead for a small number of alternatives */ + auto* result = static_cast(nullptr); + switch (ptr.index()) { + case 0: result = std::get<0>(ptr); break; + case 1: result = std::get<1>(ptr); break; + case 2: result = std::get<2>(ptr).get(); break; + case 3: result = std::get<3>(ptr).get(); break; + } + return result; + } + + // Helper function for allocating memory in constructors + static auto allocate(size_type size, + device_id_t device = 0, + MemoryType memory_type = DeviceMemory, + cudaStream_t stream = 0) + { + auto result = data_store{}; + if (memory_type == DeviceMemory) { + if constexpr (IS_GPU_BUILD) { + result = data_store{owned_d_buffer{ + device, + size, + stream, + }}; + } else { + throw TritonException(Error::Internal, + "DeviceMemory requested in CPU-only build of FIL backend"); + } + } else { + result = std::make_unique(size); + } + return result; + } + + // Helper function for copying memory in constructors, where there are + // stronger guarantees on conditions that would otherwise need to be + // checked + static void copy(data_store const& dst, data_store const& src, size_type len, cudaStream_t stream) + { + // This function will only be called in constructors, so we allow a + // const_cast here to perform the initial copy of data from a + // Buffer to a newly-created Buffer + auto raw_dst = const_cast*>(get_raw_ptr(dst)); + auto raw_src = get_raw_ptr(src); + + auto dst_mem_type = dst.index() % 2 == 0 ? HostMemory : DeviceMemory; + auto src_mem_type = src.index() % 2 == 0 ? HostMemory : DeviceMemory; + + detail::copy(raw_dst, raw_src, len, stream, dst_mem_type, src_mem_type); + } +}; + +/** + * @brief Copy data from one Buffer to another + * + * @param dst The destination buffer + * @param src The source buffer + * @param dst_begin The offset from the beginning of the destination buffer + * at which to begin copying to. + * @param src_begin The offset from the beginning of the source buffer + * at which to begin copying from. + * @param src_end The offset from the beginning of the source buffer + * before which to end copying from. + */ +template +void copy(Buffer& dst, + Buffer const& src, + typename Buffer::size_type dst_begin, + typename Buffer::size_type src_begin, + typename Buffer::size_type src_end) +{ + if (dst.stream() != src.stream()) { dst.set_stream(src.stream()); } + auto len = src_end - src_begin; + if (len < 0 || src_end > src.size() || len > dst.size() - dst_begin) { + throw TritonException(Error::Internal, "bad copy between buffers"); + } + + auto raw_dst = dst.data() + dst_begin; + auto raw_src = src.data() + src_begin; + + detail::copy(raw_dst, raw_src, len, dst.stream(), dst.mem_type(), src.mem_type()); +} + +template +void copy(Buffer& dst, Buffer const& src) +{ + copy(dst, src, 0, 0, src.size()); +} + +template +void copy(Buffer& dst, Buffer const& src, typename Buffer::size_type dst_begin) +{ + copy(dst, src, dst_begin, 0, src.size()); +} + +template +void copy(Buffer& dst, + Buffer const& src, + typename Buffer::size_type src_begin, + typename Buffer::size_type src_end) +{ + copy(dst, src, 0, src_begin, src_end); +} +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/detail/cpu_only/copy.hpp b/backend/cpp/include/triton_backend/memory/detail/cpu_only/copy.hpp new file mode 100644 index 0000000..1d4778e --- /dev/null +++ b/backend/cpp/include/triton_backend/memory/detail/cpu_only/copy.hpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include + +#ifndef TRITON_ENABLE_GPU +#include +#endif +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +namespace detail { + +template +void copy(T* dst, + T const* src, + std::size_t len, + cudaStream_t stream, + MemoryType dst_type, + MemoryType src_type) +{ + if (dst_type == DeviceMemory || src_type == DeviceMemory) { + throw TritonException(Error::Internal, "Cannot copy device memory in non-GPU build"); + } else { + std::memcpy(dst, src, len * sizeof(T)); + } +} + +} // namespace detail +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/detail/cpu_only/owned_device_buffer.hpp b/backend/cpp/include/triton_backend/memory/detail/cpu_only/owned_device_buffer.hpp new file mode 100644 index 0000000..ba7fece --- /dev/null +++ b/backend/cpp/include/triton_backend/memory/detail/cpu_only/owned_device_buffer.hpp @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +namespace detail { + +template +struct owned_device_buffer { + using non_const_T = std::remove_const_t; + owned_device_buffer(device_id_t device_id, std::size_t size, cudaStream_t stream) + { + throw TritonException(Error::Internal, + "Attempted to use device buffer in non-GPU build"); + } + + auto* get() const { return static_cast(nullptr); } +}; + +} // namespace detail +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/detail/cpu_only/resource.hpp b/backend/cpp/include/triton_backend/memory/detail/cpu_only/resource.hpp new file mode 100644 index 0000000..2a96a58 --- /dev/null +++ b/backend/cpp/include/triton_backend/memory/detail/cpu_only/resource.hpp @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +namespace detail { + +template<> +inline void setup_memory_resource(device_id_t device_id, + TRITONBACKEND_MemoryManager* triton_manager) { } + +} // namespace detail +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/detail/gpu_only/copy.hpp b/backend/cpp/include/triton_backend/memory/detail/gpu_only/copy.hpp new file mode 100644 index 0000000..b1c021e --- /dev/null +++ b/backend/cpp/include/triton_backend/memory/detail/gpu_only/copy.hpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifdef TRITON_ENABLE_GPU +#include +#include +#endif + +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +namespace detail { + +template +void copy(T* dst, + T const* src, + std::size_t len, + cudaStream_t stream, + MemoryType dst_type, + MemoryType src_type) +{ + if (dst_type == DeviceMemory || src_type == DeviceMemory) { + try { + raft::copy(dst, src, len, stream); + } catch (raft::cuda_error const& err) { + throw TritonException(Error::Internal, err.what()); + } + } else { + std::memcpy(dst, src, len * sizeof(T)); + } +} + +} // namespace detail +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/detail/gpu_only/owned_device_buffer.hpp b/backend/cpp/include/triton_backend/memory/detail/gpu_only/owned_device_buffer.hpp new file mode 100644 index 0000000..f897c58 --- /dev/null +++ b/backend/cpp/include/triton_backend/memory/detail/gpu_only/owned_device_buffer.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +namespace detail { + +template +struct owned_device_buffer { + using non_const_T = std::remove_const_t; + owned_device_buffer(device_id_t device_id, std::size_t size, cudaStream_t stream) + : data_{[&device_id, &size, &stream]() { + auto device_context = device_setter{device_id}; + return rmm::device_buffer{size * sizeof(T), rmm::cuda_stream_view{stream}}; + }()} + { + } + + auto* get() const { return reinterpret_cast(data_.data()); } + + private: + mutable rmm::device_buffer data_; +}; + +} // namespace detail +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/detail/gpu_only/resource.hpp b/backend/cpp/include/triton_backend/memory/detail/gpu_only/resource.hpp new file mode 100644 index 0000000..639572a --- /dev/null +++ b/backend/cpp/include/triton_backend/memory/detail/gpu_only/resource.hpp @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +namespace detail { + +inline auto& resource_lock() +{ + static auto lock = std::mutex{}; + return lock; +} + +/** A struct used solely to keep memory resources in-scope for the lifetime + * of the backend */ +struct resource_data { + resource_data() : base_mr_{}, triton_mrs_{} {} + auto* make_new_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* manager) + { + if (manager == nullptr && triton_mrs_.size() != 0) { + manager = triton_mrs_.back().get_triton_manager(); + } + triton_mrs_.emplace_back(manager, device_id, &base_mr_); + return &(triton_mrs_.back()); + } + + private: + rmm::mr::cuda_memory_resource base_mr_; + std::deque triton_mrs_; +}; + +inline auto& get_device_resources() +{ + static auto device_resources = resource_data{}; + return device_resources; +} + +inline auto is_triton_resource(rmm::cuda_device_id const& device_id) +{ + auto* triton_mr = + dynamic_cast(rmm::mr::get_per_device_resource(device_id)); + return (triton_mr != nullptr && triton_mr->get_triton_manager() != nullptr); +} + +template<> +inline void setup_memory_resource(device_id_t device_id, + TRITONBACKEND_MemoryManager* triton_manager) +{ + auto lock = std::lock_guard{detail::resource_lock()}; + auto rmm_device_id = rmm::cuda_device_id{device_id}; + + if (!detail::is_triton_resource(rmm_device_id)) { + auto& device_resources = detail::get_device_resources(); + rmm::mr::set_per_device_resource(rmm_device_id, + device_resources.make_new_resource(device_id, triton_manager)); + } +} + +/* inline auto* get_memory_resource(device_id_t device_id) +{ + auto rmm_device_id = rmm::cuda_device_id{device_id}; + return rmm::mr::get_per_device_resource(rmm_device_id); +} + +inline auto* get_memory_resource() { return rmm::mr::get_current_device_resource(); } */ + +} // namespace detail +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/detail/owned_device_buffer.hpp b/backend/cpp/include/triton_backend/memory/detail/owned_device_buffer.hpp new file mode 100644 index 0000000..61a470c --- /dev/null +++ b/backend/cpp/include/triton_backend/memory/detail/owned_device_buffer.hpp @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +namespace triton { +namespace backend { +namespace dev_tools { +namespace detail { + +template +struct owned_device_buffer { +}; + +} // namespace detail +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/detail/resource.hpp b/backend/cpp/include/triton_backend/memory/detail/resource.hpp new file mode 100644 index 0000000..cf922b1 --- /dev/null +++ b/backend/cpp/include/triton_backend/memory/detail/resource.hpp @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +#include + +namespace triton { +namespace backend { +namespace dev_tools { +namespace detail { + +template +inline void setup_memory_resource(device_id_t device_id, + TRITONBACKEND_MemoryManager* triton_manager = nullptr) { +} + +} // namespace detail +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/resource.hpp b/backend/cpp/include/triton_backend/memory/resource.hpp new file mode 100644 index 0000000..e95378f --- /dev/null +++ b/backend/cpp/include/triton_backend/memory/resource.hpp @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +#include +#include +#include +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif + +namespace triton { +namespace backend { +namespace dev_tools { + +inline void setup_memory_resource(device_id_t device_id, TRITONBACKEND_MemoryManager* triton_manager = nullptr) { + detail::setup_memory_resource(device_id, triton_manager); +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/memory/types.hpp b/backend/cpp/include/triton_backend/memory/types.hpp new file mode 100644 index 0000000..57386f1 --- /dev/null +++ b/backend/cpp/include/triton_backend/memory/types.hpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { +namespace backend { +namespace dev_tools { +using MemoryType = TRITONSERVER_MemoryType; +auto constexpr DeviceMemory = TRITONSERVER_MEMORY_GPU; +auto constexpr HostMemory = TRITONSERVER_MEMORY_CPU; +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/model/model.hpp b/backend/cpp/include/triton_backend/model/model.hpp new file mode 100644 index 0000000..a46151f --- /dev/null +++ b/backend/cpp/include/triton_backend/model/model.hpp @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +template +struct Model { + virtual void predict(Batch& batch) const = 0; + + virtual void load() {} + virtual void unload() {} + + /** + * @brief Return the preferred memory type in which to store data for this + * batch or std::nullopt to accept whatever Triton returns + * + * The base implementation of this method will require data on-host if the + * model itself is deployed on the host OR if this backend has not been + * compiled with GPU support. Otherwise, models deployed on device will + * receive memory on device. Overriding this method will allow derived + * model classes to select a preferred memory location based on properties + * of the batch or to simply return std::nullopt if device memory or host + * memory will do equally well. + */ + virtual std::optional preferred_mem_type(Batch& batch) const + { + return (IS_GPU_BUILD && deployment_type_ == GPUDeployment) ? DeviceMemory : HostMemory; + } + virtual std::optional preferred_mem_type_in(Batch& batch) const + { + return preferred_mem_type(batch); + } + virtual std::optional preferred_mem_type_out(Batch& batch) const + { + return preferred_mem_type(batch); + } + + /** + * @brief Retrieve a stream used to set up batches for this model + * + * The base implementation of this method simply returns the default stream + * provided by Triton for use with this model. Child classes may choose to + * override this in order to provide different streams for use with + * successive incoming batches. For instance, one might cycle through + * several streams in order to distribute batches across them, but care + * should be taken to ensure proper synchronization in this case. + */ + virtual cudaStream_t get_stream() const { return default_stream_; } + + /** + * @brief Get input tensor of a particular named input for an entire batch + */ + template + auto get_input(Batch& batch, + std::string const& name, + std::optional const& mem_type, + cudaStream_t stream) const + { + return batch.get_input(name, mem_type, device_id_, stream); + } + template + auto get_input(Batch& batch, + std::string const& name, + std::optional const& mem_type) const + { + return get_input(batch, name, mem_type, default_stream_); + } + template + auto get_input(Batch& batch, std::string const& name) const + { + return get_input(batch, name, preferred_mem_type(batch), default_stream_); + } + + /** + * @brief Get output tensor of a particular named output for an entire batch + */ + template + auto get_output(Batch& batch, + std::string const& name, + std::optional const& mem_type, + device_id_t device_id, + cudaStream_t stream) const + { + return batch.get_output(name, mem_type, device_id, stream); + } + template + auto get_output(Batch& batch, + std::string const& name, + std::optional const& mem_type, + cudaStream_t stream) const + { + return get_output(batch, name, mem_type, device_id_, stream); + } + template + auto get_output(Batch& batch, + std::string const& name, + std::optional const& mem_type) const + { + return get_output(batch, name, mem_type, device_id_, default_stream_); + } + template + auto get_output(Batch& batch, std::string const& name) const + { + return get_output(batch, name, preferred_mem_type(batch), device_id_, default_stream_); + } + + /** + * @brief Retrieve value of configuration parameter + */ + template + auto get_config_param(std::string const& name) const + { + return shared_state_->template get_config_param(name); + } + template + auto get_config_param(std::string const& name, T default_value) const + { + return shared_state_->template get_config_param(name, default_value); + } + template + auto get_config_param(char const* name) const + { + return get_config_param(std::string(name)); + } + template + auto get_config_param(char const* name, T default_value) const + { + return get_config_param(std::string(name), default_value); + } + + Model(std::shared_ptr shared_state, + device_id_t device_id, + cudaStream_t default_stream, + DeploymentType deployment_type, + std::string const& filepath) + : shared_state_{shared_state}, + device_id_{device_id}, + default_stream_{default_stream}, + deployment_type_{deployment_type}, + filepath_{filepath} + { + if constexpr (IS_GPU_BUILD) { setup_memory_resource(device_id_); } + } + + auto get_device_id() const { return device_id_; } + auto get_deployment_type() const { return deployment_type_; } + auto const& get_filepath() const { return filepath_; } + + auto get_output_shape(std::string const& name) const + { + return shared_state_->get_output_shape(name); + } + + protected: + auto get_shared_state() const { return shared_state_; } + + private: + std::shared_ptr shared_state_; + device_id_t device_id_; + cudaStream_t default_stream_; + DeploymentType deployment_type_; + std::string filepath_; +}; +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/model/shared_state.hpp b/backend/cpp/include/triton_backend/model/shared_state.hpp new file mode 100644 index 0000000..f09b3fa --- /dev/null +++ b/backend/cpp/include/triton_backend/model/shared_state.hpp @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +/** + * @brief Stores shared state for multiple instances of the same model + */ +struct SharedModelState { + virtual void load() {} + virtual void unload() {} + + explicit SharedModelState(std::unique_ptr&& config, + bool squeeze_output = false) + : config_{std::move(config)}, + max_batch_size_{get_max_batch_size(*config_)}, + output_shapes_([this, squeeze_output]() { + auto result = std::vector>>{}; + auto output_entries = triton::common::TritonJson::Value{}; + triton_check(config_->MemberAsArray("output", &output_entries)); + + result.reserve(output_entries.ArraySize()); + + // Using a raw loop because TritonJSON::Value access has no iterator interface + for (std::size_t i = 0; i < output_entries.ArraySize(); ++i) { + auto output_entry = triton::common::TritonJson::Value{}; + triton_check(output_entries.IndexAsObject(i, &output_entry)); + auto name = std::string{}; + triton_check(output_entry.MemberAsString("name", &name)); + + auto shape = std::vector{}; + auto reshape_entry = triton::common::TritonJson::Value{}; + if (output_entry.Find("reshape", &reshape_entry)) { + ParseShape(reshape_entry, "shape", &shape); + } else { + ParseShape(output_entry, "dims", &shape); + } + if (shape[0] != -1) { shape.insert(shape.begin(), -1); } + // The squeeze_output option was introduced to handle a bad choice of + // convention in the original FIL backend implementation. For legacy + // compatibility, we introduced this option into Triton's dev tools, + // but in general, new backends are advised to avoid using it and + // defer this sort of flattening operation to the consumer. + if (squeeze_output) { + shape.erase(std::remove(shape.begin(), shape.end(), std::int64_t{1}), shape.end()); + } + result.insert( + std::upper_bound(std::begin(output_shapes_), + std::end(output_shapes_), + name, + [](auto& value, auto& entry) { return value < entry.first; }), + {name, shape}); + } + + return result; + }()) + { + } + + template + auto get_config_param(std::string const& name) + { + return get_config_param(name, std::optional{}); + } + + template + auto get_config_param(std::string const& name, T default_value) + { + return get_config_param(name, std::make_optional(default_value)); + } + + auto get_output_shape(std::string const& name) const + { + auto cached_shape = std::lower_bound( + std::begin(output_shapes_), std::end(output_shapes_), name, [](auto& entry, auto& value) { + return entry.first < value; + }); + if (cached_shape == std::end(output_shapes_) || name != cached_shape->first) { + auto log_stream = std::stringstream{}; + log_stream << "No output with name " << name << " in configuration."; + throw TritonException(Error::Internal, log_stream.str()); + } else { + return cached_shape->second; + } + } + + auto get_output_names() const { + auto output_names = std::vector{}; + output_names.reserve(output_shapes_.size()); + std::transform(std::begin(output_shapes_), std::end(output_shapes_), std::back_inserter(output_names), [](auto& output_shape) { + return output_shape.first; + }); + return output_names; + } + + auto check_output_name(std::string const& name) const { + // #TODO: Figure out a way to use std::binary_search here + auto cached_shape = std::lower_bound( + std::begin(output_shapes_), std::end(output_shapes_), name, [](auto& entry, auto& value) { + return entry.first < value; + }); + return cached_shape != std::end(output_shapes_) && name == cached_shape->first; + } + + private: + std::unique_ptr config_; + Batch::size_type max_batch_size_; + std::vector>> mutable output_shapes_; + + template + auto get_config_param(std::string const& name, std::optional const& default_value) + { + auto result = T{}; + if (name == std::string("max_batch_size")) { + result = max_batch_size_; + return result; + } + auto parameters = common::TritonJson::Value{}; + auto json_value = common::TritonJson::Value{}; + if (config_->Find("parameters", ¶meters) && parameters.Find(name.c_str(), &json_value)) { + auto string_repr = std::string{}; + triton_check(json_value.MemberAsString("string_value", &string_repr)); + + auto input_stream = std::istringstream{string_repr}; + + if constexpr (std::is_same_v) { + input_stream >> std::boolalpha >> result; + } else { + input_stream >> result; + } + + if (input_stream.fail()) { + if (default_value) { + result = *default_value; + } else { + throw TritonException(Error::InvalidArg, std::string("Bad input for parameter ") + name); + } + } + } else { + if (default_value) { + result = *default_value; + } else { + throw TritonException( + Error::InvalidArg, + std::string("Required parameter ") + name + std::string(" not found in config")); + } + } + + return result; + } +}; +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/tensor/dtype.hpp b/backend/cpp/include/triton_backend/tensor/dtype.hpp new file mode 100644 index 0000000..9545541 --- /dev/null +++ b/backend/cpp/include/triton_backend/tensor/dtype.hpp @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { + +using DType = TRITONSERVER_DataType; +auto constexpr DTypeBool = TRITONSERVER_TYPE_BOOL; +auto constexpr DTypeUint8 = TRITONSERVER_TYPE_UINT8; +auto constexpr DTypeChar = DTypeUint8; +auto constexpr DTypeByte = DTypeUint8; +auto constexpr DTypeUint16 = TRITONSERVER_TYPE_UINT16; +auto constexpr DTypeUint32 = TRITONSERVER_TYPE_UINT32; +auto constexpr DTypeUint64 = TRITONSERVER_TYPE_UINT64; +auto constexpr DTypeInt8 = TRITONSERVER_TYPE_INT8; +auto constexpr DTypeInt16 = TRITONSERVER_TYPE_INT16; +auto constexpr DTypeInt32 = TRITONSERVER_TYPE_INT32; +auto constexpr DTypeInt64 = TRITONSERVER_TYPE_INT64; +auto constexpr DTypeFloat32 = TRITONSERVER_TYPE_FP32; +auto constexpr DTypeFloat64 = TRITONSERVER_TYPE_FP64; + +template +struct TritonType { +}; + +template +struct TritonDtype { +}; + +template <> +struct TritonType { + typedef bool type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeBool; +}; + +template <> +struct TritonType { + typedef std::uint8_t type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeUint8; +}; + +template <> +struct TritonType { + typedef std::uint16_t type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeUint16; +}; + +template <> +struct TritonType { + typedef std::uint32_t type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeUint32; +}; + +template <> +struct TritonType { + typedef std::uint64_t type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeUint64; +}; + +template <> +struct TritonType { + typedef std::int8_t type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeInt8; +}; + +template <> +struct TritonType { + typedef std::int16_t type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeInt16; +}; + +template <> +struct TritonType { + typedef std::int32_t type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeInt32; +}; + +template <> +struct TritonType { + typedef std::int64_t type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeInt64; +}; + +template <> +struct TritonType { + typedef float type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeFloat32; +}; + +template <> +struct TritonType { + typedef double type; +}; + +template +struct TritonDtype> { + static constexpr DType value = DTypeFloat64; +}; + +inline std::ostream& operator<<(std::ostream& out, DType const& dtype) +{ + out << TRITONSERVER_DataTypeString(dtype); + return out; +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/tensor/tensor.hpp b/backend/cpp/include/triton_backend/tensor/tensor.hpp new file mode 100644 index 0000000..c52bbe3 --- /dev/null +++ b/backend/cpp/include/triton_backend/tensor/tensor.hpp @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +template +struct BaseTensor { + using size_type = typename Buffer::size_type; + + BaseTensor() : shape_{}, buffer_{} {} + BaseTensor(std::vector const& shape, Buffer&& buffer) + : shape_(shape), buffer_{std::move(buffer)} + { + } + + virtual ~BaseTensor() = 0; + + /** + * @brief Construct a BaseTensor from a collection of buffers + * + * Given a collection of buffers, collate them all into one buffer stored in + * a new BaseTensor + */ + template + BaseTensor(std::vector const& shape, + Iter begin, + Iter end, + MemoryType mem_type, + device_id_t device, + cudaStream_t stream) + : shape_(shape), buffer_([&begin, &end, &mem_type, &device, &stream]() { + auto total_size = std::transform_reduce( + begin, end, size_type{}, std::plus<>{}, [](auto&& buffer) { return buffer.size(); }); + + auto result = Buffer(total_size, mem_type, device, stream); + + std::accumulate(begin, end, size_type{}, [&result](auto offset, auto& buffer) { + copy(result, buffer, offset); + return offset + buffer.size(); + }); + return result; + }()) + { + } + + auto const& shape() const { return shape_; } + auto size() const { return buffer_.size(); } + auto data() const { return buffer_.data(); } + auto& buffer() { return buffer_; } + + auto constexpr dtype() { return TritonDtype::value; } + auto mem_type() const { return buffer_.mem_type(); } + auto stream() const { return buffer_.stream(); } + auto device() const { return buffer_.device(); } + + void stream_synchronize() const + { + if (mem_type() == DeviceMemory) { buffer_.stream_synchronize(); } + } + + void set_stream(cudaStream_t new_stream) { buffer_.set_stream(new_stream); } + + private: + std::vector shape_; + Buffer buffer_; +}; + +template +BaseTensor::~BaseTensor() +{ +} + +template +struct Tensor final : BaseTensor { + Tensor() : BaseTensor{} {} + Tensor(std::vector::size_type> const& shape, Buffer&& buffer) + : BaseTensor(shape, std::move(buffer)) + { + } + + template + Tensor(std::vector::size_type> const& shape, + Iter begin, + Iter end, + MemoryType mem_type, + device_id_t device, + cudaStream_t stream) + : BaseTensor(shape, begin, end, mem_type, device, stream) + { + } +}; + +template +struct OutputTensor final : BaseTensor { + OutputTensor(std::vector::size_type>&& shape, + Buffer&& buffer, + std::string const& name, + std::shared_ptr responder) + : BaseTensor(std::move(shape), std::move(buffer)), name_{name}, responder_{responder} + { + } + /** + * @brief Prepare final output data from this tensor for responding to + * request + * + * This method *must* be called by triton_backend backends on all of their + * output tensors before returning from their `predict` methods. Because we + * cannot know a priori what names backends might have for their tensors + * and what types will be stored in those tensors, the triton_backend + * library cannot store references to those tensors that might otherwise be + * used to finalize them. + */ + void finalize() + { + auto& shape = BaseTensor::shape(); + auto triton_shape = std::vector{}; + triton_shape.reserve(shape.size()); + std::transform( + std::begin(shape), std::end(shape), std::back_inserter(triton_shape), [](auto& val) { + return narrow(val); + }); + + // Must call the following because BackendOutputResponder does not expose + // its stream, so we cannot be certain that our data is not being + // processed on another stream. + BaseTensor::stream_synchronize(); + responder_->ProcessTensor(name_.c_str(), + TritonDtype::value, + triton_shape, + reinterpret_cast(BaseTensor::data()), + BaseTensor::mem_type(), + BaseTensor::device()); + } + + private: + std::string name_; + std::shared_ptr responder_; +}; + +template , T>>> +void copy(BaseTensor& dst, BaseTensor& src) +{ + copy(dst.buffer(), src.buffer()); +} + +/** + * @brief Copy data from src Tensor into buffers indicated by iterators + * + * This method is provided to assist with distributing data from a single + * Tensor into many smaller buffers which have been set up to receive a part + * of the data from the src Tensor + */ +template +void copy(Iter begin, Iter end, BaseTensor& src) +{ + std::accumulate(begin, end, typename BaseTensor::size_type{}, [&src](auto offset, auto& dst) { + auto end_offset = offset + dst.size(); + copy(dst.buffer(), src.buffer(), offset, end_offset); + return end_offset; + }); +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/api/execute.hpp b/backend/cpp/include/triton_backend/triton/api/execute.hpp new file mode 100644 index 0000000..3233e89 --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/api/execute.hpp @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +namespace triton_api { +template +auto* execute(TRITONBACKEND_ModelInstance* instance, + TRITONBACKEND_Request** raw_requests, + std::size_t request_count) +{ + auto start_time = std::chrono::steady_clock::now(); + + auto* result = static_cast(nullptr); + + try { + auto* model_state = get_model_state(*get_model_from_instance(*instance)); + auto* instance_state = get_instance_state(*instance); + auto& model = instance_state->get_model(); + auto max_batch_size = model.template get_config_param("max_batch_size"); + + /* Note: It is safe to keep a reference to the model in this closure + * and a pointer to the instance in the next because the batch goes + * out of scope at the end of this block and Triton guarantees that + * the liftimes of both the instance and model extend beyond this + * function call. */ + auto output_shape_fetcher = [&model](std::string const& name, Batch::size_type batch_dim) { + auto result = std::vector{}; + auto config_shape = model.get_output_shape(name); + if (config_shape.size() > 0 && config_shape[0] < 0) { config_shape[0] = batch_dim; } + std::transform( + std::begin(config_shape), + std::end(config_shape), + std::back_inserter(result), + [](auto& coord) { + if (coord < 0) { + throw TritonException( + Error::Internal, + "Backends with variable-shape outputs must request desired output shape"); + } else { + return narrow(coord); + } + }); + return result; + }; + auto statistics_reporter = [instance](TRITONBACKEND_Request* request, + time_point req_start, + time_point req_comp_start, + time_point req_comp_end, + time_point req_end) { + report_statistics(*instance, *request, req_start, req_comp_start, req_comp_end, req_end); + }; + + auto batch = Batch(raw_requests, + request_count, + *(model_state->TritonMemoryManager()), + std::move(output_shape_fetcher), + std::move(statistics_reporter), + model_state->EnablePinnedInput(), + model_state->EnablePinnedOutput(), + max_batch_size, + model.get_stream()); + + if constexpr (IS_GPU_BUILD) { + if (model.get_deployment_type() == GPUDeployment) { + cuda_check(cudaSetDevice(model.get_device_id())); + } + } + + auto predict_err = static_cast(nullptr); + try { + model.predict(batch); + } catch (TritonException& err) { + predict_err = err.error(); + } + + auto& compute_start_time = batch.compute_start_time(); + auto compute_end_time = std::chrono::steady_clock::now(); + batch.finalize(predict_err); + auto end_time = std::chrono::steady_clock::now(); + + report_statistics( + *instance, request_count, start_time, compute_start_time, compute_end_time, end_time); + } catch (TritonException& err) { + result = err.error(); + } + + return result; +} +} // namespace triton_api +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/api/initialize.hpp b/backend/cpp/include/triton_backend/triton/api/initialize.hpp new file mode 100644 index 0000000..25dd376 --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/api/initialize.hpp @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +namespace triton_api { +inline auto* initialize(TRITONBACKEND_Backend* backend) +{ + auto* result = static_cast(nullptr); + try { + auto name = get_backend_name(*backend); + + log_info(__FILE__, __LINE__) << "TRITONBACKEND_Initialize: " << name; + + if (!check_backend_version(*backend)) { + throw TritonException{Error::Unsupported, + "triton backend API version does not support this backend"}; + } + if constexpr (IS_GPU_BUILD) { + auto device_count = int{}; + auto cuda_err = cudaGetDeviceCount(&device_count); + if (device_count > 0 && cuda_err == cudaSuccess) { + auto device_id = int{}; + cuda_check(cudaGetDevice(&device_id)); + auto* triton_manager = static_cast(nullptr); + triton_check(TRITONBACKEND_BackendMemoryManager(backend, &triton_manager)); + + setup_memory_resource(static_cast(device_id), triton_manager); + } + } + } catch (TritonException& err) { + result = err.error(); + } + return result; +} +} // namespace triton_api +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/api/instance_finalize.hpp b/backend/cpp/include/triton_backend/triton/api/instance_finalize.hpp new file mode 100644 index 0000000..10a249b --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/api/instance_finalize.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +namespace triton_api { +template +auto* instance_finalize(TRITONBACKEND_ModelInstance* instance) +{ + auto* result = static_cast(nullptr); + try { + auto* instance_state = get_instance_state(*instance); + if (instance_state != nullptr) { + instance_state->unload(); + + log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInstanceFinalize: delete instance state"; + + delete instance_state; + } + } catch (TritonException& err) { + result = err.error(); + } + + return result; +} +} // namespace triton_api +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/api/instance_initialize.hpp b/backend/cpp/include/triton_backend/triton/api/instance_initialize.hpp new file mode 100644 index 0000000..5c71084 --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/api/instance_initialize.hpp @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +namespace triton_api { +template +auto* instance_initialize(TRITONBACKEND_ModelInstance* instance) +{ + auto* result = static_cast(nullptr); + try { + auto name = get_model_instance_name(*instance); + auto device_id = get_device_id(*instance); + auto deployment_type = get_deployment_type(*instance); + if constexpr (!IS_GPU_BUILD) { + if (deployment_type == GPUDeployment) { + throw TritonException(Error::Unsupported, "KIND_GPU cannot be used in CPU-only build"); + } + } + + log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInstanceInitialize: " << name << " (" + << TRITONSERVER_InstanceGroupKindString(deployment_type) + << " device " << device_id << ")"; + + auto* triton_model = get_model_from_instance(*instance); + auto* model_state = get_model_state(*triton_model); + if constexpr (IS_GPU_BUILD) { + setup_memory_resource(device_id, model_state->TritonMemoryManager()); + } + + auto dev_tools_model = std::make_unique(*model_state, instance); + if constexpr (IS_GPU_BUILD) { + auto& model = dev_tools_model->get_model(); + if (model.get_deployment_type() == GPUDeployment) { + cuda_check(cudaSetDevice(model.get_device_id())); + } + } + dev_tools_model->load(); + + set_instance_state(*instance, std::move(dev_tools_model)); + } catch (TritonException& err) { + result = err.error(); + } + return result; +} +} // namespace triton_api +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/api/model_finalize.hpp b/backend/cpp/include/triton_backend/triton/api/model_finalize.hpp new file mode 100644 index 0000000..f385cc1 --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/api/model_finalize.hpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +namespace triton_api { +template +auto* model_finalize(TRITONBACKEND_Model* model) +{ + auto* result = static_cast(nullptr); + try { + auto model_state = get_model_state(*model); + if (model_state != nullptr) { model_state->get_shared_state()->unload(); } + + log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelFinalize: delete model state"; + + delete model_state; + } catch (TritonException& err) { + result = err.error(); + } + + return result; +} +} // namespace triton_api +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/api/model_initialize.hpp b/backend/cpp/include/triton_backend/triton/api/model_initialize.hpp new file mode 100644 index 0000000..8877b90 --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/api/model_initialize.hpp @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +namespace triton_api { +template +auto* model_initialize(TRITONBACKEND_Model* model) +{ + auto* result = static_cast(nullptr); + try { + auto name = get_model_name(*model); + + auto version = get_model_version(*model); + + log_info(__FILE__, __LINE__) << "TRITONBACKEND_ModelInitialize: " << name << " (version " + << version << ")"; + + auto dev_tools_model_state = std::make_unique(*model); + dev_tools_model_state->load(); + + set_model_state(*model, std::move(dev_tools_model_state)); + } catch (TritonException& err) { + result = err.error(); + } + + return result; +} +} // namespace triton_api +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/backend.hpp b/backend/cpp/include/triton_backend/triton/backend.hpp new file mode 100644 index 0000000..76b848f --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/backend.hpp @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +inline auto get_backend_name(TRITONBACKEND_Backend& backend) +{ + const char* cname; + triton_check(TRITONBACKEND_BackendName(&backend, &cname)); + return std::string(cname); +} + +namespace { +struct backend_version { + std::uint32_t major; + std::uint32_t minor; +}; +} // namespace + +inline auto check_backend_version(TRITONBACKEND_Backend& backend) +{ + auto version = backend_version{}; + triton_check(TRITONBACKEND_ApiVersion(&version.major, &version.minor)); + + log_info(__FILE__, __LINE__) << "Triton TRITONBACKEND API version: " << version.major << "." + << version.minor; + + auto name = get_backend_name(backend); + + log_info(__FILE__, __LINE__) << "'" << name + << "' TRITONBACKEND API version: " << TRITONBACKEND_API_VERSION_MAJOR + << "." << TRITONBACKEND_API_VERSION_MINOR; + + return ((version.major == TRITONBACKEND_API_VERSION_MAJOR) && + (version.minor >= TRITONBACKEND_API_VERSION_MINOR)); +} +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/config.hpp b/backend/cpp/include/triton_backend/triton/config.hpp new file mode 100644 index 0000000..80974a7 --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/config.hpp @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +inline auto get_max_batch_size(common::TritonJson::Value& config) +{ + auto reported = int64_t{}; + triton_check(config.MemberAsInt("max_batch_size", &reported)); + return narrow(reported); +} +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/deployment.hpp b/backend/cpp/include/triton_backend/triton/deployment.hpp new file mode 100644 index 0000000..c985d66 --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/deployment.hpp @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { +namespace backend { +namespace dev_tools { +using DeploymentType = TRITONSERVER_InstanceGroupKind; +auto constexpr GPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_GPU; +auto constexpr CPUDeployment = TRITONSERVER_INSTANCEGROUPKIND_CPU; +// Note (wphicks): We currently are not including "Auto" or "Model" because I +// am not sure exactly how those would be used in context. If there is a +// demand, they can be added. +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/device.hpp b/backend/cpp/include/triton_backend/triton/device.hpp new file mode 100644 index 0000000..480fbda --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/device.hpp @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { +namespace backend { +namespace dev_tools { +using device_id_t = std::int32_t; +} +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/input.hpp b/backend/cpp/include/triton_backend/triton/input.hpp new file mode 100644 index 0000000..0e724a7 --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/input.hpp @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) +{ + auto result = static_cast(nullptr); + triton_check(TRITONBACKEND_RequestInput(request, name.c_str(), &result)); + return result; +} + +template +auto get_triton_input_shape(Iter requests_begin, Iter requests_end, std::string const& name) +{ + auto result = std::vector{}; + + auto reported_dtype = DType{}; + auto const* input_shape = static_cast(nullptr); + auto input_dims = uint32_t{}; + + auto batch_dim = std::accumulate( + requests_begin, + requests_end, + int64_t{}, + [&reported_dtype, &input_shape, &input_dims, &name](auto total, auto& request) { + auto* input = get_triton_input(request, name); + triton_check(TRITONBACKEND_InputProperties( + input, nullptr, &reported_dtype, &input_shape, &input_dims, nullptr, nullptr)); + + if (reported_dtype != TritonDtype::value) { + auto log_stream = std::stringstream{}; + log_stream << "incorrect type " << reported_dtype << " for input with required type " + << TritonDtype::value; + throw(TritonException(Error::Internal, log_stream.str())); + } + + if (input_dims != 0) { total += *input_shape; } + return total; + }); + + result.reserve(input_dims); + std::transform(input_shape, input_shape + input_dims, std::back_inserter(result), [](auto& val) { + return narrow(val); + }); + + if (!result.empty()) { result[0] = narrow(batch_dim); } + + return result; +} +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/logging.hpp b/backend/cpp/include/triton_backend/triton/logging.hpp new file mode 100644 index 0000000..c7a4072 --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/logging.hpp @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { + +namespace { +/** Log message at indicated level */ +inline void log(TRITONSERVER_LogLevel level, char const* filename, int line, char const* message) +{ + triton_check(TRITONSERVER_LogMessage(level, filename, line, message)); +} +} // namespace + +struct log_stream : public std::ostream { + log_stream(TRITONSERVER_LogLevel level, char const* filename, int line) + : std::ostream{}, buffer_{level, filename, line} + { + rdbuf(&buffer_); + } + log_stream(TRITONSERVER_LogLevel level) : std::ostream{}, buffer_{level, __FILE__, __LINE__} + { + rdbuf(&buffer_); + } + + ~log_stream() + { + try { + flush(); + } catch (std::ios_base::failure const& ignored_err) { + // Ignore error if flush fails + } + } + + private: + struct log_buffer : public std::stringbuf { + log_buffer(TRITONSERVER_LogLevel level, char const* filename, int line) + : level_{level}, filename_{filename}, line_{line} + { + } + + virtual int sync() + { + auto msg = str(); + if (!msg.empty()) { + log(level_, filename_, line_, msg.c_str()); + str(""); + } + return 0; + } + + private: + TRITONSERVER_LogLevel level_; + char const* filename_; + int line_; + }; + + log_buffer buffer_; +}; + +/** Log message at INFO level */ +inline void log_info(char const* filename, int line, char const* message) +{ + log(TRITONSERVER_LOG_INFO, filename, line, message); +} +inline void log_info(char const* filename, int line, std::string const& message) +{ + log_info(filename, line, message.c_str()); +} +inline void log_info(char const* message) { log_info(__FILE__, __LINE__, message); } +inline void log_info(std::string const& message) { log_info(__FILE__, __LINE__, message.c_str()); } +inline auto log_info(char const* filename, int line) +{ + return log_stream(TRITONSERVER_LOG_INFO, filename, line); +} +inline auto log_info() { return log_stream(TRITONSERVER_LOG_INFO); } + +/** Log message at WARN level */ +inline void log_warn(char const* filename, int line, char const* message) +{ + log(TRITONSERVER_LOG_WARN, filename, line, message); +} +inline void log_warn(char const* filename, int line, std::string const& message) +{ + log_warn(filename, line, message.c_str()); +} +inline void log_warn(char const* message) { log_warn(__FILE__, __LINE__, message); } +inline void log_warn(std::string const& message) { log_warn(__FILE__, __LINE__, message.c_str()); } +inline auto log_warn(char const* filename, int line) +{ + return log_stream(TRITONSERVER_LOG_WARN, filename, line); +} +inline auto log_warn() { return log_stream(TRITONSERVER_LOG_WARN); } + +/** Log message at ERROR level */ +inline void log_error(char const* filename, int line, char const* message) +{ + log(TRITONSERVER_LOG_ERROR, filename, line, message); +} +inline void log_error(char const* filename, int line, std::string const& message) +{ + log_error(filename, line, message.c_str()); +} +inline void log_error(char const* message) { log_error(__FILE__, __LINE__, message); } +inline void log_error(std::string const& message) +{ + log_error(__FILE__, __LINE__, message.c_str()); +} +inline auto log_error(char const* filename, int line) +{ + return log_stream(TRITONSERVER_LOG_ERROR, filename, line); +} +inline auto log_error() { return log_stream(TRITONSERVER_LOG_ERROR); } + +/** Log message at VERBOSE level */ +inline void log_debug(char const* filename, int line, char const* message) +{ + log(TRITONSERVER_LOG_VERBOSE, filename, line, message); +} +inline void log_debug(char const* filename, int line, std::string const& message) +{ + log_debug(filename, line, message.c_str()); +} +inline void log_debug(char const* message) { log_debug(__FILE__, __LINE__, message); } +inline void log_debug(std::string const& message) +{ + log_debug(__FILE__, __LINE__, message.c_str()); +} +inline auto log_debug(char const* filename, int line) +{ + return log_stream(TRITONSERVER_LOG_VERBOSE, filename, line); +} +inline auto log_debug() { return log_stream(TRITONSERVER_LOG_VERBOSE); } + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/model.hpp b/backend/cpp/include/triton_backend/triton/model.hpp new file mode 100644 index 0000000..6f05f77 --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/model.hpp @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { + +inline auto get_model_version(TRITONBACKEND_Model& model) +{ + auto version = std::uint64_t{}; + triton_check(TRITONBACKEND_ModelVersion(&model, &version)); + return version; +} + +inline auto get_model_name(TRITONBACKEND_Model& model) +{ + auto* cname = static_cast(nullptr); + triton_check(TRITONBACKEND_ModelName(&model, &cname)); + return std::string(cname); +} + +inline auto get_model_config(TRITONBACKEND_Model& model) +{ + auto* config_message = static_cast(nullptr); + triton_check(TRITONBACKEND_ModelConfig(&model, 1, &config_message)); + + auto* buffer = static_cast(nullptr); + auto byte_size = std::size_t{}; + triton_check(TRITONSERVER_MessageSerializeToJson(config_message, &buffer, &byte_size)); + + auto model_config = std::make_unique(); + auto* err = model_config->Parse(buffer, byte_size); + auto* result = TRITONSERVER_MessageDelete(config_message); + if (err != nullptr) { throw(TritonException(err)); } + if (result != nullptr) { throw(TritonException(result)); } + return model_config; +} + +/** + * @brief Set model state (as used by Triton) to given object + * + * This function accepts a unique_ptr to an object derived from a Triton + * BackendModel object and sets it as the stored state for a model in the + * Triton server. Note that this object is not the same as a dev tools + * "SharedModelState" object. The object that Triton expects must wrap this + * SharedModelState and provide additional interface compatibility. + */ +template +void set_model_state(TRITONBACKEND_Model& model, std::unique_ptr&& model_state) +{ + triton_check(TRITONBACKEND_ModelSetState(&model, reinterpret_cast(model_state.release()))); +} + +/** Given a model, return its associated ModelState object */ +template +auto* get_model_state(TRITONBACKEND_Model& model) +{ + auto* vstate = static_cast(nullptr); + triton_check(TRITONBACKEND_ModelState(&model, &vstate)); + + auto* model_state = reinterpret_cast(vstate); + + return model_state; +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/model_instance.hpp b/backend/cpp/include/triton_backend/triton/model_instance.hpp new file mode 100644 index 0000000..220e007 --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/model_instance.hpp @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +/** Get the name of a Triton model instance from the instance itself */ +inline auto get_model_instance_name(TRITONBACKEND_ModelInstance& instance) +{ + auto* cname = static_cast(nullptr); + triton_check(TRITONBACKEND_ModelInstanceName(&instance, &cname)); + return std::string(cname); +} + +/** Get the device on which a Triton model instance is loaded + * + * If this instance is loaded on the host, 0 will be returned. Otherwise the + * GPU device id will be returned.*/ +inline auto get_device_id(TRITONBACKEND_ModelInstance& instance) +{ + auto device_id = device_id_t{}; + triton_check(TRITONBACKEND_ModelInstanceDeviceId(&instance, &device_id)); + return device_id; +} + +/** Determine how a Triton model instance is deployed + * + * Returns enum value indicating whether the instance is deployed on device + * or on the host + */ +inline auto get_deployment_type(TRITONBACKEND_ModelInstance& instance) +{ + auto kind = GPUDeployment; + triton_check(TRITONBACKEND_ModelInstanceKind(&instance, &kind)); + return kind; +} + +/** Return the Triton model from one of its instances + */ +inline auto* get_model_from_instance(TRITONBACKEND_ModelInstance& instance) +{ + auto* model = static_cast(nullptr); + triton_check(TRITONBACKEND_ModelInstanceModel(&instance, &model)); + return model; +} + +/** + * @brief Set Triton model instance state to given object + * + * This function accepts a unique_ptr to an object derived from a Triton + * BackendModelInstance object and sets it as the stored state for a model in the + * Triton server. Note that this object is not the same as a dev tools + * "Model" object. The object that Triton expects must wrap this Model and + * provide additional interface compatibility. + */ +template +void set_instance_state(TRITONBACKEND_ModelInstance& instance, + std::unique_ptr&& model_instance_state) +{ + triton_check(TRITONBACKEND_ModelInstanceSetState( + &instance, reinterpret_cast(model_instance_state.release()))); +} + +/** Get model instance state from instance */ +template +auto* get_instance_state(TRITONBACKEND_ModelInstance& instance) +{ + auto* instance_state = static_cast(nullptr); + triton_check( + TRITONBACKEND_ModelInstanceState(&instance, reinterpret_cast(&instance_state))); + return instance_state; +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/model_instance_state.hpp b/backend/cpp/include/triton_backend/triton/model_instance_state.hpp new file mode 100644 index 0000000..83d515a --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/model_instance_state.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { + +template +struct ModelInstanceState : public BackendModelInstance { + ModelInstanceState(TritonModelState& model_state, + TRITONBACKEND_ModelInstance* triton_model_instance) + : BackendModelInstance(&model_state, triton_model_instance), + model_(model_state.get_shared_state(), + dev_tools::get_device_id(*triton_model_instance), + CudaStream(), + Kind(), + JoinPath({model_state.RepositoryPath(), + std::to_string(model_state.Version()), + ArtifactFilename()})) + { + } + + auto& get_model() const { return model_; } + + void load() { model_.load(); } + void unload() { model_.unload(); } + + private: + DevToolsModel model_; +}; + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/model_state.hpp b/backend/cpp/include/triton_backend/triton/model_state.hpp new file mode 100644 index 0000000..1af6065 --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/model_state.hpp @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +template +struct TritonModelState : public BackendModel { + TritonModelState(TRITONBACKEND_Model& triton_model) + : BackendModel(&triton_model), + state_{std::make_shared(get_model_config(triton_model))} + { + } + + void load() { state_->load(); } + void unload() { state_->unload(); } + + auto get_shared_state() { return state_; } + + private: + std::shared_ptr state_; +}; + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/output.hpp b/backend/cpp/include/triton_backend/triton/output.hpp new file mode 100644 index 0000000..6e54c5e --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/output.hpp @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +inline auto* get_triton_input(TRITONBACKEND_Request* request, std::string const& name) +{ + auto* result = static_cast(nullptr); + triton_check(TRITONBACKEND_RequestInput(request, name.c_str(), &result)); + return result; +} + +template +auto get_triton_output_shape(Iter requests_begin, Iter requests_end, std::string const& name) +{ + auto result = std::vector{}; + + auto reported_dtype = DType{}; + auto const* input_shape = static_cast(nullptr); + auto input_dims = uint32_t{}; + + auto batch_dim = + std::reduce(requests_begin, + requests_end, + int64_t{}, + [&reported_dtype, &input_shape, &input_dims, &name](auto& request, auto total) { + auto* input = get_triton_input(request, name); + triton_check(TRITONBACKEND_InputProperties( + input, nullptr, &reported_dtype, &input_shape, &input_dims, nullptr, nullptr)); + + if (reported_dtype != TritonDtype::value) { + auto log_stream = std::stringstream{}; + log_stream << "incorrect type " << reported_dtype + << " for output with required type " << TritonDtype::value; + throw(TritonException(Error::Internal, log_stream.str())); + } + + if (input_dims != 0) { total += *input_shape; } + return total; + }); + + result.reserve(input_dims); + std::transform(input_shape, input_shape + input_dims, std::back_inserter(result), [](auto& val) { + return narrow(val); + }); + + if (!result.empty()) { result[0] = narrow(batch_dim); } + + return result; +} +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/requests.hpp b/backend/cpp/include/triton_backend/triton/requests.hpp new file mode 100644 index 0000000..ad1a91b --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/requests.hpp @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +using request_size_t = uint32_t; + +template +void release_requests(Iter begin, Iter end) +{ + std::for_each(begin, end, [](auto& request) { + try { + triton_check(TRITONBACKEND_RequestRelease(request, TRITONSERVER_REQUEST_RELEASE_ALL)); + } catch (TritonException& err) { + log_error(__FILE__, __LINE__, err.what()); + } + }); +} +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/responses.hpp b/backend/cpp/include/triton_backend/triton/responses.hpp new file mode 100644 index 0000000..721ce3a --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/responses.hpp @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { + +template +auto construct_responses(Iter requests_begin, Iter requests_end) +{ + auto responses = std::vector{}; + + auto requests_size = std::distance(requests_begin, requests_end); + if (!(requests_size > 0)) { + throw TritonException(Error::Internal, + "Invalid iterators for requests when constructing responses"); + } + responses.reserve(requests_size); + + std::transform(requests_begin, requests_end, std::back_inserter(responses), [](auto* request) { + auto* response = static_cast(nullptr); + triton_check(TRITONBACKEND_ResponseNew(&response, request)); + return response; + }); + return responses; +} + +template +void send_responses(Iter begin, Iter end, TRITONSERVER_Error* err) +{ + std::for_each(begin, end, [err](auto& response) { + decltype(err) err_copy; + if (err != nullptr) { + err_copy = TRITONSERVER_ErrorNew(TRITONSERVER_ErrorCode(err), TRITONSERVER_ErrorMessage(err)); + } else { + err_copy = err; + } + + if (response == nullptr) { + log_error(__FILE__, __LINE__) << "Failure in response collation"; + } else { + try { + triton_check( + TRITONBACKEND_ResponseSend(response, TRITONSERVER_RESPONSE_COMPLETE_FINAL, err_copy)); + } catch (TritonException& err) { + log_error(__FILE__, __LINE__, err.what()); + } + } + }); +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/statistics.hpp b/backend/cpp/include/triton_backend/triton/statistics.hpp new file mode 100644 index 0000000..4d25361 --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/statistics.hpp @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +using time_point = std::chrono::time_point; + +/** + * @brief Report inference statistics for a single request + * + * @param instance The Triton model instance which is processing this request + * @param request The Triton request object itself + * @param start_time The time at which the backend first received the request + * @param compute_start_time The time at which the backend began actual + * inference on the request + * @param compute_end_time The time at which the backend completed inference + * on the request + * @param end_time The time at which the backend finished all processing on + * the request, including copying out results and returning a response + */ +inline void report_statistics(TRITONBACKEND_ModelInstance& instance, + TRITONBACKEND_Request& request, + time_point start_time, + time_point compute_start_time, + time_point compute_end_time, + time_point end_time) +{ + triton_check( + TRITONBACKEND_ModelInstanceReportStatistics(&instance, + &request, + true, + start_time.time_since_epoch().count(), + compute_start_time.time_since_epoch().count(), + compute_end_time.time_since_epoch().count(), + end_time.time_since_epoch().count())); +} + +/** + * @brief Report inference statistics for a batch of requests of given size + * + * @param instance The Triton model instance which is processing this batch + * @param request_count The number of requests in this batch + * @param start_time The time at which the backend first received the batch + * @param compute_start_time The time at which the backend began actual + * inference on the batch + * @param compute_end_time The time at which the backend completed inference + * on the batch + * @param end_time The time at which the backend finished all processing on + * the batch, including copying out results and returning a response + */ +inline void report_statistics(TRITONBACKEND_ModelInstance& instance, + std::size_t request_count, + time_point start_time, + time_point compute_start_time, + time_point compute_end_time, + time_point end_time) +{ + triton_check( + TRITONBACKEND_ModelInstanceReportBatchStatistics(&instance, + request_count, + start_time.time_since_epoch().count(), + compute_start_time.time_since_epoch().count(), + compute_end_time.time_since_epoch().count(), + end_time.time_since_epoch().count())); +} +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/triton/triton_memory_resource.hpp b/backend/cpp/include/triton_backend/triton/triton_memory_resource.hpp new file mode 100644 index 0000000..4c4b197 --- /dev/null +++ b/backend/cpp/include/triton_backend/triton/triton_memory_resource.hpp @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +struct triton_memory_resource final : public rmm::mr::device_memory_resource { + triton_memory_resource(TRITONBACKEND_MemoryManager* manager, + device_id_t device_id, + rmm::mr::device_memory_resource* fallback) + : manager_{manager}, device_id_{device_id}, fallback_{fallback} + { + } + + bool supports_streams() const noexcept override { return false; } + bool supports_get_mem_info() const noexcept override { return false; } + auto* get_triton_manager() const noexcept { return manager_; } + + private: + TRITONBACKEND_MemoryManager* manager_; + std::int64_t device_id_; + rmm::mr::device_memory_resource* fallback_; + + void* do_allocate(std::size_t bytes, rmm::cuda_stream_view stream) override + { + auto* ptr = static_cast(nullptr); + if (manager_ == nullptr) { + ptr = fallback_->allocate(bytes, stream); + } else { + triton_check(TRITONBACKEND_MemoryManagerAllocate( + manager_, &ptr, TRITONSERVER_MEMORY_GPU, device_id_, static_cast(bytes))); + } + return ptr; + } + + void do_deallocate(void* ptr, std::size_t bytes, rmm::cuda_stream_view stream) + { + if (manager_ == nullptr) { + fallback_->deallocate(ptr, bytes, stream); + } else { + triton_check( + TRITONBACKEND_MemoryManagerFree(manager_, ptr, TRITONSERVER_MEMORY_GPU, device_id_)); + } + } + + bool do_is_equal(rmm::mr::device_memory_resource const& other) const noexcept override + { + auto* other_triton_mr = dynamic_cast(&other); + return (other_triton_mr != nullptr && other_triton_mr->get_triton_manager() == manager_); + } + + std::pair do_get_mem_info(rmm::cuda_stream_view stream) const override + { + return {0, 0}; + } +}; + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/utils/const_agnostic.hpp b/backend/cpp/include/triton_backend/utils/const_agnostic.hpp new file mode 100644 index 0000000..4ca9dcc --- /dev/null +++ b/backend/cpp/include/triton_backend/utils/const_agnostic.hpp @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace triton { +namespace backend { +namespace dev_tools { + +template +using const_agnostic_same_t = + std::enable_if_t, std::remove_const_t>>; + +} +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/utils/device_setter.hpp b/backend/cpp/include/triton_backend/utils/device_setter.hpp new file mode 100644 index 0000000..8ea21a3 --- /dev/null +++ b/backend/cpp/include/triton_backend/utils/device_setter.hpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#ifdef TRITON_ENABLE_GPU +#include +#endif +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { + +/** Struct for setting cuda device within a code block */ +struct device_setter { + device_setter(device_id_t device) : prev_device_{} { + if constexpr(IS_GPU_BUILD) { + cuda_check(cudaGetDevice(&prev_device_)); + cuda_check(cudaSetDevice(device)); + } else { + throw TritonException(Error::Internal, "Device setter used in non-GPU build"); + } + } + + ~device_setter() { + if constexpr(IS_GPU_BUILD) { + cudaSetDevice(prev_device_); + } + } + private: + device_id_t prev_device_; +}; + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/include/triton_backend/utils/narrow.hpp b/backend/cpp/include/triton_backend/utils/narrow.hpp new file mode 100644 index 0000000..da9e524 --- /dev/null +++ b/backend/cpp/include/triton_backend/utils/narrow.hpp @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { + +template +auto narrow(F from) +{ + auto to = static_cast(from); + + if (static_cast(to) != from || + (std::is_signed::value && !std::is_signed::value && from < F{}) || + (std::is_signed::value && !std::is_signed::value && to < T{}) || + (std::is_signed::value == std::is_signed::value && ((to < T{}) != (from < F{})))) { + throw TritonException(Error::Internal, "invalid narrowing"); + } + return to; +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/scripts/run-clang-format.py b/backend/cpp/scripts/run-clang-format.py new file mode 100755 index 0000000..614697b --- /dev/null +++ b/backend/cpp/scripts/run-clang-format.py @@ -0,0 +1,148 @@ +# Copyright (c) 2019-2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This file was copied from the rapidsai/cuml repo + +from __future__ import print_function +import sys +import re +import os +import subprocess +import argparse +import tempfile +import shutil + + +EXPECTED_VERSION = "11.0.0" +VERSION_REGEX = re.compile(r"clang-format version ([0-9.]+)") +DEFAULT_DIRS = ["cpp/src", + "cpp/include", + "cpp/test"] + + +def parse_args(): + argparser = argparse.ArgumentParser("Runs clang-format on a project") + argparser.add_argument("-dstdir", type=str, default=None, + help="Directory to store the temporary outputs of" + " clang-format. If nothing is passed for this, then" + " a temporary dir will be created using `mkdtemp`") + argparser.add_argument("-exe", type=str, default="clang-format", + help="Path to clang-format exe") + argparser.add_argument("-inplace", default=False, action="store_true", + help="Replace the source files itself.") + argparser.add_argument("-regex", type=str, + default=r"[.](cu|cuh|h|hpp|cpp)$", + help="Regex string to filter in sources") + argparser.add_argument("-ignore", type=str, default=r"cannylab/bh[.]cu$", + help="Regex used to ignore files from matched list") + argparser.add_argument("-v", dest="verbose", action="store_true", + help="Print verbose messages") + argparser.add_argument("dirs", type=str, nargs="*", + help="List of dirs where to find sources") + args = argparser.parse_args() + args.regex_compiled = re.compile(args.regex) + args.ignore_compiled = re.compile(args.ignore) + if args.dstdir is None: + args.dstdir = tempfile.mkdtemp() + ret = subprocess.check_output("%s --version" % args.exe, shell=True) + ret = ret.decode("utf-8") + version = VERSION_REGEX.match(ret) + if version is None: + raise Exception("Failed to figure out clang-format version!") + version = version.group(1) + if version != EXPECTED_VERSION: + raise Exception("clang-format exe must be v%s found '%s'" % \ + (EXPECTED_VERSION, version)) + if len(args.dirs) == 0: + args.dirs = DEFAULT_DIRS + return args + + +def list_all_src_files(file_regex, ignore_regex, srcdirs, dstdir, inplace): + allFiles = [] + for srcdir in srcdirs: + for root, dirs, files in os.walk(srcdir): + for f in files: + if re.search(file_regex, f): + src = os.path.join(root, f) + if re.search(ignore_regex, src): + continue + if inplace: + _dir = root + else: + _dir = os.path.join(dstdir, root) + dst = os.path.join(_dir, f) + allFiles.append((src, dst)) + return allFiles + + +def run_clang_format(src, dst, exe, verbose): + dstdir = os.path.dirname(dst) + if not os.path.exists(dstdir): + os.makedirs(dstdir) + # run the clang format command itself + if src == dst: + cmd = "%s -i %s" % (exe, src) + else: + cmd = "%s %s > %s" % (exe, src, dst) + try: + subprocess.check_call(cmd, shell=True) + except subprocess.CalledProcessError: + print("Failed to run clang-format! Maybe your env is not proper?") + raise + # run the diff to check if there are any formatting issues + cmd = "diff -q %s %s >/dev/null" % (src, dst) + try: + subprocess.check_call(cmd, shell=True) + if verbose: + print("%s passed" % os.path.basename(src)) + except subprocess.CalledProcessError: + print("%s failed! 'diff %s %s' will show formatting violations!" % \ + (os.path.basename(src), src, dst)) + return False + return True + + +def main(): + args = parse_args() + # Attempt to making sure that we run this script from root of repo always + if not os.path.exists(".git"): + print("Error!! This needs to always be run from the root of repo") + sys.exit(-1) + all_files = list_all_src_files(args.regex_compiled, args.ignore_compiled, + args.dirs, args.dstdir, args.inplace) + + # Check whether clang-format exists + if shutil.which("clang-format") is None: + print("clang-format not found. Exiting...") + return + + # actual format checker + status = True + for src, dst in all_files: + if not run_clang_format(src, dst, args.exe, args.verbose): + status = False + if not status: + print("clang-format failed! You have 2 options:") + print(" 1. Look at formatting differences above and fix them manually") + print(" 2. Or run the below command to bulk-fix all these at once") + print("Bulk-fix command: ") + print(" python cpp/scripts/run-clang-format.py %s -inplace" % \ + " ".join(sys.argv[1:])) + sys.exit(-1) + return + + +if __name__ == "__main__": + main() diff --git a/backend/cpp/src/CMakeLists.txt b/backend/cpp/src/CMakeLists.txt new file mode 100644 index 0000000..665b284 --- /dev/null +++ b/backend/cpp/src/CMakeLists.txt @@ -0,0 +1,68 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +# keep the files in alphabetical order! +add_library( + triton_dev_tools-identity SHARED + src/api.cc +) + +if(TRITON_ENABLE_GPU) + set_target_properties(triton_dev_tools-identity + PROPERTIES BUILD_RPATH "\$ORIGIN" + # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) +else() + set_target_properties(triton_dev_tools-identity + PROPERTIES BUILD_RPATH "\$ORIGIN" + # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) +endif() + +target_compile_options(triton_dev_tools-identity + PRIVATE "$<$:${DEV_TOOLS_CXX_FLAGS}>" + "$<$:${DEV_TOOLS_CUDA_FLAGS}>" +) + +target_include_directories(triton_dev_tools-identity + PRIVATE "$" + "${CMAKE_CURRENT_SOURCE_DIR}/src" +) + +target_link_libraries(triton_dev_tools-identity +PRIVATE + $<$:rmm::rmm> + $<$:raft::raft> + triton-core-serverstub + triton-backend-utils + "${TRITONSERVER_LIB}" + $ +) + +install( + TARGETS triton_dev_tools-identity + LIBRARY DESTINATION /opt/tritonserver/backends/dev_tools-identity +) diff --git a/backend/cpp/src/api.cc b/backend/cpp/src/api.cc new file mode 100644 index 0000000..dbeb77b --- /dev/null +++ b/backend/cpp/src/api.cc @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace NAMESPACE { + +using ModelState = dev_tools::TritonModelState; +using ModelInstanceState = + dev_tools::ModelInstanceState; + +extern "C" { + +/** Confirm that backend is compatible with Triton's backend API version + */ +TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { + return dev_tools::triton_api::initialize(backend); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { + return dev_tools::triton_api::model_initialize(model); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { + return dev_tools::triton_api::model_finalize(model); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize( + TRITONBACKEND_ModelInstance* instance) { + return dev_tools::triton_api::instance_initialize(instance); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize( + TRITONBACKEND_ModelInstance* instance) { + return dev_tools::triton_api::instance_finalize(instance); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( + TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** raw_requests, + uint32_t const request_count) { + return dev_tools::triton_api::execute( + instance, raw_requests, static_cast(request_count)); +} + +} // extern "C" + +} // namespace NAMESPACE +} // namespace backend +} // namespace triton diff --git a/backend/cpp/src/model.h b/backend/cpp/src/model.h new file mode 100644 index 0000000..a292d20 --- /dev/null +++ b/backend/cpp/src/model.h @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +#include +#include + +#include +#include +#include // dev_tools::Batch +#include // dev_tools::MemoryType +#include // dev_tools::Model +#include // dev_tools::copy +#include // dev_tools::DeploymentType +#include // dev_tools::device_id_t + +namespace triton { +namespace backend { +namespace NAMESPACE { + +/* Any logic necessary to perform inference with a model and manage its data + * should be implemented in a struct named DevToolsModel, as shown here */ + +struct DevToolsModel : dev_tools::Model { + /*************************************************************************** + * BOILERPLATE * + * *********************************************************************** * + * The following constructor can be copied directly into any model + * implementation. + **************************************************************************/ + DevToolsModel(std::shared_ptr shared_state, + dev_tools::device_id_t device_id, + cudaStream_t default_stream, + dev_tools::DeploymentType deployment_type, + std::string const& filepath) + : dev_tools::Model( + shared_state, device_id, default_stream, deployment_type, filepath) + { + } + + /*************************************************************************** + * BASIC FEATURES * + * *********************************************************************** * + * The only method that *must* be implemented for a viable model is the + * `predict` method, but the others presented here are often used for basic + * model implementations. Filling out these methods should take care of most + * use cases. + **************************************************************************/ + + /*************************************************************************** + * predict * + * *********************************************************************** * + * This method performs the actual inference step on input data. Implementing + * a predict function requires four steps: + * 1. Call `get_input` on the provided `Batch` object for each of the input + * tensors named in the config file for this backend. This provides a + * `Tensor` object containing the input data. + * 2. Call `get_output` on the provided `Batch` object for each of the output + * tensors named in the config file for this backend. This provides a + * `Tensor` object to which output values can be written. + * 3. Perform inference based on the input Tensors and store the results in + * the output Tensors. `some_tensor.data()` can be used to retrieve a raw + * pointer to the underlying data. + * 4. Call the `finalize` method on all output tensors. + **************************************************************************/ + void predict(dev_tools::Batch& batch) const + { + // 1. Acquire a tensor representing the input named "input__0" + auto input = get_input(batch, "input__0"); + // 2. Acquire a tensor representing the output named "output__0" + auto output = get_output(batch, "output__0"); + + // 3. Perform inference. In this example, we simply copy the data from the + // input to the output tensor. + dev_tools::copy(output, input); + + // 4. Call finalize on all output tensors. In this case, we have just one + // output, so we call finalize on it. + output.finalize(); + } + + /*************************************************************************** + * load / unload * + * *********************************************************************** * + * These methods can be used to perform one-time loading/unloading of + * resources when a model is created. For example, data representing the + * model may be loaded onto the GPU in the `load` method and unloaded in the + * `unload` method. This data will then remain loaded while the server is + * running. + * + * While these methods take no arguments, it is typical to read any necessary + * input from the model configuration file by using the `get_config_param` + * method. Any parameters defined in the "parameters" section of the config + * can be accessed by name in this way. The maximum batch size can also be + * retrieved using the name "max_batch_size". + * + * These methods need not be explicitly implemented if no loading/unloading + * logic is required, but we show them here for illustrative purposes. + **************************************************************************/ + void load() {} + void unload() {} + + /*************************************************************************** + * ADVANCED FEATURES * + * *********************************************************************** * + * None of the following methods are required to be implemented in order to + * create a valid model, but they are presented here for those who require + * the additional functionality they provide. + **************************************************************************/ + + /*************************************************************************** + * preferred_mem_type / preferred_mem_type_in / preferred_mem_type_out * + * *********************************************************************** * + * If implemented, `preferred_mem_type` allows for control over when input + * and output data are provided on the host versus on device. In the case + * that a model prefers to receive its input on-host but return output + * on-device (or vice versa), `preferred_mem_type_in` and + * `preferred_mem_type_out` can be used for even more precise control. + * + * In this example, we simply return `std::nullopt` to indicate that the + * model has no preference on its input/output data locations. Note that the + * Batch being processed is taken as input to this function to facilitate + * implementations that may switch their preferred memory location based on + * properties of the batch. + * + * Valid MemoryType options to return are dev_tools::HostMemory and + * dev_tools::DeviceMemory. + **************************************************************************/ + std::optional preferred_mem_type(dev_tools::Batch& batch) const + { + return std::nullopt; + } +}; + +} // namespace NAMESPACE +} // namespace backend +} // namespace triton diff --git a/backend/cpp/src/names.h b/backend/cpp/src/names.h new file mode 100644 index 0000000..3477995 --- /dev/null +++ b/backend/cpp/src/names.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +/* Triton expects certain definitions within its backend libraries to follow + * specific naming conventions. Specifically, for a backend named + * "dev_tools_identity," most definitions should appear within a namespace called + * triton::backend::dev_tools_identity. + * + * In order to facilitate this with minimal effort on the part of backend + * developers, we ask that you put the name of your backend here. This macro is + * then used to propagate the correct namespace name wherever it is needed in + * the impl and interface code. */ + +#define NAMESPACE dev_tools_identity diff --git a/backend/cpp/src/shared_state.h b/backend/cpp/src/shared_state.h new file mode 100644 index 0000000..5daccf4 --- /dev/null +++ b/backend/cpp/src/shared_state.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include + +namespace triton { +namespace backend { +namespace NAMESPACE { + +/* Triton allows multiple instances of a single model to be instantiated at the + * same time (e.g. on different GPUs). All instances of a model share access to + * an object which manages any state that can be shared across all instances. + * Any logic necessary for managing such state should be implemented in a + * struct named DevToolsSharedState, as shown here. Models may access this shared + * state object via the `get_shared_state` method, which returns a shared + * pointer to the DevToolsSharedState object. + * + * Not all backends require shared state, so leaving this implementation empty + * is entirely valid */ + +struct DevToolsSharedState : dev_tools::SharedModelState { + DevToolsSharedState(std::unique_ptr&& config) + : dev_tools::SharedModelState{std::move(config)} + { + } + void load() {} + void unload() {} +}; + +} // namespace NAMESPACE +} // namespace backend +} // namespace triton diff --git a/backend/cpp/test/CMakeLists.txt b/backend/cpp/test/CMakeLists.txt new file mode 100644 index 0000000..9d8230e --- /dev/null +++ b/backend/cpp/test/CMakeLists.txt @@ -0,0 +1,102 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +# keep the files in alphabetical order! +add_executable(test_triton_backend + test/batch/batch.cpp + test/build_control.cpp + test/exceptions.cpp + test/memory/buffer.cpp + test/memory/detail/copy.cpp + test/memory/detail/owned_device_buffer.cpp + test/memory/resource.cpp + test/memory/types.cpp + test/tensor/dtype.cpp + test/tensor/tensor.cpp + test/test.cpp + test/triton/api/execute.cpp + test/triton/api/initialize.cpp + test/triton/api/instance_finalize.cpp + test/triton/api/instance_initialize.cpp + test/triton/api/model_finalize.cpp + test/triton/api/model_initialize.cpp + test/triton/backend.cpp + test/triton/config.cpp + test/triton/deployment.cpp + test/triton/device.cpp + test/triton/input.cpp + test/triton/logging.cpp + test/triton/model.cpp + test/triton/model_instance.cpp + test/triton/requests.cpp + test/triton/responses.cpp + test/triton/statistics.cpp + test/utils/const_agnostic.cpp + test/utils/narrow.cpp +) + +IF(TRITON_ENABLE_GPU) + set_target_properties(test_triton_backend + PROPERTIES BUILD_RPATH "\$ORIGIN" + # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) +else() + set_target_properties(test_triton_backend + PROPERTIES BUILD_RPATH "\$ORIGIN" + # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) +endif() + +target_compile_options(test_triton_backend + PRIVATE "$<$:${DEV_TOOLS_CXX_FLAGS}>" + "$<$:${DEV_TOOLS_CUDA_FLAGS}>" +) + +target_include_directories(test_triton_backend + PUBLIC "$" + "$" +) + + +find_library( + TRITONSERVER_LIB + tritonserver + PATHS /opt/tritonserver/lib +) + +target_link_libraries(test_triton_backend +PRIVATE + $<$:rmm::rmm> + $<$:raft::raft> + triton-core-serverstub + triton-backend-utils + gmock + gmock_main + GTest::gtest + GTest::gtest_main + "${TRITONSERVER_LIB}" + $ +) diff --git a/backend/cpp/test/batch/batch.cpp b/backend/cpp/test/batch/batch.cpp new file mode 100644 index 0000000..61e7768 --- /dev/null +++ b/backend/cpp/test/batch/batch.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/build_control.cpp b/backend/cpp/test/build_control.cpp new file mode 100644 index 0000000..4e45cbb --- /dev/null +++ b/backend/cpp/test/build_control.cpp @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +namespace triton { +namespace backend { +namespace dev_tools { + +TEST(DevToolsTriton, build_control) +{ +#ifdef TRITON_ENABLE_GPU + ASSERT_EQ(IS_GPU_BUILD, true) << "IS_GPU_BUILD constant has wrong value\n"; +#else + ASSERT_EQ(IS_GPU_BUILD, false) << "IS_GPU_BUILD constant has wrong value\n"; +#endif +} +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/test/exceptions.cpp b/backend/cpp/test/exceptions.cpp new file mode 100644 index 0000000..7bdabfc --- /dev/null +++ b/backend/cpp/test/exceptions.cpp @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +#include + +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { + +TEST(DevToolsTriton, default_except) +{ + try { + throw TritonException(); + } catch (TritonException const& err) { + EXPECT_EQ(std::string(err.what()), std::string("encountered unknown error")); + } +} + +TEST(DevToolsTriton, msg_except) +{ + auto msg = std::string("TEST ERROR MESSAGE"); + try { + throw TritonException(Error::Internal, msg); + } catch (TritonException const& err) { + EXPECT_EQ(std::string(err.what()), msg); + } + try { + throw TritonException(Error::Internal, msg.c_str()); + } catch (TritonException const& err) { + EXPECT_EQ(std::string(err.what()), msg); + } + try { + throw TritonException(Error::Internal, msg); + } catch (TritonException const& err) { + try { + throw(TritonException(err.error())); + } catch (TritonException const& err2) { + EXPECT_EQ(std::string(err2.what()), msg); + } + } +} + +TEST(DevToolsTriton, triton_check) +{ + auto msg = std::string("TEST ERROR MESSAGE"); + EXPECT_THROW(triton_check(TRITONSERVER_ErrorNew(Error::Internal, msg.c_str())), TritonException); + triton_check(nullptr); +} + +TEST(DevToolsTriton, cuda_check) +{ +#ifdef TRITON_ENABLE_GPU + EXPECT_THROW(cuda_check(cudaError::cudaErrorMissingConfiguration), TritonException); + cuda_check(cudaError::cudaSuccess); +#else + EXPECT_THROW(cuda_check(cudaError::cudaErrorNonGpuBuild), TritonException); +#endif +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/test/memory/buffer.cpp b/backend/cpp/test/memory/buffer.cpp new file mode 100644 index 0000000..6edc3b7 --- /dev/null +++ b/backend/cpp/test/memory/buffer.cpp @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef TRITON_ENABLE_GPU +#include +#endif +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +TEST(DevToolsTriton, default_buffer) +{ + auto buffer = Buffer(); + EXPECT_EQ(buffer.mem_type(), HostMemory); + EXPECT_EQ(buffer.size(), 0); + EXPECT_EQ(buffer.data(), nullptr); + EXPECT_EQ(buffer.device(), 0); + EXPECT_EQ(buffer.stream(), cudaStream_t{}); +#ifdef TRITON_ENABLE_GPU + auto stream = cudaStream_t{}; + cudaStreamCreate(&stream); + buffer.set_stream(stream); + EXPECT_EQ(buffer.stream(), stream); + cudaStreamDestroy(stream); +#endif +} + +TEST(DevToolsTriton, device_buffer) +{ + auto data = std::vector{1, 2, 3}; +#ifdef TRITON_ENABLE_GPU + auto buffer = Buffer(data.size(), DeviceMemory, 0, 0); + + ASSERT_EQ(buffer.mem_type(), DeviceMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_NE(buffer.data(), nullptr); + + auto data_out = std::vector(data.size()); + cudaMemcpy(static_cast(buffer.data()), + static_cast(data.data()), + sizeof(int) * data.size(), + cudaMemcpyHostToDevice); + cudaMemcpy(static_cast(data_out.data()), + static_cast(buffer.data()), + sizeof(int) * data.size(), + cudaMemcpyDeviceToHost); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + +#else + EXPECT_THROW(Buffer(data.size(), DeviceMemory, 0, 0), TritonException); +#endif +} + +TEST(DevToolsTriton, non_owning_device_buffer) +{ + auto data = std::vector{1, 2, 3}; +#ifdef TRITON_ENABLE_GPU + auto* ptr_d = static_cast(nullptr); + cudaMalloc(reinterpret_cast(&ptr_d), sizeof(int) * data.size()); + cudaMemcpy(static_cast(ptr_d), + static_cast(data.data()), + sizeof(int) * data.size(), + cudaMemcpyHostToDevice); + auto buffer = Buffer(ptr_d, data.size(), DeviceMemory); + + ASSERT_EQ(buffer.mem_type(), DeviceMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_EQ(buffer.data(), ptr_d); + + auto data_out = std::vector(data.size()); + cudaMemcpy(static_cast(data_out.data()), + static_cast(buffer.data()), + sizeof(int) * data.size(), + cudaMemcpyDeviceToHost); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + + cudaFree(reinterpret_cast(ptr_d)); +#else + ASSERT_THROW(Buffer(data.data(), data.size(), DeviceMemory), TritonException); +#endif +} + +TEST(DevToolsTriton, host_buffer) +{ + auto data = std::vector{1, 2, 3}; + auto buffer = Buffer(data.size(), HostMemory, 0, 0); + + ASSERT_EQ(buffer.mem_type(), HostMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_NE(buffer.data(), nullptr); + + std::memcpy( + static_cast(buffer.data()), static_cast(data.data()), data.size() * sizeof(int)); + + auto data_out = std::vector(buffer.data(), buffer.data() + buffer.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +TEST(DevToolsTriton, non_owning_host_buffer) +{ + auto data = std::vector{1, 2, 3}; + auto buffer = Buffer(data.data(), data.size(), HostMemory); + + ASSERT_EQ(buffer.mem_type(), HostMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_EQ(buffer.data(), data.data()); + + auto data_out = std::vector(buffer.data(), buffer.data() + buffer.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +TEST(DevToolsTriton, copy_buffer) +{ + auto data = std::vector{1, 2, 3}; + auto orig_buffer = Buffer(data.data(), data.size(), HostMemory); + auto buffer = Buffer(orig_buffer); + + ASSERT_EQ(buffer.mem_type(), HostMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_NE(buffer.data(), orig_buffer.data()); + + auto data_out = std::vector(buffer.data(), buffer.data() + buffer.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +TEST(DevToolsTriton, move_buffer) +{ + auto data = std::vector{1, 2, 3}; + auto buffer = Buffer(Buffer(data.data(), data.size(), HostMemory)); + + ASSERT_EQ(buffer.mem_type(), HostMemory); + ASSERT_EQ(buffer.size(), data.size()); + ASSERT_EQ(buffer.data(), data.data()); + + auto data_out = std::vector(buffer.data(), buffer.data() + buffer.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +TEST(DevToolsTriton, move_assignment_buffer) +{ + auto data = std::vector{1, 2, 3}; + +#ifdef TRITON_ENABLE_GPU + auto buffer = Buffer{data.data(), data.size() - 1, DeviceMemory}; +#else + auto buffer = Buffer{data.data(), data.size() - 1, HostMemory}; +#endif + buffer = Buffer{data.size(), HostMemory}; + + ASSERT_EQ(buffer.mem_type(), HostMemory); + ASSERT_EQ(buffer.size(), data.size()); +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/test/memory/detail/copy.cpp b/backend/cpp/test/memory/detail/copy.cpp new file mode 100644 index 0000000..da59fa7 --- /dev/null +++ b/backend/cpp/test/memory/detail/copy.cpp @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef TRITON_ENABLE_GPU +#include +#include +#else +#include +#endif +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +TEST(DevToolsTriton, copy) +{ + auto data = std::vector{1, 2, 3}; + auto data_out = std::vector(data.size()); + detail::copy(data_out.data(), data.data(), data.size(), 0, HostMemory, HostMemory); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + + data_out = std::vector(data.size()); +#ifdef TRITON_ENABLE_GPU + auto* ptr_d = static_cast(nullptr); + cudaMalloc(reinterpret_cast(&ptr_d), sizeof(int) * data.size()); + detail::copy(ptr_d, data.data(), data.size(), 0, DeviceMemory, HostMemory); + + cudaMemcpy(static_cast(data_out.data()), + static_cast(ptr_d), + sizeof(int) * data.size(), + cudaMemcpyDeviceToHost); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + cudaFree(reinterpret_cast(ptr_d)); +#else + EXPECT_THROW(detail::copy(data_out.data(), data.data(), data.size(), 0, HostMemory, DeviceMemory), + TritonException); + EXPECT_THROW(detail::copy(data_out.data(), data.data(), data.size(), 0, DeviceMemory, HostMemory), + TritonException); +#endif +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/test/memory/detail/owned_device_buffer.cpp b/backend/cpp/test/memory/detail/owned_device_buffer.cpp new file mode 100644 index 0000000..f924a07 --- /dev/null +++ b/backend/cpp/test/memory/detail/owned_device_buffer.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef TRITON_ENABLE_GPU +#include +#include +#else +#include +#endif +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +TEST(DevToolsTriton, owned_device_buffer) +{ + auto data = std::vector{1, 2, 3}; +#ifdef TRITON_ENABLE_GPU + auto device_id = 0; + cudaGetDevice(&device_id); + auto stream = cudaStream_t{}; + cudaStreamCreate(&stream); + + auto buffer = detail::owned_device_buffer(device_id, data.size(), stream); + auto data_out = std::vector(data.size()); + cudaMemcpy(static_cast(buffer.get()), + static_cast(data.data()), + sizeof(int) * data.size(), + cudaMemcpyHostToDevice); + cudaMemcpy(static_cast(data_out.data()), + static_cast(buffer.get()), + sizeof(int) * data.size(), + cudaMemcpyDeviceToHost); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + cudaStreamDestroy(stream); +#else + // Workaround for ungraceful handling of multiple template parameters in + // EXPECT_THROW + using dev_buffer = detail::owned_device_buffer; + EXPECT_THROW(dev_buffer(0, data.size(), 0), TritonException); +#endif +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/test/memory/resource.cpp b/backend/cpp/test/memory/resource.cpp new file mode 100644 index 0000000..5f413bd --- /dev/null +++ b/backend/cpp/test/memory/resource.cpp @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef TRITON_ENABLE_GPU +#include +#include +#include +#include +#endif + +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { + +TEST(DevToolsTriton, set_memory_resource) +{ +#ifdef TRITON_ENABLE_GPU + auto device_id = int{}; + cuda_check(cudaGetDevice(&device_id)); + EXPECT_EQ(rmm::mr::get_current_device_resource()->is_equal(rmm::mr::cuda_memory_resource{}), + true); + setup_memory_resource(device_id); + EXPECT_EQ(rmm::mr::get_current_device_resource()->is_equal(rmm::mr::cuda_memory_resource{}), + false); +#else + setup_memory_resource(0); +#endif +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/test/memory/types.cpp b/backend/cpp/test/memory/types.cpp new file mode 100644 index 0000000..cc86dc2 --- /dev/null +++ b/backend/cpp/test/memory/types.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/tensor/dtype.cpp b/backend/cpp/test/tensor/dtype.cpp new file mode 100644 index 0000000..53cf17f --- /dev/null +++ b/backend/cpp/test/tensor/dtype.cpp @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +namespace triton { +namespace backend { +namespace dev_tools { + +template +void check_dtype_conversion() +{ + EXPECT_EQ(D, TritonDtype::type>::value); + EXPECT_EQ(D, TritonDtype::type const>::value); +} + +TEST(DevToolsTriton, dtype) +{ + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); + check_dtype_conversion(); +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/test/tensor/tensor.cpp b/backend/cpp/test/tensor/tensor.cpp new file mode 100644 index 0000000..6e63450 --- /dev/null +++ b/backend/cpp/test/tensor/tensor.cpp @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { + +TEST(DevToolsTriton, default_tensor) +{ + auto tensor = Tensor(); + EXPECT_EQ(tensor.buffer().size(), 0); + EXPECT_EQ(tensor.shape().size(), 0); +} + +TEST(DevToolsTriton, move_buffer_tensor) +{ + auto shape = std::vector{2, 2}; + auto data = std::vector{1, 2, 3, 4}; + auto tensor = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); + + EXPECT_EQ(data.data(), tensor.data()); + EXPECT_EQ(data.size(), tensor.size()); + EXPECT_THAT(tensor.shape(), ::testing::ElementsAreArray(shape)); + + EXPECT_EQ(tensor.dtype(), DTypeInt32); + EXPECT_EQ(tensor.mem_type(), HostMemory); + EXPECT_EQ(tensor.stream(), cudaStream_t{}); + EXPECT_EQ(tensor.device(), 0); + + auto data_out = std::vector(tensor.data(), tensor.data() + tensor.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +TEST(DevToolsTriton, multi_buffer_tensor) +{ + auto shape = std::vector{2, 2}; + auto data = std::vector{1, 2, 3, 4}; + + auto all_buffers = std::vector>{}; + all_buffers.reserve(data.size()); + auto mem_type = HostMemory; + if constexpr (IS_GPU_BUILD) { mem_type = DeviceMemory; } + std::transform(data.begin(), data.end(), std::back_inserter(all_buffers), [mem_type](auto& elem) { + return Buffer{&elem, 1, mem_type}; + }); + auto tensor = + Tensor(shape, all_buffers.begin(), all_buffers.end(), mem_type, 0, cudaStream_t{}); + + auto data_out = std::vector(data.size()); +#ifdef TRITON_ENABLE_GPU + cudaMemcpy(static_cast(data_out.data()), + static_cast(tensor.data()), + sizeof(int) * tensor.size(), + cudaMemcpyDeviceToHost); +#else + std::memcpy(static_cast(data_out.data()), + static_cast(tensor.data()), + sizeof(int) * tensor.size()); +#endif + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); +} + +TEST(DevToolsTriton, tensor_copy) +{ + auto shape = std::vector{2, 2}; + auto data = std::vector{1, 2, 3, 4}; + + auto data1 = data; + auto tensor1 = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); + auto data2 = std::vector(data1.size()); + auto tensor2 = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); + + dev_tools::copy(tensor2, tensor1); + + auto data_out = std::vector(tensor2.data(), tensor2.data() + tensor2.size()); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + + auto small_shape = std::vector{2}; + auto small_data = std::vector(2); + auto tensor3 = + Tensor(small_shape, Buffer{small_data.data(), small_data.size(), HostMemory}); + + EXPECT_THROW(dev_tools::copy(tensor3, tensor1), TritonException); +} + +TEST(DevToolsTriton, tensor_multi_copy) +{ + auto shape = std::vector{2, 2}; + auto data = std::vector{1, 2, 3, 4}; + + auto data1 = data; + auto tensor1 = Tensor(shape, Buffer{data.data(), data.size(), HostMemory}); + + auto receiver_shape = std::vector{1}; + auto receivers = std::vector>{}; + + receivers.reserve(data.size()); + std::transform( + data.begin(), data.end(), std::back_inserter(receivers), [&receiver_shape](auto& val) { + return Tensor(receiver_shape, Buffer{std::size_t{1}, HostMemory}); + }); + + dev_tools::copy(receivers.begin(), receivers.end(), tensor1); + + auto data_out = std::vector{}; + data_out.reserve(receivers.size()); + std::transform( + receivers.begin(), receivers.end(), std::back_inserter(data_out), [](auto& tensor) { + return *tensor.data(); + }); + EXPECT_THAT(data_out, ::testing::ElementsAreArray(data)); + + // Throw if trying to copy to too many outputs + receivers.emplace_back(receiver_shape, Buffer{std::size_t{1}, HostMemory}); + EXPECT_THROW(dev_tools::copy(receivers.begin(), receivers.end(), tensor1), TritonException); +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/test/test.cpp b/backend/cpp/test/test.cpp new file mode 100644 index 0000000..e6bba9c --- /dev/null +++ b/backend/cpp/test/test.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +namespace triton { +namespace backend { +namespace dev_tools { + +TEST(DevToolsTriton, installed) { std::cout << test_install() << "\n"; } +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/test/triton/api/execute.cpp b/backend/cpp/test/triton/api/execute.cpp new file mode 100644 index 0000000..f4582b8 --- /dev/null +++ b/backend/cpp/test/triton/api/execute.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/triton/api/initialize.cpp b/backend/cpp/test/triton/api/initialize.cpp new file mode 100644 index 0000000..9862e5b --- /dev/null +++ b/backend/cpp/test/triton/api/initialize.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/triton/api/instance_finalize.cpp b/backend/cpp/test/triton/api/instance_finalize.cpp new file mode 100644 index 0000000..4f71439 --- /dev/null +++ b/backend/cpp/test/triton/api/instance_finalize.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/triton/api/instance_initialize.cpp b/backend/cpp/test/triton/api/instance_initialize.cpp new file mode 100644 index 0000000..a0349d4 --- /dev/null +++ b/backend/cpp/test/triton/api/instance_initialize.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/triton/api/model_finalize.cpp b/backend/cpp/test/triton/api/model_finalize.cpp new file mode 100644 index 0000000..a7b85e1 --- /dev/null +++ b/backend/cpp/test/triton/api/model_finalize.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/triton/api/model_initialize.cpp b/backend/cpp/test/triton/api/model_initialize.cpp new file mode 100644 index 0000000..138efbf --- /dev/null +++ b/backend/cpp/test/triton/api/model_initialize.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/triton/backend.cpp b/backend/cpp/test/triton/backend.cpp new file mode 100644 index 0000000..aa5020e --- /dev/null +++ b/backend/cpp/test/triton/backend.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/triton/config.cpp b/backend/cpp/test/triton/config.cpp new file mode 100644 index 0000000..1d4291f --- /dev/null +++ b/backend/cpp/test/triton/config.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/triton/deployment.cpp b/backend/cpp/test/triton/deployment.cpp new file mode 100644 index 0000000..cb0347b --- /dev/null +++ b/backend/cpp/test/triton/deployment.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/triton/device.cpp b/backend/cpp/test/triton/device.cpp new file mode 100644 index 0000000..cefe845 --- /dev/null +++ b/backend/cpp/test/triton/device.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/triton/input.cpp b/backend/cpp/test/triton/input.cpp new file mode 100644 index 0000000..c7b31c2 --- /dev/null +++ b/backend/cpp/test/triton/input.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/triton/logging.cpp b/backend/cpp/test/triton/logging.cpp new file mode 100644 index 0000000..ca55c5f --- /dev/null +++ b/backend/cpp/test/triton/logging.cpp @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +TEST(DevToolsTriton, logging) +{ + log_debug("Debug test message"); + log_info("Info test message"); + log_warn("Warn test message"); + log_error("Error test message"); +} + +TEST(DevToolsTriton, stream_logging) +{ + log_debug() << "Streamed debug test message"; + log_info() << "Streamed info test message"; + log_warn() << "Streamed warn test message"; + log_error() << "Streamed error test message"; +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/test/triton/model.cpp b/backend/cpp/test/triton/model.cpp new file mode 100644 index 0000000..c46eac1 --- /dev/null +++ b/backend/cpp/test/triton/model.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/triton/model_instance.cpp b/backend/cpp/test/triton/model_instance.cpp new file mode 100644 index 0000000..6655785 --- /dev/null +++ b/backend/cpp/test/triton/model_instance.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/triton/requests.cpp b/backend/cpp/test/triton/requests.cpp new file mode 100644 index 0000000..bc8a7b0 --- /dev/null +++ b/backend/cpp/test/triton/requests.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/triton/responses.cpp b/backend/cpp/test/triton/responses.cpp new file mode 100644 index 0000000..8129f90 --- /dev/null +++ b/backend/cpp/test/triton/responses.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/triton/statistics.cpp b/backend/cpp/test/triton/statistics.cpp new file mode 100644 index 0000000..4c0b189 --- /dev/null +++ b/backend/cpp/test/triton/statistics.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include diff --git a/backend/cpp/test/utils/const_agnostic.cpp b/backend/cpp/test/utils/const_agnostic.cpp new file mode 100644 index 0000000..2ab28c2 --- /dev/null +++ b/backend/cpp/test/utils/const_agnostic.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +TEST(DevToolsTriton, const_agnostic) +{ + static_assert(std::is_same, void>::value); + static_assert(std::is_same, void>::value); +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/cpp/test/utils/narrow.cpp b/backend/cpp/test/utils/narrow.cpp new file mode 100644 index 0000000..9741912 --- /dev/null +++ b/backend/cpp/test/utils/narrow.cpp @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +namespace triton { +namespace backend { +namespace dev_tools { +TEST(DevToolsTriton, narrow) +{ + EXPECT_THROW(narrow(-1), TritonException); + narrow(int{5}); + EXPECT_THROW(narrow(std::numeric_limits::max()), TritonException); + narrow(std::size_t{5}); +} + +} // namespace dev_tools +} // namespace backend +} // namespace triton diff --git a/backend/docs/usage.md b/backend/docs/usage.md new file mode 100644 index 0000000..65468f0 --- /dev/null +++ b/backend/docs/usage.md @@ -0,0 +1,344 @@ + + +# Using RAPIDS-Triton + +## Getting Started +To begin developing a custom backend with RAPIDS-Triton, we strongly recommend +that you take advantage of the [rapids-triton-template +repo](https://github.com/rapidsai/rapids-triton-template), which provides a +basic template for your backend code. If this is your first time developing a +backend with RAPIDS-Triton, the easiest way to get started is to follow the +[Linear Example](https://github.com/rapidsai/rapids-triton-linear-example). +This provides a detailed walkthrough of every step in the process of creating a +backend with example code. The rest of these usage docs will provide general +information on specific features you are likely to take advantage of when +building your backend. + +## Logging +To provide logging messages in your backend, RAPIDS-Triton provides `log_info`, +`log_warn`, `log_error`, and `log_debug`. During default Triton execution, all +logging messages up to (but not including) debug level will be visible. These +functions can be invoked in two ways and can optionally include file and line +information. To add a logging message to your code, use one of the following +invocations: +```cpp +#include + +void logging_example() { + rapids::log_info() << "This is a log message."; + rapids::log_info("This is an equivalent invocation."); + rapids::log_info(__FILE__, __LINE__) << "This one has file and line info."; + rapids::log_info(__FILE__, __LINE__, "And so does this one."); +} +``` + +## Error Handling +If you encounter an error condition at any point in your backend which cannot +be otherwise handled, you should throw a `TritonException`. In most cases, this +error will be gracefully handled and passed to the Triton server in a way that +will not interfere with execution of other backends, models, or requests. + +`TritonException` objects are constructed with an error type and a message +indicating what went wrong, as shown below: +```cpp +#include + +void error_example() { + throw rapids::TritonException(rapids::Error::Internal, "Something bad happened!"); +} +``` + +Available error types are: +* `Internal`: The most common error type. Used when an unexpected condition + which is not the result of bad user input (e.g. CUDA error). +* `NotFound`: An error type returned when a named resource (e.g. named CUDA IPC + memory block) cannot be found. +* `InvalidArg`: An error type returned when the user has provided invalid input + in a configuration file or request. +* `Unavailable`: An error returned when a resource exists but is currently + unavailable. +* `Unsupported`: An error which indicates that a requested functionality is not + implemented by this backend (e.g. GPU execution for a CPU-only backend). +* `AlreadyExists`: An error which indicates that a resource which is being + created has already been created. +* `Unknown`: The type of the error cannot be established. This type should be + avoided wherever possible. + +The `cuda_check` function is provided to help facilitate error handling of +direct invocations of the CUDA API. If such an invocation fails, `cuda_check` +will throw an appropriate `TritonException`: + +```cpp +#include + +void cuda_check_example() { + rapids::cuda_check(cudaSetDevice(0)); +} +``` + +If a `TritonException` is thrown while a backend is being loaded, Triton's +server logs will indicate the failure and include the error message. If a +`TritonException` is thrown while a model is being loaded, Triton's server logs +will display the error message in the loading logs for that model. If a +`TritonException` is thrown during handling of a request, the client will +receive an indication that the request failed along with the error message, but +the model can continue to process other requests. + +## CPU-Only Builds +Most Triton backends include support for builds intended to support only CPU +execution. While this is not required, RAPIDS-Triton includes a compile-time +constant which can be useful for facilitating this: + +```cpp +#include + +void do_a_gpu_thing() { + if constexpr (rapids::IS_GPU_BUILD) { + rapids::log_info("Executing on GPU..."); + } else { + rapids::log_error("Can't do that! This is a CPU-only build."); + } +} +``` + +You can also make use of the preprocessor identifier `TRITON_ENABLE_GPU` for +conditional inclusion of headers: +```cpp +#ifdef TRITON_ENABLE_GPU +#include +#endif +``` + +Sometimes, having a CUDA symbol available in a CPU-only build can avoid layers +of indirection which would otherwise be required to allow for compilation of +both GPU and CPU versions of particular code. RAPIDS-Triton includes a header +which has some placeholders for CUDA symbols used internally by the library, +and which may be useful for backends which implement CPU-only builds as well. +Note that all placeholder symbols are namespaced within +`triton::backend::rapids`. Note that not all symbols from the CUDA runtime API +are included, but additional symbols will be added over time. All placeholder +symbols will be implemented in a way that is consistent with similar +placeholders in the main Triton codebase. A typical usage is shown below: +```cpp +#ifdef TRITON_ENABLE_GPU +#include +#else +#include +#endif +// E.g. cudaStream_t is now defined regardless of whether or not this is a +// CPU-only build. +``` + +## Buffers +Within a backend, it is often useful to process data in a way that is agnostic +to whether the underlying memory is on the host or on device and whether that +memory is owned by the backend or provided by Triton. For instance, a backend +may receive input data from Triton on the host and conditionally transfer it to +the GPU before processing. In this case, owned memory must be allocated on the +GPU to store the data, but after that point, the backend will treat the data +exactly the same as if Triton had provided it on device in the first place. + +In order to handle such situations, RAPIDS-Triton provides the `Buffer` object. +When the `Buffer` is non-owning, it provides a lightweight wrapper to the +underlying memory. When it is owning, `Buffer` will handle any necessary +deallocation (on host or device). These objects can also be extremely useful +for passing data back and forth between host and device. The following examples +show ways in which `Buffer` objects can be constructed and used: + +```cpp +#include +#include +#include // rapids::HostMemory and rapids::DeviceMemory +#include // rapids::Buffer + +void buffer_examples() { + auto data = std::vector{0, 1, 2, 3, 4}; + + // This buffer is a lightweight wrapper around the data stored in the `data` + // vector. Because this constructor takes an `int*` pointer as its first + // argument, it is assumed that the lifecycle of the underlying memory is + // separately managed. + auto non_owning_host_buffer = rapids::Buffer(data.data(), rapids::HostMemory); + + // This buffer owns its own memory on the host, with space for 5 ints. When + // it goes out of scope, the memory will be appropriately deallocated. + auto owning_host_buffer = rapids::Buffer(5, rapids::HostMemory); + + // This buffer is constructed as a copy of `non_owning_host_buffer`. Because + // its requested memory type is `DeviceMemory`, the data will be copied to a + // new (owned) GPU allocation. Device and stream can also be specified in the + // constructor. + auto owning_device_buffer = rapids::Buffer(non_owning_host_buffer, rapids::DeviceMemory); + + // Once again, because this constructor takes an `int*` pointer, it will + // simply be a lightweight wrapper around the memory that is actually managed + // by `owning_device_buffer`. Here we have omitted the memory type argument, + // since it defaults to `DeviceMemory`. This constructor can also accept + // device and stream arguments, and care should be taken to ensure that the + // right device is specified when the buffer does not allocate its own + // memory. + auto non_owning_device_buffer = rapids::Buffer(owning_host_buffer.data()); + + auto base_buffer1 = rapids::Buffer(data.data(), rapids::HostMemory); + // Because this buffer is on the host, just like the (moved-from) buffer it + // is being constructed from, it remains non-owning + auto non_owning_moved_buffer = rapids::Buffer(std::move(base_buffer1), rapids::HostMemory); + + auto base_buffer2 = rapids::Buffer(data.data(), rapids::HostMemory); + // Because this buffer is on the device, unlike the (moved-from) buffer it is + // being constructed from, memory must be allocated on-device, and the new + // buffer becomes owning. + auto owning_moved_buffer = rapids::Buffer(std::move(base_buffer2), rapids::DeviceMemory); +} +``` + +### Useful Methods +* `data()`: Return a raw pointer to the buffer's data +* `size()`: Return the number of elements contained by the buffer +* `mem_type()`: Return the type of memory (`HostMemory` or `DeviceMemory`) + contained by the buffer +* `device()`: Return the id of the device on which this buffer resides (always + 0 for host buffers) +* `stream()`: Return the CUDA stream associated with this buffer. +* `stream_synchronize()`: Perform a stream synchronization on this buffer's + stream. +* `set_stream(cudaStream_t new_stream)`: Synchronize on the current stream and + then switch buffer to the new stream. + +## Tensors +`Tensor` objects are wrappers around `Buffers` with some additional metadata +and functionality. All `Tensor` objects have a shape which can be retrieved as +a `std::vector` using the `shape()` method. A reference to the underlying +buffer can also be retrieved with the `buffer()` method. + +`OutputTensor` objects are used to store data which will eventually be returned +as part of Triton's response to a client request. Their `finalize` methods are +used to actually marshal their underlying data into a response. + +In general, `OutputTensor` objects should not be constructed directly but +should instead be retrieved using the `get_output` method of a `Model` +(described later). + +## Moving Data: `rapids::copy` +Moving data around between host and device or simply between buffers of the +same type can be one of the more error-prone tasks outside of actual model +execution in a backend. To help make this process easier, RAPIDS-Triton +provides a number of overrides of the `rapids::copy` function, which provides a +safe way to mode data between buffers or tensors. Assuming the size attribute +of the buffer or tensor has not been corrupted, `rapids::copy` should never +result in segfaults or invalid memory access on device. + +Additional overrides of `rapids::copy` exist, but we will describe the most +common uses of it here. Note that you need not worry about where the underlying +data is located (on host or device) when invoking `rapids::copy`. The function +will take care of detecting and handling this. `Tensor` overrides are in +`rapids_triton/tensor/tensor.hpp` and `Buffer` overrides are in +`rapids_triton/memory/buffer.hpp`. + +### Between two buffers or tensors... +If you wish to simply copy the entire contents of one buffer into another or +one tensor into another, `rapids::copy` can be invoked as follows: +```cpp +rapids::copy(destination_buffer, source_buffer); +rapids::copy(destination_tensor, source_tensor); +``` +If the destination is too small to contain the data from the source, a +`TritonException` will be thrown. + +### From one tensor to many... +To distribute data from one tensor to many, the following override is +available: +```cpp +rapids::copy(iterator_to_first_destination, iterator_to_last_destination, source); +``` +Note that destination tensors can be of different sizes. If the destination +buffers cannot contain all data from the source, a `TritonException` will be +thrown. Destination tensors can also be a mixture of device and host tensors if +desired. + +### From part of one buffer to part of another... +To move data from part of one buffer to part of another, you can use another +override as in the following example: +```cpp +rapids::copy(destination_buffer, source_buffer, 10, 3, 6); +``` +The extra arguments here provide the offset from the beginning of the +destination buffer to which data should be copied, the index of the beginning +element to be copied from the source, and the index one past the final element to +be copied from the source. Thus, this invocation will copy the third, fourth, +and fifth elements of the source buffer to the tenth, eleventh, and twelfth +elements of the destination. If the destination buffer only had room for (e.g.) +eleven elements, a `TritonException` would be thrown. + +## `Model` +For a thorough introduction to developing a RAPIDS-Triton `Model` for your +backend, see the [Linear Example +repo](https://github.com/rapidsai/rapids-triton-linear-example). Here, we will +just briefly summarize some of the useful methods of `Model` objects. + +### Non-Virtual Methods +* `get_input`: Used to retrieve an input tensor of a particular name from + Triton +* `get_output`: Used to retrieve an output tensor of a particular name from + Triton +* `get_config_param`: Used to retrieve a named parameter from the configuration + file for this model +* `get_device_id`: The device on which this model is deployed (0 for host + deployments) +* `get_deployment_type`: One of `GPUDeployment` or `CPUDeployment` depending on + whether this model is configured to be deployed on device or host + +### Virtual Methods +* `predict`: The method which performs actual inference on input data and + stores it to the output location +* `load`: A method which can be overridden to load resources that will be used + for the lifetime of the model +* `unload`: A method used to unload any resources loaded in `load` if + necessary +* `preferred_mem_type`, `preferred_mem_type_in`, and `preferred_mem_type_out`: + The location (device or host) where input and output data should be stored. + The latter two methods can be overridden if input and output data should be + stored differently. Otherwise, `preferred_mem_type` will be used for both. +* `get_stream`: A method which can be overridden to provide different streams + for handling successive batches. Otherwise, the default stream associated + with this model will be used. + +## `SharedState` +Multiple instances of a RAPIDS-Triton model may need to share some data between +them (or may choose to do so for efficiency). `SharedState` objects facilitate +this. For a thorough introduction to developing a RAPIDS-Triton `SharedState` +for your backend, see the [Linear Example +repo](https://github.com/rapidsai/rapids-triton-linear-example). Just like the +`Model` objects which share a particular `SharedState` object, configuration +parameters can be retrieved using `SharedState`'s `get_config_param` method. +Otherwise, most additional functionality is defined by the backend +implementation, including `load` and `unload` methods for any necessary +loading/unloading of resources that will be used for the lifetime of the shared +state. + +Note that just one shared state is constructed by the server regardless of how +many instances of a given model are created. + +## Other Memory Allocations +For most device memory allocations, it is strongly recommended that you simply +construct a `Buffer` of the correct size and type. However, if you absolutely +cannot use a `Buffer` in a particular context, you are encouraged to allocate +and deallocate device memory using [RMM](https://github.com/rapidsai/rmm). Any +memory managed in this way will make use of Triton's CUDA memory pool, which +will be faster than performing individual allocations. It is strongly +recommended that you not change the RMM device resource in your backend, since +doing so will cause allocations to no longer make use of Triton's memory pool. diff --git a/backend/python/pyproject.toml b/backend/python/pyproject.toml new file mode 100644 index 0000000..9787c3b --- /dev/null +++ b/backend/python/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/backend/python/rapids_triton/__init__.py b/backend/python/rapids_triton/__init__.py new file mode 100644 index 0000000..5c99549 --- /dev/null +++ b/backend/python/rapids_triton/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from rapids_triton.client import Client +from rapids_triton.logging import logger diff --git a/backend/python/rapids_triton/client.py b/backend/python/rapids_triton/client.py new file mode 100644 index 0000000..5da497b --- /dev/null +++ b/backend/python/rapids_triton/client.py @@ -0,0 +1,268 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import namedtuple +import concurrent.futures +import time + +from rapids_triton.triton.client import get_triton_client +from rapids_triton.triton.io import ( + create_triton_input, create_triton_output, destroy_shared_memory_region +) + +from rapids_triton.triton.dtype import dtype_to_triton_name +from rapids_triton.triton.response import get_response_data +from tritonclient import utils as triton_utils + + +# TODO(wphicks): Propagate device ids for cuda shared memory + +MultiModelOutput = namedtuple('MultiModelOutput', ('name', 'version', 'output')) + + +class Client(object): + def __init__( + self, + protocol='grpc', + host='localhost', + port=None, + concurrency=4): + self.triton_client = get_triton_client( + protocol=protocol, + host=host, + port=port, + concurrency=concurrency + ) + self._protocol = protocol + + @property + def protocol(self): + return self._protocol + + def create_inputs(self, array_inputs, shared_mem=None): + return [ + create_triton_input( + self.triton_client, + arr, + name, + dtype_to_triton_name(arr.dtype), + protocol=self.protocol, + shared_mem=shared_mem + ) + for name, arr in array_inputs.items() + ] + + def create_outputs(self, output_sizes, shared_mem=None): + return { + name: create_triton_output( + self.triton_client, + size, + name, + protocol=self.protocol, + shared_mem=shared_mem + ) for name, size in output_sizes.items() + } + + def wait_for_server(self, timeout): + server_wait_start = time.time() + while True: + try: + if self.triton_client.is_server_ready(): + break + except triton_utils.InferenceServerException: + pass + if time.time() - server_wait_start > timeout: + raise RuntimeError("Server startup timeout expired") + time.sleep(1) + + def clear_shared_memory(self): + self.triton_client.unregister_cuda_shared_memory() + self.triton_client.unregister_system_shared_memory() + + def release_io(self, io_objs): + for io_ in io_objs: + if io_.name is not None: + self.triton_client.unregister_cuda_shared_memory( + name=io_.name + ) + destroy_shared_memory_region( + io_.handle, shared_mem='cuda' + ) + + def get_model_config(self, model_name): + return self.triton_client.get_model_config(model_name).config + + def predict( + self, + model_name, + input_data, + output_sizes, + model_version='1', + shared_mem=None, + attempts=1): + model_version = str(model_version) + + try: + inputs = self.create_inputs(input_data, shared_mem=shared_mem) + outputs = self.create_outputs(output_sizes, shared_mem=shared_mem) + + response = self.triton_client.infer( + model_name, + model_version=model_version, + inputs=[input_.input for input_ in inputs], + outputs=[output_.output for output_ in outputs.values()] + ) + result = { + name: get_response_data(response, handle, name) + for name, (_, handle, _) in outputs.items() + } + self.release_io(inputs) + self.release_io(outputs.values()) + + except triton_utils.InferenceServerException: + if attempts > 1: + return self.predict( + model_name, + input_data, + output_sizes, + model_version=model_version, + shared_mem=shared_mem, + attempts=attempts - 1 + ) + raise + return result + + def predict_async( + self, + model_name, + input_data, + output_sizes, + model_version='1', + shared_mem=None, + attempts=1): + + model_version = str(model_version) + + inputs = self.create_inputs(input_data, shared_mem=shared_mem) + outputs = self.create_outputs(output_sizes, shared_mem=shared_mem) + + future_result = concurrent.futures.Future() + def callback(result, error): + if error is None: + output_arrays = { + name: get_response_data(result, handle, name) + for name, (_, handle, _) in outputs.items() + } + + future_result.set_result(output_arrays) + + self.release_io(outputs.values()) + else: + if isinstance(error, triton_utils.InferenceServerException): + if attempts > 1: + future_result.set_result(self.predict( + model_name, + input_data, + output_sizes, + model_version=model_version, + shared_mem=shared_mem, + attempts=attempts - 1 + )) + future_result.set_exception(error) + + self.triton_client.async_infer( + model_name, + model_version=model_version, + inputs=[input_.input for input_ in inputs], + outputs=[output_.output for output_ in outputs.values()], + callback=callback + ) + + if shared_mem is not None: + def release_callback(fut): + self.release_io(inputs) + + future_result.add_done_callback(release_callback) + return future_result + + def predict_multimodel_async( + self, + model_names, + input_data, + output_sizes, + model_versions=('1',), + shared_mem=None, + executor=None, + attempts=1): + + all_models = [ + (name, str(version)) + for name in model_names + for version in model_versions + ] + + inputs = self.create_inputs(input_data, shared_mem=shared_mem) + + all_future_results = [] + for model_name, version in all_models: + outputs = self.create_outputs(output_sizes, shared_mem=shared_mem) + + def create_callback(future_result, outputs): + def callback(result, error): + if error is None: + output_arrays = { + name: get_response_data(result, handle, name) + for name, (_, handle, _) in outputs.items() + } + + future_result.set_result( + MultiModelOutput( + name=model_name, version=version, output=output_arrays + ) + ) + + self.release_io(outputs.values()) + else: + if isinstance(error, triton_utils.InferenceServerException): + if attempts > 1: + future_result.set_result(self.predict( + model_name, + input_data, + output_sizes, + model_version=version, + shared_mem=shared_mem, + attempts=attempts - 1 + )) + future_result.set_exception(error) + return callback + + all_future_results.append(concurrent.futures.Future()) + self.triton_client.async_infer( + model_name, + model_version=version, + inputs=[input_.input for input_ in inputs], + outputs=[output_.output for output_ in outputs.values()], + callback=create_callback(all_future_results[-1], outputs) + ) + + def wait_for_all(future_results, releasable_inputs): + concurrent.futures.wait(future_results) + self.release_io(releasable_inputs) + return [fut.result() for fut in future_results] + + if executor is None: + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + return executor.submit(wait_for_all, all_future_results, inputs) + else: + return executor.submit(wait_for_all, all_future_results, inputs) diff --git a/backend/python/rapids_triton/exceptions.py b/backend/python/rapids_triton/exceptions.py new file mode 100644 index 0000000..dc7f8c5 --- /dev/null +++ b/backend/python/rapids_triton/exceptions.py @@ -0,0 +1,18 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class IncompatibleSharedMemory(Exception): + """Error thrown if operation cannot be completed with given shared memory + type""" diff --git a/backend/python/rapids_triton/logging.py b/backend/python/rapids_triton/logging.py new file mode 100644 index 0000000..9b8e2c4 --- /dev/null +++ b/backend/python/rapids_triton/logging.py @@ -0,0 +1,18 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +logger = logging.getLogger('rapids_triton') +logger.setLevel(logging.INFO) diff --git a/backend/python/rapids_triton/testing.py b/backend/python/rapids_triton/testing.py new file mode 100644 index 0000000..ece5e6a --- /dev/null +++ b/backend/python/rapids_triton/testing.py @@ -0,0 +1,123 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import numpy as np + +from rapids_triton.logging import logger +from rapids_triton.triton.client import STANDARD_PORTS +from rapids_triton.client import Client + + +def arrays_close( + a, + b, + atol=None, + rtol=None, + total_atol=None, + total_rtol=None, + assert_close=False): + """ + Compare numpy arrays for approximate equality + + :param numpy.array a: The array to compare against a reference value + :param numpy.array b: The reference array to compare against + :param float atol: The maximum absolute difference allowed between an + element in a and an element in b before they are considered non-close. + If both atol and rtol are set to None, atol is assumed to be 0. If atol + is set to None and rtol is not None, no absolute threshold is used in + comparisons. + :param float rtol: The maximum relative difference allowed between an + element in a and an element in b before they are considered non-close. + If rtol is set to None, no relative threshold is used in comparisons. + :param int total_atol: The maximum number of elements allowed to be + non-close before the arrays are considered non-close. + :param float total_rtol: The maximum proportion of elements allowed to be + non-close before the arrays are considered non-close. + """ + + if np.any(a.shape != b.shape): + if assert_close: + raise AssertionError( + "Arrays have different shapes:\n{} vs. {}".format( + a.shape, b.shape + ) + ) + return False + + if a.size == 0 and b.size == 0: + return True + + if atol is None and rtol is None: + atol = 0 + if total_atol is None and total_rtol is None: + total_atol = 0 + + diff_mask = np.ones(a.shape, dtype='bool') + + diff = np.abs(a-b) + + if atol is not None: + diff_mask = np.logical_and(diff_mask, diff > atol) + + if rtol is not None: + diff_mask = np.logical_and(diff_mask, diff > rtol * np.abs(b)) + + is_close = True + + mismatch_count = np.sum(diff_mask) + + if total_atol is not None and mismatch_count > total_atol: + is_close = False + + mismatch_proportion = mismatch_count / a.size + if total_rtol is not None and mismatch_proportion > total_rtol: + is_close = False + + if assert_close and not is_close: + total_tol_desc = [] + if total_atol is not None: + total_tol_desc.append(str(int(total_atol))) + if total_rtol is not None: + total_tol_desc.append( + "{:.2f} %".format(total_rtol * 100) + ) + total_tol_desc = " or ".join(total_tol_desc) + + msg = """Arrays have more than {} mismatched elements. + +Mismatch in {} ({:.2f} %) elements + a: {} + b: {} + + Mismatched indices: {}""".format( + total_tol_desc, mismatch_count, mismatch_proportion * 100, a, b, + np.transpose(np.nonzero(diff_mask))) + raise AssertionError(msg) + return is_close + + +def get_random_seed(): + """Provide random seed to allow for easer reproduction of testing failures + + Note: Code taken directly from cuML testing infrastructure""" + current_random_seed = os.getenv('PYTEST_RANDOM_SEED') + if current_random_seed is not None and current_random_seed.isdigit(): + random_seed = int(current_random_seed) + else: + random_seed = np.random.randint(0, 1e6) + os.environ['PYTEST_RANDOM_SEED'] = str(random_seed) + logger.info("Random seed value: %d", random_seed) + return random_seed diff --git a/backend/python/rapids_triton/triton/__init__.py b/backend/python/rapids_triton/triton/__init__.py new file mode 100644 index 0000000..3291348 --- /dev/null +++ b/backend/python/rapids_triton/triton/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/backend/python/rapids_triton/triton/client.py b/backend/python/rapids_triton/triton/client.py new file mode 100644 index 0000000..eaa86e3 --- /dev/null +++ b/backend/python/rapids_triton/triton/client.py @@ -0,0 +1,48 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tritonclient.http as triton_http +import tritonclient.grpc as triton_grpc + +STANDARD_PORTS = { + 'http': 8000, + 'grpc': 8001 +} + + +def get_triton_client( + protocol="grpc", + host='localhost', + port=None, + concurrency=4): + """Get Triton client instance of desired type """ + + if port is None: + port = STANDARD_PORTS[protocol] + + if protocol == 'grpc': + client = triton_grpc.InferenceServerClient( + url=f'{host}:{port}', + verbose=False + ) + elif protocol == 'http': + client = triton_http.InferenceServerClient( + url=f'{host}:{port}', + verbose=False, + concurrency=concurrency + ) + else: + raise RuntimeError('Bad protocol: "{}"'.format(protocol)) + + return client diff --git a/backend/python/rapids_triton/triton/dtype.py b/backend/python/rapids_triton/triton/dtype.py new file mode 100644 index 0000000..e3077a9 --- /dev/null +++ b/backend/python/rapids_triton/triton/dtype.py @@ -0,0 +1,34 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np + +DTYPE_NAMES = { + np.dtype('bool').str: 'BOOL', + np.dtype('uint8').str: 'UINT8', + np.dtype('uint16').str: 'UINT16', + np.dtype('uint32').str: 'UINT32', + np.dtype('uint64').str: 'UINT64', + np.dtype('int8').str: 'INT8', + np.dtype('int16').str: 'INT16', + np.dtype('int32').str: 'INT32', + np.dtype('int64').str: 'INT64', + np.dtype('float16').str: 'FP16', + np.dtype('float32').str: 'FP32', + np.dtype('float64').str: 'FP64' +} + +def dtype_to_triton_name(dtype): + dtype = np.dtype(dtype).str + return DTYPE_NAMES.get(dtype, 'BYTES') diff --git a/backend/python/rapids_triton/triton/io.py b/backend/python/rapids_triton/triton/io.py new file mode 100644 index 0000000..b86097d --- /dev/null +++ b/backend/python/rapids_triton/triton/io.py @@ -0,0 +1,170 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import namedtuple +from uuid import uuid4 + +import tritonclient.http as triton_http +import tritonclient.grpc as triton_grpc +from rapids_triton.utils.safe_import import ImportReplacement +from rapids_triton.exceptions import IncompatibleSharedMemory +from tritonclient import utils as triton_utils +try: + import tritonclient.utils.cuda_shared_memory as shm +except OSError: # CUDA libraries not available + shm = ImportReplacement('tritonclient.utils.cuda_shared_memory') + + +TritonInput = namedtuple('TritonInput', ('name', 'handle', 'input')) +TritonOutput = namedtuple('TritonOutput', ('name', 'handle', 'output')) + +def set_unshared_input_data(triton_input, data, protocol='grpc'): + if protocol == 'grpc': + triton_input.set_data_from_numpy(data) + else: + triton_input.set_data_from_numpy(data, binary_data=True) + + return TritonInput(None, None, triton_input) + + +def set_shared_input_data(triton_client, triton_input, data, protocol='grpc'): + input_size = data.size * data.itemsize + + input_name = 'input_{}'.format(uuid4().hex) + + input_handle = shm.create_shared_memory_region( + input_name, input_size, 0 + ) + + shm.set_shared_memory_region(input_handle, [data]) + + triton_client.register_cuda_shared_memory( + input_name, shm.get_raw_handle(input_handle), 0, input_size + ) + + triton_input.set_shared_memory(input_name, input_size) + + return TritonInput(input_name, input_handle, triton_input) + + +def set_input_data( + triton_client, + triton_input, + data, + protocol='grpc', + shared_mem=None): + if shared_mem is None: + return set_unshared_input_data( + triton_input, data, protocol=protocol + ) + if shared_mem == 'cuda': + return set_shared_input_data( + triton_client, triton_input, data, protocol=protocol + ) + raise RuntimeError("Unsupported shared memory type") + + +def create_triton_input( + triton_client, data, name, dtype, protocol='grpc', shared_mem=None): + if protocol == 'grpc': + triton_input = triton_grpc.InferInput(name, data.shape, dtype) + else: + triton_input = triton_http.InferInput(name, data.shape, dtype) + + return set_input_data( + triton_client, + triton_input, + data, + protocol=protocol, + shared_mem=shared_mem + ) + + +def create_output_handle(triton_client, triton_output, size, shared_mem=None): + if shared_mem is None: + return (None, None) + + output_name = 'output_{}'.format(uuid4().hex) + output_handle = shm.create_shared_memory_region( + output_name, size, 0 + ) + + triton_client.register_cuda_shared_memory( + output_name, shm.get_raw_handle(output_handle), 0, size + ) + + triton_output.set_shared_memory(output_name, size) + + return output_name, output_handle + + +def create_triton_output( + triton_client, size, name, protocol='grpc', shared_mem=None): + """Set up output memory in Triton + + Parameters + ---------- + triton_client : Triton client object + The client used to set output parameters + size : int + The size of the output in bytes + name : str + The model-defined name for this output + protocol : 'grpc' or 'http' + The protocol used for communication with the server + """ + if protocol == 'grpc': + triton_output = triton_grpc.InferRequestedOutput(name) + else: + triton_output = triton_grpc.InferRequestedOutput( + name, binary_data=True + ) + + output_name, output_handle = create_output_handle( + triton_client, triton_output, size, shared_mem=shared_mem + ) + + return TritonOutput( + name=output_name, + handle=output_handle, + output=triton_output + ) + + +def destroy_shared_memory_region(handle, shared_mem='cuda'): + """Release memory from a given shared memory handle + + Parameters + ---------- + handle : c_void_p + The handle (as returned by the Triton client) for the region to be + released. + shared_mem : 'cuda' or 'system' or None + The type of shared memory region to release. If None, an exception will + be thrown. + """ + if shared_mem is None: + raise IncompatibleSharedMemory( + "Attempting to release non-shared memory" + ) + elif shared_mem == 'system': + raise NotImplementedError( + "System shared memory not yet supported" + ) + elif shared_mem == 'cuda': + shm.destroy_shared_memory_region(handle) + else: + raise NotImplementedError( + f"Unrecognized memory type {shared_mem}" + ) diff --git a/backend/python/rapids_triton/triton/message.py b/backend/python/rapids_triton/triton/message.py new file mode 100644 index 0000000..ae9df1e --- /dev/null +++ b/backend/python/rapids_triton/triton/message.py @@ -0,0 +1,29 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class TritonMessage: + """Adapter to read output from both GRPC and HTTP responses""" + def __init__(self, message): + self.message = message + + def __getattr__(self, attr): + try: + return getattr(self.message, attr) + except AttributeError: + try: + return self.message[attr] + except Exception: # Re-raise AttributeError + pass + raise diff --git a/backend/python/rapids_triton/triton/response.py b/backend/python/rapids_triton/triton/response.py new file mode 100644 index 0000000..47332ce --- /dev/null +++ b/backend/python/rapids_triton/triton/response.py @@ -0,0 +1,37 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tritonclient import utils as triton_utils + +from rapids_triton.triton.message import TritonMessage +from rapids_triton.utils.safe_import import ImportReplacement +try: + import tritonclient.utils.cuda_shared_memory as shm +except OSError: # CUDA libraries not available + shm = ImportReplacement('tritonclient.utils.cuda_shared_memory') + + +def get_response_data(response, output_handle, output_name): + """Convert Triton response to NumPy array""" + if output_handle is None: + return response.as_numpy(output_name) + else: + network_result = TritonMessage( + response.get_output(output_name) + ) + return shm.get_contents_as_numpy( + output_handle, + triton_utils.triton_to_np_dtype(network_result.datatype), + network_result.shape + ) diff --git a/backend/python/rapids_triton/utils/__init__.py b/backend/python/rapids_triton/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/python/rapids_triton/utils/safe_import.py b/backend/python/rapids_triton/utils/safe_import.py new file mode 100644 index 0000000..1738600 --- /dev/null +++ b/backend/python/rapids_triton/utils/safe_import.py @@ -0,0 +1,38 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class ImportUnavailableError(Exception): + '''Error thrown if a symbol is unavailable due to an issue importing it''' + + +class ImportReplacement: + """A class to be used in place of an importable symbol if that symbol + cannot be imported + + Parameters + ---------- + symbol: str + The name or import path to be used in error messages when attempting to + make use of this symbol. E.g. "some_pkg.func" would result in an + exception with message "some_pkg.func could not be imported" + """ + def __init__(self, symbol): + self._msg = f'{symbol} could not be imported' + + def __getattr__(self, name): + raise ImportUnavailableError(self._msg) + + def __call__(self, *args, **kwargs): + raise ImportUnavailableError(self._msg) diff --git a/backend/python/setup.py b/backend/python/setup.py new file mode 100644 index 0000000..2cfa713 --- /dev/null +++ b/backend/python/setup.py @@ -0,0 +1,28 @@ +# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import setup, find_packages + +setup( + name='rapids_triton', + description="Tools for clients to RAPIDS-Triton backends", + version='22.02.00', # TODO(wphicks): versioneer + author='NVIDIA Corporation', + license='Apache', + packages=find_packages(), + install_requires=[ + 'numpy', + 'tritonclient[all]' + ] +) diff --git a/backend/qa/L0_e2e/cpu_model_repository/identity/1/empty.model b/backend/qa/L0_e2e/cpu_model_repository/identity/1/empty.model new file mode 100644 index 0000000..e69de29 diff --git a/backend/qa/L0_e2e/cpu_model_repository/identity/2/empty.model b/backend/qa/L0_e2e/cpu_model_repository/identity/2/empty.model new file mode 100644 index 0000000..e69de29 diff --git a/backend/qa/L0_e2e/cpu_model_repository/identity/config.pbtxt b/backend/qa/L0_e2e/cpu_model_repository/identity/config.pbtxt new file mode 100644 index 0000000..86193e9 --- /dev/null +++ b/backend/qa/L0_e2e/cpu_model_repository/identity/config.pbtxt @@ -0,0 +1,20 @@ +backend: "dev_tools-identity" +max_batch_size: 32768 +input [ + { + name: "input__0" + data_type: TYPE_FP32 + dims: [ 1 ] + } +] +output [ + { + name: "output__0" + data_type: TYPE_FP32 + dims: [ 1 ] + } +] +version_policy: { all { }} +instance_group [{ kind: KIND_CPU }] +parameters [ ] +dynamic_batching { } diff --git a/backend/qa/L0_e2e/model_repository/identity/1/empty.model b/backend/qa/L0_e2e/model_repository/identity/1/empty.model new file mode 100644 index 0000000..e69de29 diff --git a/backend/qa/L0_e2e/model_repository/identity/2/empty.model b/backend/qa/L0_e2e/model_repository/identity/2/empty.model new file mode 100644 index 0000000..e69de29 diff --git a/backend/qa/L0_e2e/model_repository/identity/config.pbtxt b/backend/qa/L0_e2e/model_repository/identity/config.pbtxt new file mode 100644 index 0000000..40b4974 --- /dev/null +++ b/backend/qa/L0_e2e/model_repository/identity/config.pbtxt @@ -0,0 +1,20 @@ +backend: "dev_tools-identity" +max_batch_size: 32768 +input [ + { + name: "input__0" + data_type: TYPE_FP32 + dims: [ 1 ] + } +] +output [ + { + name: "output__0" + data_type: TYPE_FP32 + dims: [ 1 ] + } +] +version_policy: { all { }} +instance_group [{ kind: KIND_GPU }] +parameters [ ] +dynamic_batching { } diff --git a/backend/qa/L0_e2e/test_model.py b/backend/qa/L0_e2e/test_model.py new file mode 100644 index 0000000..cd13d62 --- /dev/null +++ b/backend/qa/L0_e2e/test_model.py @@ -0,0 +1,133 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import concurrent +import os + +import numpy as np +import pytest + +from rapids_triton import Client +from rapids_triton.testing import get_random_seed, arrays_close + +TOTAL_SAMPLES = 8192 + +def valid_shm_modes(): + modes = [None] + if os.environ.get('CPU_ONLY', 0) == 0: + modes.append('cuda') + return modes + + +@pytest.fixture(scope='session') +def model_versions(): + return (1, 2) + +@pytest.fixture(scope='session') +def client(): + client = Client() + client.wait_for_server(60) + return client + +@pytest.fixture +def model_inputs(): + np.random.seed(get_random_seed()) + return { + input_name: + np.random.rand(TOTAL_SAMPLES, 1).astype('float32') + for input_name in ('input__0',) + } + +@pytest.fixture +def model_output_sizes(): + return {'output__0': TOTAL_SAMPLES * np.dtype('float32').itemsize} + +def get_ground_truth(inputs): + return {'output__0': inputs['input__0']} + + +@pytest.mark.parametrize("model_name", ['identity']) +@pytest.mark.parametrize("shared_mem", valid_shm_modes()) +def test_model(client, model_name, shared_mem, model_inputs, model_output_sizes): + result = client.predict( + model_name, model_inputs, model_output_sizes, shared_mem=shared_mem + ) + ground_truth = get_ground_truth(model_inputs) + + for output_name in sorted(ground_truth.keys()): + arrays_close( + result[output_name], + ground_truth[output_name], + atol=1e-5, + assert_close=True + ) + + +@pytest.mark.parametrize("model_name", ['identity']) +@pytest.mark.parametrize("shared_mem", valid_shm_modes()) +@pytest.mark.parametrize( + "batch_size", + [1, TOTAL_SAMPLES // 3, TOTAL_SAMPLES // 2] +) +def test_predict_async(client, model_name, shared_mem, model_inputs, batch_size): + results = [] + gt_results = [] + for i in range( + 0, + TOTAL_SAMPLES // batch_size + int(bool(TOTAL_SAMPLES % batch_size)) + ): + min_index = i * batch_size + max_index = min((i + 1) * batch_size, TOTAL_SAMPLES) + cur_input = {name: arr[min_index: max_index] for name, arr in + model_inputs.items()} + cur_output_size = { + 'output__0': (max_index - min_index) * np.dtype('float32').itemsize + } + results.append(client.predict_async( + model_name, cur_input, cur_output_size, shared_mem=shared_mem + )) + gt_results.append(get_ground_truth(cur_input)) + concurrent.futures.wait(results, timeout=60) + results = [result_.result() for result_ in results] + + for result, ground_truth in zip(results, gt_results): + for output_name in sorted(ground_truth.keys()): + arrays_close( + result[output_name], + ground_truth[output_name], + atol=1e-5, + assert_close=True + ) + + +@pytest.mark.parametrize("model_name", ['identity']) +@pytest.mark.parametrize("shared_mem", valid_shm_modes()) +def test_predict_multimodel_async( + client, model_name, shared_mem, model_inputs, model_output_sizes, + model_versions): + all_results = client.predict_multimodel_async( + [model_name], model_inputs, model_output_sizes, + model_versions=model_versions, shared_mem=shared_mem + ) + ground_truth = get_ground_truth(model_inputs) + + all_results = all_results.result(timeout=60) + for result in all_results: + for output_name in sorted(ground_truth.keys()): + arrays_close( + result.output[output_name], + ground_truth[output_name], + atol=1e-5, + assert_close=True + ) diff --git a/backend/qa/entrypoint.sh b/backend/qa/entrypoint.sh new file mode 100755 index 0000000..046f8a0 --- /dev/null +++ b/backend/qa/entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +QA_DIR=$(cd $(dirname $0); pwd) +TEST_SCRIPT="$QA_DIR/run_tests.sh" + +if [[ $TRITON_ENABLE_GPU != "OFF" ]] +then + MODEL_REPO="${QA_DIR}/L0_e2e/model_repository" "$TEST_SCRIPT" + export TEST_EXE='' # Only run unit tests once + MODEL_REPO="${QA_DIR}/L0_e2e/cpu_model_repository" "$TEST_SCRIPT" +fi + +CPU_ONLY=1 MODEL_REPO="${QA_DIR}/L0_e2e/cpu_model_repository" "$TEST_SCRIPT" diff --git a/backend/qa/run_tests.sh b/backend/qa/run_tests.sh new file mode 100755 index 0000000..69a3d0d --- /dev/null +++ b/backend/qa/run_tests.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +QA_DIR=$(cd $(dirname $0); pwd) +SERVER_ARGS="" +UUID="$(cat /proc/sys/kernel/random/uuid)" +CONTAINER_NAME="rapids_triton-ci-$UUID" +DOCKER_RUN=0 +DOCKER_ARGS="-d -p 8000:8000 -p 8001:8001 -p 8002:8002 --name ${CONTAINER_NAME}" +TRITON_PID="" +LOG_DIR="${QA_DIR}/logs" +SERVER_LOG="${LOG_DIR}/${UUID}-server.log" + +if [ ! -d "${LOG_DIR}" ] +then + mkdir -p "${LOG_DIR}" +fi + +if [ -z $MODEL_REPO ] +then + MODEL_REPO="${QA_DIR}/L0_e2e/model_repository" +fi +MODEL_REPO="$(readlink -f $MODEL_REPO)" + +DOCKER_ARGS="${DOCKER_ARGS} -v ${MODEL_REPO}:/models" + +if [ -z $CPU_ONLY ] || [ $CPU_ONLY -eq 0 ] +then + if [[ -v CUDA_VISIBLE_DEVICES ]] + then + if [ -z $CUDA_VISIBLE_DEVICES ] + then + CPU_ONLY=1 + else + DOCKER_ARGS="${DOCKER_ARGS} --gpus ${CUDA_VISIBLE_DEVICES}" + fi + else + DOCKER_ARGS="${DOCKER_ARGS} --gpus all" + fi +else + export CUDA_VISIBLE_DEVICES="" +fi + +# If a Triton Docker image has been provided or no tritonserver executable is +# available, run the server via Docker +if [ ! -z $TRITON_IMAGE ] || ! command -v tritonserver +then + DOCKER_RUN=1 + TRITON_IMAGE=${TRITON_IMAGE:-rapids_triton_identity} + SERVER_ARGS="${SERVER_ARGS} --model-repository=/models" +else + SERVER_ARGS="${SERVER_ARGS} --model-repository=${MODEL_REPO}" +fi + +start_server() { + if [ $DOCKER_RUN -eq 1 ] + then + docker run $DOCKER_ARGS $TRITON_IMAGE > /dev/null + else + tritonserver $SERVER_ARGS > $SERVER_LOG 2>&1 & + TRITON_PID="$!" + fi +} + +start_server + +if [ -z $TEST_EXE ] +then + echo 'No TEST_EXE variable defined; skipping unit tests' +else + if [ $DOCKER_RUN -eq 1 ] + then + docker exec $CONTAINER_NAME "$TEST_EXE" + else + "$TEST_EXE" + fi +fi + +finally() { + if [ -z $TRITON_PID ] + then + docker logs $CONTAINER_NAME > $SERVER_LOG 2>&1 + docker rm -f $CONTAINER_NAME > /dev/null 2>&1 + else + kill -15 $TRITON_PID + wait + fi +} + +trap finally EXIT + +pytest "$QA_DIR" diff --git a/examples/backend_linear/CMakeLists.txt b/examples/backend_linear/CMakeLists.txt new file mode 100644 index 0000000..25c0558 --- /dev/null +++ b/examples/backend_linear/CMakeLists.txt @@ -0,0 +1,158 @@ +#============================================================================= +# Copyright (c) 2020-2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +cmake_minimum_required(VERSION 3.21 FATAL_ERROR) + +############################################################################## +# - Target names ------------------------------------------------------------- +set(BACKEND_NAME "rapids_linear") +set(BACKEND_TARGET "triton_${BACKEND_NAME}") + + +############################################################################## +# - Prepare rapids-cmake ----------------------------------------------------- +file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.12/RAPIDS.cmake + ${CMAKE_BINARY_DIR}/RAPIDS.cmake) +include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) +include(rapids-cmake) +include(rapids-cpm) +include(rapids-cuda) +include(rapids-export) +include(rapids-find) + +rapids_cuda_init_architectures(TRITON_LINEAR_EXAMPLE) + +project(TRITON_LINEAR_EXAMPLE VERSION 21.10.00 LANGUAGES CXX CUDA) + +############################################################################## +# - build type --------------------------------------------------------------- + +# Set a default build type if none was specified +rapids_cmake_build_type(Release) + +# this is needed for clang-tidy runs +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +############################################################################## +# - User Options ------------------------------------------------------------ + +option(BUILD_BACKEND_TESTS "Build rapids_triton_linear unit-tests" ON) +option(CUDA_ENABLE_KERNEL_INFO "Enable kernel resource usage info" OFF) +option(CUDA_ENABLE_LINE_INFO "Enable lineinfo in nvcc" OFF) +option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" ON) +option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) +option(NVTX "Enable nvtx markers" OFF) + +message(VERBOSE "TRITON_LINEAR_EXAMPLE: Enabling detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") +message(VERBOSE "TRITON_LINEAR_EXAMPLE: Enabling kernelinfo in nvcc: ${CUDA_ENABLE_KERNEL_INFO}") +message(VERBOSE "TRITON_LINEAR_EXAMPLE: Enabling lineinfo in nvcc: ${CUDA_ENABLE_LINE_INFO}") +message(VERBOSE "TRITON_LINEAR_EXAMPLE: Enabling nvtx markers: ${NVTX}") +message(VERBOSE "TRITON_LINEAR_EXAMPLE: Build TRITON_LINEAR_EXAMPLE unit-tests: ${BUILD_TESTS}") + +# Set RMM logging level +set(RMM_LOGGING_LEVEL "INFO" CACHE STRING "Choose the logging level.") +set_property(CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF") +message(VERBOSE "TRITON_LINEAR_EXAMPLE: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") + +############################################################################## +# - Conda environment detection ---------------------------------------------- + +if(DETECT_CONDA_ENV) + rapids_cmake_support_conda_env( conda_env MODIFY_PREFIX_PATH ) + if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED ENV{CONDA_PREFIX}) + message(STATUS "CUML: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}") + set(CMAKE_INSTALL_PREFIX "$ENV{CONDA_PREFIX}") + endif() +endif() + +############################################################################## +# - compiler options --------------------------------------------------------- + +# * find CUDAToolkit package +# * determine GPU architectures +# * enable the CMake CUDA language +# * set other CUDA compilation flags +rapids_find_package(CUDAToolkit REQUIRED + BUILD_EXPORT_SET triton-linear-example-exports + INSTALL_EXPORT_SET triton-linear-example-exports +) +include(cmake/modules/ConfigureCUDA.cmake) + +############################################################################## +# - Requirements ------------------------------------------------------------- + +# add third party dependencies using CPM +rapids_cpm_init() + +# TODO(wphicks) +include(cmake/thirdparty/get_rapids-triton.cmake) + +if(BUILD_TESTS) + include(cmake/thirdparty/get_gtest.cmake) +endif() + + +############################################################################## +# - install targets----------------------------------------------------------- + +add_library( + ${BACKEND_TARGET} SHARED + src/api.cc + src/gpu_infer.cu +) + +set_target_properties(${BACKEND_TARGET} +PROPERTIES BUILD_RPATH "\$ORIGIN" + # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON +) + +target_compile_options(${BACKEND_TARGET} + PRIVATE "$<$:${RAPIDS_TRITON_BACKEND_CXX_FLAGS}>" + "$<$:${RAPIDS_TRITON_BACKEND_CUDA_FLAGS}>" +) + +target_include_directories(${BACKEND_TARGET} + PRIVATE "$" + "${CMAKE_CURRENT_SOURCE_DIR}/src" +) + +target_link_libraries(${BACKEND_TARGET} +PRIVATE + triton_backend::triton_backend + triton-core-serverstub + triton-backend-utils + "${TRITONSERVER_LIB}" + $ +) + +install( + TARGETS ${BACKEND_TARGET} + LIBRARY DESTINATION /opt/tritonserver/backends/${BACKEND_NAME} +) + +############################################################################## +# - build test executable ---------------------------------------------------- + +# TODO (wphicks) +# if(BUILD_TESTS) +# include(test/CMakeLists.txt) +# endif() diff --git a/examples/backend_linear/Dockerfile b/examples/backend_linear/Dockerfile new file mode 100644 index 0000000..c92ed94 --- /dev/null +++ b/examples/backend_linear/Dockerfile @@ -0,0 +1,94 @@ +########################################################################################### +# Arguments for controlling build details +########################################################################################### +# Version of Triton to use +ARG TRITON_VERSION=22.11 +# Base container image +ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:${TRITON_VERSION}-py3 +# Whether or not to build indicated components +ARG BUILD_TESTS=OFF +ARG BUILD_EXAMPLE=ON + +FROM ${BASE_IMAGE} as base + +ENV PATH="/root/miniconda3/bin:${PATH}" + +RUN apt-get update \ + && apt-get install --no-install-recommends -y wget patchelf \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +ENV PYTHONDONTWRITEBYTECODE=true + +RUN wget \ + https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + && mkdir /root/.conda \ + && bash Miniconda3-latest-Linux-x86_64.sh -b \ + && rm -f Miniconda3-latest-Linux-x86_64.sh + +COPY ./conda/environments/rapids_triton_dev_cuda11.4.yml /environment.yml + +RUN conda env update -f /environment.yml \ + && rm /environment.yml \ + && conda clean -afy \ + && find /root/miniconda3/ -follow -type f -name '*.pyc' -delete \ + && find /root/miniconda3/ -follow -type f -name '*.js.map' -delete + +ENV PYTHONDONTWRITEBYTECODE=false + +RUN mkdir /rapids_triton + +COPY ./src /rapids_triton/src +COPY ./CMakeLists.txt /rapids_triton +COPY ./cmake /rapids_triton/cmake + +WORKDIR /rapids_triton + +SHELL ["conda", "run", "--no-capture-output", "-n", "rapids_triton_dev", "/bin/bash", "-c"] + +FROM base as build-stage + +ARG TRITON_VERSION +ENV TRITON_VERSION=$TRITON_VERSION + +ARG BUILD_TYPE=Release +ENV BUILD_TYPE=$BUILD_TYPE +ARG BUILD_TESTS +ENV BUILD_TESTS=$BUILD_TESTS +ARG BUILD_EXAMPLE +ENV BUILD_EXAMPLE=$BUILD_EXAMPLE + +RUN mkdir /rapids_triton/build /rapids_triton/install + +WORKDIR /rapids_triton/build + +RUN cmake \ + -GNinja \ + -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ + -DBUILD_TESTS="${BUILD_TESTS}" \ + -DCMAKE_INSTALL_PREFIX=/rapids_triton/install \ + -DTRITON_COMMON_REPO_TAG="r${TRITON_VERSION}" \ + -DTRITON_CORE_REPO_TAG="r${TRITON_VERSION}" \ + -DTRITON_BACKEND_REPO_TAG="r${TRITON_VERSION}" \ + .. + +RUN ninja install + +FROM ${BASE_IMAGE} + +ARG BACKEND_NAME +ENV BACKEND_NAME=$BACKEND_NAME + +RUN mkdir /models + +# Remove existing backend install +RUN if [ -d /opt/tritonserver/backends/${BACKEND_NAME} ]; \ + then \ + rm -rf /opt/tritonserver/backends/${BACKEND_NAME}/*; \ + fi + +COPY --from=build-stage \ + /opt/tritonserver/backends/$BACKEND_NAME \ + /opt/tritonserver/backends/$BACKEND_NAME + +ENTRYPOINT ["tritonserver", "--model-repository=/models"] diff --git a/examples/backend_linear/README.md b/examples/backend_linear/README.md new file mode 100644 index 0000000..78bd193 --- /dev/null +++ b/examples/backend_linear/README.md @@ -0,0 +1,556 @@ + + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +# The RAPIDS-Triton Linear Example + +This repository offers an annotated example of how to create a custom Triton +backend using the RAPIDS-Triton library. In the following, we will demonstrate +step-by-step how to create a backend with RAPIDS-Triton that, when given two +vectors (**u** and **v**) as input will return a vector **r** according to the +following equation: + +**r** = α * **u** + **v** + **c** + +where α is a scalar constant read from a configuration file and **c** is a +constant vector read from a "model" file. Along the way, we will illustrate a +variety of useful operations in RAPIDS-Triton, including retrieving data from a +configuration file and loading model resources. + +This example is intended to provide a fair amount of depth about backend +development with RAPIDS-Triton. For a simpler example, check out the +pass-through backend in the main [RAPIDS-Triton +repo](https://github.com/rapidsai/rapids-triton#simple-example). +For even more detail on RAPIDS-Triton features introduced here, check out the +API documentation. + +All of the following steps are written as if you were starting from scratch in +creating this backend, but you can also just browse the files in this repo to +see how the final version of the backend might look. + +## 1. Getting Started + +It is strongly recommended that you start from the [RAPIDS-Triton template +repo](https://github.com/rapidsai/rapids-triton-template) whenever you begin +developing a custom backend. This repo provides the boilerplate that would +otherwise be necessary to get started with your custom backend. Go ahead and +[create a new +repository](https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/creating-a-repository-on-github/creating-a-repository-from-a-template) based on this template. + +## 2. Pick a Name + +Your custom backend will need a name that uniquely identifies it to the Triton +server. For this example, we will use the name `rapids_linear`. We will need to +provide this name in two places: + + +### 2.1 Update `names.h` +In the `src/names.h` file, adjust the definition of NAMESPACE to read + +```cpp +#define NAMESPACE rapids_linear +``` + +Triton conventions require that backend definition be placed in a namespace of +the form `triton::backend::NAME_OF_BACKEND`, so we define the namespace name +here and use it where required in the `rapids-triton-template` code. + +### 2.2 Update `CMakeLists.txt` +Near the top of `CMakeLists.txt` in the section labeled "Target names", there +is an option to provide a `BACKEND_NAME`. In this case, we will set this as +follows: + +```cmake +set(BACKEND_NAME "rapids_linear") +``` + +## 3. Create an Example Configuration + +[Configuration +files](https://github.com/triton-inference-server/server/blob/main/docs/model_configuration.md) +provide settings for model deployments which determine how +the model will behave throughout its entire deployment (as opposed to values +which vary on a request-by-request basis). It is often helpful to think through +what your configuration file will look like before actually writing any code +for your custom backend. In the present example, we need to specify a few +different things: + +1. The name of the backend which will be used to serve this model (the name + chosen in step 2) +2. The names of the input vectors +3. The value of α +4. Where to load **c** from + +With this in mind, an example configuration file for the `rapids_linear` +backend might look something like this: + +```protobuf +name: "linear_example" +backend: "rapids_linear" +max_batch_size: 32768 +input [ + { + name: "u" + data_type: TYPE_FP32 + dims: [ 4 ] + }, + { + name: "v" + data_type: TYPE_FP32 + dims: [ 4 ] + } +] +output [ + { + name: "r" + data_type: TYPE_FP32 + dims: [ 4 ] + } +] +instance_group [{ kind: KIND_GPU }] +parameters [ + { + key: "alpha" + value: { string_value: "2.0" } + } +] +dynamic_batching { + max_queue_delay_microseconds: 100 +} +``` + +Let's review the pieces of this configuration file in a bit more detail, since +they will introduce several important concepts for development of our backend. + +### `name` +This provides a name for an individual model. Remember that a backend is used +to serve a particular *kind* of model, but Triton can serve many different +models of this kind. The name given here specifies the specific model being +served in order to allow clients to submit requests to that model. + +### `backend` +This is the name of the custom backend we are developing, in this case +`rapids_linear`. + +### `max_batch_size` +All RAPIDS-Triton backends require that models specify some maximum batch size, +although this value can be arbitrarily large. + +### `input` +This section specifies the input tensors used for this backend. Each input +tensor has a name (which we will use later in the actual custom backend code), +a +[datatype](https://github.com/triton-inference-server/server/blob/main/docs/model_configuration.md#datatypes) +(we use 32-bit floats in this example), and a shape. In this case, the **u** +and **v** vectors must be of the same shape as one another and as **c**, so we +will arbitrarily choose to make them of dimension 4 for our example +configuration. + +### `output` +This section follows the same conventions as `input`. Again, for this +particular example, we must match the dimensions of the inputs and **c**. + +### `instance_group` +This section determines the hardware on which this model can be deployed (GPU +or CPU). For this example backend, we will demonstrate how to ensure that +models can perform inference on both GPUs and CPUs, buy we will start with GPU +inference. + +### `parameters` +This section allows us to specify any other settings that may be required for +our model. Note that we specify all settings as `string_value` entries, but +RAPIDS-Triton will allow us to easily parse those values into bools, floats, +ints, or any other basic type. + +### `dynamic_batching` +This section specifies how Triton's dynamic batcher will be used to combine +requests into batches for this model. By setting `max_queue_delay_microseconds` +to 100, we are allowing Triton to gather requests in a window of up to 100 +microseconds before passing them to the backend. Triton's [model +analyzer](https://github.com/triton-inference-server/model_analyzer) can help +determine the optimal value for this window, but we will arbitrarily set it to +100 for this example. + +Note that batching can happen both client-side and server-side. If a client +were to submit a request with 5x4 input tensors, for instance, the Triton +server will correctly interpret this as 5 different **u** and **v** vectors. It +can then batch this request with the next, which may carry 7x4 inputs or 1x4 +inputs or any other valid mini-batch shape. + +## 4. Define RapidsSharedState + +Triton allows multiple copies or "instances" of a model to be loaded at the +same time. One obvious use case for this feature is to load the same model on +multiple GPUs, but it can also be used to "oversubscribe" hardware when this is +beneficial for meeting e.g. throughput or latency requirements. When multiple +instances of a model are loaded, they all have access to an object which +manages state that is relevant to all instances. + +The most basic state that is shared among instances of a model is the model +configuration itself (as defined in step 3), but we can also use it to share +data that is specifically required by our backend. In our particular case, it +would be useful to cache the value of α so that we do not have to retrieve +it from the configuration each time (which may involve additional parsing). + +All RAPIDS-Triton backends store their shared state in a class called +`RapidsSharedState` in their own namespace, which inherits from +`dev_tools::SharedModelState`. A basic implementation of this class for the +current backend might look something like the following: + +```cpp +struct RapidsSharedState : dev_tools::SharedModelState { + RapidsSharedState(std::unique_ptr&& config) + : dev_tools::SharedModelState{std::move(config)} {} + void load() {} + void unload() {} + + float alpha = 1.0f; +}; +``` + +You may safely ignore the constructor in this example. It is boilerplate code +that should be included for all `RapidsSharedState` definitions and will not +change between backends. Take a look at `src/shared_state.h` to see this +implementation in context. + +Note that we have added a public member variables to this class definition +which will be used to store α. One could equally well have made +these private members with getter functions or added arbitrarily complex logic +to this class definition, but we will leave them as is for simplicity. + +### Accessing configuration parameters + +Next, we need to actually load a value into this newly-defined member. We can +do this by filling out the logic for our `load` method. For example, in order +to load α, we could implement something like: +```cpp +void load() { alpha = get_config_param("alpha"); } +``` + +Here, the `get_config_param` function allows us to directly access the `alpha` +parameter we defined in our example configuration file. By invoking the `float` +instantiation of this template, we ensure that we will retrieve the value as a +float. + +### Unloading resources + +In this particular case, our shared state does not include any resources that +need to be unloaded, but the `unload` method is available to do precisely that. +We will instead use it here simply to illustrate the use of RAPIDS-Triton +logging functions. Logging functions are defined in +`rapids_triton/triton/logging.hpp` and may be invoked as follows: +```cpp +void unload() { dev_tools::log_info(__FILE__, __LINE__) << "Unloading shared state..."; } +``` +The arguments to `log_info` and related functions may be omitted, but if +included may provide some use in debugging. + +## 5. Define RapidsModel + +The real heart of any RAPIDS-Triton backend is its RapidsModel definition. This +class implements the actual logic to deserialize a model and use it to perform +inference. + +### 5.1 Deserialize the model +Most backends will need to deserialize their models from some file on-disk. +In our case, we are using the **c** vector as a stand-in for some more +interesting model structure. For simplicity, we will assume that **c** is +just stored as space-separated floats in a text file. + +A natural question is why we did not perform this deserialization in +`RapidsSharedState::load` since **c** is the same for all instances of a model. +In general, instances may be initialized on different GPUs, or some may be on +GPUs while others are on CPUs, so we defer model deserialization until we know +where the model should be deserialized *to*. + +Given that **c** could be stored on either host or device, the question of how +to represent it as a member variable becomes a little more fraught. We could +represent it as a raw pointer, but this requires a great deal of manual +tracking to determine whether to use `std::malloc` or `cudaMalloc` for the +initial allocation in the model `load()` method and `std::free` or `cudaFree` +in `unload()`. + +To help simplify how situations like this are handled, RAPIDS-Triton introduces +a lightweight `Buffer` class, which can provide unified RAII access to memory +on host or device **or** a non-owning view on host/device memory that was +allocated elsewhere. We'll introduce a `Buffer` member to RapidsModel to store **c**: + +```cpp +dev_tools::Buffer c{}; +``` + +Now, we are ready to actually load **c** from its file representation. +`dev_tools::Model`, the abstract parent class of `RapidsModel` implementations, +defines several methods to help with model deserialization, including: +- `get_filepath`, which returns the path to the model file if it is specified + in the config or the model directory if it is not +- `get_deployment_type`, which returns whether this model should be deployed on + CPU or GPU +- `get_device_id`, which returns the id of the device on which the model should + be deployed (always 0 for CPU deployments) + +Using these functions, we can define our `load()` function as follows. First, +we determine the full filepath to the model file, defaulting to a name of +`c.txt` if it is not specified in the config file: +```cpp +if (std::filesystem::is_directory(path)) { + path /= "c.txt"; +} +``` + +Next, we actually read the values of **c** into a temporary vector from its +text file representation: +```cpp + auto model_vec = std::vector{}; + auto model_file = std::ifstream(path.string()); + auto input_line = std::string{}; + std::getline(model_file, input_line); + auto input_stream = std::stringstream{input_line}; + auto value = 0.0f; + while (input_stream >> value) { + model_vec.push_back(value); + } +``` + +We then query the model to figure out exactly what sort of Buffer will be +needed to store **c**: +```cpp +auto memory_type = dev_tools::MemoryType{}; +if (get_deployment_type() == dev_tools::GPUDeployment) { + memory_type = dev_tools::DeviceMemory; +} else { + memory_type = dev_tools::HostMemory; +} +c = dev_tools::Buffer(model_vec.size(), memory_type, get_device_id()); +``` + +Finally, we use the helper function `dev_tools::copy` to copy the values of **c** +from a Buffer-based view of `model_vec` to the `c` Buffer itself: +```cpp +dev_tools::copy(c, dev_tools::Buffer(model_vec.data(), model_vec.size(), + dev_tools::HostMemory)); +``` + +By taking advantage of Buffer's RAII semantics, we eliminate the need to +explicitly implement an `unload` function, but we could do so if necessary. + +### 5.2 Write the predict function +Now all that remains is to use our loaded "model" for inference. The `predict` +method of a RapidsModel implementation takes in a `Batch` object as argument, +which can be used to retrieve the input and output tensors that we will operate +on during inference. We can retrieve these tensors using the names we +originally specified in the config file: + +```cpp +auto u = get_input(batch, "u"); +auto v = get_input(batch, "v"); +auto r = get_output(batch, "r"); +``` + +By default, the location of the returned tensors (host or device) is determined +by whether the model is deployed on host or device and whether or not the +backend was compiled with GPU-support enabled. You may choose to override the +`preferred_mem_type` method of your RapidsModel implementation in order to +specify a different general rule, or you can optionally pass a `MemoryType` to +`get_input` and `get_output` for even finer-grained control. Here, we will +simply accept the default behavior and use the `mem_type` method of the +returned tensor to determine how inference will proceed. + +For tensors on the host, our inference logic might look something like: + +```cpp +if (u.mem_type() == dev_tools::HostMemory) { + auto alpha = get_shared_state()->alpha; + for (std::size_t i{}; i < u.size(); ++i) { + r.data()[i] = + alpha * u.data()[i] + v.data()[i] + c.data()[i % c.size()]; + } +} +``` + +We'll define the logic for GPU inference in a separate `.cu` file like so: +```cuda +__global__ void cu_gpu_infer(float* r, float const* u, float const* v, + float* c, float alpha, std::size_t features, + std::size_t length) { + auto id = blockIdx.x * blockDim.x + threadIdx.x; + if (id < length) { + r[id] = alpha * u[id] + v[id] + c[id % features]; + } +} + +void gpu_infer(float* r, float const* u, float const* v, float* c, float alpha, + std::size_t features, std::size_t length, cudaStream_t stream) { + auto constexpr block_size = 1024; + auto grid_size = static_cast(std::max(1.0f, std::ceil(length / + static_cast(block_size))));; + cu_gpu_infer<<>>(r, u, v, c, alpha, features, length); +} +``` + +and then call it within our RapidsModel `predict` method via: + +```cpp +gpu_infer(r.data(), u.data(), v.data(), c.data(), alpha, c.size(), + u.size(), r.stream()); +``` + +After the actual inference has been performed, the one remaining task is to +call the `finalize` method of all output tensors. In this example, we have +exactly one, so the final line of our `predict` method is just: + +```cpp +r.finalize(); +``` + +To see all of this in context, check out the `src/gpu_infer.h` and +`src/gpu_infer.cu` files where GPU inference has been implemented as well as +`src/model.h` where it is used. When introducing a new source file, don't +forget to add it to CMakeLists.txt so that it will be included in the build. + +## 6. Build the backend +Having defined all the necessary logic for serving our model, we can now +actually build the server container with the new backend included. To do so, +run the following command from the base of the repository: + +```bash +docker build --build-arg BACKEND_NAME=rapids_linear -t rapids_linear . +``` + +## 7. Test + +All that remains is to test that the backend performs correctly. In this repo, +we have provided two model directories in `qa/L0_e2e/model_repository`. These +models are identical except that one is deployed on CPU and the other on GPU. +The `config.pbtxt` files are laid out exactly as discussed in step 3. + +To start the server with these models, run the following: + +```bash +docker run \ + --gpus=all \ + --rm \ + -p 8000:8000 \ + -p 8001:8001 \ + -p 8002:8002 \ + -v $PWD/qa/L0_e2e/model_repository:/models + rapids_linear +``` + +You can now submit inference requests via any Triton client. For convenience, +we will use the client provided by the `rapids_triton` package in the main +RAPIDS-Triton repo. This package is primarily designed to assist with writing +end-to-end tests for RAPIDS-Triton backends. We can install it into a conda +environment as follows: + +```bash +conda env create -f conda/environments/rapids_triton_test.yml +conda activate rapids_triton_test +python -m pip install git+https://github.com/rapidsai/rapids-triton.git#subdirectory=python +``` + +To use it for a basic test, we might execute something like the following + +```python +from rapids_triton import Client + +client = Client() + +u = np.array([[2, 2, 3, 3]], dtype='float32') +v = np.array([[-1, 1, 1, -1]], dtype='float32') +# The value of the c vector specified in c.txt +c = np.array([[1, 2, 3, 4]], dtype='float32') +alpha = 2 + +ground_truth = alpha * u + v + np.repeat(c, u.shape[0], axis=0) + +print({'r': ground_truth}) + +print(client.predict( + # Specify name of model to use for prediction + 'linear_example', + # Provide input arrays + {'u': u, 'v': v}, + # Provide size in bytes of expected output(s) + {'r': u.shape[0] * u.shape[1] * np.dtype('float32').itemsize}, + # Optionally submit request with Triton's shared memory mode + shared_mem='cuda' +)) +print(client.predict( + 'linear_example_cpu', + {'u': u, 'v': v}, + {'r': u.shape[0] * u.shape[1] * np.dtype('float32').itemsize} +)) +``` +which should give us the following output: +``` +{'r': array([[ 4., 7., 10., 9.]], dtype=float32)} +{'r': array([[ 4., 7., 10., 9.]], dtype=float32)} +{'r': array([[ 4., 7., 10., 9.]], dtype=float32)} +``` + +While this suggests that the backend is operating correctly, we probably want +to set up a more robust test with larger input data for use in CI and +development testing. See `qa/L0_e2e/test_model.py` for an example of how such a +test might be created. + +## Conclusion +This walkthrough has provided an in-depth look at how to create a Triton +backend using RAPIDS-Triton, from initial description of the backend behavior +to end-to-end testing of models deployed using this backend. Following similar +steps, you should be able to integrate almost any algorithm for deployment with +Triton. + +While we have tried to cover a wide variety of possible use cases with this +example, there is more to explore in the [RAPIDS-Triton +documentation](https://github.com/rapidsai/rapids-triton/blob/main/docs/usage.md) +itself. If there is something you would like to do with RAPIDS-Triton which +does not seem to be covered by the available API or if something is not working +as expected, please submit a feature request or bug report to the +[RAPIDS-Triton issue +tracker](https://github.com/rapidsai/rapids-triton/issues). If you think this +example could be improved or expanded in some way, please [submit a pull +request](https://github.com/rapidsai/rapids-triton-linear-example/pulls) or +[issue](https://github.com/rapidsai/rapids-triton-linear-example/issues) to +this repo. + +For additional information about using and deploying Triton after creating a +backend like this, check out the [main Triton +repo](https://github.com/triton-inference-server/server/). There you will find +information about many more tools to help you get the most out of Triton, +including: +- [`perf_analyzer`](https://github.com/triton-inference-server/server/blob/main/docs/perf_analyzer.md): + A tool to help measure throughput and latency for models deployed with Triton +- [`model_analyzer`](https://github.com/triton-inference-server/server/blob/main/docs/model_analyzer.md): + A tool to help determine what parameters will optimize throughput, latency, + or any other metric for your deployed models +- [Helm + charts](https://ngc.nvidia.com/catalog/helm-charts/nvidia:tritoninferenceserver/) + and other information to help you easily deploy Triton in any cloud service + or orchestration environment diff --git a/examples/backend_linear/cmake/modules/ConfigureCUDA.cmake b/examples/backend_linear/cmake/modules/ConfigureCUDA.cmake new file mode 100644 index 0000000..f170102 --- /dev/null +++ b/examples/backend_linear/cmake/modules/ConfigureCUDA.cmake @@ -0,0 +1,43 @@ +#============================================================================= +# Copyright (c) 2018-2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +if(DISABLE_DEPRECATION_WARNINGS) + list(APPEND RAPIDS_TRITON_CXX_FLAGS -Wno-deprecated-declarations) + list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Xcompiler=-Wno-deprecated-declarations) +endif() + +if(CMAKE_COMPILER_IS_GNUCXX) + list(APPEND RAPIDS_TRITON_CXX_FLAGS -Wall -Werror -Wno-unknown-pragmas -Wno-error=deprecated-declarations) +endif() + +list(APPEND RAPIDS_TRITON_CUDA_FLAGS --expt-extended-lambda --expt-relaxed-constexpr) + +# set warnings as errors +if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 11.2.0) + list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Werror=all-warnings) +endif() +list(APPEND RAPIDS_TRITON_CUDA_FLAGS -Xcompiler=-Wall,-Werror,-Wno-error=deprecated-declarations) + +# Option to enable line info in CUDA device compilation to allow introspection when profiling / memchecking +if(CUDA_ENABLE_LINEINFO) + list(APPEND RAPIDS_TRITON_CUDA_FLAGS -lineinfo) +endif() + +# Debug options +if(CMAKE_BUILD_TYPE MATCHES Debug) + message(VERBOSE "RAPIDS_TRITON: Building with debugging flags") + list(APPEND RAPIDS_TRITON_CUDA_FLAGS -G -Xcompiler=-rdynamic) +endif() diff --git a/examples/backend_linear/cmake/thirdparty/get_gtest.cmake b/examples/backend_linear/cmake/thirdparty/get_gtest.cmake new file mode 100644 index 0000000..02cf9cc --- /dev/null +++ b/examples/backend_linear/cmake/thirdparty/get_gtest.cmake @@ -0,0 +1,43 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_gtest VERSION) + + if(TARGET GTest::gtest) + return() + endif() + + rapids_cpm_find(GTest ${VERSION} + GLOBAL_TARGETS gtest gtest_main GTest::gtest GTest::gtest_main gmock gmock_main + CPM_ARGS + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG release-${VERSION} + GIT_SHALLOW TRUE + OPTIONS "INSTALL_GTEST OFF" + # googletest >= 1.10.0 provides a cmake config file -- use it if it exists + FIND_PACKAGE_ARGUMENTS "CONFIG" + ) + + if(NOT TARGET GTest::gtest) + add_library(GTest::gtest ALIAS gtest) + add_library(GTest::gtest_main ALIAS gtest_main) + endif() + +endfunction() + +set(RAFT_MIN_VERSION_gtest 1.10.0) + +find_and_configure_gtest(${RAFT_MIN_VERSION_gtest}) diff --git a/examples/backend_linear/cmake/thirdparty/get_rapids-triton.cmake b/examples/backend_linear/cmake/thirdparty/get_rapids-triton.cmake new file mode 100644 index 0000000..54ad51b --- /dev/null +++ b/examples/backend_linear/cmake/thirdparty/get_rapids-triton.cmake @@ -0,0 +1,46 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_backend_dev_tools) + + set(oneValueArgs VERSION FORK PINNED_TAG) + cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN} ) + + rapids_cpm_find(triton_backend ${PKG_VERSION} + GLOBAL_TARGETS triton_backend::triton_backend + BUILD_EXPORT_SET triton_backend-exports + INSTALL_EXPORT_SET triton_backend-exports + CPM_ARGS + GIT_REPOSITORY https://github.com/${PKG_FORK}/triton_developer_tools.git + GIT_TAG ${PKG_PINNED_TAG} + SOURCE_SUBDIR backend/cpp + OPTIONS + "BUILD_TESTS OFF" + "BUILD_EXAMPLE OFF" + ) + + message(VERBOSE "RAPIDS_TRITON_LINEAR: Using RAPIDS-Triton located in ${rapids_triton_SOURCE_DIR}") + +endfunction() + +# Change pinned tag here to test a commit in CI +# To use a different RAFT locally, set the CMake variable +# CPM_raft_SOURCE=/path/to/local/raft +find_and_configure_backend_dev_tools(VERSION 23.01 + FORK wphicks + PINNED_TAG fea-backend + ) diff --git a/examples/backend_linear/conda/environments/rapids_triton_dev_cuda11.4.yml b/examples/backend_linear/conda/environments/rapids_triton_dev_cuda11.4.yml new file mode 100644 index 0000000..fc39c82 --- /dev/null +++ b/examples/backend_linear/conda/environments/rapids_triton_dev_cuda11.4.yml @@ -0,0 +1,13 @@ +--- +name: rapids_triton_dev +channels: + - nvidia + - conda-forge +dependencies: + - ccache + - cmake>=3.21,!=3.25.0 + - cudatoolkit=11.5 + - libstdcxx-ng<=11.2.0 + - libgcc-ng<=11.2.0 + - ninja + - rapidjson diff --git a/examples/backend_linear/conda/environments/rapids_triton_test.yml b/examples/backend_linear/conda/environments/rapids_triton_test.yml new file mode 100644 index 0000000..8a4c4c0 --- /dev/null +++ b/examples/backend_linear/conda/environments/rapids_triton_test.yml @@ -0,0 +1,12 @@ +--- +name: rapids_triton_test +channels: + - conda-forge +dependencies: + - flake8 + - pip + - python + - pytest + - numpy + - pip: + - nvidia-pyindex diff --git a/examples/backend_linear/src/api.cc b/examples/backend_linear/src/api.cc new file mode 100644 index 0000000..00f5780 --- /dev/null +++ b/examples/backend_linear/src/api.cc @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace NAMESPACE { + +using ModelState = dev_tools::TritonModelState; +using ModelInstanceState = + dev_tools::ModelInstanceState; + +extern "C" { + +/** Confirm that backend is compatible with Triton's backend API version + */ +TRITONSERVER_Error* TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend) { + return dev_tools::triton_api::initialize(backend); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelInitialize(TRITONBACKEND_Model* model) { + return dev_tools::triton_api::model_initialize(model); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelFinalize(TRITONBACKEND_Model* model) { + return dev_tools::triton_api::model_finalize(model); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelInstanceInitialize( + TRITONBACKEND_ModelInstance* instance) { + return dev_tools::triton_api::instance_initialize(instance); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelInstanceFinalize( + TRITONBACKEND_ModelInstance* instance) { + return dev_tools::triton_api::instance_finalize(instance); +} + +TRITONSERVER_Error* TRITONBACKEND_ModelInstanceExecute( + TRITONBACKEND_ModelInstance* instance, TRITONBACKEND_Request** raw_requests, + uint32_t const request_count) { + return dev_tools::triton_api::execute( + instance, raw_requests, static_cast(request_count)); +} + +} // extern "C" + +} // namespace NAMESPACE +} // namespace backend +} // namespace triton diff --git a/examples/backend_linear/src/gpu_infer.cu b/examples/backend_linear/src/gpu_infer.cu new file mode 100644 index 0000000..e2feb6f --- /dev/null +++ b/examples/backend_linear/src/gpu_infer.cu @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include + +namespace triton { namespace backend { namespace NAMESPACE { + +namespace { +__global__ void cu_gpu_infer(float* r, float const* u, float const* v, + float* c, float alpha, std::size_t features, + std::size_t length) { + auto id = blockIdx.x * blockDim.x + threadIdx.x; + if (id < length) { + r[id] = alpha * u[id] + v[id] + c[id % features]; + } +} +} + +void gpu_infer(float* r, float const* u, float const* v, float* c, float alpha, + std::size_t features, std::size_t length, cudaStream_t stream) { + auto constexpr block_size = 1024; + auto grid_size = static_cast(std::max(1.0f, std::ceil(length / + static_cast(block_size))));; + cu_gpu_infer<<>>(r, u, v, c, alpha, features, length); +} + +}}} diff --git a/examples/backend_linear/src/gpu_infer.h b/examples/backend_linear/src/gpu_infer.h new file mode 100644 index 0000000..3400b97 --- /dev/null +++ b/examples/backend_linear/src/gpu_infer.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include +#include +#include + +namespace triton { +namespace backend { +namespace NAMESPACE { + +void gpu_infer(float* r, float const* u, float const* v, float* c, float alpha, + std::size_t features, std::size_t length, cudaStream_t stream); +} +} // namespace backend +} // namespace triton diff --git a/examples/backend_linear/src/model.h b/examples/backend_linear/src/model.h new file mode 100644 index 0000000..173918d --- /dev/null +++ b/examples/backend_linear/src/model.h @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include // dev_tools::Batch +#include // dev_tools::Buffer, dev_tools::copy +#include // dev_tools::MemoryType +#include // dev_tools::Model +#include // dev_tools::copy +#include // dev_tools::DeploymentType +#include // dev_tools::device_id_t +#include // dev_tools::log_info +#include + +namespace triton { +namespace backend { +namespace NAMESPACE { + +struct RapidsModel : dev_tools::Model { + RapidsModel(std::shared_ptr shared_state, + dev_tools::device_id_t device_id, cudaStream_t default_stream, + dev_tools::DeploymentType deployment_type, + std::string const& filepath) + : dev_tools::Model(shared_state, device_id, + default_stream, deployment_type, + filepath) {} + + void predict(dev_tools::Batch& batch) const { + auto u = get_input(batch, "u"); + auto v = get_input(batch, "v"); + + auto r = get_output(batch, "r"); + + auto alpha = get_shared_state()->alpha; + if (u.mem_type() == dev_tools::HostMemory) { + for (auto i = std::size_t{}; i < u.size(); ++i) { + r.data()[i] = + alpha * u.data()[i] + v.data()[i] + c.data()[i % c.size()]; + } + } else { + gpu_infer(r.data(), u.data(), v.data(), c.data(), alpha, c.size(), + u.size(), r.stream()); + } + + r.finalize(); + } + + void load() { + auto path = std::filesystem::path(get_filepath()); + /* If the config file does not specify a filepath for the model, + * get_filepath returns the directory where the serialized model should be + * found. It is generally good practice to provide logic to allow the use + * of a default filename so that model configurations do not always have to + * specify a path to their model */ + if (std::filesystem::is_directory(path)) { + path /= "c.txt"; + } + + // Read space-separated text file into a vector of floats + auto model_vec = std::vector{}; + auto model_file = std::ifstream(path.string()); + auto input_line = std::string{}; + std::getline(model_file, input_line); + auto input_stream = std::stringstream{input_line}; + auto value = 0.0f; + while (input_stream >> value) { + model_vec.push_back(value); + } + + // Construct buffer to hold c based on details of this model deployment + auto memory_type = dev_tools::MemoryType{}; + if constexpr (dev_tools::IS_GPU_BUILD) { + if (get_deployment_type() == dev_tools::GPUDeployment) { + memory_type = dev_tools::DeviceMemory; + } else { + memory_type = dev_tools::HostMemory; + } + } else { + memory_type = dev_tools::HostMemory; + } + + c = dev_tools::Buffer(model_vec.size(), memory_type, get_device_id(), + get_stream()); + + /* Use a Buffer view on model_vec to safely copy data to its final + * location. Making use of dev_tools::copy here provides additional safety + * checks to avoid buffer overruns. Note that the destination buffer comes + * first in dev_tools::copy calls, so we are copying *into* c */ + dev_tools::copy(c, dev_tools::Buffer(model_vec.data(), model_vec.size(), + dev_tools::HostMemory)); + } + + private: + dev_tools::Buffer c{}; +}; + +} // namespace NAMESPACE +} // namespace backend +} // namespace triton diff --git a/examples/backend_linear/src/names.h b/examples/backend_linear/src/names.h new file mode 100644 index 0000000..b134abc --- /dev/null +++ b/examples/backend_linear/src/names.h @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#define NAMESPACE rapids_linear diff --git a/examples/backend_linear/src/shared_state.h b/examples/backend_linear/src/shared_state.h new file mode 100644 index 0000000..d81e8c6 --- /dev/null +++ b/examples/backend_linear/src/shared_state.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include +#include +#include + +namespace triton { +namespace backend { +namespace NAMESPACE { + +struct RapidsSharedState : dev_tools::SharedModelState { + RapidsSharedState(std::unique_ptr&& config) + : dev_tools::SharedModelState{std::move(config)} {} + void load() { alpha = get_config_param("alpha"); } + void unload() { + dev_tools::log_info(__FILE__, __LINE__) << "Unloading shared state..."; + } + + float alpha = 1.0f; +}; + +} // namespace NAMESPACE +} // namespace backend +} // namespace triton diff --git a/qa/L0_e2e/model_repository/linear_example/1/c.txt b/qa/L0_e2e/model_repository/linear_example/1/c.txt new file mode 100644 index 0000000..a91c0af --- /dev/null +++ b/qa/L0_e2e/model_repository/linear_example/1/c.txt @@ -0,0 +1 @@ +1.0 2.0 3.0 4.0 diff --git a/qa/L0_e2e/model_repository/linear_example/config.pbtxt b/qa/L0_e2e/model_repository/linear_example/config.pbtxt new file mode 100644 index 0000000..8dbfe20 --- /dev/null +++ b/qa/L0_e2e/model_repository/linear_example/config.pbtxt @@ -0,0 +1,32 @@ +name: "linear_example" +backend: "rapids_linear" +max_batch_size: 32768 +input [ + { + name: "u" + data_type: TYPE_FP32 + dims: [ 4 ] + }, + { + name: "v" + data_type: TYPE_FP32 + dims: [ 4 ] + } +] +output [ + { + name: "r" + data_type: TYPE_FP32 + dims: [ 4 ] + } +] +instance_group [{ kind: KIND_GPU }] +parameters [ + { + key: "alpha" + value: { string_value: "2.0" } + } +] +dynamic_batching { + max_queue_delay_microseconds: 100 +} diff --git a/qa/L0_e2e/model_repository/linear_example_cpu/1/c.txt b/qa/L0_e2e/model_repository/linear_example_cpu/1/c.txt new file mode 100644 index 0000000..a91c0af --- /dev/null +++ b/qa/L0_e2e/model_repository/linear_example_cpu/1/c.txt @@ -0,0 +1 @@ +1.0 2.0 3.0 4.0 diff --git a/qa/L0_e2e/model_repository/linear_example_cpu/config.pbtxt b/qa/L0_e2e/model_repository/linear_example_cpu/config.pbtxt new file mode 100644 index 0000000..5bf4d75 --- /dev/null +++ b/qa/L0_e2e/model_repository/linear_example_cpu/config.pbtxt @@ -0,0 +1,32 @@ +name: "linear_example_cpu" +backend: "rapids_linear" +max_batch_size: 32768 +input [ + { + name: "u" + data_type: TYPE_FP32 + dims: [ 4 ] + }, + { + name: "v" + data_type: TYPE_FP32 + dims: [ 4 ] + } +] +output [ + { + name: "r" + data_type: TYPE_FP32 + dims: [ 4 ] + } +] +instance_group [{ kind: KIND_CPU }] +parameters [ + { + key: "alpha" + value: { string_value: "2.0" } + } +] +dynamic_batching { + max_queue_delay_microseconds: 100 +} diff --git a/qa/L0_e2e/test_model.py b/qa/L0_e2e/test_model.py new file mode 100644 index 0000000..bb9b87c --- /dev/null +++ b/qa/L0_e2e/test_model.py @@ -0,0 +1,68 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import pytest + +from rapids_triton import Client +from rapids_triton.testing import get_random_seed, arrays_close + +TOTAL_SAMPLES = 8192 +FEATURE_COUNT = 4 +ALPHA = 2 +C = np.array([[1, 2, 3, 4]], dtype='float32') + +@pytest.fixture +def model_inputs(): + np.random.seed(get_random_seed()) + return { + input_name: + np.random.rand(TOTAL_SAMPLES, FEATURE_COUNT).astype('float32') + for input_name in ('u', 'v') + } + +@pytest.fixture +def model_output_sizes(): + return {'r': TOTAL_SAMPLES * FEATURE_COUNT * np.dtype('float32').itemsize} + +def get_ground_truth(inputs): + u = inputs['u'] + v = inputs['v'] + return {'r': ALPHA * u + v + np.repeat(C, u.shape[0], axis=0)} + + +@pytest.mark.parametrize( + "model_name", ['linear_example', 'linear_example_cpu'] +) +def test_model(model_name, model_inputs, model_output_sizes): + client = Client() + result = client.predict(model_name, model_inputs, model_output_sizes) + # shm_result = client.predict( + # model_name, model_inputs, model_output_sizes, shared_mem='cuda' + # ) + ground_truth = get_ground_truth(model_inputs) + + for output_name in sorted(ground_truth.keys()): + arrays_close( + result[output_name], + ground_truth[output_name], + atol=1e-5, + assert_close=True + ) + # arrays_close( + # shm_result[output_name], + # ground_truth[output_name], + # atol=1e-5, + # assert_close=True + # )