diff --git a/.tekton/lightspeed-stack-0-5-pull-request.yaml b/.tekton/lightspeed-stack-0-5-pull-request.yaml new file mode 100644 index 000000000..8b2a890e9 --- /dev/null +++ b/.tekton/lightspeed-stack-0-5-pull-request.yaml @@ -0,0 +1,662 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/lightspeed-core/lightspeed-stack?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/cancel-in-progress: "true" + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch + == "release/0.5" + labels: + appstudio.openshift.io/application: lightspeed-core-0-5 + appstudio.openshift.io/component: lightspeed-stack-0-5 + pipelines.appstudio.openshift.io/type: build + name: lightspeed-stack-0-5-on-pull-request + namespace: lightspeed-core-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: output-image + value: quay.io/redhat-user-workloads/lightspeed-core-tenant/lightspeed-stack-0-5:on-pr-{{revision}} + - name: image-expires-after + value: 5d + - name: build-platforms + value: + - linux/x86_64 + - linux-c6gd2xlarge/arm64 + - name: build-source-image + value: 'true' + - name: prefetch-input + value: | + [ + { + "type": "rpm", + "path": "." + }, + { + "type": "generic", + "path": "." + }, + { + "type": "pip", + "path": ".", + "requirements_files": [ + "requirements.hashes.wheel.txt", + "requirements.hashes.source.txt", + "requirements.hermetic.txt" + ], + "requirements_build_files": ["requirements-build.txt"], + "binary": { + "packages": "aiohappyeyeballs,aiohttp,aiosignal,aiosqlite,annotated-doc,annotated-types,anyio,asyncpg,cffi,chevron,click,cryptography,datasets,dill,distro,dnspython,docstring-parser,durationpy,einops,email-validator,faiss-cpu,fire,frozenlist,fsspec,google-cloud-core,google-crc32c,google-genai,google-resumable-media,grpc-google-iam-v1,grpcio,grpcio-status,h11,hf-xet,httpcore,httpx,httpx-sse,idna,importlib-metadata,jinja2,jiter,joblib,jsonschema,jsonschema-specifications,kubernetes,lxml,markdown-it-py,mcp,mdurl,mpmath,multidict,networkx,numpy,oauthlib,packaging,pandas,peft,pillow,prometheus-client,prompt-toolkit,propcache,psycopg2-binary,pyarrow,pyasn1-modules,pycparser,pydantic,pydantic-core,pygments,python-dateutil,python-multipart,pyyaml,referencing,requests-oauthlib,rpds-py,safetensors,scikit-learn,scipy,setuptools,six,sniffio,sqlalchemy,sympy,termcolor,threadpoolctl,tiktoken,tokenizers,torch,tqdm,transformers,tree-sitter,triton,typing-extensions,typing-inspection,tzdata,urllib3,websocket-client,websockets,wrapt,xxhash,yarl,zipp,uv,pip,maturin", + "os": "linux", + "arch": "x86_64,aarch64", + "py_version": 312 + } + } + ] + - name: hermetic + value: 'true' + - name: dockerfile + value: Containerfile + - name: build-args-file + value: build-args-konflux.conf + pipelineSpec: + description: | + This pipeline is ideal for building multi-arch container images from a Containerfile while maintaining trust after pipeline customization. + + _Uses `buildah` to create a multi-platform container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. This pipeline requires that the [multi platform controller](https://github.com/konflux-ci/multi-platform-controller) is deployed and configured on your Konflux instance. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta?tab=tags)_ + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: Path to the source code of an application's component from where to build image. + name: path-context + type: string + - default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter path-context + name: dockerfile + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched + name: prefetch-input + type: string + - default: "" + description: Image tag expiration time, time values could be something like 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + type: string + - default: "false" + description: Build a source image. + name: build-source-image + type: string + - default: "true" + description: Add built image into an OCI image index + name: build-image-index + type: string + - default: docker + description: The format for the resulting image's mediaType. Valid values are oci or docker. + name: buildah-format + type: string + - default: [] + description: Array of --build-arg values ("arg=value" strings) for buildah + name: build-args + type: array + - default: "" + description: Path to a file with build arguments for buildah, see https://www.mankier.com/1/buildah-build#--build-arg-file + name: build-args-file + type: string + - default: "false" + description: Whether to enable privileged mode, should be used only with remote VMs + name: privileged-nested + type: string + - default: + - linux/x86_64 + description: List of platforms to build the container images on. The available set of values is determined by the configuration of the multi-platform-controller. + name: build-platforms + type: array + - name: enable-cache-proxy + default: 'false' + description: Enable cache proxy configuration + type: string + - name: enable-package-registry-proxy + default: 'true' + description: Use the package registry proxy when prefetching dependencies + type: string + - name: sast-target-dirs + type: string + default: . + description: Target directories to scan with SAST tools. Multiple values should be separated with commas. + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + tasks: + - name: init + params: + - name: enable-cache-proxy + value: $(params.enable-cache-proxy) + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-init:0.4@sha256:5a423246792ac501ea279229b42ee57da9927da441c04b5c9ff86817b0856b08 + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + - name: ociStorage + value: $(params.output-image).git + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:d30f13dd15daf89dd6dc645243b3444d35570d13f7840c3fd65e366022515205 + - name: kind + value: task + resolver: bundles + workspaces: + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) + - name: ociStorage + value: $(params.output-image).prefetch + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + - name: enable-package-registry-proxy + value: $(params.enable-package-registry-proxy) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.3@sha256:3dc78afbf3a441e0280067433cb28ea3d2d0088ec214c73bf063f145b4f273ef + - name: kind + value: task + resolver: bundles + workspaces: + - name: git-basic-auth + workspace: git-auth + - name: netrc + workspace: netrc + - matrix: + params: + - name: PLATFORM + value: + - $(params.build-platforms) + name: build-images + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: BUILD_ARGS + value: + - $(params.build-args[*]) + - name: BUILD_ARGS_FILE + value: $(params.build-args-file) + - name: PRIVILEGED_NESTED + value: $(params.privileged-nested) + - name: SOURCE_URL + value: $(tasks.clone-repository.results.url) + - name: BUILDAH_FORMAT + value: $(params.buildah-format) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: IMAGE_APPEND_PLATFORM + value: "true" + - name: HTTP_PROXY + value: $(tasks.init.results.http-proxy) + - name: NO_PROXY + value: $(tasks.init.results.no-proxy) + runAfter: + - prefetch-dependencies + taskRef: + params: + - name: name + value: buildah-remote-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.9@sha256:77007259cc87f32d63d2c201226aadaab98313cfd4e02b46abc243c4d2cc27bd + - name: kind + value: task + resolver: bundles + - name: build-image-index + params: + - name: IMAGE + value: $(params.output-image) + - name: ALWAYS_BUILD_INDEX + value: $(params.build-image-index) + - name: IMAGES + value: + - $(tasks.build-images.results.IMAGE_REF[*]) + - name: BUILDAH_FORMAT + value: $(params.buildah-format) + runAfter: + - build-images + taskRef: + params: + - name: name + value: build-image-index + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.3@sha256:b33bfa8dc27dbf459f0779598ba45dcaa490bcc9f8efe1652bcf360ec8cb5582 + - name: kind + value: task + resolver: bundles + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: BINARY_IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: source-build-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:8567bb7bf8fa9147c96b297533336fa7079ecf972cb86c09ccdd6bddedb25711 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.build-source-image) + operator: in + values: + - "true" + - name: deprecated-base-image-check + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:e78d0d3baf3c8cfc1a5ad278196b74032d9568b143a87c7a79ab780fedfb296e + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - matrix: + params: + - name: image-platform + value: + - $(params.build-platforms) + name: clair-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:8fad4c2e2f470f82ee43d6b2ac72327b4d9c6e9cb514a678911c1c9359c29894 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - matrix: + params: + - name: platform + value: + - $(params.build-platforms) + name: ecosystem-cert-preflight-checks + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: ecosystem-cert-preflight-checks + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:88f4fd6d7812a3c46f120f3035974f5fb8cb06b5e3e927badf6e8370f1516a88 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: ARGS + value: --project-name=lightspeed-stack --report --org=dca2ca89-7e51-4a3a-b7a5-6ad5633057b8 + - name: TARGET_DIRS + value: $(params.sast-target-dirs) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-snyk-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:0ebf28a0abd5a167438d4628938a74ade6f00a44a4b7ed1cfa9cfc57a5b24748 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - matrix: + params: + - name: image-arch + value: + - $(params.build-platforms) + name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:567cb66bd2e1f4b58b9d4d756f3317fc62479e0b40aa0de66094b1f12d296cfc + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-coverity-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: BUILD_ARGS + value: + - $(params.build-args[*]) + - name: BUILD_ARGS_FILE + value: $(params.build-args-file) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) + runAfter: + - coverity-availability-check + taskRef: + params: + - name: name + value: sast-coverity-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-coverity-check-oci-ta:0.3@sha256:e92d00ed858233d0096627861192d3e4fc013cf1559c0d0b0ea0657d3377ce75 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - input: $(tasks.coverity-availability-check.results.STATUS) + operator: in + values: + - success + - name: coverity-availability-check + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: coverity-availability-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-coverity-availability-check:0.2@sha256:8b501440a960aec446db2ebc6625a49d0317a9fc7bf0f7bd9b18cb63052db7de + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-shell-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-shell-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:3cbb3535af6e7d4396858179a6427caaffb2e68775594795692fc01f28ae313f + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-unicode-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-unicode-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:223812001607b07f0e07d56bef7b7d619144e660c0c57f21ddd44ce0c8c4785b + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: apply-tags + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: apply-tags + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3@sha256:a291081de7fb27f832c6fc3c4b078acf7e6162ca4c085db38b118ca87e8b5b66 + - name: kind + value: task + resolver: bundles + - name: push-dockerfile + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: push-dockerfile-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.3@sha256:7855471abfe87de080b914f2f3ca27c59e64f6448a7c2435e51435b764494c71 + - name: kind + value: task + resolver: bundles + - name: rpms-signature-scan + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: rpms-signature-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:237c54b069d16c3785d1302f19be309aa6c0ae2313d446e30cb74671e07ca676 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: git-auth + optional: true + - name: netrc + optional: true + timeouts: + pipeline: 4h + tasks: 4h + taskRunTemplate: + serviceAccountName: build-pipeline-lightspeed-stack-0-5 + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/.tekton/lightspeed-stack-0-5-push.yaml b/.tekton/lightspeed-stack-0-5-push.yaml new file mode 100644 index 000000000..d7521d360 --- /dev/null +++ b/.tekton/lightspeed-stack-0-5-push.yaml @@ -0,0 +1,651 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/lightspeed-core/lightspeed-stack?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/cancel-in-progress: "false" + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch + == "release/0.5" + labels: + appstudio.openshift.io/application: lightspeed-core-0-5 + appstudio.openshift.io/component: lightspeed-stack-0-5 + pipelines.appstudio.openshift.io/type: build + name: lightspeed-stack-0-5-on-push + namespace: lightspeed-core-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: output-image + value: quay.io/redhat-user-workloads/lightspeed-core-tenant/lightspeed-stack-0-5:{{revision}} + - name: build-platforms + value: + - linux/x86_64 + - linux-c6gd2xlarge/arm64 + - name: build-source-image + value: 'true' + - name: prefetch-input + value: | + [ + { + "type": "rpm", + "path": "." + }, + { + "type": "generic", + "path": "." + }, + { + "type": "pip", + "path": ".", + "requirements_files": [ + "requirements.hashes.wheel.txt", + "requirements.hashes.source.txt", + "requirements.hermetic.txt" + ], + "requirements_build_files": ["requirements-build.txt"], + "binary": { + "packages": "aiohappyeyeballs,aiohttp,aiosignal,aiosqlite,annotated-doc,annotated-types,anyio,asyncpg,cffi,chevron,click,cryptography,datasets,dill,distro,dnspython,docstring-parser,durationpy,einops,email-validator,faiss-cpu,fire,frozenlist,fsspec,google-cloud-core,google-crc32c,google-genai,google-resumable-media,grpc-google-iam-v1,grpcio,grpcio-status,h11,hf-xet,httpcore,httpx,httpx-sse,idna,importlib-metadata,jinja2,jiter,joblib,jsonschema,jsonschema-specifications,kubernetes,lxml,markdown-it-py,mcp,mdurl,mpmath,multidict,networkx,numpy,oauthlib,packaging,pandas,peft,pillow,prometheus-client,prompt-toolkit,propcache,psycopg2-binary,pyarrow,pyasn1-modules,pycparser,pydantic,pydantic-core,pygments,python-dateutil,python-multipart,pyyaml,referencing,requests-oauthlib,rpds-py,safetensors,scikit-learn,scipy,setuptools,six,sniffio,sqlalchemy,sympy,termcolor,threadpoolctl,tiktoken,tokenizers,torch,tqdm,transformers,tree-sitter,triton,typing-extensions,typing-inspection,tzdata,urllib3,websocket-client,websockets,wrapt,xxhash,yarl,zipp,uv,pip,maturin", + "os": "linux", + "arch": "x86_64,aarch64", + "py_version": 312 + } + } + ] + - name: hermetic + value: 'true' + - name: dockerfile + value: Containerfile + - name: build-args-file + value: build-args-konflux.conf + pipelineSpec: + description: | + This pipeline is ideal for building multi-arch container images from a Containerfile while maintaining trust after pipeline customization. + + _Uses `buildah` to create a multi-platform container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. This pipeline requires that the [multi platform controller](https://github.com/konflux-ci/multi-platform-controller) is deployed and configured on your Konflux instance. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta?tab=tags)_ + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: Path to the source code of an application's component from where to build image. + name: path-context + type: string + - default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter path-context + name: dockerfile + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched + name: prefetch-input + type: string + - default: "" + description: Image tag expiration time, time values could be something like 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + type: string + - default: "false" + description: Build a source image. + name: build-source-image + type: string + - default: "true" + description: Add built image into an OCI image index + name: build-image-index + type: string + - default: docker + description: The format for the resulting image's mediaType. Valid values are oci or docker. + name: buildah-format + type: string + - default: [] + description: Array of --build-arg values ("arg=value" strings) for buildah + name: build-args + type: array + - default: "" + description: Path to a file with build arguments for buildah, see https://www.mankier.com/1/buildah-build#--build-arg-file + name: build-args-file + type: string + - default: "false" + description: Whether to enable privileged mode, should be used only with remote VMs + name: privileged-nested + type: string + - default: + - linux/x86_64 + description: List of platforms to build the container images on. The available set of values is determined by the configuration of the multi-platform-controller. + name: build-platforms + type: array + - name: enable-package-registry-proxy + default: 'true' + description: Use the package registry proxy when prefetching dependencies + type: string + - name: sast-target-dirs + type: string + default: . + description: Target directories to scan with SAST tools. Multiple values should be separated with commas. + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + tasks: + - name: init + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-init:0.4@sha256:5a423246792ac501ea279229b42ee57da9927da441c04b5c9ff86817b0856b08 + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + - name: ociStorage + value: $(params.output-image).git + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:d30f13dd15daf89dd6dc645243b3444d35570d13f7840c3fd65e366022515205 + - name: kind + value: task + resolver: bundles + workspaces: + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) + - name: ociStorage + value: $(params.output-image).prefetch + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + - name: enable-package-registry-proxy + value: $(params.enable-package-registry-proxy) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.3@sha256:3dc78afbf3a441e0280067433cb28ea3d2d0088ec214c73bf063f145b4f273ef + - name: kind + value: task + resolver: bundles + workspaces: + - name: git-basic-auth + workspace: git-auth + - name: netrc + workspace: netrc + - matrix: + params: + - name: PLATFORM + value: + - $(params.build-platforms) + name: build-images + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: BUILD_ARGS + value: + - $(params.build-args[*]) + - name: BUILD_ARGS_FILE + value: $(params.build-args-file) + - name: PRIVILEGED_NESTED + value: $(params.privileged-nested) + - name: SOURCE_URL + value: $(tasks.clone-repository.results.url) + - name: BUILDAH_FORMAT + value: $(params.buildah-format) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: IMAGE_APPEND_PLATFORM + value: "true" + runAfter: + - prefetch-dependencies + taskRef: + params: + - name: name + value: buildah-remote-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.9@sha256:77007259cc87f32d63d2c201226aadaab98313cfd4e02b46abc243c4d2cc27bd + - name: kind + value: task + resolver: bundles + - name: build-image-index + params: + - name: IMAGE + value: $(params.output-image) + - name: ALWAYS_BUILD_INDEX + value: $(params.build-image-index) + - name: IMAGES + value: + - $(tasks.build-images.results.IMAGE_REF[*]) + - name: BUILDAH_FORMAT + value: $(params.buildah-format) + runAfter: + - build-images + taskRef: + params: + - name: name + value: build-image-index + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.3@sha256:b33bfa8dc27dbf459f0779598ba45dcaa490bcc9f8efe1652bcf360ec8cb5582 + - name: kind + value: task + resolver: bundles + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: BINARY_IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: source-build-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:8567bb7bf8fa9147c96b297533336fa7079ecf972cb86c09ccdd6bddedb25711 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.build-source-image) + operator: in + values: + - "true" + - name: deprecated-base-image-check + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:e78d0d3baf3c8cfc1a5ad278196b74032d9568b143a87c7a79ab780fedfb296e + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - matrix: + params: + - name: image-platform + value: + - $(params.build-platforms) + name: clair-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:8fad4c2e2f470f82ee43d6b2ac72327b4d9c6e9cb514a678911c1c9359c29894 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - matrix: + params: + - name: platform + value: + - $(params.build-platforms) + name: ecosystem-cert-preflight-checks + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: ecosystem-cert-preflight-checks + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:88f4fd6d7812a3c46f120f3035974f5fb8cb06b5e3e927badf6e8370f1516a88 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: ARGS + value: --project-name=lightspeed-stack --report --org=dca2ca89-7e51-4a3a-b7a5-6ad5633057b8 + - name: TARGET_DIRS + value: $(params.sast-target-dirs) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-snyk-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:0ebf28a0abd5a167438d4628938a74ade6f00a44a4b7ed1cfa9cfc57a5b24748 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - matrix: + params: + - name: image-arch + value: + - $(params.build-platforms) + name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:567cb66bd2e1f4b58b9d4d756f3317fc62479e0b40aa0de66094b1f12d296cfc + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-coverity-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: BUILD_ARGS + value: + - $(params.build-args[*]) + - name: BUILD_ARGS_FILE + value: $(params.build-args-file) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) + runAfter: + - coverity-availability-check + taskRef: + params: + - name: name + value: sast-coverity-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-coverity-check-oci-ta:0.3@sha256:e92d00ed858233d0096627861192d3e4fc013cf1559c0d0b0ea0657d3377ce75 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - input: $(tasks.coverity-availability-check.results.STATUS) + operator: in + values: + - success + - name: coverity-availability-check + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: coverity-availability-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-coverity-availability-check:0.2@sha256:8b501440a960aec446db2ebc6625a49d0317a9fc7bf0f7bd9b18cb63052db7de + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-shell-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-shell-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:3cbb3535af6e7d4396858179a6427caaffb2e68775594795692fc01f28ae313f + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-unicode-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-unicode-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:223812001607b07f0e07d56bef7b7d619144e660c0c57f21ddd44ce0c8c4785b + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: apply-tags + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: ADDITIONAL_TAGS + value: + - $(tasks.clone-repository.results.short-commit) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: apply-tags + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3@sha256:a291081de7fb27f832c6fc3c4b078acf7e6162ca4c085db38b118ca87e8b5b66 + - name: kind + value: task + resolver: bundles + - name: push-dockerfile + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: push-dockerfile-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.3@sha256:7855471abfe87de080b914f2f3ca27c59e64f6448a7c2435e51435b764494c71 + - name: kind + value: task + resolver: bundles + - name: rpms-signature-scan + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: rpms-signature-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:237c54b069d16c3785d1302f19be309aa6c0ae2313d446e30cb74671e07ca676 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: git-auth + optional: true + - name: netrc + optional: true + timeouts: + pipeline: 4h + tasks: 4h + taskRunTemplate: + serviceAccountName: build-pipeline-lightspeed-stack-0-5 + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/.tekton/lightspeed-stack-pull-request.yaml b/.tekton/lightspeed-stack-pull-request.yaml index a98fe68b1..3067e5eb9 100644 --- a/.tekton/lightspeed-stack-pull-request.yaml +++ b/.tekton/lightspeed-stack-pull-request.yaml @@ -145,6 +145,14 @@ spec: default: 'false' description: Enable cache proxy configuration type: string + - name: enable-package-registry-proxy + default: 'true' + description: Use the package registry proxy when prefetching dependencies + type: string + - name: sast-target-dirs + type: string + default: . + description: Target directories to scan with SAST tools. Multiple values should be separated with commas. results: - description: "" name: IMAGE_URL @@ -168,7 +176,7 @@ spec: - name: name value: init - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-init:0.4@sha256:288f3106118edc1d0f0c79a89c960abf5841a4dd8bc3f38feb10527253105b19 + value: quay.io/konflux-ci/tekton-catalog/task-init:0.4@sha256:5a423246792ac501ea279229b42ee57da9927da441c04b5c9ff86817b0856b08 - name: kind value: task resolver: bundles @@ -189,7 +197,7 @@ spec: - name: name value: git-clone-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:2c388d28651457db60bb90287e7d8c3680303197196e4476878d98d81e8b6dc9 + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:d30f13dd15daf89dd6dc645243b3444d35570d13f7840c3fd65e366022515205 - name: kind value: task resolver: bundles @@ -206,6 +214,8 @@ spec: value: $(params.output-image).prefetch - name: ociArtifactExpiresAfter value: $(params.image-expires-after) + - name: enable-package-registry-proxy + value: $(params.enable-package-registry-proxy) runAfter: - clone-repository taskRef: @@ -213,7 +223,7 @@ spec: - name: name value: prefetch-dependencies-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.3@sha256:2229dbc5e15acc0a6d8aec526465aeb0ad54e269c311ac3d0aba88013845e308 + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.3@sha256:3dc78afbf3a441e0280067433cb28ea3d2d0088ec214c73bf063f145b4f273ef - name: kind value: task resolver: bundles @@ -271,7 +281,7 @@ spec: - name: name value: buildah-remote-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.9@sha256:a9ca472e297388d6ef8d1f51ee205abee6076aed7c5356ec0df84f14a2e78ad8 + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.9@sha256:77007259cc87f32d63d2c201226aadaab98313cfd4e02b46abc243c4d2cc27bd - name: kind value: task resolver: bundles @@ -279,10 +289,6 @@ spec: params: - name: IMAGE value: $(params.output-image) - - name: COMMIT_SHA - value: $(tasks.clone-repository.results.commit) - - name: IMAGE_EXPIRES_AFTER - value: $(params.image-expires-after) - name: ALWAYS_BUILD_INDEX value: $(params.build-image-index) - name: IMAGES @@ -297,7 +303,7 @@ spec: - name: name value: build-image-index - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.2@sha256:c7b0f7e1f743040d99a3532abbdfddc9484f80fd559a75171c97499c3eb5d163 + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.3@sha256:b33bfa8dc27dbf459f0779598ba45dcaa490bcc9f8efe1652bcf360ec8cb5582 - name: kind value: task resolver: bundles @@ -318,7 +324,7 @@ spec: - name: name value: source-build-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:362f0475df00e7dfb5f15dea0481d1b68b287f60411718d70a23da3c059a5613 + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:8567bb7bf8fa9147c96b297533336fa7079ecf972cb86c09ccdd6bddedb25711 - name: kind value: task resolver: bundles @@ -340,7 +346,7 @@ spec: - name: name value: deprecated-image-check - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:5ff16b7e6b4a8aa1adb352e74b9f831f77ff97bafd1b89ddb0038d63335f1a67 + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:e78d0d3baf3c8cfc1a5ad278196b74032d9568b143a87c7a79ab780fedfb296e - name: kind value: task resolver: bundles @@ -367,7 +373,7 @@ spec: - name: name value: clair-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:3fa03be0280f33d7070ea53f26d53e727199737a7a2b9a59a95071ae40a999ac + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:8fad4c2e2f470f82ee43d6b2ac72327b4d9c6e9cb514a678911c1c9359c29894 - name: kind value: task resolver: bundles @@ -392,7 +398,7 @@ spec: - name: name value: ecosystem-cert-preflight-checks - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:b4ac586edea81dcd25dfc17f1bd57899825be2b443e48d572cd05ce058f153bb + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:88f4fd6d7812a3c46f120f3035974f5fb8cb06b5e3e927badf6e8370f1516a88 - name: kind value: task resolver: bundles @@ -413,6 +419,8 @@ spec: value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - name: ARGS value: --project-name=lightspeed-stack --report --org=dca2ca89-7e51-4a3a-b7a5-6ad5633057b8 + - name: TARGET_DIRS + value: $(params.sast-target-dirs) runAfter: - build-image-index taskRef: @@ -420,7 +428,7 @@ spec: - name: name value: sast-snyk-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:d83becbfefe2aa39971c3d37bdc23489b745e22fd86cf4872455a133f8cb274f + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:0ebf28a0abd5a167438d4628938a74ade6f00a44a4b7ed1cfa9cfc57a5b24748 - name: kind value: task resolver: bundles @@ -447,7 +455,7 @@ spec: - name: name value: clamav-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:9f18b216ce71a66909e7cb17d9b34526c02d73cf12884ba32d1f10614f7b9f5a + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:567cb66bd2e1f4b58b9d4d756f3317fc62479e0b40aa0de66094b1f12d296cfc - name: kind value: task resolver: bundles @@ -485,6 +493,8 @@ spec: value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - name: CACHI2_ARTIFACT value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) runAfter: - coverity-availability-check taskRef: @@ -492,7 +502,7 @@ spec: - name: name value: sast-coverity-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-coverity-check-oci-ta:0.3@sha256:47f4e2d0881ac8c43a1ea1e2375bb2591dff34b5aa8c7366a043652d1eed499c + value: quay.io/konflux-ci/tekton-catalog/task-sast-coverity-check-oci-ta:0.3@sha256:e92d00ed858233d0096627861192d3e4fc013cf1559c0d0b0ea0657d3377ce75 - name: kind value: task resolver: bundles @@ -513,7 +523,7 @@ spec: - name: name value: coverity-availability-check - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-coverity-availability-check:0.2@sha256:de35caf2f090e3275cfd1019ea50d9662422e904fb4aebd6ea29fb53a1ad57f5 + value: quay.io/konflux-ci/tekton-catalog/task-coverity-availability-check:0.2@sha256:8b501440a960aec446db2ebc6625a49d0317a9fc7bf0f7bd9b18cb63052db7de - name: kind value: task resolver: bundles @@ -532,6 +542,8 @@ spec: value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - name: CACHI2_ARTIFACT value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) runAfter: - build-image-index taskRef: @@ -539,7 +551,7 @@ spec: - name: name value: sast-shell-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:6f047f52c04ee6e4d2cb25af46e3ea92b235f6c5e02da540fb7ef0b90718bc0a + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:3cbb3535af6e7d4396858179a6427caaffb2e68775594795692fc01f28ae313f - name: kind value: task resolver: bundles @@ -558,6 +570,8 @@ spec: value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - name: CACHI2_ARTIFACT value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) runAfter: - build-image-index taskRef: @@ -565,7 +579,7 @@ spec: - name: name value: sast-unicode-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:55006815522c57c1f83451dc0cba723ff7427dbac48553538b75cda7bf886d79 + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:223812001607b07f0e07d56bef7b7d619144e660c0c57f21ddd44ce0c8c4785b - name: kind value: task resolver: bundles @@ -587,7 +601,7 @@ spec: - name: name value: apply-tags - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3@sha256:aa62b41861c09e2e59c69cc6e9a1f740bf0c81e6a1eb03f57f59dfda0f65840e + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3@sha256:a291081de7fb27f832c6fc3c4b078acf7e6162ca4c085db38b118ca87e8b5b66 - name: kind value: task resolver: bundles @@ -610,7 +624,7 @@ spec: - name: name value: push-dockerfile-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.3@sha256:1bc2d0f26b89259db090a47bb38217c82c05e335d626653d184adf1d196ca131 + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.3@sha256:7855471abfe87de080b914f2f3ca27c59e64f6448a7c2435e51435b764494c71 - name: kind value: task resolver: bundles @@ -627,7 +641,7 @@ spec: - name: name value: rpms-signature-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:3d48fa2fcc898bae9ce730a71789c34758a767b44bc36fd24938779deef4ee96 + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:237c54b069d16c3785d1302f19be309aa6c0ae2313d446e30cb74671e07ca676 - name: kind value: task resolver: bundles diff --git a/.tekton/lightspeed-stack-push.yaml b/.tekton/lightspeed-stack-push.yaml index 69f29bad6..a1c646d3c 100644 --- a/.tekton/lightspeed-stack-push.yaml +++ b/.tekton/lightspeed-stack-push.yaml @@ -133,6 +133,14 @@ spec: description: List of platforms to build the container images on. The available set of values is determined by the configuration of the multi-platform-controller. name: build-platforms type: array + - name: enable-package-registry-proxy + default: 'true' + description: Use the package registry proxy when prefetching dependencies + type: string + - name: sast-target-dirs + type: string + default: . + description: Target directories to scan with SAST tools. Multiple values should be separated with commas. results: - description: "" name: IMAGE_URL @@ -153,7 +161,7 @@ spec: - name: name value: init - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-init:0.4@sha256:288f3106118edc1d0f0c79a89c960abf5841a4dd8bc3f38feb10527253105b19 + value: quay.io/konflux-ci/tekton-catalog/task-init:0.4@sha256:5a423246792ac501ea279229b42ee57da9927da441c04b5c9ff86817b0856b08 - name: kind value: task resolver: bundles @@ -174,7 +182,7 @@ spec: - name: name value: git-clone-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:2c388d28651457db60bb90287e7d8c3680303197196e4476878d98d81e8b6dc9 + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:d30f13dd15daf89dd6dc645243b3444d35570d13f7840c3fd65e366022515205 - name: kind value: task resolver: bundles @@ -191,6 +199,8 @@ spec: value: $(params.output-image).prefetch - name: ociArtifactExpiresAfter value: $(params.image-expires-after) + - name: enable-package-registry-proxy + value: $(params.enable-package-registry-proxy) runAfter: - clone-repository taskRef: @@ -198,7 +208,7 @@ spec: - name: name value: prefetch-dependencies-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.3@sha256:2229dbc5e15acc0a6d8aec526465aeb0ad54e269c311ac3d0aba88013845e308 + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.3@sha256:3dc78afbf3a441e0280067433cb28ea3d2d0088ec214c73bf063f145b4f273ef - name: kind value: task resolver: bundles @@ -252,7 +262,7 @@ spec: - name: name value: buildah-remote-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.9@sha256:a9ca472e297388d6ef8d1f51ee205abee6076aed7c5356ec0df84f14a2e78ad8 + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.9@sha256:77007259cc87f32d63d2c201226aadaab98313cfd4e02b46abc243c4d2cc27bd - name: kind value: task resolver: bundles @@ -260,10 +270,6 @@ spec: params: - name: IMAGE value: $(params.output-image) - - name: COMMIT_SHA - value: $(tasks.clone-repository.results.commit) - - name: IMAGE_EXPIRES_AFTER - value: $(params.image-expires-after) - name: ALWAYS_BUILD_INDEX value: $(params.build-image-index) - name: IMAGES @@ -278,7 +284,7 @@ spec: - name: name value: build-image-index - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.2@sha256:c7b0f7e1f743040d99a3532abbdfddc9484f80fd559a75171c97499c3eb5d163 + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.3@sha256:b33bfa8dc27dbf459f0779598ba45dcaa490bcc9f8efe1652bcf360ec8cb5582 - name: kind value: task resolver: bundles @@ -299,7 +305,7 @@ spec: - name: name value: source-build-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:362f0475df00e7dfb5f15dea0481d1b68b287f60411718d70a23da3c059a5613 + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:8567bb7bf8fa9147c96b297533336fa7079ecf972cb86c09ccdd6bddedb25711 - name: kind value: task resolver: bundles @@ -321,7 +327,7 @@ spec: - name: name value: deprecated-image-check - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:5ff16b7e6b4a8aa1adb352e74b9f831f77ff97bafd1b89ddb0038d63335f1a67 + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:e78d0d3baf3c8cfc1a5ad278196b74032d9568b143a87c7a79ab780fedfb296e - name: kind value: task resolver: bundles @@ -348,7 +354,7 @@ spec: - name: name value: clair-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:3fa03be0280f33d7070ea53f26d53e727199737a7a2b9a59a95071ae40a999ac + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:8fad4c2e2f470f82ee43d6b2ac72327b4d9c6e9cb514a678911c1c9359c29894 - name: kind value: task resolver: bundles @@ -373,7 +379,7 @@ spec: - name: name value: ecosystem-cert-preflight-checks - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:b4ac586edea81dcd25dfc17f1bd57899825be2b443e48d572cd05ce058f153bb + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:88f4fd6d7812a3c46f120f3035974f5fb8cb06b5e3e927badf6e8370f1516a88 - name: kind value: task resolver: bundles @@ -394,6 +400,8 @@ spec: value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) - name: ARGS value: --project-name=lightspeed-stack --report --org=dca2ca89-7e51-4a3a-b7a5-6ad5633057b8 + - name: TARGET_DIRS + value: $(params.sast-target-dirs) runAfter: - build-image-index taskRef: @@ -401,7 +409,7 @@ spec: - name: name value: sast-snyk-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:d83becbfefe2aa39971c3d37bdc23489b745e22fd86cf4872455a133f8cb274f + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:0ebf28a0abd5a167438d4628938a74ade6f00a44a4b7ed1cfa9cfc57a5b24748 - name: kind value: task resolver: bundles @@ -428,7 +436,7 @@ spec: - name: name value: clamav-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:9f18b216ce71a66909e7cb17d9b34526c02d73cf12884ba32d1f10614f7b9f5a + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:567cb66bd2e1f4b58b9d4d756f3317fc62479e0b40aa0de66094b1f12d296cfc - name: kind value: task resolver: bundles @@ -466,6 +474,8 @@ spec: value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - name: CACHI2_ARTIFACT value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) runAfter: - coverity-availability-check taskRef: @@ -473,7 +483,7 @@ spec: - name: name value: sast-coverity-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-coverity-check-oci-ta:0.3@sha256:47f4e2d0881ac8c43a1ea1e2375bb2591dff34b5aa8c7366a043652d1eed499c + value: quay.io/konflux-ci/tekton-catalog/task-sast-coverity-check-oci-ta:0.3@sha256:e92d00ed858233d0096627861192d3e4fc013cf1559c0d0b0ea0657d3377ce75 - name: kind value: task resolver: bundles @@ -494,7 +504,7 @@ spec: - name: name value: coverity-availability-check - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-coverity-availability-check:0.2@sha256:de35caf2f090e3275cfd1019ea50d9662422e904fb4aebd6ea29fb53a1ad57f5 + value: quay.io/konflux-ci/tekton-catalog/task-coverity-availability-check:0.2@sha256:8b501440a960aec446db2ebc6625a49d0317a9fc7bf0f7bd9b18cb63052db7de - name: kind value: task resolver: bundles @@ -513,6 +523,8 @@ spec: value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - name: CACHI2_ARTIFACT value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) runAfter: - build-image-index taskRef: @@ -520,7 +532,7 @@ spec: - name: name value: sast-shell-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:6f047f52c04ee6e4d2cb25af46e3ea92b235f6c5e02da540fb7ef0b90718bc0a + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:3cbb3535af6e7d4396858179a6427caaffb2e68775594795692fc01f28ae313f - name: kind value: task resolver: bundles @@ -539,6 +551,8 @@ spec: value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) - name: CACHI2_ARTIFACT value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) runAfter: - build-image-index taskRef: @@ -546,7 +560,7 @@ spec: - name: name value: sast-unicode-check-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:55006815522c57c1f83451dc0cba723ff7427dbac48553538b75cda7bf886d79 + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:223812001607b07f0e07d56bef7b7d619144e660c0c57f21ddd44ce0c8c4785b - name: kind value: task resolver: bundles @@ -571,7 +585,7 @@ spec: - name: name value: apply-tags - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3@sha256:aa62b41861c09e2e59c69cc6e9a1f740bf0c81e6a1eb03f57f59dfda0f65840e + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3@sha256:a291081de7fb27f832c6fc3c4b078acf7e6162ca4c085db38b118ca87e8b5b66 - name: kind value: task resolver: bundles @@ -594,7 +608,7 @@ spec: - name: name value: push-dockerfile-oci-ta - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.3@sha256:1bc2d0f26b89259db090a47bb38217c82c05e335d626653d184adf1d196ca131 + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.3@sha256:7855471abfe87de080b914f2f3ca27c59e64f6448a7c2435e51435b764494c71 - name: kind value: task resolver: bundles @@ -611,7 +625,7 @@ spec: - name: name value: rpms-signature-scan - name: bundle - value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:3d48fa2fcc898bae9ce730a71789c34758a767b44bc36fd24938779deef4ee96 + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:237c54b069d16c3785d1302f19be309aa6c0ae2313d446e30cb74671e07ca676 - name: kind value: task resolver: bundles diff --git a/.tekton/lightspeed-stack-release-0-5-pull-request.yaml b/.tekton/lightspeed-stack-release-0-5-pull-request.yaml new file mode 100644 index 000000000..843b8c215 --- /dev/null +++ b/.tekton/lightspeed-stack-release-0-5-pull-request.yaml @@ -0,0 +1,602 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/lightspeed-core/lightspeed-stack?rev={{revision}} + build.appstudio.redhat.com/commit_sha: "{{revision}}" + build.appstudio.redhat.com/pull_request_number: "{{pull_request_number}}" + build.appstudio.redhat.com/target_branch: "{{target_branch}}" + pipelinesascode.tekton.dev/cancel-in-progress: "true" + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: + event == "pull_request" && target_branch + == "release/0.5" + creationTimestamp: null + labels: + appstudio.openshift.io/application: lightspeed-stack + appstudio.openshift.io/component: lightspeed-stack-release-0-5 + pipelines.appstudio.openshift.io/type: build + name: lightspeed-stack-release-0-5-on-pull-request + namespace: lightspeed-core-tenant +spec: + params: + - name: git-url + value: "{{source_url}}" + - name: revision + value: "{{revision}}" + - name: output-image + value: quay.io/redhat-user-workloads/lightspeed-core-tenant/lightspeed-stack-release-0-5:on-pr-{{revision}} + - name: image-expires-after + value: 5d + - name: build-platforms + value: + - linux/x86_64 + - linux-c6gd2xlarge/arm64 + - name: build-source-image + value: "true" + - name: prefetch-input + value: | + [ + { + "type": "rpm", + "path": "." + }, + { + "type": "generic", + "path": "." + }, + { + "type": "pip", + "path": ".", + "requirements_files": [ + "requirements.hashes.wheel.txt", + "requirements.hashes.source.txt", + "requirements.hermetic.txt" + ], + "requirements_build_files": ["requirements-build.txt"], + "binary": { + "packages": "aiohappyeyeballs,aiohttp,aiosignal,aiosqlite,annotated-doc,annotated-types,anyio,asyncpg,cffi,chevron,click,cryptography,datasets,dill,distro,dnspython,docstring-parser,durationpy,einops,email-validator,faiss-cpu,fire,frozenlist,fsspec,google-cloud-core,google-crc32c,google-genai,google-resumable-media,grpc-google-iam-v1,grpcio,grpcio-status,h11,hf-xet,httpcore,httpx,httpx-sse,idna,importlib-metadata,jinja2,jiter,joblib,jsonschema,jsonschema-specifications,kubernetes,lxml,markdown-it-py,mcp,mdurl,mpmath,multidict,networkx,numpy,oauthlib,packaging,pandas,peft,pillow,prometheus-client,prompt-toolkit,propcache,psycopg2-binary,pyarrow,pyasn1-modules,pycparser,pydantic,pydantic-core,pygments,python-dateutil,python-multipart,pyyaml,referencing,requests-oauthlib,rpds-py,safetensors,scikit-learn,scipy,setuptools,six,sniffio,sqlalchemy,sympy,termcolor,threadpoolctl,tiktoken,tokenizers,torch,tqdm,transformers,tree-sitter,triton,typing-extensions,typing-inspection,tzdata,urllib3,websocket-client,websockets,wrapt,xxhash,yarl,zipp,uv,pip,maturin", + "os": "linux", + "arch": "x86_64,aarch64", + "py_version": 312 + } + } + ] + - name: hermetic + value: "true" + - name: dockerfile + value: Containerfile + - name: build-args-file + value: build-args-konflux.conf + pipelineSpec: + description: | + This pipeline is ideal for building multi-arch container images from a Containerfile while maintaining trust after pipeline customization. + + _Uses `buildah` to create a multi-platform container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. This pipeline requires that the [multi platform controller](https://github.com/konflux-ci/multi-platform-controller) is deployed and configured on your Konflux instance. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta?tab=tags)_ + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: + Path to the source code of an application's component from where + to build image. + name: path-context + type: string + - default: Dockerfile + description: + Path to the Dockerfile inside the context specified by parameter + path-context + name: dockerfile + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched + name: prefetch-input + type: string + - default: "" + description: + Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + type: string + - default: "false" + description: Build a source image. + name: build-source-image + type: string + - default: "true" + description: Add built image into an OCI image index + name: build-image-index + type: string + - default: docker + description: + The format for the resulting image's mediaType. Valid values are + oci or docker. + name: buildah-format + type: string + - default: "false" + description: Enable cache proxy configuration + name: enable-cache-proxy + - default: [] + description: Array of --build-arg values ("arg=value" strings) for buildah + name: build-args + type: array + - default: "" + description: Path to a file with build arguments for buildah, see https://www.mankier.com/1/buildah-build#--build-arg-file + name: build-args-file + type: string + - default: "false" + description: + Whether to enable privileged mode, should be used only with remote + VMs + name: privileged-nested + type: string + - default: + - linux/x86_64 + description: + List of platforms to build the container images on. The available + set of values is determined by the configuration of the multi-platform-controller. + name: build-platforms + type: array + - name: enable-package-registry-proxy + default: 'true' + description: Use the package registry proxy when prefetching dependencies + type: string + - name: sast-target-dirs + type: string + default: . + description: Target directories to scan with SAST tools. Multiple values should be separated with commas. + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + tasks: + - name: init + params: + - name: enable-cache-proxy + value: $(params.enable-cache-proxy) + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-init:0.4@sha256:5a423246792ac501ea279229b42ee57da9927da441c04b5c9ff86817b0856b08 + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + - name: ociStorage + value: $(params.output-image).git + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:d30f13dd15daf89dd6dc645243b3444d35570d13f7840c3fd65e366022515205 + - name: kind + value: task + resolver: bundles + workspaces: + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) + - name: ociStorage + value: $(params.output-image).prefetch + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + - name: enable-package-registry-proxy + value: $(params.enable-package-registry-proxy) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.3@sha256:3dc78afbf3a441e0280067433cb28ea3d2d0088ec214c73bf063f145b4f273ef + - name: kind + value: task + resolver: bundles + workspaces: + - name: git-basic-auth + workspace: git-auth + - name: netrc + workspace: netrc + - matrix: + params: + - name: PLATFORM + value: + - $(params.build-platforms) + name: build-images + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: BUILD_ARGS + value: + - $(params.build-args[*]) + - name: BUILD_ARGS_FILE + value: $(params.build-args-file) + - name: PRIVILEGED_NESTED + value: $(params.privileged-nested) + - name: SOURCE_URL + value: $(tasks.clone-repository.results.url) + - name: BUILDAH_FORMAT + value: $(params.buildah-format) + - name: HTTP_PROXY + value: $(tasks.init.results.http-proxy) + - name: NO_PROXY + value: $(tasks.init.results.no-proxy) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: IMAGE_APPEND_PLATFORM + value: "true" + runAfter: + - prefetch-dependencies + taskRef: + params: + - name: name + value: buildah-remote-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.9@sha256:77007259cc87f32d63d2c201226aadaab98313cfd4e02b46abc243c4d2cc27bd + - name: kind + value: task + resolver: bundles + - name: build-image-index + params: + - name: IMAGE + value: $(params.output-image) + - name: ALWAYS_BUILD_INDEX + value: $(params.build-image-index) + - name: IMAGES + value: + - $(tasks.build-images.results.IMAGE_REF[*]) + - name: BUILDAH_FORMAT + value: $(params.buildah-format) + runAfter: + - build-images + taskRef: + params: + - name: name + value: build-image-index + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.3@sha256:b33bfa8dc27dbf459f0779598ba45dcaa490bcc9f8efe1652bcf360ec8cb5582 + - name: kind + value: task + resolver: bundles + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: BINARY_IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: source-build-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:8567bb7bf8fa9147c96b297533336fa7079ecf972cb86c09ccdd6bddedb25711 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.build-source-image) + operator: in + values: + - "true" + - name: deprecated-base-image-check + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:e78d0d3baf3c8cfc1a5ad278196b74032d9568b143a87c7a79ab780fedfb296e + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - matrix: + params: + - name: image-platform + value: + - $(params.build-platforms) + name: clair-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:8fad4c2e2f470f82ee43d6b2ac72327b4d9c6e9cb514a678911c1c9359c29894 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - matrix: + params: + - name: platform + value: + - $(params.build-platforms) + name: ecosystem-cert-preflight-checks + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: ecosystem-cert-preflight-checks + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:88f4fd6d7812a3c46f120f3035974f5fb8cb06b5e3e927badf6e8370f1516a88 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-snyk-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:0ebf28a0abd5a167438d4628938a74ade6f00a44a4b7ed1cfa9cfc57a5b24748 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - matrix: + params: + - name: image-arch + value: + - $(params.build-platforms) + name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:567cb66bd2e1f4b58b9d4d756f3317fc62479e0b40aa0de66094b1f12d296cfc + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-shell-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-shell-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:3cbb3535af6e7d4396858179a6427caaffb2e68775594795692fc01f28ae313f + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-unicode-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-unicode-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:223812001607b07f0e07d56bef7b7d619144e660c0c57f21ddd44ce0c8c4785b + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: apply-tags + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: apply-tags + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3@sha256:a291081de7fb27f832c6fc3c4b078acf7e6162ca4c085db38b118ca87e8b5b66 + - name: kind + value: task + resolver: bundles + - name: push-dockerfile + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: push-dockerfile-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.3@sha256:7855471abfe87de080b914f2f3ca27c59e64f6448a7c2435e51435b764494c71 + - name: kind + value: task + resolver: bundles + - name: rpms-signature-scan + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: rpms-signature-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:237c54b069d16c3785d1302f19be309aa6c0ae2313d446e30cb74671e07ca676 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: git-auth + optional: true + - name: netrc + optional: true + taskRunTemplate: + serviceAccountName: build-pipeline-lightspeed-stack-release-0-5 + workspaces: + - name: git-auth + secret: + secretName: "{{ git_auth_secret }}" +status: {} diff --git a/.tekton/lightspeed-stack-release-0-5-push.yaml b/.tekton/lightspeed-stack-release-0-5-push.yaml new file mode 100644 index 000000000..219ef6015 --- /dev/null +++ b/.tekton/lightspeed-stack-release-0-5-push.yaml @@ -0,0 +1,599 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/lightspeed-core/lightspeed-stack?rev={{revision}} + build.appstudio.redhat.com/commit_sha: "{{revision}}" + build.appstudio.redhat.com/target_branch: "{{target_branch}}" + pipelinesascode.tekton.dev/cancel-in-progress: "false" + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: + event == "push" && target_branch + == "release/0.5" + creationTimestamp: null + labels: + appstudio.openshift.io/application: lightspeed-stack + appstudio.openshift.io/component: lightspeed-stack-release-0-5 + pipelines.appstudio.openshift.io/type: build + name: lightspeed-stack-release-0-5-on-push + namespace: lightspeed-core-tenant +spec: + params: + - name: git-url + value: "{{source_url}}" + - name: revision + value: "{{revision}}" + - name: output-image + value: quay.io/redhat-user-workloads/lightspeed-core-tenant/lightspeed-stack-release-0-5:{{revision}} + - name: build-platforms + value: + - linux/x86_64 + - linux-c6gd2xlarge/arm64 + - name: dockerfile + value: Containerfile + - name: build-source-image + value: "true" + - name: prefetch-input + value: | + [ + { + "type": "rpm", + "path": "." + }, + { + "type": "generic", + "path": "." + }, + { + "type": "pip", + "path": ".", + "requirements_files": [ + "requirements.hashes.wheel.txt", + "requirements.hashes.source.txt", + "requirements.hermetic.txt" + ], + "requirements_build_files": ["requirements-build.txt"], + "binary": { + "packages": "aiohappyeyeballs,aiohttp,aiosignal,aiosqlite,annotated-doc,annotated-types,anyio,asyncpg,cffi,chevron,click,cryptography,datasets,dill,distro,dnspython,docstring-parser,durationpy,einops,email-validator,faiss-cpu,fire,frozenlist,fsspec,google-cloud-core,google-crc32c,google-genai,google-resumable-media,grpc-google-iam-v1,grpcio,grpcio-status,h11,hf-xet,httpcore,httpx,httpx-sse,idna,importlib-metadata,jinja2,jiter,joblib,jsonschema,jsonschema-specifications,kubernetes,lxml,markdown-it-py,mcp,mdurl,mpmath,multidict,networkx,numpy,oauthlib,packaging,pandas,peft,pillow,prometheus-client,prompt-toolkit,propcache,psycopg2-binary,pyarrow,pyasn1-modules,pycparser,pydantic,pydantic-core,pygments,python-dateutil,python-multipart,pyyaml,referencing,requests-oauthlib,rpds-py,safetensors,scikit-learn,scipy,setuptools,six,sniffio,sqlalchemy,sympy,termcolor,threadpoolctl,tiktoken,tokenizers,torch,tqdm,transformers,tree-sitter,triton,typing-extensions,typing-inspection,tzdata,urllib3,websocket-client,websockets,wrapt,xxhash,yarl,zipp,uv,pip,maturin", + "os": "linux", + "arch": "x86_64,aarch64", + "py_version": 312 + } + } + ] + - name: hermetic + value: "true" + - name: build-args-file + value: build-args-konflux.conf + pipelineSpec: + description: | + This pipeline is ideal for building multi-arch container images from a Containerfile while maintaining trust after pipeline customization. + + _Uses `buildah` to create a multi-platform container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. This pipeline requires that the [multi platform controller](https://github.com/konflux-ci/multi-platform-controller) is deployed and configured on your Konflux instance. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://conforma.dev/docs/policy/packages/release_trusted_task.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta?tab=tags)_ + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: + Path to the source code of an application's component from where + to build image. + name: path-context + type: string + - default: Dockerfile + description: + Path to the Dockerfile inside the context specified by parameter + path-context + name: dockerfile + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched + name: prefetch-input + type: string + - default: "" + description: + Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + type: string + - default: "false" + description: Build a source image. + name: build-source-image + type: string + - default: "true" + description: Add built image into an OCI image index + name: build-image-index + type: string + - default: docker + description: + The format for the resulting image's mediaType. Valid values are + oci or docker. + name: buildah-format + type: string + - default: "false" + description: Enable cache proxy configuration + name: enable-cache-proxy + - default: [] + description: Array of --build-arg values ("arg=value" strings) for buildah + name: build-args + type: array + - default: "" + description: Path to a file with build arguments for buildah, see https://www.mankier.com/1/buildah-build#--build-arg-file + name: build-args-file + type: string + - default: "false" + description: + Whether to enable privileged mode, should be used only with remote + VMs + name: privileged-nested + type: string + - default: + - linux/x86_64 + description: + List of platforms to build the container images on. The available + set of values is determined by the configuration of the multi-platform-controller. + name: build-platforms + type: array + - name: enable-package-registry-proxy + default: 'true' + description: Use the package registry proxy when prefetching dependencies + type: string + - name: sast-target-dirs + type: string + default: . + description: Target directories to scan with SAST tools. Multiple values should be separated with commas. + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + tasks: + - name: init + params: + - name: enable-cache-proxy + value: $(params.enable-cache-proxy) + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-init:0.4@sha256:5a423246792ac501ea279229b42ee57da9927da441c04b5c9ff86817b0856b08 + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + - name: ociStorage + value: $(params.output-image).git + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:d30f13dd15daf89dd6dc645243b3444d35570d13f7840c3fd65e366022515205 + - name: kind + value: task + resolver: bundles + workspaces: + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) + - name: ociStorage + value: $(params.output-image).prefetch + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + - name: enable-package-registry-proxy + value: $(params.enable-package-registry-proxy) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.3@sha256:3dc78afbf3a441e0280067433cb28ea3d2d0088ec214c73bf063f145b4f273ef + - name: kind + value: task + resolver: bundles + workspaces: + - name: git-basic-auth + workspace: git-auth + - name: netrc + workspace: netrc + - matrix: + params: + - name: PLATFORM + value: + - $(params.build-platforms) + name: build-images + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: BUILD_ARGS + value: + - $(params.build-args[*]) + - name: BUILD_ARGS_FILE + value: $(params.build-args-file) + - name: PRIVILEGED_NESTED + value: $(params.privileged-nested) + - name: SOURCE_URL + value: $(tasks.clone-repository.results.url) + - name: BUILDAH_FORMAT + value: $(params.buildah-format) + - name: HTTP_PROXY + value: $(tasks.init.results.http-proxy) + - name: NO_PROXY + value: $(tasks.init.results.no-proxy) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: IMAGE_APPEND_PLATFORM + value: "true" + runAfter: + - prefetch-dependencies + taskRef: + params: + - name: name + value: buildah-remote-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-buildah-remote-oci-ta:0.9@sha256:77007259cc87f32d63d2c201226aadaab98313cfd4e02b46abc243c4d2cc27bd + - name: kind + value: task + resolver: bundles + - name: build-image-index + params: + - name: IMAGE + value: $(params.output-image) + - name: ALWAYS_BUILD_INDEX + value: $(params.build-image-index) + - name: IMAGES + value: + - $(tasks.build-images.results.IMAGE_REF[*]) + - name: BUILDAH_FORMAT + value: $(params.buildah-format) + runAfter: + - build-images + taskRef: + params: + - name: name + value: build-image-index + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.3@sha256:b33bfa8dc27dbf459f0779598ba45dcaa490bcc9f8efe1652bcf360ec8cb5582 + - name: kind + value: task + resolver: bundles + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: BINARY_IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: source-build-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.3@sha256:8567bb7bf8fa9147c96b297533336fa7079ecf972cb86c09ccdd6bddedb25711 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.build-source-image) + operator: in + values: + - "true" + - name: deprecated-base-image-check + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.5@sha256:e78d0d3baf3c8cfc1a5ad278196b74032d9568b143a87c7a79ab780fedfb296e + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - matrix: + params: + - name: image-platform + value: + - $(params.build-platforms) + name: clair-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.3@sha256:8fad4c2e2f470f82ee43d6b2ac72327b4d9c6e9cb514a678911c1c9359c29894 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - matrix: + params: + - name: platform + value: + - $(params.build-platforms) + name: ecosystem-cert-preflight-checks + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: ecosystem-cert-preflight-checks + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.2@sha256:88f4fd6d7812a3c46f120f3035974f5fb8cb06b5e3e927badf6e8370f1516a88 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-snyk-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.4@sha256:0ebf28a0abd5a167438d4628938a74ade6f00a44a4b7ed1cfa9cfc57a5b24748 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - matrix: + params: + - name: image-arch + value: + - $(params.build-platforms) + name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.3@sha256:567cb66bd2e1f4b58b9d4d756f3317fc62479e0b40aa0de66094b1f12d296cfc + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-shell-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-shell-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-shell-check-oci-ta:0.1@sha256:3cbb3535af6e7d4396858179a6427caaffb2e68775594795692fc01f28ae313f + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-unicode-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + - name: TARGET_DIRS + value: $(params.sast-target-dirs) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-unicode-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-unicode-check-oci-ta:0.4@sha256:223812001607b07f0e07d56bef7b7d619144e660c0c57f21ddd44ce0c8c4785b + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: apply-tags + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: apply-tags + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.3@sha256:a291081de7fb27f832c6fc3c4b078acf7e6162ca4c085db38b118ca87e8b5b66 + - name: kind + value: task + resolver: bundles + - name: push-dockerfile + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: push-dockerfile-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.3@sha256:7855471abfe87de080b914f2f3ca27c59e64f6448a7c2435e51435b764494c71 + - name: kind + value: task + resolver: bundles + - name: rpms-signature-scan + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: rpms-signature-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:237c54b069d16c3785d1302f19be309aa6c0ae2313d446e30cb74671e07ca676 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: git-auth + optional: true + - name: netrc + optional: true + taskRunTemplate: + serviceAccountName: build-pipeline-lightspeed-stack-release-0-5 + workspaces: + - name: git-auth + secret: + secretName: "{{ git_auth_secret }}" +status: {} diff --git a/Containerfile b/Containerfile index 134604471..00fbf8a44 100644 --- a/Containerfile +++ b/Containerfile @@ -1,7 +1,7 @@ # vim: set filetype=dockerfile -ARG BUILDER_BASE_IMAGE=registry.access.redhat.com/ubi9/python-312 +ARG BUILDER_BASE_IMAGE=registry.access.redhat.com/ubi9/python-312:9.8-1781023618 ARG BUILDER_DNF_COMMAND=dnf -ARG RUNTIME_BASE_IMAGE=registry.access.redhat.com/ubi9/python-312-minimal +ARG RUNTIME_BASE_IMAGE=registry.access.redhat.com/ubi9/python-312-minimal:9.8-1781061228 ARG RUNTIME_DNF_COMMAND=microdnf FROM ${BUILDER_BASE_IMAGE} AS builder @@ -24,7 +24,7 @@ USER root # Install gcc - required by polyleven python package on aarch64 # (dependency of autoevals, no pre-built binary wheels for linux on aarch64) # cmake and cargo are required by fastuuid, maturin -RUN ${BUILDER_DNF_COMMAND} install -y --nodocs --setopt=keepcache=0 --setopt=tsflags=nodocs gcc gcc-c++ cmake cargo +RUN ${BUILDER_DNF_COMMAND} install -y --nodocs --setopt=keepcache=0 --setopt=tsflags=nodocs gcc g++ cmake cargo # Install uv package manager RUN pip3.12 install "uv>=0.8.15" diff --git a/README.md b/README.md index d14954828..27b08c8ac 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![License](https://img.shields.io/badge/license-Apache-blue)](https://github.com/lightspeed-core/lightspeed-stack/blob/main/LICENSE) [![made-with-python](https://img.shields.io/badge/Made%20with-Python-1f425f.svg)](https://www.python.org/) [![Required Python version](https://img.shields.io/python/required-version-toml?tomlFilePath=https%3A%2F%2Fraw.githubusercontent.com%2Flightspeed-core%2Flightspeed-stack%2Frefs%2Fheads%2Fmain%2Fpyproject.toml)](https://www.python.org/) -[![Tag](https://img.shields.io/github/v/tag/lightspeed-core/lightspeed-stack)](https://github.com/lightspeed-core/lightspeed-stack/releases/tag/0.5.0) +[![Tag](https://img.shields.io/github/v/tag/lightspeed-core/lightspeed-stack)](https://github.com/lightspeed-core/lightspeed-stack/releases/tag/0.5.1) Lightspeed Core Stack (LCS) is an AI-powered assistant that provides answers to product questions using backend LLM services, agents, and RAG databases. diff --git a/docs/openapi.json b/docs/openapi.json index f24a1a545..a83093ce0 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -13,7 +13,7 @@ "name": "Apache 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0.html" }, - "version": "0.5.0" + "version": "0.5.1" }, "servers": [ { diff --git a/docs/splunk.md b/docs/splunk.md index 65de8fbbd..4b23908f8 100644 --- a/docs/splunk.md +++ b/docs/splunk.md @@ -85,7 +85,7 @@ Events follow the rlsapi telemetry format for consistency with existing analytic "system_id": "abc-def-123", "total_llm_tokens": 0, "request_id": "req_xyz789", - "cla_version": "CLA/0.5.0", + "cla_version": "CLA/0.5.1", "system_os": "RHEL", "system_version": "9.3", "system_arch": "x86_64" diff --git a/redhat.repo b/redhat.repo deleted file mode 100644 index 54247d627..000000000 --- a/redhat.repo +++ /dev/null @@ -1,69 +0,0 @@ -[codeready-builder-for-rhel-9-$basearch-eus-rpms] -name = Red Hat CodeReady Linux Builder for RHEL 9 $basearch - Extended Update Support (RPMs) -baseurl = https://cdn.redhat.com/content/eus/rhel9/9.6/$basearch/codeready-builder/os -enabled = 1 -gpgcheck = 1 -gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release -sslverify = 1 -sslcacert = /etc/rhsm/ca/redhat-uep.pem -sslverifystatus = 1 -metadata_expire = 86400 -enabled_metadata = 0 -sslclientkey = $SSL_CLIENT_KEY -sslclientcert = $SSL_CLIENT_CERT - -[rhel-9-for-$basearch-appstream-eus-rpms] -name = Red Hat Enterprise Linux 9 for $basearch - AppStream - Extended Update Support (RPMs) -baseurl = https://cdn.redhat.com/content/eus/rhel9/9.6/$basearch/appstream/os -enabled = 1 -gpgcheck = 1 -gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release -sslverify = 1 -sslcacert = /etc/rhsm/ca/redhat-uep.pem -sslverifystatus = 1 -metadata_expire = 86400 -enabled_metadata = 0 -sslclientkey = $SSL_CLIENT_KEY -sslclientcert = $SSL_CLIENT_CERT - -[rhel-9-for-$basearch-baseos-eus-rpms] -name = Red Hat Enterprise Linux 9 for $basearch - BaseOS - Extended Update Support (RPMs) -baseurl = https://cdn.redhat.com/content/eus/rhel9/9.6/$basearch/baseos/os -enabled = 1 -gpgcheck = 1 -gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release -sslverify = 1 -sslcacert = /etc/rhsm/ca/redhat-uep.pem -sslverifystatus = 1 -metadata_expire = 86400 -enabled_metadata = 0 -sslclientkey = $SSL_CLIENT_KEY -sslclientcert = $SSL_CLIENT_CERT - -[rhocp-4.17-for-rhel-9-$basearch-rpms] -name = Red Hat OpenShift Container Platform 4.17 for RHEL 9 $basearch (RPMs) -baseurl = https://cdn.redhat.com/content/dist/layered/rhel9/$basearch/rhocp/4.17/os -enabled = 0 -gpgcheck = 1 -gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release -sslverify = 1 -sslcacert = /etc/rhsm/ca/redhat-uep.pem -sslverifystatus = 1 -metadata_expire = 86400 -enabled_metadata = 0 -sslclientkey = $SSL_CLIENT_KEY -sslclientcert = $SSL_CLIENT_CERT - -[rhocp-4.17-for-rhel-9-$basearch-source-rpms] -name = Red Hat OpenShift Container Platform 4.17 for RHEL 9 $basearch (Source RPMs) -baseurl = https://cdn.redhat.com/content/dist/layered/rhel9/$basearch/rhocp/4.17/source/SRPMS -enabled = 0 -gpgcheck = 1 -gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release -sslverify = 1 -sslcacert = /etc/rhsm/ca/redhat-uep.pem -sslverifystatus = 1 -metadata_expire = 86400 -enabled_metadata = 0 -sslclientkey = $SSL_CLIENT_KEY -sslclientcert = $SSL_CLIENT_CERT diff --git a/rpms.in.yaml b/rpms.in.yaml index ca64f8cf2..d7312f69d 100644 --- a/rpms.in.yaml +++ b/rpms.in.yaml @@ -1,12 +1,12 @@ packages: [ gcc, - gcc-c++, + g++, jq, patch, cmake, cargo, ] contentOrigin: - repofiles: ["./redhat.repo"] + repofiles: ["./ubi.repo"] arches: [x86_64, aarch64] diff --git a/rpms.lock.yaml b/rpms.lock.yaml index 2ca7ed624..7afd0807f 100644 --- a/rpms.lock.yaml +++ b/rpms.lock.yaml @@ -4,177 +4,121 @@ lockfileVendor: redhat arches: - arch: aarch64 packages: - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/aarch64/appstream/os/Packages/c/cargo-1.84.1-1.el9.aarch64.rpm - repoid: rhel-9-for-aarch64-appstream-eus-rpms - size: 7744425 - checksum: sha256:5db626d49748f31fb02916c24fa1a7e5759ce7b905ac3e781d42079fba8fa1c4 + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/appstream/os/Packages/c/cargo-1.92.0-1.el9.aarch64.rpm + repoid: ubi-9-for-aarch64-appstream-rpms + size: 8530651 + checksum: sha256:14ac2d264369b0ac4442d6b773224765413282ea996dac29cf576c176c8cdcba name: cargo - evr: 1.84.1-1.el9 - sourcerpm: rust-1.84.1-1.el9.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/aarch64/appstream/os/Packages/c/cmake-3.26.5-2.el9.aarch64.rpm - repoid: rhel-9-for-aarch64-appstream-eus-rpms - size: 7432689 - checksum: sha256:6ac0e5e9a4fd761f8688678ac83580c7eebeacf6c241bd8089d72c4a477b22c3 + evr: 1.92.0-1.el9 + sourcerpm: rust-1.92.0-1.el9.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/appstream/os/Packages/c/cmake-3.31.8-3.el9.aarch64.rpm + repoid: ubi-9-for-aarch64-appstream-rpms + size: 11655593 + checksum: sha256:2a77fe1c3784083dcdebdd548c16d823b3e4a5adb8d6c00e36841aa633eab53b name: cmake - evr: 3.26.5-2.el9 - sourcerpm: cmake-3.26.5-2.el9.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/aarch64/appstream/os/Packages/c/cmake-data-3.26.5-2.el9.noarch.rpm - repoid: rhel-9-for-aarch64-appstream-eus-rpms - size: 2488227 - checksum: sha256:84da65a7b8921f031d15903d91c5967022620f9e96b7493c8ab8024014755ee7 + evr: 3.31.8-3.el9 + sourcerpm: cmake-3.31.8-3.el9.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/appstream/os/Packages/c/cmake-data-3.31.8-3.el9.noarch.rpm + repoid: ubi-9-for-aarch64-appstream-rpms + size: 2829291 + checksum: sha256:1cdc2e88a996c575b750483c8f562674e4b50a6ab414c7bbe6f6b641c1db7bd9 name: cmake-data - evr: 3.26.5-2.el9 - sourcerpm: cmake-3.26.5-2.el9.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/aarch64/appstream/os/Packages/c/cmake-rpm-macros-3.26.5-2.el9.noarch.rpm - repoid: rhel-9-for-aarch64-appstream-eus-rpms - size: 12250 - checksum: sha256:1c74969c8a4f21851f5b89f25ac55c689b75bed1318d0435fc3a14a49c39d0e3 - name: cmake-rpm-macros - evr: 3.26.5-2.el9 - sourcerpm: cmake-3.26.5-2.el9.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/aarch64/appstream/os/Packages/g/gcc-c++-11.5.0-5.el9_5.aarch64.rpm - repoid: rhel-9-for-aarch64-appstream-eus-rpms - size: 12999288 - checksum: sha256:a9ff0bd2a2b3483e07dcf87f8137a6358f36f5300c934b90500f119f884e3463 - name: gcc-c++ - evr: 11.5.0-5.el9_5 - sourcerpm: gcc-11.5.0-5.el9_5.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/aarch64/appstream/os/Packages/l/libstdc++-devel-11.5.0-5.el9_5.aarch64.rpm - repoid: rhel-9-for-aarch64-appstream-eus-rpms - size: 2526795 - checksum: sha256:83a2006137335a9b17a05a02a54481abcdfd295b280b924c51caaacd7bf07ad6 - name: libstdc++-devel - evr: 11.5.0-5.el9_5 - sourcerpm: gcc-11.5.0-5.el9_5.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/aarch64/appstream/os/Packages/l/libuv-1.42.0-2.el9_4.aarch64.rpm - repoid: rhel-9-for-aarch64-appstream-eus-rpms + evr: 3.31.8-3.el9 + sourcerpm: cmake-3.31.8-3.el9.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/appstream/os/Packages/l/libuv-1.42.0-2.el9_4.aarch64.rpm + repoid: ubi-9-for-aarch64-appstream-rpms size: 150129 checksum: sha256:4dc8a40da74e0f9823356460ee11f183c70f382953700fffef0c448198a677cc name: libuv evr: 1:1.42.0-2.el9_4 sourcerpm: libuv-1.42.0-2.el9_4.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/aarch64/appstream/os/Packages/p/patch-2.7.6-16.el9.aarch64.rpm - repoid: rhel-9-for-aarch64-appstream-eus-rpms - size: 129037 - checksum: sha256:335c720da3caa41822737dd431d91a4adc79c85dedbe4483ecaf58bc83767610 - name: patch - evr: 2.7.6-16.el9 - sourcerpm: patch-2.7.6-16.el9.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/aarch64/appstream/os/Packages/r/rust-1.84.1-1.el9.aarch64.rpm - repoid: rhel-9-for-aarch64-appstream-eus-rpms - size: 26093725 - checksum: sha256:5be9185a7d684022bc0686049c22ef901c4df6dce2822bdec16a1a47c46b6861 + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/appstream/os/Packages/o/oniguruma-6.9.6-1.el9.5.aarch64.rpm + repoid: ubi-9-for-aarch64-appstream-rpms + size: 222582 + checksum: sha256:bc2305dad655ddb94f966158112efd6cefa6824d5aa2e80f63881f16cee74598 + name: oniguruma + evr: 6.9.6-1.el9.5 + sourcerpm: oniguruma-6.9.6-1.el9.5.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/appstream/os/Packages/r/rust-1.92.0-1.el9.aarch64.rpm + repoid: ubi-9-for-aarch64-appstream-rpms + size: 29421295 + checksum: sha256:43a1b0f5168a39ceea3fd90d42b23bc05d006d639cd35cfdad82a4f94aa58453 name: rust - evr: 1.84.1-1.el9 - sourcerpm: rust-1.84.1-1.el9.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/aarch64/appstream/os/Packages/r/rust-std-static-1.84.1-1.el9.aarch64.rpm - repoid: rhel-9-for-aarch64-appstream-eus-rpms - size: 39259196 - checksum: sha256:5889bced81144c4ea201085e5bfd040300c56048e5d7987e9eb69d4d252f87bf + evr: 1.92.0-1.el9 + sourcerpm: rust-1.92.0-1.el9.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/appstream/os/Packages/r/rust-std-static-1.92.0-1.el9.aarch64.rpm + repoid: ubi-9-for-aarch64-appstream-rpms + size: 41493155 + checksum: sha256:3b3a70a8e11f53559c4b84927ebd0565a073052bac2c53edda6cb328bfb28090 name: rust-std-static - evr: 1.84.1-1.el9 - sourcerpm: rust-1.84.1-1.el9.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/aarch64/baseos/os/Packages/e/ed-1.14.2-12.el9.aarch64.rpm - repoid: rhel-9-for-aarch64-baseos-eus-rpms - size: 78931 - checksum: sha256:3bce4ce6243886c448e58f589b79e3ac829fcde53d1ff13d5906a8cdc22be091 - name: ed - evr: 1.14.2-12.el9 - sourcerpm: ed-1.14.2-12.el9.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/aarch64/baseos/os/Packages/i/info-6.7-15.el9.aarch64.rpm - repoid: rhel-9-for-aarch64-baseos-eus-rpms - size: 230301 - checksum: sha256:c5ae65876c73c6f4e240081431745f5ba0a91d10a4bfb8a5d162ca3d6f039202 - name: info - evr: 6.7-15.el9 - sourcerpm: texinfo-6.7-15.el9.src.rpm + evr: 1.92.0-1.el9 + sourcerpm: rust-1.92.0-1.el9.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/j/jq-1.6-19.el9_8.2.aarch64.rpm + repoid: ubi-9-for-aarch64-baseos-rpms + size: 191195 + checksum: sha256:633aaf3e87b19d4a591bd9f47cd81fde8ec49629d3f58932addfc8a134b7949d + name: jq + evr: 1.6-19.el9_8.2 + sourcerpm: jq-1.6-19.el9_8.2.src.rpm source: [] module_metadata: [] - arch: x86_64 packages: - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/x86_64/appstream/os/Packages/c/cargo-1.84.1-1.el9.x86_64.rpm - repoid: rhel-9-for-x86_64-appstream-eus-rpms - size: 8292467 - checksum: sha256:7dd011cd79a635654ade4e3186c5f7545d692de81157d1ce1d42656eaa6993b2 + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/c/cargo-1.92.0-1.el9.x86_64.rpm + repoid: ubi-9-for-x86_64-appstream-rpms + size: 9100626 + checksum: sha256:3c4afc2cb56734d01aaaaa8d0c317688f6fe143ad239079342f4fe9b631ded0f name: cargo - evr: 1.84.1-1.el9 - sourcerpm: rust-1.84.1-1.el9.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/x86_64/appstream/os/Packages/c/cmake-3.26.5-2.el9.x86_64.rpm - repoid: rhel-9-for-x86_64-appstream-eus-rpms - size: 9159462 - checksum: sha256:f553370cb02b87e7388697468256556e765b102c2fcb56be6bc250cb2351e8ad + evr: 1.92.0-1.el9 + sourcerpm: rust-1.92.0-1.el9.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/c/cmake-3.31.8-3.el9.x86_64.rpm + repoid: ubi-9-for-x86_64-appstream-rpms + size: 13989883 + checksum: sha256:e67ea7aef1edd470e4ec22982e97871655abcdc0990754d4e8f147d4e7de317a name: cmake - evr: 3.26.5-2.el9 - sourcerpm: cmake-3.26.5-2.el9.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/x86_64/appstream/os/Packages/c/cmake-data-3.26.5-2.el9.noarch.rpm - repoid: rhel-9-for-x86_64-appstream-eus-rpms - size: 2488227 - checksum: sha256:84da65a7b8921f031d15903d91c5967022620f9e96b7493c8ab8024014755ee7 + evr: 3.31.8-3.el9 + sourcerpm: cmake-3.31.8-3.el9.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/c/cmake-data-3.31.8-3.el9.noarch.rpm + repoid: ubi-9-for-x86_64-appstream-rpms + size: 2829291 + checksum: sha256:1cdc2e88a996c575b750483c8f562674e4b50a6ab414c7bbe6f6b641c1db7bd9 name: cmake-data - evr: 3.26.5-2.el9 - sourcerpm: cmake-3.26.5-2.el9.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/x86_64/appstream/os/Packages/c/cmake-rpm-macros-3.26.5-2.el9.noarch.rpm - repoid: rhel-9-for-x86_64-appstream-eus-rpms - size: 12250 - checksum: sha256:1c74969c8a4f21851f5b89f25ac55c689b75bed1318d0435fc3a14a49c39d0e3 - name: cmake-rpm-macros - evr: 3.26.5-2.el9 - sourcerpm: cmake-3.26.5-2.el9.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/x86_64/appstream/os/Packages/g/gcc-c++-11.5.0-5.el9_5.x86_64.rpm - repoid: rhel-9-for-x86_64-appstream-eus-rpms - size: 13479598 - checksum: sha256:b8392274e302d665bc132aee4ed023f8a777d9c446531679ede18150d7867189 - name: gcc-c++ - evr: 11.5.0-5.el9_5 - sourcerpm: gcc-11.5.0-5.el9_5.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/x86_64/appstream/os/Packages/l/libstdc++-devel-11.5.0-5.el9_5.x86_64.rpm - repoid: rhel-9-for-x86_64-appstream-eus-rpms - size: 2531717 - checksum: sha256:84695eeeb1daa8ff74baf7efd9fc57fb136bec7e8a2ca56c105be6d83ec22d07 - name: libstdc++-devel - evr: 11.5.0-5.el9_5 - sourcerpm: gcc-11.5.0-5.el9_5.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/x86_64/appstream/os/Packages/l/libuv-1.42.0-2.el9_4.x86_64.rpm - repoid: rhel-9-for-x86_64-appstream-eus-rpms + evr: 3.31.8-3.el9 + sourcerpm: cmake-3.31.8-3.el9.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/l/libuv-1.42.0-2.el9_4.x86_64.rpm + repoid: ubi-9-for-x86_64-appstream-rpms size: 154427 checksum: sha256:e1fab39251239ccaad2fb4dbe6c55ec1ae60f76d4ae81582b06e6a58e30879b2 name: libuv evr: 1:1.42.0-2.el9_4 sourcerpm: libuv-1.42.0-2.el9_4.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/x86_64/appstream/os/Packages/p/patch-2.7.6-16.el9.x86_64.rpm - repoid: rhel-9-for-x86_64-appstream-eus-rpms - size: 133240 - checksum: sha256:d2e0307a2d1d4eff0c2db406841030461b35864926916f2a92244427d89316be - name: patch - evr: 2.7.6-16.el9 - sourcerpm: patch-2.7.6-16.el9.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/x86_64/appstream/os/Packages/r/rust-1.84.1-1.el9.x86_64.rpm - repoid: rhel-9-for-x86_64-appstream-eus-rpms - size: 28050444 - checksum: sha256:9ba3c53fd811af2f294e31360d75e33e4cb89893130c7b3fe0c6191e20a09f3e + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/o/oniguruma-6.9.6-1.el9.5.x86_64.rpm + repoid: ubi-9-for-x86_64-appstream-rpms + size: 226331 + checksum: sha256:6c884cc2216e5b4699ebd8cde27b39e99532520b367f645ed6cc660d081916dc + name: oniguruma + evr: 6.9.6-1.el9.5 + sourcerpm: oniguruma-6.9.6-1.el9.5.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/r/rust-1.92.0-1.el9.x86_64.rpm + repoid: ubi-9-for-x86_64-appstream-rpms + size: 31511504 + checksum: sha256:fbc70b11f38999206d70a104789d1f7f21ca9f9090c7a73d6db337bff4f5205b name: rust - evr: 1.84.1-1.el9 - sourcerpm: rust-1.84.1-1.el9.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/x86_64/appstream/os/Packages/r/rust-std-static-1.84.1-1.el9.x86_64.rpm - repoid: rhel-9-for-x86_64-appstream-eus-rpms - size: 41211472 - checksum: sha256:73bb90884432e2b43758f1043f107a570b5d54b38f17d5d0af51bac103ceb4f5 + evr: 1.92.0-1.el9 + sourcerpm: rust-1.92.0-1.el9.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/r/rust-std-static-1.92.0-1.el9.x86_64.rpm + repoid: ubi-9-for-x86_64-appstream-rpms + size: 42991765 + checksum: sha256:d286394aaa75a796a06db130d2a980bee8e6ab4cbaa38a3b84e12379fbae4671 name: rust-std-static - evr: 1.84.1-1.el9 - sourcerpm: rust-1.84.1-1.el9.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/x86_64/baseos/os/Packages/e/ed-1.14.2-12.el9.x86_64.rpm - repoid: rhel-9-for-x86_64-baseos-eus-rpms - size: 79993 - checksum: sha256:5fb3c625fd1ace94f133522bdaf4768abd78f029e20886b8e4aed2d6d1aac664 - name: ed - evr: 1.14.2-12.el9 - sourcerpm: ed-1.14.2-12.el9.src.rpm - - url: https://cdn.redhat.com/content/eus/rhel9/9.6/x86_64/baseos/os/Packages/i/info-6.7-15.el9.x86_64.rpm - repoid: rhel-9-for-x86_64-baseos-eus-rpms - size: 233806 - checksum: sha256:3643f98b45cc973073096608aaa45976d722fe284590ff7c1d5f93ad77ba0f8b - name: info - evr: 6.7-15.el9 - sourcerpm: texinfo-6.7-15.el9.src.rpm + evr: 1.92.0-1.el9 + sourcerpm: rust-1.92.0-1.el9.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/j/jq-1.6-19.el9_8.2.x86_64.rpm + repoid: ubi-9-for-x86_64-baseos-rpms + size: 197453 + checksum: sha256:9793a39a4746a09ba89c3d9ccc70150ac6c878286deee26d7e3aabede4666417 + name: jq + evr: 1.6-19.el9_8.2 + sourcerpm: jq-1.6-19.el9_8.2.src.rpm source: [] module_metadata: [] diff --git a/src/app/endpoints/vector_stores.py b/src/app/endpoints/vector_stores.py new file mode 100644 index 000000000..91892f520 --- /dev/null +++ b/src/app/endpoints/vector_stores.py @@ -0,0 +1,840 @@ +"""Handler for REST API calls to manage vector stores and files.""" + +import asyncio +import os +from io import BytesIO +from typing import Annotated, Any + +from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile, status +from llama_stack_client import ( + APIConnectionError, + BadRequestError, +) +from llama_stack_client import ( + APIStatusError as LLSApiStatusError, +) +from openai._exceptions import APIStatusError as OpenAIAPIStatusError + +from authentication import get_auth_dependency +from authentication.interface import AuthTuple +from authorization.middleware import authorize +from client import AsyncLlamaStackClientHolder +from configuration import configuration +from constants import DEFAULT_MAX_FILE_UPLOAD_SIZE +from log import get_logger +from models.config import Action +from models.requests import ( + VectorStoreCreateRequest, + VectorStoreFileCreateRequest, + VectorStoreUpdateRequest, +) +from models.responses import ( + BadRequestResponse, + FileResponse, + FileTooLargeResponse, + ForbiddenResponse, + InternalServerErrorResponse, + NotFoundResponse, + ServiceUnavailableResponse, + UnauthorizedResponse, + VectorStoreFileResponse, + VectorStoreFilesListResponse, + VectorStoreResponse, + VectorStoresListResponse, +) +from utils.endpoints import check_configuration_loaded +from utils.query import handle_known_apistatus_errors + +logger = get_logger(__name__) +router = APIRouter(tags=["vector-stores"]) + + +# Response schemas for OpenAPI documentation +vector_stores_list_responses: dict[int | str, dict[str, Any]] = { + 200: VectorStoresListResponse.openapi_response(), + 401: UnauthorizedResponse.openapi_response( + examples=["missing header", "missing token"] + ), + 403: ForbiddenResponse.openapi_response(examples=["endpoint"]), + 500: InternalServerErrorResponse.openapi_response(examples=["configuration"]), + 503: ServiceUnavailableResponse.openapi_response(), +} + +vector_store_responses: dict[int | str, dict[str, Any]] = { + 200: VectorStoreResponse.openapi_response(), + 401: UnauthorizedResponse.openapi_response( + examples=["missing header", "missing token"] + ), + 403: ForbiddenResponse.openapi_response(examples=["endpoint"]), + 404: NotFoundResponse.openapi_response(examples=["vector store"]), + 500: InternalServerErrorResponse.openapi_response(examples=["configuration"]), + 503: ServiceUnavailableResponse.openapi_response(), +} + +file_responses: dict[int | str, dict[str, Any]] = { + 200: FileResponse.openapi_response(), + 400: BadRequestResponse.openapi_response(examples=["file_upload"]), + 413: FileTooLargeResponse.openapi_response(), + 401: UnauthorizedResponse.openapi_response( + examples=["missing header", "missing token"] + ), + 403: ForbiddenResponse.openapi_response(examples=["endpoint"]), + 500: InternalServerErrorResponse.openapi_response(examples=["configuration"]), + 503: ServiceUnavailableResponse.openapi_response(), +} + +vector_store_file_responses: dict[int | str, dict[str, Any]] = { + 200: VectorStoreFileResponse.openapi_response(), + 401: UnauthorizedResponse.openapi_response( + examples=["missing header", "missing token"] + ), + 403: ForbiddenResponse.openapi_response(examples=["endpoint"]), + 404: NotFoundResponse.openapi_response(examples=["file"]), + 500: InternalServerErrorResponse.openapi_response(examples=["configuration"]), + 503: ServiceUnavailableResponse.openapi_response(), +} + +vector_store_files_list_responses: dict[int | str, dict[str, Any]] = { + 200: VectorStoreFilesListResponse.openapi_response(), + 401: UnauthorizedResponse.openapi_response( + examples=["missing header", "missing token"] + ), + 403: ForbiddenResponse.openapi_response(examples=["endpoint"]), + 404: NotFoundResponse.openapi_response(examples=["vector store"]), + 500: InternalServerErrorResponse.openapi_response(examples=["configuration"]), + 503: ServiceUnavailableResponse.openapi_response(), +} + + +@router.post("/vector-stores", responses=vector_store_responses) +@authorize(Action.MANAGE_VECTOR_STORES) +async def create_vector_store( + request: Request, + auth: Annotated[AuthTuple, Depends(get_auth_dependency())], + body: VectorStoreCreateRequest, +) -> VectorStoreResponse: + """Create a new vector store. + + Parameters: + request: The incoming HTTP request. + auth: Authentication tuple from the auth dependency. + body: Vector store creation parameters. + + Returns: + VectorStoreResponse: The created vector store object. + + Raises: + HTTPException: + - 401: Authentication failed + - 403: Authorization failed + - 500: Lightspeed Stack configuration not loaded + - 503: Unable to connect to Llama Stack + """ + _ = auth + _ = request + + check_configuration_loaded(configuration) + + try: + client = AsyncLlamaStackClientHolder().get_client() + + # Extract provider_id for extra_body (not a direct client parameter) + body_dict = body.model_dump(exclude_none=True) + extra_body = {} + if "provider_id" in body_dict: + extra_body["provider_id"] = body_dict.pop("provider_id") + if "embedding_model" in body_dict: + extra_body["embedding_model"] = body_dict.pop("embedding_model") + if "embedding_dimension" in body_dict: + extra_body["embedding_dimension"] = body_dict.pop("embedding_dimension") + + logger.debug( + "Creating vector store - body_dict: %s, extra_body: %s", + body_dict, + extra_body, + ) + + vector_store = await client.vector_stores.create( + **body_dict, + extra_body=extra_body, + ) + + return VectorStoreResponse( + id=vector_store.id, + name=vector_store.name, + created_at=vector_store.created_at, + last_active_at=vector_store.last_active_at, + expires_at=vector_store.expires_at, + status=vector_store.status or "unknown", + usage_bytes=vector_store.usage_bytes or 0, + metadata=vector_store.metadata, + ) + except APIConnectionError as e: + logger.error("Unable to connect to Llama Stack: %s", e) + response = ServiceUnavailableResponse(backend_name="Llama Stack", cause=str(e)) + raise HTTPException(**response.model_dump()) from e + except (LLSApiStatusError, OpenAIAPIStatusError) as e: + logger.error("API status error while creating vector store: %s", e) + error_response = handle_known_apistatus_errors(e, "llama-stack") + raise HTTPException(**error_response.model_dump()) from e + + +@router.get("/vector-stores", responses=vector_stores_list_responses) +@authorize(Action.READ_VECTOR_STORES) +async def list_vector_stores( + request: Request, + auth: Annotated[AuthTuple, Depends(get_auth_dependency())], +) -> VectorStoresListResponse: + """List all vector stores. + + Parameters: + request: The incoming HTTP request. + auth: Authentication tuple from the auth dependency. + + Returns: + VectorStoresListResponse: List of all vector stores. + + Raises: + HTTPException: + - 401: Authentication failed + - 403: Authorization failed + - 500: Lightspeed Stack configuration not loaded + - 503: Unable to connect to Llama Stack + """ + _ = auth + _ = request + + check_configuration_loaded(configuration) + + try: + client = AsyncLlamaStackClientHolder().get_client() + vector_stores = await client.vector_stores.list() + + data = [ + VectorStoreResponse( + id=vs.id, + name=vs.name, + created_at=vs.created_at, + last_active_at=vs.last_active_at, + expires_at=vs.expires_at or None, + status=vs.status or "unknown", + usage_bytes=vs.usage_bytes or 0, + metadata=vs.metadata, + ) + for vs in vector_stores.data + ] + + return VectorStoresListResponse(data=data) + except APIConnectionError as e: + logger.error("Unable to connect to Llama Stack: %s", e) + response = ServiceUnavailableResponse(backend_name="Llama Stack", cause=str(e)) + raise HTTPException(**response.model_dump()) from e + except (LLSApiStatusError, OpenAIAPIStatusError) as e: + logger.error("API status error while listing vector stores: %s", e) + error_response = handle_known_apistatus_errors(e, "llama-stack") + raise HTTPException(**error_response.model_dump()) from e + + +@router.get("/vector-stores/{vector_store_id}", responses=vector_store_responses) +@authorize(Action.READ_VECTOR_STORES) +async def get_vector_store( + request: Request, + vector_store_id: str, + auth: Annotated[AuthTuple, Depends(get_auth_dependency())], +) -> VectorStoreResponse: + """Retrieve a vector store by ID. + + Parameters: + request: The incoming HTTP request. + vector_store_id: ID of the vector store to retrieve. + auth: Authentication tuple from the auth dependency. + + Returns: + VectorStoreResponse: The vector store object. + + Raises: + HTTPException: + - 401: Authentication failed + - 403: Authorization failed + - 404: Vector store not found + - 500: Lightspeed Stack configuration not loaded + - 503: Unable to connect to Llama Stack + """ + _ = auth + _ = request + + check_configuration_loaded(configuration) + + try: + client = AsyncLlamaStackClientHolder().get_client() + vector_store = await client.vector_stores.retrieve(vector_store_id) + + return VectorStoreResponse( + id=vector_store.id, + name=vector_store.name, + created_at=vector_store.created_at, + last_active_at=vector_store.last_active_at, + expires_at=vector_store.expires_at, + status=vector_store.status or "unknown", + usage_bytes=vector_store.usage_bytes or 0, + metadata=vector_store.metadata, + ) + except APIConnectionError as e: + logger.error("Unable to connect to Llama Stack: %s", e) + response = ServiceUnavailableResponse(backend_name="Llama Stack", cause=str(e)) + raise HTTPException(**response.model_dump()) from e + except BadRequestError as e: + logger.error("Vector store not found: %s", e) + response = NotFoundResponse( + resource="vector_store", resource_id=vector_store_id + ) + raise HTTPException(**response.model_dump()) from e + except (LLSApiStatusError, OpenAIAPIStatusError) as e: + logger.error("API status error while getting vector store: %s", e) + error_response = handle_known_apistatus_errors(e, "llama-stack") + raise HTTPException(**error_response.model_dump()) from e + + +@router.put("/vector-stores/{vector_store_id}", responses=vector_store_responses) +@authorize(Action.MANAGE_VECTOR_STORES) +async def update_vector_store( + request: Request, + vector_store_id: str, + auth: Annotated[AuthTuple, Depends(get_auth_dependency())], + body: VectorStoreUpdateRequest, +) -> VectorStoreResponse: + """Update a vector store. + + Parameters: + request: The incoming HTTP request. + vector_store_id: ID of the vector store to update. + auth: Authentication tuple from the auth dependency. + body: Vector store update parameters. + + Returns: + VectorStoreResponse: The updated vector store object. + + Raises: + HTTPException: + - 401: Authentication failed + - 403: Authorization failed + - 404: Vector store not found + - 500: Lightspeed Stack configuration not loaded + - 503: Unable to connect to Llama Stack + """ + _ = auth + _ = request + + check_configuration_loaded(configuration) + + try: + client = AsyncLlamaStackClientHolder().get_client() + vector_store = await client.vector_stores.update( + vector_store_id, **body.model_dump(exclude_none=True) + ) + + return VectorStoreResponse( + id=vector_store.id, + name=vector_store.name, + created_at=vector_store.created_at, + last_active_at=vector_store.last_active_at, + expires_at=vector_store.expires_at, + status=vector_store.status or "unknown", + usage_bytes=vector_store.usage_bytes or 0, + metadata=vector_store.metadata or None, + ) + except APIConnectionError as e: + logger.error("Unable to connect to Llama Stack: %s", e) + response = ServiceUnavailableResponse(backend_name="Llama Stack", cause=str(e)) + raise HTTPException(**response.model_dump()) from e + except BadRequestError as e: + logger.error("Vector store not found: %s", e) + response = NotFoundResponse( + resource="vector_store", resource_id=vector_store_id + ) + raise HTTPException(**response.model_dump()) from e + except (LLSApiStatusError, OpenAIAPIStatusError) as e: + logger.error("API status error while updating vector store: %s", e) + error_response = handle_known_apistatus_errors(e, "llama-stack") + raise HTTPException(**error_response.model_dump()) from e + + +@router.delete( + "/vector-stores/{vector_store_id}", + responses={"204": {"description": "Vector store deleted"}}, + status_code=status.HTTP_204_NO_CONTENT, +) +@authorize(Action.MANAGE_VECTOR_STORES) +async def delete_vector_store( + request: Request, + vector_store_id: str, + auth: Annotated[AuthTuple, Depends(get_auth_dependency())], +) -> None: + """Delete a vector store. + + Parameters: + request: The incoming HTTP request. + vector_store_id: ID of the vector store to delete. + auth: Authentication tuple from the auth dependency. + + Raises: + HTTPException: + - 401: Authentication failed + - 403: Authorization failed + - 404: Vector store not found + - 500: Lightspeed Stack configuration not loaded + - 503: Unable to connect to Llama Stack + """ + _ = auth + _ = request + + check_configuration_loaded(configuration) + + try: + client = AsyncLlamaStackClientHolder().get_client() + await client.vector_stores.delete(vector_store_id) + except APIConnectionError as e: + logger.error("Unable to connect to Llama Stack: %s", e) + response = ServiceUnavailableResponse(backend_name="Llama Stack", cause=str(e)) + raise HTTPException(**response.model_dump()) from e + except BadRequestError as e: + logger.error("Vector store not found: %s", e) + response = NotFoundResponse( + resource="vector_store", resource_id=vector_store_id + ) + raise HTTPException(**response.model_dump()) from e + except (LLSApiStatusError, OpenAIAPIStatusError) as e: + logger.error("API status error while deleting vector store: %s", e) + error_response = handle_known_apistatus_errors(e, "llama-stack") + raise HTTPException(**error_response.model_dump()) from e + + +@router.post("/files", responses=file_responses) +@authorize(Action.MANAGE_FILES) +async def create_file( # pylint: disable=too-many-branches,too-many-statements + request: Request, + auth: Annotated[AuthTuple, Depends(get_auth_dependency())], + file: UploadFile = File(...), +) -> FileResponse: + """Upload a file. + + Parameters: + request: The incoming HTTP request. + auth: Authentication tuple from the auth dependency. + file: The file to upload. + + Returns: + FileResponse: The uploaded file object. + + Raises: + HTTPException: + - 400: Bad request (e.g., file too large, invalid format) + - 401: Authentication failed + - 403: Authorization failed + - 500: Lightspeed Stack configuration not loaded + - 503: Unable to connect to Llama Stack + """ + _ = auth + + check_configuration_loaded(configuration) + + # Check Content-Length header BEFORE reading to prevent DoS via memory exhaustion + content_length = request.headers.get("content-length") + if content_length: + try: + size = int(content_length) + if size > DEFAULT_MAX_FILE_UPLOAD_SIZE: + response = FileTooLargeResponse( + file_size=size, + max_size=DEFAULT_MAX_FILE_UPLOAD_SIZE, + ) + raise HTTPException(**response.model_dump()) + except ValueError: + # Invalid Content-Length header, continue and validate after reading + pass + + # file.size attribute if available + if hasattr(file, "size") and file.size is not None: + if file.size > DEFAULT_MAX_FILE_UPLOAD_SIZE: + response = FileTooLargeResponse( + file_size=file.size, + max_size=DEFAULT_MAX_FILE_UPLOAD_SIZE, + ) + raise HTTPException(**response.model_dump()) + + try: + client = AsyncLlamaStackClientHolder().get_client() + + # Read file content once + content = await file.read() + + # Verify actual size after reading + if len(content) > DEFAULT_MAX_FILE_UPLOAD_SIZE: + response = FileTooLargeResponse( + file_size=len(content), + max_size=DEFAULT_MAX_FILE_UPLOAD_SIZE, + ) + raise HTTPException(**response.model_dump()) + + filename = file.filename or "uploaded_file" + + # Add .txt extension if no extension present + # (since parsed PDFs/URLs are sent as plain text) + if not os.path.splitext(filename)[1]: + filename = f"{filename}.txt" + + logger.info( + "Uploading file - filename: %s, size: %d bytes", + filename, + len(content), + ) + + file_bytes = BytesIO(content) + file_bytes.name = filename + + file_obj = await client.files.create( + file=file_bytes, + purpose="assistants", + ) + + return FileResponse( + id=file_obj.id, + filename=file_obj.filename or filename, + bytes=file_obj.bytes or len(content), + created_at=file_obj.created_at, + purpose=file_obj.purpose or "assistants", + object=file_obj.object or "file", + ) + except APIConnectionError as e: + logger.error("Unable to connect to Llama Stack: %s", e) + response = ServiceUnavailableResponse(backend_name="Llama Stack", cause=str(e)) + raise HTTPException(**response.model_dump()) from e + except BadRequestError as e: + logger.error("Bad request for file upload: %s", e) + # Check if backend rejected due to file size + error_msg = str(e).lower() + if "too large" in error_msg or "size" in error_msg or "exceeds" in error_msg: + response = FileTooLargeResponse( + response="Invalid file upload", + cause=f"File upload rejected by Llama Stack: {str(e)}", + ) + else: + response = InternalServerErrorResponse.query_failed( + cause=f"File upload rejected by Llama Stack: {str(e)}" + ) + # Override to use 400 status code since it's a client error + response.status_code = status.HTTP_400_BAD_REQUEST + response.detail.response = "Invalid file upload" + raise HTTPException(**response.model_dump()) from e + except (LLSApiStatusError, OpenAIAPIStatusError) as e: + logger.error("API status error while uploading file: %s", e) + error_response = handle_known_apistatus_errors(e, "llama-stack") + raise HTTPException(**error_response.model_dump()) from e + + +@router.post( + "/vector-stores/{vector_store_id}/files", responses=vector_store_file_responses +) +@authorize(Action.MANAGE_VECTOR_STORES) +async def add_file_to_vector_store( # pylint: disable=too-many-locals,too-many-statements + request: Request, + vector_store_id: str, + auth: Annotated[AuthTuple, Depends(get_auth_dependency())], + body: VectorStoreFileCreateRequest, +) -> VectorStoreFileResponse: + """Add a file to a vector store. + + Parameters: + request: The incoming HTTP request. + vector_store_id: ID of the vector store. + auth: Authentication tuple from the auth dependency. + body: File addition parameters. + + Returns: + VectorStoreFileResponse: The vector store file object. + + Raises: + HTTPException: + - 401: Authentication failed + - 403: Authorization failed + - 404: Vector store or file not found + - 500: Lightspeed Stack configuration not loaded + - 503: Unable to connect to Llama Stack + """ + _ = auth + _ = request + + check_configuration_loaded(configuration) + + try: + client = AsyncLlamaStackClientHolder().get_client() + + # Retry logic for database lock errors + max_retries = 3 + retry_delay = 0.5 # seconds + vs_file = None + last_lock_error: Exception | None = None + + for attempt in range(max_retries): + try: + vs_file = await client.vector_stores.files.create( + vector_store_id=vector_store_id, + **body.model_dump(exclude_none=True), + ) + break # Success, exit retry loop + except Exception as retry_error: # pylint: disable=broad-exception-caught + error_msg = str(retry_error).lower() + is_lock_error = ( + "database is locked" in error_msg or "locked" in error_msg + ) + is_last_attempt = attempt == max_retries - 1 + + if is_lock_error: + last_lock_error = retry_error + if not is_last_attempt: + logger.warning( + "Database locked while adding file to vector store, " + "retrying in %s seconds (attempt %d/%d)", + retry_delay, + attempt + 1, + max_retries, + ) + await asyncio.sleep(retry_delay) + retry_delay *= 2 # Exponential backoff + continue + break + raise # Re-raise if not a lock error + if vs_file is None: + if last_lock_error is not None: + # Use standard error response model for consistency + response = InternalServerErrorResponse( + response="Failed to create vector store file", + cause="All retry attempts failed to create the vector store file", + ) + raise HTTPException(**response.model_dump()) from last_lock_error + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) + logger.info( + "Vector store file created - ID: %s, status: %s, last_error: %s", + vs_file.id, + vs_file.status, + vs_file.last_error if vs_file.last_error else "None", + ) + + return VectorStoreFileResponse( + id=vs_file.id, + vector_store_id=vs_file.vector_store_id or vector_store_id, + status=vs_file.status or "unknown", + attributes=vs_file.attributes, + last_error=( + vs_file.last_error.message + if vs_file.last_error and hasattr(vs_file.last_error, "message") + else None + ), + object=vs_file.object or "vector_store.file", + ) + except APIConnectionError as e: + logger.error("Unable to connect to Llama Stack: %s", e) + response = ServiceUnavailableResponse(backend_name="Llama Stack", cause=str(e)) + raise HTTPException(**response.model_dump()) from e + except BadRequestError as e: + logger.error("Vector store file operation failed: %s", e) + # Don't assume which resource is missing - could be vector_store_id OR file_id + response = NotFoundResponse( + resource="vector_store_or_file", + resource_id=f"vector_store={vector_store_id}, file={body.file_id}", + ) + raise HTTPException(**response.model_dump()) from e + except (LLSApiStatusError, OpenAIAPIStatusError) as e: + logger.error("API status error while adding file to vector store: %s", e) + error_response = handle_known_apistatus_errors(e, "llama-stack") + raise HTTPException(**error_response.model_dump()) from e + + +@router.get( + "/vector-stores/{vector_store_id}/files", + responses=vector_store_files_list_responses, +) +@authorize(Action.READ_VECTOR_STORES) +async def list_vector_store_files( + request: Request, + vector_store_id: str, + auth: Annotated[AuthTuple, Depends(get_auth_dependency())], +) -> VectorStoreFilesListResponse: + """List files in a vector store. + + Parameters: + request: The incoming HTTP request. + vector_store_id: ID of the vector store. + auth: Authentication tuple from the auth dependency. + + Returns: + VectorStoreFilesListResponse: List of files in the vector store. + + Raises: + HTTPException: + - 401: Authentication failed + - 403: Authorization failed + - 404: Vector store not found + - 500: Lightspeed Stack configuration not loaded + - 503: Unable to connect to Llama Stack + """ + _ = auth + _ = request + + check_configuration_loaded(configuration) + + try: + client = AsyncLlamaStackClientHolder().get_client() + files = await client.vector_stores.files.list(vector_store_id=vector_store_id) + + data = [ + VectorStoreFileResponse( + id=f.id, + vector_store_id=f.vector_store_id or vector_store_id, + status=f.status or "unknown", + attributes=f.attributes, + last_error=( + f.last_error.message + if f.last_error and hasattr(f.last_error, "message") + else None + ), + object=f.object or "vector_store.file", + ) + for f in files.data + ] + return VectorStoreFilesListResponse(data=data) + except APIConnectionError as e: + logger.error("Unable to connect to Llama Stack: %s", e) + response = ServiceUnavailableResponse(backend_name="Llama Stack", cause=str(e)) + raise HTTPException(**response.model_dump()) from e + except BadRequestError as e: + logger.error("Vector store not found: %s", e) + response = NotFoundResponse( + resource="vector_store", resource_id=vector_store_id + ) + raise HTTPException(**response.model_dump()) from e + except (LLSApiStatusError, OpenAIAPIStatusError) as e: + logger.error("API status error while listing vector store files: %s", e) + error_response = handle_known_apistatus_errors(e, "llama-stack") + raise HTTPException(**error_response.model_dump()) from e + + +@router.get( + "/vector-stores/{vector_store_id}/files/{file_id}", + responses=vector_store_file_responses, +) +@authorize(Action.READ_VECTOR_STORES) +async def get_vector_store_file( + request: Request, + vector_store_id: str, + file_id: str, + auth: Annotated[AuthTuple, Depends(get_auth_dependency())], +) -> VectorStoreFileResponse: + """Retrieve a file from a vector store. + + Parameters: + request: The incoming HTTP request. + vector_store_id: ID of the vector store. + file_id: ID of the file. + auth: Authentication tuple from the auth dependency. + + Returns: + VectorStoreFileResponse: The vector store file object. + + Raises: + HTTPException: + - 401: Authentication failed + - 403: Authorization failed + - 404: File not found in vector store + - 500: Lightspeed Stack configuration not loaded + - 503: Unable to connect to Llama Stack + """ + _ = auth + _ = request + + check_configuration_loaded(configuration) + + try: + client = AsyncLlamaStackClientHolder().get_client() + vs_file = await client.vector_stores.files.retrieve( + vector_store_id=vector_store_id, + file_id=file_id, + ) + + return VectorStoreFileResponse( + id=vs_file.id, + vector_store_id=vs_file.vector_store_id or vector_store_id, + status=vs_file.status or "unknown", + attributes=vs_file.attributes, + last_error=( + vs_file.last_error.message + if vs_file.last_error and hasattr(vs_file.last_error, "message") + else None + ), + object=vs_file.object or "vector_store.file", + ) + except APIConnectionError as e: + logger.error("Unable to connect to Llama Stack: %s", e) + response = ServiceUnavailableResponse(backend_name="Llama Stack", cause=str(e)) + raise HTTPException(**response.model_dump()) from e + except BadRequestError as e: + logger.error("Vector store file not found: %s", e) + response = NotFoundResponse(resource="file", resource_id=file_id) + raise HTTPException(**response.model_dump()) from e + except (LLSApiStatusError, OpenAIAPIStatusError) as e: + logger.error("API status error while getting vector store file: %s", e) + error_response = handle_known_apistatus_errors(e, "llama-stack") + raise HTTPException(**error_response.model_dump()) from e + + +@router.delete( + "/vector-stores/{vector_store_id}/files/{file_id}", + responses={"204": {"description": "File deleted from vector store"}}, + status_code=status.HTTP_204_NO_CONTENT, +) +@authorize(Action.MANAGE_VECTOR_STORES) +async def delete_vector_store_file( + request: Request, + vector_store_id: str, + file_id: str, + auth: Annotated[AuthTuple, Depends(get_auth_dependency())], +) -> None: + """Delete a file from a vector store. + + Parameters: + request: The incoming HTTP request. + vector_store_id: ID of the vector store. + file_id: ID of the file to delete. + auth: Authentication tuple from the auth dependency. + + Raises: + HTTPException: + - 401: Authentication failed + - 403: Authorization failed + - 404: File not found in vector store + - 500: Lightspeed Stack configuration not loaded + - 503: Unable to connect to Llama Stack + """ + _ = auth + _ = request + + check_configuration_loaded(configuration) + + try: + client = AsyncLlamaStackClientHolder().get_client() + await client.vector_stores.files.delete( + vector_store_id=vector_store_id, + file_id=file_id, + ) + except APIConnectionError as e: + logger.error("Unable to connect to Llama Stack: %s", e) + response = ServiceUnavailableResponse(backend_name="Llama Stack", cause=str(e)) + raise HTTPException(**response.model_dump()) from e + except BadRequestError as e: + logger.error("Vector store file not found: %s", e) + response = NotFoundResponse(resource="file", resource_id=file_id) + raise HTTPException(**response.model_dump()) from e + except (LLSApiStatusError, OpenAIAPIStatusError) as e: + logger.error("API status error while deleting vector store file: %s", e) + error_response = handle_known_apistatus_errors(e, "llama-stack") + raise HTTPException(**error_response.model_dump()) from e diff --git a/src/app/routers.py b/src/app/routers.py index 1841d1fc4..391864b4f 100644 --- a/src/app/routers.py +++ b/src/app/routers.py @@ -28,6 +28,7 @@ stream_interrupt, streaming_query, tools, + vector_stores, ) @@ -53,6 +54,7 @@ def include_routers(app: FastAPI) -> None: app.include_router(shields.router, prefix="/v1") app.include_router(providers.router, prefix="/v1") app.include_router(rags.router, prefix="/v1") + app.include_router(vector_stores.router, prefix="/v1") # Query endpoints app.include_router(query.router, prefix="/v1") app.include_router(streaming_query.router, prefix="/v1") diff --git a/src/constants.py b/src/constants.py index 5f05431c6..b571d56c6 100644 --- a/src/constants.py +++ b/src/constants.py @@ -128,6 +128,10 @@ DEFAULT_AUTHENTICATION_MODULE = AUTH_MOD_NOOP # Maximum allowed size for base64-encoded x-rh-identity header (bytes) DEFAULT_RH_IDENTITY_MAX_HEADER_SIZE = 8192 + +# Maximum allowed file upload size (bytes) - 100MB default +# Protects against DoS attacks via large file uploads +DEFAULT_MAX_FILE_UPLOAD_SIZE = 100 * 1024 * 1024 # 100 MB DEFAULT_JWT_UID_CLAIM = "user_id" DEFAULT_JWT_USER_NAME_CLAIM = "username" diff --git a/src/models/config.py b/src/models/config.py index 1b86e5437..3d69ecf2c 100644 --- a/src/models/config.py +++ b/src/models/config.py @@ -1030,6 +1030,11 @@ class Action(str, Enum): A2A_MESSAGE = "a2a_message" A2A_JSONRPC = "a2a_jsonrpc" + # Vector store management + MANAGE_VECTOR_STORES = "manage_vector_stores" + READ_VECTOR_STORES = "read_vector_stores" + MANAGE_FILES = "manage_files" + class AccessRule(ConfigurationBase): """Rule defining what actions a role can perform.""" diff --git a/src/models/requests.py b/src/models/requests.py index 7a8aba99c..635c1740a 100644 --- a/src/models/requests.py +++ b/src/models/requests.py @@ -1,5 +1,7 @@ """Models for REST API requests.""" +# pylint: disable=too-many-lines + from enum import Enum from typing import Any, Optional, Self @@ -935,3 +937,213 @@ def validate_authorization_header_values( "File-path based secrets are only supported in static YAML config." ) return value + + +class VectorStoreCreateRequest(BaseModel): + """Model representing a request to create a vector store. + + Attributes: + name: Name of the vector store. + embedding_model: Optional embedding model to use. + embedding_dimension: Optional embedding dimension. + chunking_strategy: Optional chunking strategy configuration. + provider_id: Optional vector store provider identifier. + metadata: Optional metadata dictionary for storing session information. + """ + + name: str = Field( + ..., + description="Name of the vector store", + examples=["my_vector_store"], + min_length=1, + max_length=256, + ) + + embedding_model: Optional[str] = Field( + None, + description="Embedding model to use for the vector store", + examples=["text-embedding-ada-002"], + ) + + embedding_dimension: Optional[int] = Field( + None, + description="Dimension of the embedding vectors", + examples=[1536], + gt=0, + ) + + chunking_strategy: Optional[dict[str, Any]] = Field( + None, + description="Chunking strategy configuration", + examples=[{"type": "fixed", "chunk_size": 512, "chunk_overlap": 50}], + ) + + provider_id: Optional[str] = Field( + None, + description="Vector store provider identifier", + examples=["rhdh-docs"], + ) + + metadata: Optional[dict[str, Any]] = Field( + None, + description="Metadata dictionary for storing session information", + examples=[{"user_id": "user123", "session_id": "sess456"}], + ) + + model_config = { + "extra": "forbid", + "json_schema_extra": { + "examples": [ + { + "name": "my_vector_store", + "embedding_model": "text-embedding-ada-002", + "embedding_dimension": 1536, + "provider_id": "rhdh-docs", + "metadata": {"user_id": "user123"}, + }, + ] + }, + } + + +class VectorStoreUpdateRequest(BaseModel): + """Model representing a request to update a vector store. + + Attributes: + name: New name for the vector store. + expires_at: Optional expiration timestamp. + metadata: Optional metadata dictionary for storing session information. + """ + + name: Optional[str] = Field( + None, + description="New name for the vector store", + examples=["updated_vector_store"], + min_length=1, + max_length=256, + ) + + expires_at: Optional[int] = Field( + None, + description="Unix timestamp when the vector store should expire", + examples=[1735689600], + gt=0, + ) + + metadata: Optional[dict[str, Any]] = Field( + None, + description="Metadata dictionary for storing session information", + examples=[{"user_id": "user123", "session_id": "sess456"}], + ) + + model_config = { + "extra": "forbid", + "json_schema_extra": { + "examples": [ + { + "name": "updated_vector_store", + "expires_at": 1735689600, + "metadata": {"user_id": "user123"}, + }, + ] + }, + } + + @model_validator(mode="after") + def check_at_least_one_field(self) -> Self: + """Ensure at least one field is provided for update. + + Raises: + ValueError: If all fields are None (empty update). + + Returns: + Self: The validated model instance. + """ + if self.name is None and self.expires_at is None and self.metadata is None: + raise ValueError( + "At least one field must be provided: name, expires_at, or metadata" + ) + return self + + +class VectorStoreFileCreateRequest(BaseModel): + """Model representing a request to add a file to a vector store. + + Attributes: + file_id: ID of the file to add to the vector store. + attributes: Optional metadata key-value pairs (max 16 pairs). + chunking_strategy: Optional chunking strategy configuration. + """ + + file_id: str = Field( + ..., + description="ID of the file to add to the vector store", + examples=["file-abc123"], + min_length=1, + ) + + attributes: Optional[dict[str, str | float | bool]] = Field( + None, + description=( + "Set of up to 16 key-value pairs for storing additional information. " + "Keys: strings (max 64 chars). Values: strings (max 512 chars), booleans, or numbers." + ), + examples=[ + {"created_at": "2026-04-04T15:20:00Z", "updated_at": "2026-04-04T15:20:00Z"} + ], + ) + + chunking_strategy: Optional[dict[str, Any]] = Field( + None, + description="Chunking strategy configuration for this file", + examples=[{"type": "fixed", "chunk_size": 512, "chunk_overlap": 50}], + ) + + model_config = { + "extra": "forbid", + "json_schema_extra": { + "examples": [ + { + "file_id": "file-abc123", + "attributes": {"created_at": "2026-04-04T15:20:00Z"}, + "chunking_strategy": {"type": "fixed", "chunk_size": 512}, + }, + ] + }, + } + + @field_validator("attributes") + @classmethod + def validate_attributes( + cls, value: Optional[dict[str, str | float | bool]] + ) -> Optional[dict[str, str | float | bool]]: + """Validate attributes field constraints. + + Ensures: + - Maximum 16 key-value pairs + - Keys are max 64 characters + - String values are max 512 characters + + Parameters: + value: The attributes dictionary to validate. + + Raises: + ValueError: If constraints are violated. + + Returns: + The validated attributes dictionary. + """ + if value is None: + return value + + if len(value) > 16: + raise ValueError(f"attributes can have at most 16 pairs, got {len(value)}") + + for key, val in value.items(): + if len(key) > 64: + raise ValueError(f"attribute key '{key}' exceeds 64 characters") + + if isinstance(val, str) and len(val) > 512: + raise ValueError(f"attribute value for '{key}' exceeds 512 characters") + + return value diff --git a/src/models/responses.py b/src/models/responses.py index 5c2e974cc..71e55934e 100644 --- a/src/models/responses.py +++ b/src/models/responses.py @@ -1804,7 +1804,14 @@ class BadRequestResponse(AbstractErrorResponse): "123e4567-e89b-12d3-a456-426614174000 has invalid format." ), }, - } + }, + { + "label": "file_upload", + "detail": { + "response": "Invalid file upload", + "cause": "File upload rejected: Invalid file format", + }, + }, ] } } @@ -2115,6 +2122,20 @@ class NotFoundResponse(AbstractErrorResponse): "cause": "Mcp Server with ID test-mcp-server does not exist", }, }, + { + "label": "vector store", + "detail": { + "response": "Vector Store not found", + "cause": "Vector Store with ID vs_abc123 does not exist", + }, + }, + { + "label": "file", + "detail": { + "response": "File not found", + "cause": "File with ID file_abc123 does not exist", + }, + }, ] } } @@ -2221,6 +2242,66 @@ def __init__( ) +class FileTooLargeResponse(AbstractErrorResponse): + """413 Content Too Large - File upload exceeds size limit.""" + + description: ClassVar[str] = "File upload exceeds size limit" + model_config = { + "json_schema_extra": { + "examples": [ + { + "label": "file upload", + "detail": { + "response": "File too large", + "cause": ( + "File size 150000000 bytes exceeds maximum " + "allowed size of 104857600 bytes (100 MB)" + ), + }, + }, + { + "label": "backend rejection", + "detail": { + "response": "Invalid file upload", + "cause": "File upload rejected by Llama Stack: File size exceeds limit", + }, + }, + ] + } + } + + def __init__( + self, + *, + response: str = "File too large", + cause: str | None = None, + file_size: int | None = None, + max_size: int | None = None, + ) -> None: + """Initialize a FileTooLargeResponse. + + Args: + response: Short summary of the error. Defaults to "File too large". + cause: Detailed explanation. If not provided, will be generated from + file_size and max_size. + file_size: The size of the uploaded file in bytes. + max_size: The maximum allowed file size in bytes. + """ + if cause is None and file_size is not None and max_size is not None: + cause = ( + f"File size {file_size} bytes exceeds maximum allowed " + f"size of {max_size} bytes ({max_size // (1024 * 1024)} MB)" + ) + elif cause is None: + cause = "The uploaded file exceeds the maximum allowed size." + + super().__init__( + response=response, + cause=cause, + status_code=status.HTTP_413_CONTENT_TOO_LARGE, + ) + + class UnprocessableEntityResponse(AbstractErrorResponse): """422 Unprocessable Entity - Request validation failed.""" @@ -2604,3 +2685,228 @@ def __init__(self, *, backend_name: str, cause: str): cause=cause, status_code=status.HTTP_503_SERVICE_UNAVAILABLE, ) + + +class VectorStoreResponse(AbstractSuccessfulResponse): + """Response model containing a single vector store. + + Attributes: + id: Vector store ID. + name: Vector store name. + created_at: Unix timestamp when created. + last_active_at: Unix timestamp of last activity. + expires_at: Optional Unix timestamp when it expires. + status: Vector store status. + usage_bytes: Storage usage in bytes. + metadata: Optional metadata dictionary for storing session information. + """ + + id: str = Field(..., description="Vector store ID") + name: str = Field(..., description="Vector store name") + created_at: int = Field(..., description="Unix timestamp when created") + last_active_at: Optional[int] = Field( + None, description="Unix timestamp of last activity" + ) + expires_at: Optional[int] = Field( + None, description="Unix timestamp when it expires" + ) + status: str = Field(..., description="Vector store status") + usage_bytes: int = Field(default=0, description="Storage usage in bytes") + metadata: Optional[dict[str, Any]] = Field( + None, + description="Metadata dictionary for storing session information", + examples=[ + {"conversation_id": "conv_123", "document_ids": ["doc_456", "doc_789"]} + ], + ) + + model_config = { + "extra": "forbid", + "json_schema_extra": { + "examples": [ + { + "id": "vs_abc123", + "name": "customer_support_docs", + "created_at": 1704067200, + "last_active_at": 1704153600, + "expires_at": None, + "status": "active", + "usage_bytes": 1048576, + "metadata": { + "conversation_id": "conv_123", + "document_ids": ["doc_456", "doc_789"], + }, + } + ] + }, + } + + +class VectorStoresListResponse(AbstractSuccessfulResponse): + """Response model containing a list of vector stores. + + Attributes: + data: List of vector store objects. + object: Object type (always "list"). + """ + + data: list[VectorStoreResponse] = Field( + default_factory=list, description="List of vector stores" + ) + object: str = Field(default="list", description="Object type") + + model_config = { + "extra": "forbid", + "json_schema_extra": { + "examples": [ + { + "data": [ + { + "id": "vs_abc123", + "name": "customer_support_docs", + "created_at": 1704067200, + "last_active_at": 1704153600, + "expires_at": None, + "status": "active", + "usage_bytes": 1048576, + "metadata": {"conversation_id": "conv_123"}, + }, + { + "id": "vs_def456", + "name": "product_documentation", + "created_at": 1704070800, + "last_active_at": 1704157200, + "expires_at": None, + "status": "active", + "usage_bytes": 2097152, + "metadata": None, + }, + ], + "object": "list", + } + ] + }, + } + + +class FileResponse(AbstractSuccessfulResponse): + """Response model containing a file object. + + Attributes: + id: File ID. + filename: File name. + bytes: File size in bytes. + created_at: Unix timestamp when created. + purpose: File purpose. + object: Object type (always "file"). + """ + + id: str = Field(..., description="File ID") + filename: str = Field(..., description="File name") + bytes: int = Field(..., description="File size in bytes") + created_at: int = Field(..., description="Unix timestamp when created") + purpose: str = Field(default="assistants", description="File purpose") + object: str = Field(default="file", description="Object type") + + model_config = { + "extra": "forbid", + "json_schema_extra": { + "examples": [ + { + "id": "file_abc123", + "filename": "documentation.pdf", + "bytes": 524288, + "created_at": 1704067200, + "purpose": "assistants", + "object": "file", + } + ] + }, + } + + +class VectorStoreFileResponse(AbstractSuccessfulResponse): + """Response model containing a vector store file object. + + Attributes: + id: Vector store file ID. + vector_store_id: ID of the vector store. + status: File processing status. + attributes: Optional metadata key-value pairs. + last_error: Optional error message if processing failed. + object: Object type (always "vector_store.file"). + """ + + id: str = Field(..., description="Vector store file ID") + vector_store_id: str = Field(..., description="ID of the vector store") + status: str = Field(..., description="File processing status") + attributes: Optional[dict[str, str | float | bool]] = Field( + None, + description=( + "Set of up to 16 key-value pairs for storing additional information. " + "Keys: strings (max 64 chars). Values: strings (max 512 chars), booleans, or numbers." + ), + ) + last_error: Optional[str] = Field( + None, description="Error message if processing failed" + ) + object: str = Field(default="vector_store.file", description="Object type") + + model_config = { + "extra": "forbid", + "json_schema_extra": { + "examples": [ + { + "id": "file_abc123", + "vector_store_id": "vs_abc123", + "status": "completed", + "attributes": {"chunk_size": "512", "indexed": True}, + "last_error": None, + "object": "vector_store.file", + } + ] + }, + } + + +class VectorStoreFilesListResponse(AbstractSuccessfulResponse): + """Response model containing a list of vector store files. + + Attributes: + data: List of vector store file objects. + object: Object type (always "list"). + """ + + data: list[VectorStoreFileResponse] = Field( + default_factory=list, description="List of vector store files" + ) + object: str = Field(default="list", description="Object type") + + model_config = { + "extra": "forbid", + "json_schema_extra": { + "examples": [ + { + "data": [ + { + "id": "file_abc123", + "vector_store_id": "vs_abc123", + "status": "completed", + "attributes": {"chunk_size": "512"}, + "last_error": None, + "object": "vector_store.file", + }, + { + "id": "file_def456", + "vector_store_id": "vs_abc123", + "status": "processing", + "attributes": None, + "last_error": None, + "object": "vector_store.file", + }, + ], + "object": "list", + } + ] + }, + } diff --git a/src/observability/README.md b/src/observability/README.md index 0232571da..d574eab03 100644 --- a/src/observability/README.md +++ b/src/observability/README.md @@ -30,7 +30,7 @@ event_data = InferenceEventData( org_id="12345678", system_id="abc-def-123", request_id="req_xyz789", - cla_version="CLA/0.5.0", + cla_version="CLA/0.5.1", system_os="RHEL", system_version="9.3", system_arch="x86_64", diff --git a/src/version.py b/src/version.py index e863b7c7c..a96388aeb 100644 --- a/src/version.py +++ b/src/version.py @@ -9,4 +9,4 @@ # [tool.pdm.version] # source = "file" # path = "src/version.py" -__version__ = "0.5.0" +__version__ = "0.5.1" diff --git a/tests/e2e/features/info.feature b/tests/e2e/features/info.feature index 8c1431790..3d53fa2d4 100644 --- a/tests/e2e/features/info.feature +++ b/tests/e2e/features/info.feature @@ -15,7 +15,7 @@ Feature: Info tests Given The system is in default state When I access REST API endpoint "info" using HTTP GET method Then The status code of the response is 200 - And The body of the response has proper name Lightspeed Core Service (LCS) and version 0.5.0 + And The body of the response has proper name Lightspeed Core Service (LCS) and version 0.5.1 And The body of the response has llama-stack version 0.5.2 @skip-in-library-mode diff --git a/tests/integration/endpoints/test_rlsapi_v1_integration.py b/tests/integration/endpoints/test_rlsapi_v1_integration.py index ae59acaab..4a8ec32a3 100644 --- a/tests/integration/endpoints/test_rlsapi_v1_integration.py +++ b/tests/integration/endpoints/test_rlsapi_v1_integration.py @@ -44,7 +44,7 @@ def _create_mock_request(mocker: MockerFixture) -> Any: mock_request = mocker.Mock() # Use spec=[] to create a Mock with no attributes, simulating absent rh_identity_data mock_request.state = mocker.Mock(spec=[]) - mock_request.headers = {"User-Agent": "CLA/0.5.0"} + mock_request.headers = {"User-Agent": "CLA/0.5.1"} return mock_request diff --git a/tests/unit/app/endpoints/conftest.py b/tests/unit/app/endpoints/conftest.py index 56e3b821e..5def43273 100644 --- a/tests/unit/app/endpoints/conftest.py +++ b/tests/unit/app/endpoints/conftest.py @@ -17,7 +17,7 @@ def mock_request_factory_fixture(mocker: MockerFixture) -> Callable[..., Any]: def _create(rh_identity: Any = None) -> Any: mock_request = mocker.Mock() - mock_request.headers = {"User-Agent": "CLA/0.5.0"} + mock_request.headers = {"User-Agent": "CLA/0.5.1"} if rh_identity is not None: mock_request.state = mocker.Mock() diff --git a/tests/unit/app/endpoints/test_vector_stores.py b/tests/unit/app/endpoints/test_vector_stores.py new file mode 100644 index 000000000..f257f68d3 --- /dev/null +++ b/tests/unit/app/endpoints/test_vector_stores.py @@ -0,0 +1,1179 @@ +"""Unit tests for the /vector-stores REST API endpoints.""" + +# pylint: disable=too-many-lines + +from typing import Any + +import pytest +from fastapi import HTTPException, Request, status +from llama_stack_client import APIConnectionError, BadRequestError +from pytest_mock import MockerFixture + +from app.endpoints.vector_stores import ( + add_file_to_vector_store, + create_file, + create_vector_store, + delete_vector_store, + delete_vector_store_file, + get_vector_store, + get_vector_store_file, + list_vector_store_files, + list_vector_stores, + update_vector_store, +) +from authentication.interface import AuthTuple +from configuration import AppConfig +from models.requests import ( + VectorStoreCreateRequest, + VectorStoreFileCreateRequest, + VectorStoreUpdateRequest, +) +from tests.unit.utils.auth_helpers import mock_authorization_resolvers + + +# pylint: disable=R0903,R0902 +class VectorStore: + """Mock vector store object.""" + + def __init__( + self, + vs_id: str, + name: str, + created_at: int = 1735689600, + vs_status: str = "active", + ) -> None: + """Initialize vector store mock.""" + self.id = vs_id + self.name = name + self.created_at = created_at + self.last_active_at = created_at + self.expires_at = None + self.object = "vector_store" + self.status = vs_status + self.usage_bytes = 0 + self.metadata = None + + +# pylint: disable=R0903 +class VectorStoresList: + """Mock vector stores list.""" + + def __init__(self, stores: list[VectorStore]) -> None: + """Initialize vector stores list mock.""" + self.data = stores + + +# pylint: disable=R0903 +class File: + """Mock file object.""" + + def __init__(self, file_id: str, filename: str, file_bytes: int = 1024) -> None: + """Initialize file mock.""" + self.id = file_id + self.filename = filename + self.bytes = file_bytes + self.created_at = 1735689600 + self.purpose = "assistants" + self.object = "file" + + +# pylint: disable=R0903 +class VectorStoreFile: + """Mock vector store file object.""" + + def __init__( + self, file_id: str, vector_store_id: str, file_status: str = "completed" + ) -> None: + """Initialize vector store file mock.""" + self.id = file_id + self.vector_store_id = vector_store_id + self.created_at = 1735689600 + self.status = file_status + self.attributes = None + self.last_error = None + self.object = "vector_store.file" + + +# pylint: disable=R0903 +class VectorStoreFilesList: + """Mock vector store files list.""" + + def __init__(self, files: list[VectorStoreFile]) -> None: + """Initialize vector store files list mock.""" + self.data = files + + +def get_test_config() -> dict[str, Any]: + """Get test configuration dictionary. + + Returns: + Test configuration dictionary. + """ + return { + "name": "foo", + "service": { + "host": "localhost", + "port": 8080, + "auth_enabled": False, + "workers": 1, + "color_log": True, + "access_log": True, + }, + "llama_stack": { + "api_key": "xyzzy", + "url": "http://x.y.com:1234", + "use_as_library_client": False, + }, + "user_data_collection": { + "feedback_enabled": False, + }, + "customization": None, + "authorization": {"access_rules": []}, + "authentication": {"module": "noop"}, + } + + +def get_test_request() -> Request: + """Get test request object. + + Returns: + Test request object. + """ + return Request( + scope={ + "type": "http", + "headers": [(b"authorization", b"Bearer test-token")], + } + ) + + +def get_test_auth() -> AuthTuple: + """Get test auth tuple. + + Returns: + Test auth tuple. + """ + return ("test_user_id", "test_user", True, "test_token") + + +@pytest.mark.asyncio +async def test_create_vector_store_configuration_not_loaded( + mocker: MockerFixture, +) -> None: + """Test create vector store endpoint if configuration is not loaded.""" + mock_authorization_resolvers(mocker) + + mock_config = AppConfig() + mocker.patch("app.endpoints.vector_stores.configuration", mock_config) + + request = get_test_request() + auth = get_test_auth() + body = VectorStoreCreateRequest(name="test_store") + + with pytest.raises(HTTPException) as e: + await create_vector_store(request=request, auth=auth, body=body) + + assert e.value.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + assert e.value.detail["response"] == "Configuration is not loaded" # type: ignore + + +@pytest.mark.asyncio +async def test_create_vector_store_success(mocker: MockerFixture) -> None: + """Test successful vector store creation.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.vector_stores.create.return_value = VectorStore("vs_123", "test_store") + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + body = VectorStoreCreateRequest(name="test_store") + + response = await create_vector_store(request=request, auth=auth, body=body) + assert response is not None + assert response.id == "vs_123" + assert response.name == "test_store" + assert response.status == "active" + + +@pytest.mark.asyncio +async def test_create_vector_store_connection_error(mocker: MockerFixture) -> None: + """Test create vector store with connection error.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.vector_stores.create.side_effect = APIConnectionError(request=None) # type: ignore + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + body = VectorStoreCreateRequest(name="test_store") + + with pytest.raises(HTTPException) as e: + await create_vector_store(request=request, auth=auth, body=body) + + assert e.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE + assert e.value.detail["response"] == "Unable to connect to Llama Stack" # type: ignore + + +@pytest.mark.asyncio +async def test_list_vector_stores_success(mocker: MockerFixture) -> None: + """Test successful vector stores list.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.vector_stores.list.return_value = VectorStoresList( + [VectorStore("vs_1", "store1"), VectorStore("vs_2", "store2")] + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + response = await list_vector_stores(request=request, auth=auth) + assert response is not None + assert len(response.data) == 2 + assert response.data[0].id == "vs_1" + assert response.data[1].id == "vs_2" + + +@pytest.mark.asyncio +async def test_get_vector_store_success(mocker: MockerFixture) -> None: + """Test successful vector store retrieval.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.vector_stores.retrieve.return_value = VectorStore( + "vs_123", "test_store" + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + response = await get_vector_store( + request=request, vector_store_id="vs_123", auth=auth + ) + assert response is not None + assert response.id == "vs_123" + assert response.name == "test_store" + + +@pytest.mark.asyncio +async def test_get_vector_store_not_found(mocker: MockerFixture) -> None: + """Test vector store retrieval with not found error.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + # Create a mock response for BadRequestError + mock_response = mocker.Mock() + mock_response.request = mocker.Mock() + mock_client.vector_stores.retrieve.side_effect = BadRequestError( + message="Not found", response=mock_response, body=None + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + with pytest.raises(HTTPException) as e: + await get_vector_store(request=request, vector_store_id="vs_999", auth=auth) + assert e.value.status_code == status.HTTP_404_NOT_FOUND + + +@pytest.mark.asyncio +async def test_update_vector_store_success(mocker: MockerFixture) -> None: + """Test successful vector store update.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.vector_stores.update.return_value = VectorStore( + "vs_123", "updated_store" + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + body = VectorStoreUpdateRequest(name="updated_store") + + response = await update_vector_store( + request=request, vector_store_id="vs_123", auth=auth, body=body + ) + assert response is not None + assert response.id == "vs_123" + assert response.name == "updated_store" + + +@pytest.mark.asyncio +async def test_delete_vector_store_success(mocker: MockerFixture) -> None: + """Test successful vector store deletion.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.vector_stores.delete.return_value = None + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + response = await delete_vector_store( + request=request, vector_store_id="vs_123", auth=auth + ) + assert response is None + + +@pytest.mark.asyncio +async def test_create_file_success(mocker: MockerFixture) -> None: + """Test successful file upload.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.files.create.return_value = File("file_123", "test.txt", 1024) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + # Mock UploadFile + mock_file = mocker.AsyncMock() + mock_file.filename = "test.txt" + mock_file.size = 12 # Size of "test content" + mock_file.read.return_value = b"test content" + + response = await create_file(request=request, auth=auth, file=mock_file) + assert response is not None + assert response.id == "file_123" + assert response.filename == "test.txt" + assert response.bytes == 1024 + + +@pytest.mark.asyncio +async def test_add_file_to_vector_store_success(mocker: MockerFixture) -> None: + """Test successfully adding file to vector store.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.vector_stores.files.create.return_value = VectorStoreFile( + "file_123", "vs_123" + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + body = VectorStoreFileCreateRequest(file_id="file_123") + + response = await add_file_to_vector_store( + request=request, vector_store_id="vs_123", auth=auth, body=body + ) + assert response is not None + assert response.id == "file_123" + assert response.vector_store_id == "vs_123" + assert response.status == "completed" + + +@pytest.mark.asyncio +async def test_add_file_to_vector_store_retry_on_database_lock( + mocker: MockerFixture, +) -> None: + """Test retry logic when database lock error occurs.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + # First call raises database lock error, second call succeeds + mock_client.vector_stores.files.create.side_effect = [ + Exception("database is locked"), + VectorStoreFile("file_123", "vs_123"), + ] + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + # Mock asyncio.sleep to avoid actual delays in tests + mock_sleep = mocker.patch("app.endpoints.vector_stores.asyncio.sleep") + + request = get_test_request() + auth = get_test_auth() + body = VectorStoreFileCreateRequest(file_id="file_123") + + response = await add_file_to_vector_store( + request=request, vector_store_id="vs_123", auth=auth, body=body + ) + assert response is not None + assert response.id == "file_123" + assert response.vector_store_id == "vs_123" + assert response.status == "completed" + + # Verify retry logic was triggered + assert mock_client.vector_stores.files.create.call_count == 2 + # Verify sleep was called once with 0.5 seconds (first retry delay) + mock_sleep.assert_called_once_with(0.5) + + +@pytest.mark.asyncio +async def test_add_file_to_vector_store_max_retries_exceeded( + mocker: MockerFixture, +) -> None: + """Test that max retries are respected when database lock persists.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + # All attempts fail with database lock error + mock_client.vector_stores.files.create.side_effect = Exception("database is locked") + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + # Mock asyncio.sleep to avoid actual delays in tests + mock_sleep = mocker.patch("app.endpoints.vector_stores.asyncio.sleep") + + request = get_test_request() + auth = get_test_auth() + body = VectorStoreFileCreateRequest(file_id="file_123") + + with pytest.raises(HTTPException) as e: + await add_file_to_vector_store( + request=request, vector_store_id="vs_123", auth=auth, body=body + ) + assert e.value.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + + # Verify all 3 retry attempts were made + assert mock_client.vector_stores.files.create.call_count == 3 + # Verify exponential backoff: 0.5s, then 1s (0.5 * 2) + assert mock_sleep.call_count == 2 + assert mock_sleep.call_args_list[0][0][0] == 0.5 + assert mock_sleep.call_args_list[1][0][0] == 1.0 + + +@pytest.mark.asyncio +async def test_add_file_to_vector_store_non_lock_error_no_retry( + mocker: MockerFixture, +) -> None: + """Test that non-lock errors are not retried. + + Non-lock errors are re-raised and handled by global exception middleware. + """ + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + # Raise a non-lock error + mock_client.vector_stores.files.create.side_effect = Exception("Some other error") + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + # Mock asyncio.sleep to verify it's not called + mock_sleep = mocker.patch("app.endpoints.vector_stores.asyncio.sleep") + + request = get_test_request() + auth = get_test_auth() + body = VectorStoreFileCreateRequest(file_id="file_123") + + # Non-lock errors are re-raised as-is (global middleware handles them) + with pytest.raises(Exception, match="Some other error"): + await add_file_to_vector_store( + request=request, vector_store_id="vs_123", auth=auth, body=body + ) + + # Verify only one attempt was made (no retries for non-lock errors) + assert mock_client.vector_stores.files.create.call_count == 1 + # Verify sleep was not called (no retry) + mock_sleep.assert_not_called() + + +@pytest.mark.asyncio +async def test_list_vector_store_files_success(mocker: MockerFixture) -> None: + """Test successfully listing files in vector store.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.vector_stores.files.list.return_value = VectorStoreFilesList( + [ + VectorStoreFile("file_1", "vs_123"), + VectorStoreFile("file_2", "vs_123"), + ] + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + response = await list_vector_store_files( + request=request, vector_store_id="vs_123", auth=auth + ) + assert response is not None + assert len(response.data) == 2 + assert response.data[0].id == "file_1" + assert response.data[1].id == "file_2" + + +@pytest.mark.asyncio +async def test_get_vector_store_file_success(mocker: MockerFixture) -> None: + """Test successfully retrieving file from vector store.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.vector_stores.files.retrieve.return_value = VectorStoreFile( + "file_123", "vs_123" + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + response = await get_vector_store_file( + request=request, vector_store_id="vs_123", file_id="file_123", auth=auth + ) + assert response is not None + assert response.id == "file_123" + assert response.vector_store_id == "vs_123" + + +@pytest.mark.asyncio +async def test_delete_vector_store_file_success(mocker: MockerFixture) -> None: + """Test successfully deleting file from vector store.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.vector_stores.files.delete.return_value = None + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + response = await delete_vector_store_file( + request=request, vector_store_id="vs_123", file_id="file_123", auth=auth + ) + assert response is None + + +# Additional error path tests + + +@pytest.mark.asyncio +async def test_list_vector_stores_connection_error(mocker: MockerFixture) -> None: + """Test list vector stores with connection error.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.vector_stores.list.side_effect = APIConnectionError(request=None) # type: ignore + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + with pytest.raises(HTTPException) as e: + await list_vector_stores(request=request, auth=auth) + assert e.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE + + +@pytest.mark.asyncio +async def test_update_vector_store_connection_error(mocker: MockerFixture) -> None: + """Test update vector store with connection error.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.vector_stores.update.side_effect = APIConnectionError(request=None) # type: ignore + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + body = VectorStoreUpdateRequest(name="updated_store") + + with pytest.raises(HTTPException) as e: + await update_vector_store( + request=request, vector_store_id="vs_123", auth=auth, body=body + ) + assert e.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE + + +@pytest.mark.asyncio +async def test_update_vector_store_not_found(mocker: MockerFixture) -> None: + """Test update vector store with not found error.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_response = mocker.Mock() + mock_response.request = mocker.Mock() + mock_client.vector_stores.update.side_effect = BadRequestError( + message="Not found", response=mock_response, body=None + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + body = VectorStoreUpdateRequest(name="updated_store") + + with pytest.raises(HTTPException) as e: + await update_vector_store( + request=request, vector_store_id="vs_999", auth=auth, body=body + ) + assert e.value.status_code == status.HTTP_404_NOT_FOUND + + +@pytest.mark.asyncio +async def test_delete_vector_store_connection_error(mocker: MockerFixture) -> None: + """Test delete vector store with connection error.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.vector_stores.delete.side_effect = APIConnectionError(request=None) # type: ignore + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + with pytest.raises(HTTPException) as e: + await delete_vector_store(request=request, vector_store_id="vs_123", auth=auth) + assert e.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE + + +@pytest.mark.asyncio +async def test_delete_vector_store_not_found(mocker: MockerFixture) -> None: + """Test delete vector store with not found error.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_response = mocker.Mock() + mock_response.request = mocker.Mock() + mock_client.vector_stores.delete.side_effect = BadRequestError( + message="Not found", response=mock_response, body=None + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + with pytest.raises(HTTPException) as e: + await delete_vector_store(request=request, vector_store_id="vs_999", auth=auth) + assert e.value.status_code == status.HTTP_404_NOT_FOUND + + +@pytest.mark.asyncio +async def test_create_file_connection_error(mocker: MockerFixture) -> None: + """Test create file with connection error.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.files.create.side_effect = APIConnectionError(request=None) # type: ignore + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + mock_file = mocker.AsyncMock() + mock_file.filename = "test.txt" + mock_file.size = 12 # Size of "test content" + mock_file.read.return_value = b"test content" + + with pytest.raises(HTTPException) as e: + await create_file(request=request, auth=auth, file=mock_file) + assert e.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE + + +@pytest.mark.asyncio +async def test_create_file_bad_request(mocker: MockerFixture) -> None: + """Test create file with bad request error.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_response = mocker.Mock() + mock_response.request = mocker.Mock() + mock_client.files.create.side_effect = BadRequestError( + message="File too large", response=mock_response, body=None + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + mock_file = mocker.AsyncMock() + mock_file.filename = "test.txt" + mock_file.size = 12 # Size of "test content" + mock_file.read.return_value = b"test content" + + with pytest.raises(HTTPException) as e: + await create_file(request=request, auth=auth, file=mock_file) + + assert e.value.status_code == status.HTTP_413_CONTENT_TOO_LARGE + + +@pytest.mark.asyncio +async def test_create_file_too_large(mocker: MockerFixture) -> None: + """Test create file with file size exceeding limit.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + # Create a mock file that exceeds the size limit + mock_file = mocker.AsyncMock() + mock_file.filename = "large_file.pdf" + mock_file.size = 200 * 1024 * 1024 # 200 MB (exceeds 100 MB limit) + mock_file.read.side_effect = AssertionError("File too large") + + with pytest.raises(HTTPException) as e: + await create_file(request=request, auth=auth, file=mock_file) + + assert e.value.status_code == status.HTTP_413_CONTENT_TOO_LARGE + assert "too large" in str(e.value.detail).lower() + + +@pytest.mark.asyncio +async def test_create_file_content_length_too_large(mocker: MockerFixture) -> None: + """Test create file with Content-Length header exceeding limit.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + # Create request with large Content-Length header + request = Request( + scope={ + "type": "http", + "headers": [ + (b"authorization", b"Bearer test-token"), + (b"content-length", b"209715200"), # 200 MB + ], + } + ) + auth = get_test_auth() + + # Create a mock file + mock_file = mocker.AsyncMock() + mock_file.filename = "large_file.pdf" + mock_file.size = None # No size attribute + + with pytest.raises(HTTPException) as e: + await create_file(request=request, auth=auth, file=mock_file) + + assert e.value.status_code == status.HTTP_413_CONTENT_TOO_LARGE + assert "too large" in str(e.value.detail).lower() + + +@pytest.mark.asyncio +async def test_add_file_to_vector_store_connection_error( + mocker: MockerFixture, +) -> None: + """Test add file to vector store with connection error.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.vector_stores.files.create.side_effect = APIConnectionError( + request=None # type: ignore + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + body = VectorStoreFileCreateRequest(file_id="file_123") + + with pytest.raises(HTTPException) as e: + await add_file_to_vector_store( + request=request, vector_store_id="vs_123", auth=auth, body=body + ) + assert e.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE + + +@pytest.mark.asyncio +async def test_add_file_to_vector_store_not_found(mocker: MockerFixture) -> None: + """Test add file to vector store with not found error.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_response = mocker.Mock() + mock_response.request = mocker.Mock() + mock_client.vector_stores.files.create.side_effect = BadRequestError( + message="File not found", response=mock_response, body=None + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + body = VectorStoreFileCreateRequest(file_id="file_999") + + with pytest.raises(HTTPException) as e: + await add_file_to_vector_store( + request=request, vector_store_id="vs_123", auth=auth, body=body + ) + assert e.value.status_code == status.HTTP_404_NOT_FOUND + + +@pytest.mark.asyncio +async def test_list_vector_store_files_connection_error( + mocker: MockerFixture, +) -> None: + """Test list vector store files with connection error.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.vector_stores.files.list.side_effect = APIConnectionError( + request=None # type: ignore + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + with pytest.raises(HTTPException) as e: + await list_vector_store_files( + request=request, vector_store_id="vs_123", auth=auth + ) + assert e.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE + + +@pytest.mark.asyncio +async def test_list_vector_store_files_not_found(mocker: MockerFixture) -> None: + """Test list vector store files with invalid vector store ID.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_response = mocker.Mock() + mock_response.request = mocker.Mock() + mock_client.vector_stores.files.list.side_effect = BadRequestError( + message="Vector store not found", response=mock_response, body=None + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + with pytest.raises(HTTPException) as e: + await list_vector_store_files( + request=request, vector_store_id="vs_999", auth=auth + ) + + assert e.value.status_code == status.HTTP_404_NOT_FOUND + + +@pytest.mark.asyncio +async def test_get_vector_store_file_connection_error(mocker: MockerFixture) -> None: + """Test get vector store file with connection error.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.vector_stores.files.retrieve.side_effect = APIConnectionError( + request=None # type: ignore + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + with pytest.raises(HTTPException) as e: + await get_vector_store_file( + request=request, vector_store_id="vs_123", file_id="file_123", auth=auth + ) + assert e.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE + + +@pytest.mark.asyncio +async def test_get_vector_store_file_not_found(mocker: MockerFixture) -> None: + """Test get vector store file with not found error.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_response = mocker.Mock() + mock_response.request = mocker.Mock() + mock_client.vector_stores.files.retrieve.side_effect = BadRequestError( + message="File not found", response=mock_response, body=None + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + with pytest.raises(HTTPException) as e: + await get_vector_store_file( + request=request, vector_store_id="vs_123", file_id="file_999", auth=auth + ) + assert e.value.status_code == status.HTTP_404_NOT_FOUND + + +@pytest.mark.asyncio +async def test_delete_vector_store_file_connection_error( + mocker: MockerFixture, +) -> None: + """Test delete vector store file with connection error.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_client.vector_stores.files.delete.side_effect = APIConnectionError( + request=None # type: ignore + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + with pytest.raises(HTTPException) as e: + await delete_vector_store_file( + request=request, vector_store_id="vs_123", file_id="file_123", auth=auth + ) + assert e.value.status_code == status.HTTP_503_SERVICE_UNAVAILABLE + + +@pytest.mark.asyncio +async def test_delete_vector_store_file_not_found(mocker: MockerFixture) -> None: + """Test delete vector store file with not found error.""" + mock_authorization_resolvers(mocker) + + config_dict = get_test_config() + cfg = AppConfig() + cfg.init_from_dict(config_dict) + + mock_client = mocker.AsyncMock() + mock_response = mocker.Mock() + mock_response.request = mocker.Mock() + mock_client.vector_stores.files.delete.side_effect = BadRequestError( + message="File not found", response=mock_response, body=None + ) + mock_lsc = mocker.patch( + "app.endpoints.vector_stores.AsyncLlamaStackClientHolder.get_client" + ) + mock_lsc.return_value = mock_client + mocker.patch("app.endpoints.vector_stores.configuration", cfg) + + request = get_test_request() + auth = get_test_auth() + + with pytest.raises(HTTPException) as e: + await delete_vector_store_file( + request=request, vector_store_id="vs_123", file_id="file_999", auth=auth + ) + assert e.value.status_code == status.HTTP_404_NOT_FOUND diff --git a/tests/unit/app/test_routers.py b/tests/unit/app/test_routers.py index 7ac178881..7e1f455a6 100644 --- a/tests/unit/app/test_routers.py +++ b/tests/unit/app/test_routers.py @@ -28,6 +28,7 @@ stream_interrupt, streaming_query, tools, + vector_stores, ) from app.routers import include_routers @@ -109,7 +110,7 @@ def test_include_routers() -> None: include_routers(app) # are all routers added? - assert len(app.routers) == 22 + assert len(app.routers) == 23 assert root.router in app.get_routers() assert info.router in app.get_routers() assert models.router in app.get_routers() @@ -132,6 +133,7 @@ def test_include_routers() -> None: assert a2a.router in app.get_routers() assert stream_interrupt.router in app.get_routers() assert responses.router in app.get_routers() + assert vector_stores.router in app.get_routers() def test_check_prefixes() -> None: @@ -139,7 +141,7 @@ def test_check_prefixes() -> None: Verify that include_routers registers the expected routers with their configured URL prefixes. - Asserts that 21 routers are registered on a MockFastAPI instance and that + Asserts that 23 routers are registered on a MockFastAPI instance and that each router's prefix matches the expected value (e.g., root, health, authorized, metrics use an empty prefix; most API routers use "/v1"; conversations_v2 uses "/v2"). @@ -148,7 +150,7 @@ def test_check_prefixes() -> None: include_routers(app) # are all routers added? - assert len(app.routers) == 22 + assert len(app.routers) == 23 assert app.get_router_prefix(root.router) == "" assert app.get_router_prefix(info.router) == "/v1" assert app.get_router_prefix(models.router) == "/v1" @@ -172,3 +174,4 @@ def test_check_prefixes() -> None: assert app.get_router_prefix(a2a.router) == "" assert app.get_router_prefix(stream_interrupt.router) == "/v1" assert app.get_router_prefix(responses.router) == "/v1" + assert app.get_router_prefix(vector_stores.router) == "/v1" diff --git a/tests/unit/models/requests/test_vector_store_requests.py b/tests/unit/models/requests/test_vector_store_requests.py new file mode 100644 index 000000000..c666023f7 --- /dev/null +++ b/tests/unit/models/requests/test_vector_store_requests.py @@ -0,0 +1,158 @@ +"""Unit tests for Vector Store request models.""" + +import pytest +from pydantic import ValidationError + +from models.requests import VectorStoreFileCreateRequest, VectorStoreUpdateRequest + + +class TestVectorStoreUpdateRequest: + """Test cases for the VectorStoreUpdateRequest model.""" + + def test_valid_update_with_name(self) -> None: + """Test valid update request with name field.""" + request = VectorStoreUpdateRequest(name="updated_store") + assert request.name == "updated_store" + assert request.expires_at is None + assert request.metadata is None + + def test_valid_update_with_expires_at(self) -> None: + """Test valid update request with expires_at field.""" + request = VectorStoreUpdateRequest(expires_at=1735689600) + assert request.name is None + assert request.expires_at == 1735689600 + assert request.metadata is None + + def test_valid_update_with_metadata(self) -> None: + """Test valid update request with metadata field.""" + metadata = {"user_id": "user123"} + request = VectorStoreUpdateRequest(metadata=metadata) + assert request.name is None + assert request.expires_at is None + assert request.metadata == metadata + + def test_valid_update_with_multiple_fields(self) -> None: + """Test valid update request with multiple fields.""" + request = VectorStoreUpdateRequest( + name="updated_store", + expires_at=1735689600, + metadata={"user_id": "user123"}, + ) + assert request.name == "updated_store" + assert request.expires_at == 1735689600 + assert request.metadata == {"user_id": "user123"} + + def test_empty_update_rejected(self) -> None: + """Test that empty update request is rejected.""" + with pytest.raises( + ValueError, + match="At least one field must be provided: name, expires_at, or metadata", + ): + VectorStoreUpdateRequest() + + +class TestVectorStoreFileCreateRequest: + """Test cases for the VectorStoreFileCreateRequest model.""" + + def test_valid_request_basic(self) -> None: + """Test valid request with only file_id.""" + request = VectorStoreFileCreateRequest(file_id="file-abc123") + assert request.file_id == "file-abc123" + assert request.attributes is None + assert request.chunking_strategy is None + + def test_valid_attributes_basic(self) -> None: + """Test valid request with attributes.""" + attributes = {"key1": "value1", "key2": "value2"} + request = VectorStoreFileCreateRequest( + file_id="file-abc123", attributes=attributes + ) + assert request.file_id == "file-abc123" + assert request.attributes == attributes + + def test_attributes_max_16_pairs(self) -> None: + """Test that attributes can have exactly 16 pairs.""" + attributes = {f"key{i}": f"value{i}" for i in range(16)} + request = VectorStoreFileCreateRequest( + file_id="file-abc123", attributes=attributes + ) + assert len(request.attributes) == 16 # type: ignore + + def test_attributes_exceeds_16_pairs(self) -> None: + """Test that attributes with more than 16 pairs is rejected.""" + attributes = {f"key{i}": f"value{i}" for i in range(17)} + with pytest.raises( + ValueError, match="attributes can have at most 16 pairs, got 17" + ): + VectorStoreFileCreateRequest(file_id="file-abc123", attributes=attributes) + + def test_attributes_key_max_64_chars(self) -> None: + """Test that attribute keys can be exactly 64 characters.""" + key_64_chars = "a" * 64 + attributes = {key_64_chars: "value"} + request = VectorStoreFileCreateRequest( + file_id="file-abc123", attributes=attributes + ) + assert key_64_chars in request.attributes # type: ignore + + def test_attributes_key_exceeds_64_chars(self) -> None: + """Test that attribute keys exceeding 64 characters are rejected.""" + key_65_chars = "a" * 65 + attributes = {key_65_chars: "value"} + with pytest.raises(ValueError, match="exceeds 64 characters"): + VectorStoreFileCreateRequest(file_id="file-abc123", attributes=attributes) + + def test_attributes_string_value_max_512_chars(self) -> None: + """Test that string attribute values can be exactly 512 characters.""" + value_512_chars = "b" * 512 + attributes = {"key": value_512_chars} + request = VectorStoreFileCreateRequest( + file_id="file-abc123", attributes=attributes + ) + assert request.attributes["key"] == value_512_chars # type: ignore + + def test_attributes_string_value_exceeds_512_chars(self) -> None: + """Test that string attribute values exceeding 512 characters are rejected.""" + value_513_chars = "b" * 513 + attributes = {"key": value_513_chars} + with pytest.raises(ValueError, match="exceeds 512 characters"): + VectorStoreFileCreateRequest(file_id="file-abc123", attributes=attributes) + + def test_attributes_non_string_values_allowed(self) -> None: + """Test that non-string attribute values (numbers, booleans) are not length-checked.""" + attributes = { + "bool_key": True, + "int_key": 12345, + "float_key": 3.14159, + } + request = VectorStoreFileCreateRequest( + file_id="file-abc123", attributes=attributes + ) + assert request.attributes == attributes + + def test_attributes_mixed_value_types(self) -> None: + """Test that mixed value types in attributes are validated correctly.""" + attributes = { + "string_key": "value", + "bool_key": False, + "number_key": 42, + } + request = VectorStoreFileCreateRequest( + file_id="file-abc123", attributes=attributes + ) + assert request.attributes == attributes + + def test_attributes_none_is_valid(self) -> None: + """Test that None attributes is valid (optional field).""" + request = VectorStoreFileCreateRequest(file_id="file-abc123", attributes=None) + assert request.attributes is None + + def test_file_id_required(self) -> None: + """Test that file_id is required.""" + with pytest.raises(ValidationError): + VectorStoreFileCreateRequest() # type: ignore + + def test_file_id_cannot_be_empty(self) -> None: + """Test that file_id cannot be an empty string.""" + with pytest.raises(ValidationError, match="at least 1 character"): + VectorStoreFileCreateRequest(file_id="") diff --git a/tests/unit/models/responses/test_error_responses.py b/tests/unit/models/responses/test_error_responses.py index 306daeb6d..f3661f66f 100644 --- a/tests/unit/models/responses/test_error_responses.py +++ b/tests/unit/models/responses/test_error_responses.py @@ -81,7 +81,7 @@ def test_openapi_response(self) -> None: # Verify example count matches schema examples count assert len(examples) == expected_count - assert expected_count == 1 + assert expected_count == 2 # Verify example structure assert "conversation_id" in examples @@ -472,7 +472,7 @@ def test_openapi_response(self) -> None: # Verify example count matches schema examples count assert len(examples) == expected_count - assert expected_count == 6 + assert expected_count == 8 # Verify all labeled examples are present assert "conversation" in examples @@ -481,6 +481,8 @@ def test_openapi_response(self) -> None: assert "rag" in examples assert "streaming request" in examples assert "mcp server" in examples + assert "vector store" in examples + assert "file" in examples # Verify example structure for one example conversation_example = examples["conversation"] diff --git a/tests/unit/observability/formats/test_rlsapi.py b/tests/unit/observability/formats/test_rlsapi.py index 4ea8624ac..a88d62bdb 100644 --- a/tests/unit/observability/formats/test_rlsapi.py +++ b/tests/unit/observability/formats/test_rlsapi.py @@ -17,7 +17,7 @@ def sample_event_data_fixture() -> InferenceEventData: org_id="12345678", system_id="abc-def-123", request_id="req_xyz789", - cla_version="CLA/0.5.0", + cla_version="CLA/0.5.1", system_os="RHEL", system_version="9.3", system_arch="x86_64", @@ -40,7 +40,7 @@ def test_builds_event_with_all_fields( assert event["org_id"] == "12345678" assert event["system_id"] == "abc-def-123" assert event["request_id"] == "req_xyz789" - assert event["cla_version"] == "CLA/0.5.0" + assert event["cla_version"] == "CLA/0.5.1" assert event["system_os"] == "RHEL" assert event["system_version"] == "9.3" assert event["system_arch"] == "x86_64" diff --git a/ubi.repo b/ubi.repo new file mode 100644 index 000000000..27aebe353 --- /dev/null +++ b/ubi.repo @@ -0,0 +1,62 @@ +[ubi-9-for-$basearch-baseos-rpms] +name = Red Hat Universal Base Image 9 (RPMs) - BaseOS +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/baseos/os +enabled = 1 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1 + +[ubi-9-for-$basearch-baseos-debug-rpms] +name = Red Hat Universal Base Image 9 (Debug RPMs) - BaseOS +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/baseos/debug +enabled = 0 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1 + +[ubi-9-for-$basearch-baseos-source-rpms] +name = Red Hat Universal Base Image 9 (Source RPMs) - BaseOS +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/baseos/source/SRPMS +enabled = 0 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1 + +[ubi-9-for-$basearch-appstream-rpms] +name = Red Hat Universal Base Image 9 (RPMs) - AppStream +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/appstream/os +enabled = 1 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1 + +[ubi-9-for-$basearch-appstream-debug-rpms] +name = Red Hat Universal Base Image 9 (Debug RPMs) - AppStream +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/appstream/debug +enabled = 0 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1 + +[ubi-9-for-$basearch-appstream-source-rpms] +name = Red Hat Universal Base Image 9 (Source RPMs) - AppStream +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/appstream/source/SRPMS +enabled = 0 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1 + +[codeready-builder-for-ubi-9-$basearch-rpms] +name = Red Hat Universal Base Image 9 (RPMs) - CodeReady Builder +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/codeready-builder/os +enabled = 1 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1 + +[codeready-builder-for-ubi-9-$basearch-debug-rpms] +name = Red Hat Universal Base Image 9 (Debug RPMs) - CodeReady Builder +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/codeready-builder/debug +enabled = 0 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1 + +[codeready-builder-for-ubi-9-$basearch-source-rpms] +name = Red Hat Universal Base Image 9 (Source RPMs) - CodeReady Builder +baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/codeready-builder/source/SRPMS +enabled = 0 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release +gpgcheck = 1