konflux: hermetic build

Signed-off-by: Haoyu Sun <hasun@redhat.com>

Author: raptorsun

.tekton/lightspeed-stack-pull-request.yaml

@@ -27,9 +27,17 @@ spec:
     value: quay.io/redhat-user-workloads/lightspeed-core-tenant/lightspeed-stack:on-pr-{{revision}}
   - name: image-expires-after
     value: 5d
+  # todo: add arm64. refer to https://konflux.pages.redhat.com/docs/users/getting-started/multi-platform-builds.html#arm64-2
   - name: build-platforms
     value:
     - linux/x86_64
+  # todo: change on-push pipeline,too
+  - name: build-source-image
+    value: 'true'
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "."}, {"type": "pip", "path": ".", "allow_binary": "true"}]'
+  - name: hermetic
+    value: 'true'
   - name: dockerfile
     value: Containerfile
   pipelineSpec:

Containerfile

@@ -19,13 +19,24 @@ RUN dnf --disablerepo="*" --enablerepo="ubi-9-appstream-rpms" --enablerepo="ubi-
 # Install uv package manager
 RUN pip3.12 install "uv==0.8.15"
 
+# Install pdm and pdm-backend for hermetic builds
+RUN if [ -f /cachi2/cachi2.env ]; then pip3.12 install "pdm>=2.21.0" "pdm-backend"; fi
+
 # Add explicit files and directories
 # (avoid accidental inclusion of local directories or env files or credentials)
 COPY ${LSC_SOURCE_DIR}/src ./src
-COPY ${LSC_SOURCE_DIR}/pyproject.toml ${LSC_SOURCE_DIR}/LICENSE ${LSC_SOURCE_DIR}/README.md ${LSC_SOURCE_DIR}/uv.lock ./
+COPY ${LSC_SOURCE_DIR}/pyproject.toml ${LSC_SOURCE_DIR}/LICENSE ${LSC_SOURCE_DIR}/README.md ${LSC_SOURCE_DIR}/uv.lock ${LSC_SOURCE_DIR}/requirements.txt ./
 
 # Bundle additional dependencies for library mode.
-RUN uv sync --locked --no-dev --group llslibdev
+# Source cachi2 environment for hermetic builds if available, otherwise use normal installation
+# cachi2.env has these env vars:
+# PIP_FIND_LINKS=/cachi2/output/deps/pip
+# PIP_NO_INDEX=true
+RUN if [ -f /cachi2/cachi2.env ]; then \
+        . /cachi2/cachi2.env && uv sync --locked --no-dev --group llslibdev-hermetic --no-index --find-links ${PIP_FIND_LINKS} --python 3.12 -v; \
+    else \
+        uv sync --locked --no-dev --group llslibdev; \
+    fi
 
 # Explicitly remove some packages to mitigate some CVEs
 # - GHSA-wj6h-64fc-37mp: python-ecdsa package won't fix it upstream.

pyproject.toml

@@ -86,8 +86,23 @@ Issues = "https://github.com/lightspeed-core/lightspeed-stack/issues"
 name = "pytorch-cpu"
 url = "https://download.pytorch.org/whl/cpu"
 explicit = true
+
+# [[tool.uv.index]]
+# name = "pypi-default"
+# url = "https://pypi.org/simple"
+# explicit = true
 [tool.uv.sources]
-torch = [{ index = "pytorch-cpu" }]
+torch = [
+    { index = "pytorch-cpu", group = "llslibdev" },
+    # { index = "pypi-default", group = "llslibdev-hermetic" }
+]
+# [tool.uv]
+# conflicts = [
+#     [
+#       { group = "llslibdev" },
+#       { group = "llslibdev-hermetic" },
+#     ],
+# ]
 
 [dependency-groups]
 dev = [
@@ -160,6 +175,55 @@ llslibdev = [
     "blobfile>=3.0.0",
     "psutil>=7.0.0",
 ]
+# lslibdev-hermetic = [
+#    # the same as llslibdev, just using default index.
+#    "matplotlib>=3.10.0",
+#    "pillow>=11.1.0",
+#    "pandas>=2.2.3",
+#    "scikit-learn>=1.5.2",
+#    "psycopg2-binary>=2.9.10",
+#    # API eval: inline::meta-reference
+#    "tree_sitter>=0.24.0",
+#    "pythainlp>=3.0.10",
+#    "langdetect>=1.0.9",
+#    "emoji>=2.1.0",
+#    "nltk>=3.8.1",
+#    # API inference: remote::gemini
+#    "litellm>=1.75.5.post1",
+#    # API inference: inline::sentence-transformers
+#    "sentence-transformers>=5.0.0",
+#    # API vector_io: inline::faiss
+#    "faiss-cpu>=1.11.0",
+#    # API scoring: inline::basic
+#    "requests>=2.32.4",
+#    # API datasetio: inline::localfs
+#    "aiosqlite>=0.21.0",
+#    # API datasetio: remote::huggingface
+#    "datasets>=3.6.0",
+#    # API telemetry: inline::meta-reference
+#    "opentelemetry-sdk>=1.34.1",
+#    "opentelemetry-exporter-otlp>=1.34.1",
+#    # API tool_runtime: inline::rag-runtime
+#    "transformers>=4.34.0",
+#    "numpy==2.2.6",
+#    # API tool_runtime: remote::model-context-protocol
+#    "mcp>=1.9.4",
+#    # API post_training: inline::huggingface
+#    "torch==2.7.1",
+#    "trl>=0.18.2",
+#    "peft>=0.15.2",
+#    # Other
+#    "autoevals>=0.0.129",
+#    "fire>=0.7.0",
+#    "opentelemetry-instrumentation>=0.55b0",
+#    "blobfile>=3.0.0",
+#    "psutil>=7.0.0",
+# ]
+tool-hermetic = [
+    "uv==0.8.15",
+    "pdm>=2.21.0",
+    "pdm-backend==2.4.5",
+]
 
 build = [
     "build>=1.2.2.post1",

requirements.hermetic.txt

@@ -0,0 +1,3 @@
+uv==0.8.15
+pdm>=2.21.0
+pdm-backend

requirements.txt

@@ -0,0 +1,4 @@
+packages: [gcc, jq, patch]
+contentOrigin:
+  repofiles: ["./ubi.repo"]
+arches: [x86_64, aarch64]