diff --git a/apps/site/understack-cdn.yaml b/apps/site/understack-cdn.yaml new file mode 100644 index 000000000..9aeae0cb9 --- /dev/null +++ b/apps/site/understack-cdn.yaml @@ -0,0 +1,11 @@ +--- +component: understack-cdn +sources: + - ref: understack + path: 'components/understack-cdn' + helm: + valueFiles: + - $understack/components/understack-cdn/values.yaml + - $deploy/{{.name}}/understack-cdn/values.yaml + ignoreMissingValueFiles: true + - ref: deploy diff --git a/charts/argocd-understack/templates/application-cdn.yaml b/charts/argocd-understack/templates/application-cdn.yaml new file mode 100644 index 000000000..9194e320a --- /dev/null +++ b/charts/argocd-understack/templates/application-cdn.yaml @@ -0,0 +1,41 @@ +{{- if eq (include "understack.isEnabled" (list $.Values.site "understack_cdn")) "true" }} +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: {{ printf "%s-%s" $.Release.Name "understack-cdn" }} + finalizers: + - resources-finalizer.argocd.argoproj.io + annotations: + argocd.argoproj.io/compare-options: ServerSideDiff=true,IncludeMutationWebhook=true +spec: + destination: + namespace: understack-cdn + server: {{ $.Values.cluster_server }} + project: understack + sources: + - helm: + ignoreMissingValueFiles: true + valueFiles: + - $understack/components/understack-cdn/values.yaml + - $deploy/{{ include "understack.deploy_path" $ }}/understack-cdn/values.yaml + path: components/understack-cdn + ref: understack + repoURL: {{ include "understack.understack_url" $ }} + targetRevision: {{ include "understack.understack_ref" $ }} + - ref: deploy + repoURL: {{ include "understack.deploy_url" $ }} + targetRevision: {{ include "understack.deploy_ref" $ }} + syncPolicy: + automated: + prune: true + selfHeal: true + managedNamespaceMetadata: + annotations: + argocd.argoproj.io/sync-options: Delete=false + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + - ApplyOutOfSyncOnly=true +{{- end }} diff --git a/components/understack-cdn/Chart.yaml b/components/understack-cdn/Chart.yaml new file mode 100644 index 000000000..97450feb9 --- /dev/null +++ b/components/understack-cdn/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: understack-cdn +description: HTTP service to serve firmware images from central store +type: application +version: 0.1.0 diff --git a/components/understack-cdn/README.md b/components/understack-cdn/README.md new file mode 100644 index 000000000..9348397ca --- /dev/null +++ b/components/understack-cdn/README.md @@ -0,0 +1,61 @@ +# Poor-man's CDN for serving firmware images + +Images are stored in Object Store + +Caching reverse-proxies at each fabric will fetch the images from Object Store +and make them available via HTTPS. + +This allows a device to access firmware images via an HTTPS request to a +cluster-local tendot IP address. + +## Proxy configuration + +The proxy edge service caches files locally on a persistent volume. + +All files are proxied to our object bucket. Anonymous credentials are used, +therefore we need to make the files in our bucket readable by anonymous if they +are to be accessible via HTTP. + +## Uploading file to object storage + +We have an ObjectBucket provisioned via rook/ceph. The operator for our +ObjectBucketClaim creates credentials for us to access this bucket. + +Our credentials and bucket info is in a secret and a configmap both named after +the bucketclaim: + +``` sh +KEY_ID=`kubectl -n understack-cdn get secrets firmware-images -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d` +KEY=`kubectl -n understack-cdn get secrets firmware-images -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' | base64 -d` +``` + +I was able to manage the bucket using the minio CLI client called "mc". + +``` sh +mc alias set myrook https://object-storage.dev.undercloud.rackspace.net/firmware-images/ $KEY_ID $KEY +mc anonymous set download myrook/firmware-images +mc cp DELL/R7615/BIOS_H3TGJ_WN64_1.15.3.EXE myrook/firmware-images/DELL/R7615/ +mc anonymous set download myrook/firmware-images/DELL/R7615/BIOS_H3TGJ_WN64_1.15.3.EXE +``` + +## Testing with curl + +curl https://cdn.dev.undercloud.rackspace.net/firmware-images/DELL/R7615/BIOS_H3TGJ_WN64_1.15.3.EXE | shasum + +## See nginx logs to check that it is Caching + +``` sh +⇒ kubectl -n understack-cdn logs deployments/cdn-edge +Defaulted container "nginx" out of: nginx, cache-dir-init (init) +/docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration +/docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/ +/docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh +10-listen-on-ipv6-by-default.sh: info: can not modify /etc/nginx/conf.d/default.conf (read-only file system?) +/docker-entrypoint.sh: Sourcing /docker-entrypoint.d/15-local-resolvers.envsh +/docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh +/docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh +/docker-entrypoint.sh: Configuration complete; ready for start up +10.64.49.118 - - [26/Feb/2026:12:36:47 +0000] "GET /DELL/R7615/BIOS_H3TGJ_WN64_1.15.3.EXE HTTP/1.1" 200 2523429 "-" "curl/8.14.1" "10.64.50.136" cache=EXPIRED +10.64.49.118 - - [26/Feb/2026:12:36:56 +0000] "GET /DELL/R7615/BIOS_H3TGJ_WN64_1.15.3.EXE HTTP/1.1" 200 32591328 "-" "curl/8.14.1" "10.64.50.136" cache=HIT +10.64.49.118 - - [26/Feb/2026:12:45:18 +0000] "GET /DELL/R7615/BIOS_H3TGJ_WN64_1.15.3.EXE HTTP/1.1" 200 32591328 "-" "curl/8.14.1" "10.64.50.136" cache=HIT +``` diff --git a/components/understack-cdn/templates/deployment.yaml.tpl b/components/understack-cdn/templates/deployment.yaml.tpl new file mode 100644 index 000000000..bc62ce960 --- /dev/null +++ b/components/understack-cdn/templates/deployment.yaml.tpl @@ -0,0 +1,90 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cdn-edge + labels: + app: cdn-edge +spec: + replicas: 1 + selector: + matchLabels: + app: cdn-edge + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: + app: cdn-edge + spec: + # Init container to set correct permissions on cache dir + initContainers: + - name: cache-dir-init + image: busybox:1.36 + command: ["sh", "-c", "mkdir -p /var/cache/nginx/cdn && chown -R 101:101 /var/cache/nginx"] + volumeMounts: + - name: nginx-cache + mountPath: /var/cache/nginx + + containers: + - name: nginx + image: nginx:1.27-alpine + ports: + - containerPort: 8080 + name: http + + resources: + requests: + cpu: "250m" + memory: "256Mi" + limits: + cpu: "2" + memory: "1Gi" + + volumeMounts: + - name: nginx-config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + - name: nginx-config + mountPath: /etc/nginx/conf.d/default.conf + subPath: default.conf + - name: nginx-cache + mountPath: /var/cache/nginx + + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + + readinessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 3 + periodSeconds: 5 + + # Graceful shutdown — let in-flight transfers complete + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "sleep 5 && nginx -s quit"] + + securityContext: + runAsNonRoot: true + runAsUser: 101 # nginx user in nginx:alpine + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + + volumes: + - name: nginx-config + configMap: + name: nginx-config + - name: nginx-cache + persistentVolumeClaim: + claimName: nginx-cache + + terminationGracePeriodSeconds: 30 diff --git a/components/understack-cdn/templates/nginx-config.yaml.tpl b/components/understack-cdn/templates/nginx-config.yaml.tpl new file mode 100644 index 000000000..4c09742f1 --- /dev/null +++ b/components/understack-cdn/templates/nginx-config.yaml.tpl @@ -0,0 +1,145 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-config +data: + nginx.conf: | + worker_processes auto; + error_log /var/log/nginx/error.log warn; + pid /var/cache/nginx/nginx.pid; + + # Tune for large file serving + worker_rlimit_nofile 65535; + + events { + worker_connections 4096; + use epoll; + multi_accept on; + } + + http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for" ' + 'cache=$upstream_cache_status'; + + access_log /var/log/nginx/access.log main; + + # Large file optimisations + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + + # Proxy cache zone configuration: + # keys_zone=cdn_cache:50m — 50MB for cache keys/metadata (~400k keys) + # max_size=50g — on-disk cache (adjust to your PVC size) + # inactive=30d — evict if not accessed in this time + # use_temp_path=off — write directly to cache dir (avoids extra copy) + proxy_cache_path /var/cache/nginx/cdn + levels=1:2 + keys_zone=cdn_cache:50m + max_size={{ .Values.cdn.cacheSize }} + inactive={{ .Values.cdn.inactive }} + use_temp_path=off; + + # Don't buffer large files to disk before sending — stream them + proxy_buffering on; + proxy_request_buffering off; + + # Increase timeouts for large file transfers + proxy_connect_timeout 10s; + proxy_send_timeout 300s; + proxy_read_timeout 300s; + send_timeout 300s; + + # Hide upstream headers we don't want to leak + proxy_hide_header x-amz-request-id; + proxy_hide_header x-amz-id-2; + + include /etc/nginx/conf.d/*.conf; + } + default.conf: | + upstream s3_origin { + server {{ .Values.cdn.objectStorageServerHostname }}:443; + keepalive 32; + } + + server { + listen 8080; + server_name _; + + # TLS — cert mounted from a k8s secret via ingress or directly + #ssl_certificate /etc/nginx/tls/tls.crt; + #ssl_certificate_key /etc/nginx/tls/tls.key; + #ssl_protocols TLSv1.2 TLSv1.3; + #ssl_ciphers HIGH:!aNULL:!MD5; + + proxy_cache cdn_cache; + proxy_cache_valid 200 206 7d; # Cache 200 and partial content + proxy_cache_valid 404 1m; # Don't cache 404s for long + proxy_cache_use_stale error timeout updating http_500 http_502 http_503; + proxy_cache_lock on; # Collapse simultaneous requests for the same file + proxy_cache_lock_timeout 10s; + + proxy_cache_key "$scheme$proxy_host$uri"; + + add_header X-Cache-Status $upstream_cache_status always; + add_header X-Served-By $hostname always; + + add_header X-Content-Type-Options nosniff always; + add_header X-Frame-Options DENY always; + + proxy_cache_revalidate on; + proxy_cache_bypass 0; + proxy_no_cache 0; + proxy_ignore_headers Cache-Control Expires Set-Cookie; + + location /{{ .Values.cdn.bucketName }}/ { + # Forward to Object Storage. + # S3-compatible API expects requests in the form: /bucket-name/key + # Our clients are using URL paths in the exact same format. + proxy_pass https://s3_origin$request_uri; + + proxy_http_version 1.1; + proxy_set_header Connection ""; # keepalive to upstream + proxy_set_header Host rook-ceph-rgw-ceph-objectstore.rook-ceph.svc; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # Don't forward auth headers downstream + proxy_set_header Authorization ""; + + # SSL settings for upstream connection + proxy_ssl_server_name on; + proxy_ssl_protocols TLSv1.2 TLSv1.3; + + # Tell clients files are immutable — they should cache forever + add_header Cache-Control "public, max-age=31536000, immutable" always; + + # Support resumable downloads + proxy_force_ranges on; + + # Stream large files rather than buffering defeats the cache, so keep buffering on: + proxy_buffering on; + } + + # Health check endpoint (used by k8s liveness/readiness probes) + location /healthz { + access_log off; + return 200 "ok\n"; + add_header Content-Type text/plain; + } + + # Expose basic cache stats (restrict to internal) + location /nginx_status { + stub_status; + allow 10.0.0.0/8; + allow 172.16.0.0/12; + allow 192.168.0.0/16; + deny all; + } + } diff --git a/components/understack-cdn/templates/object-bucket-claim.yaml.tpl b/components/understack-cdn/templates/object-bucket-claim.yaml.tpl new file mode 100644 index 000000000..79ba65c53 --- /dev/null +++ b/components/understack-cdn/templates/object-bucket-claim.yaml.tpl @@ -0,0 +1,12 @@ +{{- if .Values.cdn.objectBucketIsLocal }} +apiVersion: objectbucket.io/v1alpha1 +kind: ObjectBucketClaim +metadata: + name: {{ .Values.cdn.bucketName }} +spec: + bucketName: {{ .Values.cdn.bucketName }} + storageClassName: ceph-bucket + additionalConfig: + maxObjects: "1000" + maxSize: {{ .Values.cdn.bucketMaxSize }} +{{- end }} diff --git a/components/understack-cdn/templates/pvc.yaml.tpl b/components/understack-cdn/templates/pvc.yaml.tpl new file mode 100644 index 000000000..cd37f581e --- /dev/null +++ b/components/understack-cdn/templates/pvc.yaml.tpl @@ -0,0 +1,12 @@ +# Persistent volume for the Nginx cache. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nginx-cache +spec: + accessModes: + - ReadWriteOnce + storageClassName: {{ .Values.cdn.CacheStorageClassName }} + resources: + requests: + storage: {{ .Values.cdn.cacheSize }} diff --git a/components/understack-cdn/templates/service.yaml.tpl b/components/understack-cdn/templates/service.yaml.tpl new file mode 100644 index 000000000..8d221d050 --- /dev/null +++ b/components/understack-cdn/templates/service.yaml.tpl @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: cdn-edge +spec: + selector: + app: cdn-edge + ports: + - name: http + port: 80 + targetPort: 8080 + type: ClusterIP diff --git a/components/understack-cdn/values.yaml b/components/understack-cdn/values.yaml new file mode 100644 index 000000000..7fbd4fe44 --- /dev/null +++ b/components/understack-cdn/values.yaml @@ -0,0 +1,18 @@ +cdn: + # Cache is limited to this size. We will provision a volume this size so + # changing it would require the filesystem to be rebuilt/resized: + cacheSize: 10g + # Files are evicted from the cache if they have not been accessed in: + inactive: 7d + # where to create our PersistentVolume for the cache + CacheStorageClassName: openebs-lvm + + # The location of our object bucket (s3-compatible object store) providing us + # with anonymous access to our image files: + objectStorageServerHostname: object-storage.dev.undercloud.rackspace.net + bucketName: firmware-images + + # Whether to create a local ObjectBucketClaim on this cluster + objectBucketIsLocal: false + # If above is true, how much space to claim for this object bucket: + bucketMaxSize: 40g