Upgrade .net version and packages

.github/workflows/build.yaml

@@ -22,7 +22,7 @@ jobs:
         run: dotnet restore
       - name: Build
         run: dotnet build --no-restore
-      - name: Test
+      - name: Run unit tests
         working-directory: ./test/SnD.ApiClient.Tests
         run: |
           dotnet add package coverlet.msbuild &&
@@ -33,3 +33,19 @@ jobs:
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           lcov-file: ./test/Coverage/coverage.info
+
+      - name: Setup just
+        uses: extractions/setup-just@v3
+
+      - name: Setup integration tests
+        run: just up
+
+      - name: Run acceptance tests
+        working-directory: ./test/SnD.ApiClient.Tests.Acceptance
+        run: |
+          dotnet test SnD.ApiClient.Tests.Acceptance.csproj --configuration Debug --runtime linux-x64
+
+      - name: Get logs from application pods
+        if: always()
+        run: |
+          kubectl describe pods --namespace default || true

integration_tests/kind.yaml

@@ -0,0 +1,14 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+    extraPortMappings:
+      - containerPort: 80
+        hostPort: 5555
+        protocol: TCP
+      - containerPort: 30043
+        hostPort: 9000
+        protocol: TCP
+      - containerPort: 30042
+        hostPort: 9042
+        protocol: TCP

integration_tests/manifests/hello-world-algorithm.yaml

@@ -0,0 +1,68 @@
+apiVersion: science.sneaksanddata.com/v1
+kind: NexusAlgorithmTemplate
+metadata:
+  name: hello-world
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "0"
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  args:
+    - main.py
+    - '--sas-uri=%s'
+    - '--request-id=%s'
+  command: python
+  computeResources:
+    cpuLimit: 1000m
+    memoryLimit: 1000Mi
+  container:
+    image: nexus-hello
+    registry: ghcr.io/sneaksanddata
+    serviceAccountName: nexus
+    versionTag: 0.1.2
+  datadogIntegrationSettings:
+    mountDatadogSocket: true
+  errorHandlingBehaviour:
+    fatalExitCodes:
+      - 137
+    transientExitCodes:
+      - 139
+  runtimeEnvironment:
+    deadlineSeconds: 120
+    environmentVariables:
+      - name: DD_AGENT_HOST
+        valueFrom:
+          fieldRef:
+            fieldPath: status.hostIP
+      - name: PROTEUS__DD_SITE
+        value: datadoghq.eu
+      - name: NEXUS__ALGORITHM_NAME
+        value: nexus-hello
+      - name: NEXUS__SCHEDULER_URL
+        value: http://nexus.nexus.svc.cluster.local:8080
+      - name: NEXUS__RECEIVER_URL
+        value: http://dockerhost.nexus.svc.cluster.local:8081
+      - name: NEXUS__STORAGE_CLIENT_CLASS
+        value: adapta.storage.blob.s3_storage_client.S3StorageClient
+      - name: ALGORITHM_STORAGE__BASE_PATH
+        value: s3a://nexus/storage
+      - name: NEXUS__ALGORITHM_OUTPUT_PATH
+        value: s3a://nexus/result
+      - name: NEXUS__TELEMETRY_PATH
+        value: s3a://nexus/telemetry
+      - name: ALGORITHM_STORAGE_TYPE
+        value: S3
+      - name: PROTEUS__AWS_REGION
+        value: us-east-1
+      - name: PROTEUS__AWS_ENDPOINT
+        value: http://dockerhost.nexus.svc.cluster.local:9000
+      - name: PROTEUS__AWS_SECRET_ACCESS_KEY
+        value: minioadmin
+      - name: PROTEUS__AWS_ACCESS_KEY_ID
+        value: minioadmin
+    mappedEnvironmentVariables: []
+    maximumRetries: 1
+  workgroupRef:
+    group: science.sneaksanddata.com
+    kind: NexusAlgorithmWorkgroup
+    name: default

integration_tests/manifests/hello-world-workgroup.yaml

@@ -0,0 +1,13 @@
+apiVersion: science.sneaksanddata.com/v1
+kind: NexusAlgorithmWorkgroup
+metadata:
+  name: default
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  affinity: {}
+  cluster: kind-nexus-shard-0
+  description: Default workgroup
+  tolerations: []

integration_tests/manifests/ingress.yaml

@@ -0,0 +1,39 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: receiver
+  annotations:
+    nginx.ingress.kubernetes.io/use-regex: "true"
+    nginx.ingress.kubernetes.io/rewrite-target: /$2
+spec:
+  ingressClassName: nginx
+  rules:
+    - http:
+        paths:
+          - pathType: ImplementationSpecific
+            path: /receiver(/|$)(.*)
+            backend:
+              service:
+                name: nexus-receiver
+                port:
+                  number: 8080
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: scheduler
+  annotations:
+    nginx.ingress.kubernetes.io/use-regex: "true"
+    nginx.ingress.kubernetes.io/rewrite-target: /$2
+spec:
+  ingressClassName: nginx
+  rules:
+    - http:
+        paths:
+          - pathType: ImplementationSpecific
+            path: /scheduler(/|$)(.*)
+            backend:
+              service:
+                name: nexus
+                port:
+                  number: 8080

integration_tests/manifests/minio.yaml

@@ -0,0 +1,116 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: minio
+  labels:
+    app: minio
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: minio
+  template:
+    metadata:
+      labels:
+        app: minio
+    spec:
+      hostname: minio-e2e
+      containers:
+        - name: minio
+          image: quay.io/minio/minio
+          args:
+            - server
+            - /data
+            - "--console-address"
+            - ":9123"
+          ports:
+            - containerPort: 9000
+            - containerPort: 9123
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 9123
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 10
+            failureThreshold: 5
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 9123
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 10
+            failureThreshold: 5
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio
+spec:
+  selector:
+    app: minio
+  ports:
+    - name: api
+      port: 9000
+      targetPort: 9000
+    - name: console
+      port: 9123
+      targetPort: 9123
+  type: ClusterIP
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: minio-setup
+spec:
+  backoffLimit: 5
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: minio-setup
+          image: quay.io/minio/minio
+          command:
+            - /bin/sh
+            - -c
+            - |
+              echo "Waiting for Minio to be available via mc..."
+              max_retries=30
+              attempt=1
+              until mc alias set e2e "http://minio.default.svc.cluster.local:9000" minioadmin minioadmin >/dev/null 2>&1; do
+              if (( attempt >= max_retries )); then
+                echo "minio did not become available after $max_retries attempts."
+              exit 1
+              fi
+              sleep 1
+              ((attempt++))
+              done
+              echo "mc is available."
+              mc admin info e2e
+              mc mb --ignore-existing e2e/tmp
+              mc mb --ignore-existing e2e/nexus
+              mc mb --ignore-existing e2e/nexus-sdk-tests
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio-node-port
+spec:
+  selector:
+    app: minio
+  ports:
+    - port: 9000
+      targetPort: 9000
+      nodePort: 30043
+  type: NodePort
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: minio-credentials
+stringData:
+  NEXUS__S3_BUFFER__ACCESS_KEY_ID: "minioadmin"
+  NEXUS__S3_BUFFER__SECRET_ACCESS_KEY: "minioadmin"
+  NEXUS__S3_BUFFER__ENDPOINT: "http://minio.default.svc.cluster.local:9000"
+  NEXUS__S3_BUFFER__REGION: "us-east-1"

integration_tests/manifests/nexus-algorithm-sa.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: nexus:serviceaccount:nexus
+subjects:
+  - kind: ServiceAccount
+    name: nexus
+    namespace: default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: admin

integration_tests/manifests/scylladb.yaml

@@ -0,0 +1,85 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: scylla
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: scylla
+  template:
+    metadata:
+      labels:
+        app: scylla
+    spec:
+      initContainers:
+        - name: create-scylla-yaml
+          image: busybox
+          command: ["/bin/sh", "-c", "touch /scylla-conf/scylla.yaml"]
+          volumeMounts:
+            - name: scylla-conf
+              mountPath: /scylla-conf
+      containers:
+        - name: scylla
+          image: scylladb/scylla
+          command: ["scylla"]
+          args:
+            - "--rpc-address"
+            - "0.0.0.0"
+            - "--broadcast-rpc-address"
+            - "127.0.0.1"
+            - "--smp"
+            - "1"
+            - "--developer-mode=1"
+          livenessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - '[ "$(nodetool statusgossip)" = "running" ]'
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 10
+            failureThreshold: 5
+          volumeMounts:
+            - name: scylla-conf
+              mountPath: /conf/scylla.yaml
+              subPath: scylla.yaml
+      volumes:
+        - name: scylla-conf
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: scylla
+spec:
+  selector:
+    app: scylla
+  ports:
+    - name: cql
+      port: 9042
+      targetPort: 9042
+  type: ClusterIP
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cassandra-credentials
+stringData:
+  NEXUS__SCYLLA_CQL_STORE__HOSTS: "scylla.default.svc.cluster.local"
+  NEXUS__SCYLLA_CQL_STORE__USER: "cassandra"
+  NEXUS__SCYLLA_CQL_STORE__PASSWORD: "cassandra"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: scylla-node-port
+spec:
+  selector:
+    app: scylla
+  ports:
+    - port: 9042
+      targetPort: 9042
+      nodePort: 30042
+  type: NodePort