Remove endpoint /executionLogs from orchestrator (#1265)

* reused path from backend for sandbox storage

Author: nulls

docker-compose.yaml

@@ -25,7 +25,6 @@ services:
       - "5100:5100"
     volumes:
       - "/var/run/docker.sock:/var/run/docker.sock"
-      - /home/saveu/executionLogs:/home/cnb/executionLogs/
       - /home/saveu/configs/orchestrator:/home/cnb/config
       - save-tmp-resources:/tmp
     extra_hosts:

save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/AgentConfiguration.kt

@@ -71,13 +71,11 @@ data class HeartbeatConfig(
  *
  * @property url URL of orchestrator
  * @property heartbeatEndpoint endpoint to post heartbeats to
- * @property executionLogsEndpoint endpoint to post executionLogs to
  */
 @Serializable
 data class OrchestratorConfig(
     val url: String,
     val heartbeatEndpoint: String = "/heartbeat",
-    val executionLogsEndpoint: String = "/executionLogs",
 )
 
 /**

save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/SaveAgent.kt

@@ -24,8 +24,6 @@ import generated.SAVE_CLOUD_VERSION
 import io.ktor.client.*
 import io.ktor.client.call.body
 import io.ktor.client.request.*
-import io.ktor.client.request.forms.*
-import io.ktor.client.statement.*
 import io.ktor.http.*
 import io.ktor.utils.io.core.*
 import okio.FileSystem
@@ -269,11 +267,11 @@ class SaveAgent(private val config: AgentConfiguration,
         logInfoCustom("SAVE has completed execution with status ${executionResult.code}")
 
         val saveCliLogFilePath = config.logFilePath
-        val byteArray = FileSystem.SYSTEM.source(saveCliLogFilePath.toPath())
+        val saveCliLogData = FileSystem.SYSTEM.source(saveCliLogFilePath.toPath())
             .buffer()
             .readByteArray()
-        val saveCliLogData = String(byteArray).split("\n")
-        launchLogSendingJob(byteArray)
+            .let { String(it) }
+            .split("\n")
         logDebugCustom("SAVE has completed execution, execution logs:")
         saveCliLogData.forEach {
             logDebugCustom("[SAVE] $it")
@@ -358,16 +356,6 @@ class SaveAgent(private val config: AgentConfiguration,
         }
     }
 
-    private fun CoroutineScope.launchLogSendingJob(byteArray: ByteArray): Job = launch {
-        runCatching {
-            sendLogs(byteArray)
-        }
-            .exceptionOrNull()
-            ?.let {
-                logErrorCustom("Couldn't send logs, reason: ${it.message}")
-            }
-    }
-
     private fun CoroutineScope.handleSuccessfulExit(): Job {
         val jsonReport = "${config.save.reportDir}/save.out.json"
         val result = runCatching {
@@ -395,24 +383,6 @@ class SaveAgent(private val config: AgentConfiguration,
         }
     }
 
-    /**
-     * @param byteArray byte array with logs of CLI execution progress that will be sent in a message
-     */
-    private suspend fun sendLogs(byteArray: ByteArray): HttpResponse =
-            httpClient.post {
-                url("${config.orchestrator.url}${config.orchestrator.executionLogsEndpoint}")
-                setBody(MultiPartFormDataContent(formData {
-                    append(
-                        "executionLogs",
-                        byteArray,
-                        Headers.build {
-                            append(HttpHeaders.ContentType, ContentType.MultiPart.FormData)
-                            append(HttpHeaders.ContentDisposition, "filename=${config.id}")
-                        }
-                    )
-                }))
-            }
-
     private suspend fun sendReport(testResultDebugInfo: TestResultDebugInfo) = httpClient.post {
         url("${config.backend.url}${config.backend.debugInfoEndpoint}?executionId=${executionId.value}")
         contentType(ContentType.Application.Json)

save-agent/src/linuxX64Test/kotlin/com/saveourtool/save/agent/SaveAgentTest.kt

@@ -46,8 +46,6 @@ class SaveAgentTest {
                         HttpStatusCode.OK,
                         headersOf(HttpHeaders.ContentType, ContentType.Application.Json.toString())
                     )
-                    "/executionData" -> respond("", status = HttpStatusCode.OK)
-                    "/executionLogs" -> respond("", status = HttpStatusCode.OK)
                     else -> error("Unhandled ${request.url}")
                 }
             }

save-cloud-charts/save-cloud/templates/orchestrator-deployment.yaml

@@ -47,8 +47,6 @@ spec:
                   fieldPath: spec.nodeName
           volumeMounts:
             - {{ include "spring-boot.config-volume-mount" . | indent 14 | trim }}
-            - name: execution-logs-storage
-              mountPath: /home/cnb/executionLogs/
           {{- include "spring-boot.management" .Values.orchestrator | nindent 10 }}
           resources:
             limits:
@@ -57,6 +55,3 @@ spec:
               memory: 600M
       volumes:
         - {{ include "spring-boot.config-volume" (dict "service" .Values.orchestrator) | indent 10 | trim }}
-        - name: execution-logs-storage
-          # FixMe: Do we still need logs storage? All logs should be stored in Loki.
-          emptyDir: {}

save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/config/ConfigProperties.kt

@@ -16,7 +16,6 @@ import org.springframework.boot.context.properties.ConstructorBinding
  * @property kubernetes configuration for setup in Kubernetes
  * @property dockerResourcesLifetime time, after which resources (images, containers, etc) should be released
  * @property agentsCount a number of agents to start for every [Execution]
- * @property executionLogs path to folder to store cli logs
  * @property shutdown configuration related to process of shutting down groups of agents for executions
  * @property aptExtraFlags additional flags that will be passed to `apt-get` when building image for tests
  * @property adjustResourceOwner whether Linux user that will be set as owner of resources copied into docker build directory
@@ -35,7 +34,6 @@ data class ConfigProperties(
     val kubernetes: KubernetesSettings?,
     val dockerResourcesLifetime: String = "720h",
     val agentsCount: Int,
-    val executionLogs: String,
     val shutdown: ShutdownSettings,
     val aptExtraFlags: String = "",
     val adjustResourceOwner: Boolean = true,

save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/controller/AgentsController.kt

@@ -15,22 +15,16 @@ import com.github.dockerjava.api.exception.DockerException
 import io.fabric8.kubernetes.client.KubernetesClientException
 import org.slf4j.LoggerFactory
 import org.springframework.http.HttpStatus
-import org.springframework.http.MediaType
 import org.springframework.http.ResponseEntity
-import org.springframework.http.codec.multipart.FilePart
 import org.springframework.web.bind.annotation.PostMapping
 import org.springframework.web.bind.annotation.RequestBody
 import org.springframework.web.bind.annotation.RequestParam
-import org.springframework.web.bind.annotation.RequestPart
 import org.springframework.web.bind.annotation.RestController
 import org.springframework.web.reactive.function.client.WebClientResponseException
 import org.springframework.web.server.ResponseStatusException
 import reactor.core.publisher.Mono
 import reactor.kotlin.core.publisher.doOnError
 
-import java.io.File
-import java.io.FileOutputStream
-
 /**
  * Controller used to start agents with needed information
  */
@@ -108,38 +102,6 @@ class AgentsController(
         dockerService.stopAgents(agentIds)
     }
 
-    /**
-     * @param executionLogs ExecutionLogs
-     */
-    @PostMapping("/executionLogs", consumes = [MediaType.MULTIPART_FORM_DATA_VALUE])
-    fun saveAgentsLog(@RequestPart(required = true) executionLogs: FilePart) {
-        // File name is equals to agent id
-        val agentId = executionLogs.filename()
-        val logDir = File(configProperties.executionLogs)
-        if (!logDir.exists()) {
-            log.info("Folder to store logs from agents was created: ${logDir.name}")
-            logDir.mkdirs()
-        }
-        val logFile = File(logDir.path + File.separator + "$agentId.log")
-        if (!logFile.exists()) {
-            logFile.createNewFile()
-            log.info("Log file for $agentId agent was created")
-        }
-        executionLogs.content()
-            .map { dtBuffer ->
-                FileOutputStream(logFile, true).use { os ->
-                    dtBuffer.asInputStream().use {
-                        it.copyTo(os)
-                    }
-                }
-            }
-            .collectList()
-            .doOnSuccess {
-                log.info("Logs of agent with id = $agentId were written")
-            }
-            .subscribe()
-    }
-
     /**
      * Delete containers and images associated with execution [executionId]
      *

save-orchestrator/src/main/resources/application-mac.properties

@@ -1,3 +1,2 @@
-orchestrator.executionLogs=/Users/Shared/.save-cloud/cnb/executionLogs/
 orchestrator.test-resources.tmp-path=/Users/Shared/.save-cloud/cnb/tmp
 orchestrator.adjustResourceOwner=false

save-orchestrator/src/main/resources/application-win.properties

@@ -1,4 +1,3 @@
 orchestrator.backend-url=http://localhost:5800/internal
-orchestrator.execution-logs=${user.home}/.save-cloud/executionLogs/
 orchestrator.test-resources.tmp-path=${user.home}/.save-cloud/tmp
 orchestrator.adjust-resource-owner=false

save-orchestrator/src/main/resources/application.properties

@@ -4,7 +4,6 @@ spring.codec.max-in-memory-size=100MB
 spring.jmx.enabled=false
 management.endpoints.web.exposure.include=health,info,prometheus
 orchestrator.backend-url=http://backend:5800/internal
-orchestrator.execution-logs=/home/cnb/executionLogs/
 orchestrator.test-resources.tmp-path=/tmp
 orchestrator.shutdown.checks-interval-millis=5000
 orchestrator.shutdown.graceful-timeout-seconds=60