Merge branch 'release/3.0.0-RC2'

Author: To-om

CHANGELOG.md

@@ -1,7 +1,16 @@
 # Change Log
 
-## [3.0.0-RC1](https://github.com/TheHive-Project/Cortex/tree/3.0.0-RC1) (2019-04-05)
+## [3.0.0-RC2](https://github.com/TheHive-Project/Cortex/tree/3.0.0-RC2) (2019-05-03)
+
+[Full Changelog](https://github.com/TheHive-Project/Cortex/compare/3.0.0-RC1...3.0.0-RC2)
+
+**Fixed bugs:**
+
+- Unable to load Analyzers with 3.0.0 [\#185](https://github.com/TheHive-Project/Cortex/issues/185)
+- Cortex will fail to run analyzers [\#182](https://github.com/TheHive-Project/Cortex/issues/182)
+- Docker container exposes tcp/9000 instead of tcp/9001 [\#166](https://github.com/TheHive-Project/Cortex/issues/166)
 
+## [3.0.0-RC1](https://github.com/TheHive-Project/Cortex/tree/3.0.0-RC1) (2019-04-05)
 [Full Changelog](https://github.com/TheHive-Project/Cortex/compare/2.1.3...3.0.0-RC1)
 
 **Implemented enhancements:**

app/org/thp/cortex/models/Job.scala

@@ -36,6 +36,7 @@ trait JobAttributes {
   val fromCache = optionalAttribute("fromCache", F.booleanFmt, "Indicates if cache is used", O.form)
   val tpe = attribute("type", F.enumFmt(WorkerType), "", O.readonly)
   val lbel = optionalAttribute("label", F.stringFmt, "Label of the job")
+  val cacheTag = optionalAttribute("cacheTag", F.stringFmt, "hash of job discriminant, used for cache", O.readonly)
 }
 
 @Singleton

app/org/thp/cortex/services/JobSrv.scala

@@ -20,6 +20,7 @@ import org.thp.cortex.models._
 import org.elastic4play._
 import org.elastic4play.controllers._
 import org.elastic4play.services._
+import org.elastic4play.utils.Hasher
 
 @Singleton
 class JobSrv(
@@ -243,11 +244,10 @@ class JobSrv(
       parameters: JsObject,
       label: Option[String],
       force: Boolean)(implicit authContext: AuthContext): Future[Job] = {
-    val previousJob = if (force) Future.successful(None)
-    else findSimilarJob(worker, dataType, dataAttachment, tlp, parameters)
+    val previousJob = findSimilarJob(worker, dataType, dataAttachment, tlp, parameters, force)
     previousJob.flatMap {
-      case Some(job) ⇒ Future.successful(job)
-      case None ⇒ isUnderRateLimit(worker).flatMap {
+      case Right(job) ⇒ Future.successful(job)
+      case Left(cacheTag) ⇒ isUnderRateLimit(worker).flatMap {
         case true ⇒
           val fields = Fields(Json.obj(
             "workerDefinitionId" → worker.workerDefinitionId(),
@@ -260,7 +260,8 @@ class JobSrv(
             "pap" → pap,
             "message" → message,
             "parameters" → parameters.toString,
-            "type" → worker.tpe()))
+            "type" → worker.tpe(),
+            "cacheTag" → cacheTag))
             .set("label", label.map(JsString.apply))
           val fieldWithData = dataAttachment match {
             case Left(data)        ⇒ fields.set("data", data)
@@ -298,28 +299,27 @@ class JobSrv(
       .getOrElse(Future.successful(true))
   }
 
-  def findSimilarJob(worker: Worker, dataType: String, dataAttachment: Either[String, Attachment], tlp: Long, parameters: JsObject): Future[Option[Job]] = {
-    val cache = worker.jobCache().fold(jobCache)(_.minutes)
-    if (cache.length == 0 || worker.tpe() == WorkerType.responder) {
+  def findSimilarJob(worker: Worker, dataType: String, dataAttachment: Either[String, Attachment], tlp: Long, parameters: JsObject, force: Boolean): Future[Either[String, Job]] = {
+    val cacheTag = Hasher("MD5").fromString(s"${worker.id}|$dataType|$tlp|${dataAttachment.fold(data ⇒ data, attachment ⇒ attachment.id)}|$parameters").head.toString()
+    lazy val cache = worker.jobCache().fold(jobCache)(_.minutes)
+    if (force || cache.length == 0 || worker.tpe() == WorkerType.responder) {
       logger.info("Job cache is disabled")
-      Future.successful(None)
+      Future.successful(Left(cacheTag))
     }
     else {
       import org.elastic4play.services.QueryDSL._
       logger.info(s"Looking for similar job in the last ${cache.toMinutes} minutes (worker=${worker.id}, dataType=$dataType, data=$dataAttachment, tlp=$tlp, parameters=$parameters)")
+
       val now = new Date().getTime
       find(and(
-        "workerId" ~= worker.id,
+        "cacheTag" ~= cacheTag,
         "status" ~!= JobStatus.Failure,
         "status" ~!= JobStatus.Deleted,
-        "startDate" ~>= (now - cache.toMillis),
-        "dataType" ~= dataType,
-        "tlp" ~= tlp,
-        dataAttachment.fold(data ⇒ "data" ~= data, attachment ⇒ "attachment.id" ~= attachment.id),
-        "parameters" ~= parameters.toString), Some("0-1"), Seq("-createdAt"))
+        "startDate" ~>= (now - cache.toMillis)), Some("0-1"), Seq("-createdAt"))
         ._1
         .map(j ⇒ new Job(jobModel, j.attributes + ("fromCache" → JsBoolean(true))))
         .runWith(Sink.headOption)
+        .map(_.toRight(cacheTag))
     }
   }
 

app/org/thp/cortex/services/WorkerSrv.scala

@@ -160,6 +160,7 @@ class WorkerSrv @Inject() (
     def readDirectory(path: Path, workerType: WorkerType.Type): Seq[WorkerDefinition] = {
       for {
         workerDir ← Files.newDirectoryStream(path).asScala.toSeq
+        if Files.isDirectory(workerDir)
         infoFile ← Files.newDirectoryStream(workerDir, "*.json").asScala
         workerDefinition ← readFile(infoFile, workerType)
       } yield workerDefinition

docker.sbt

@@ -11,9 +11,9 @@ version in Docker := {
 }
 defaultLinuxInstallLocation in Docker := "/opt/cortex"
 dockerRepository := Some("thehiveproject")
-dockerUpdateLatest := !version.value.toUpperCase.contains("RC")
+dockerUpdateLatest := !version.value.toUpperCase.contains("RC") && !version.value.contains("SNAPSHOT")
 dockerEntrypoint := Seq("/opt/cortex/entrypoint")
-dockerExposedPorts := Seq(9000)
+dockerExposedPorts := Seq(9001)
 mappings in Docker ++= Seq(
   file("package/docker/entrypoint") -> "/opt/cortex/entrypoint",
   file("package/logback.xml") -> "/etc/cortex/logback.xml",

version.sbt

@@ -1 +1 @@
-version in ThisBuild := "3.0.0-RC1"
+version in ThisBuild := "3.0.0-RC2"

www/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cortex",
-  "version": "3.0.0-RC1",
+  "version": "3.0.0-RC2",
   "description": "A powerfull observable analysis engine",
   "license": "AGPL-v3",
   "homepage": "https://github.com/TheHive-Project/Cortex",