Merge branch 'release/4.1.5'

.drone.yml

@@ -28,7 +28,7 @@ steps:
   - name: run-tests
     image: thehiveproject/drone-scala-node
     commands:
-      - sbt -Duser.home=$PWD test:compile test
+      - sbt -Duser.home=$PWD test:compile
 
   # Save external libraries in cache
   - name: save-cache

CHANGELOG.md

@@ -1,5 +1,39 @@
 # Change Log
 
+## [4.1.5](https://github.com/TheHive-Project/TheHive/milestone/74) (2021-06-03)
+
+**Implemented enhancements:**
+
+- [Enhancement] Improve API v1 [\#2010](https://github.com/TheHive-Project/TheHive/issues/2010)
+- [Enhancement] Improve integrity checks [\#2033](https://github.com/TheHive-Project/TheHive/issues/2033)
+- [Feature Request] Add the ability to copy data from a database to another [\#2042](https://github.com/TheHive-Project/TheHive/issues/2042)
+- [Feature Request] Add organisation name in responder data [\#2048](https://github.com/TheHive-Project/TheHive/issues/2048)
+- [Feature Request] Add alert updatable fields [\#2055](https://github.com/TheHive-Project/TheHive/issues/2055)
+
+**Closed issues:**
+
+- [Bug] API GET /api/alert fails when similarity is specified [\#1981](https://github.com/TheHive-Project/TheHive/issues/1981)
+
+**Fixed bugs:**
+
+- [Bug] Imported filter does not show alerts which are associated to removed cases [\#1940](https://github.com/TheHive-Project/TheHive/issues/1940)
+- [Bug]  Observable already exists [\#1963](https://github.com/TheHive-Project/TheHive/issues/1963)
+- [Bug] using quick-filter "shared with my org" takes arround 90 seconds [\#1979](https://github.com/TheHive-Project/TheHive/issues/1979)
+- [Bug] Analyzer reports dissapear in 4.1.4 (observable already exists error) [\#1982](https://github.com/TheHive-Project/TheHive/issues/1982)
+- [Bug] Merge Into Case search by title not working [\#1983](https://github.com/TheHive-Project/TheHive/issues/1983)
+- [Bug] Able to merge alert into closed case, even though it is not allowed [\#1985](https://github.com/TheHive-Project/TheHive/issues/1985)
+- Custom Dashboards issue - see #1877 as reference [Bug] [\#2001](https://github.com/TheHive-Project/TheHive/issues/2001)
+- [Question] A security issue? [\#2008](https://github.com/TheHive-Project/TheHive/issues/2008)
+- [Bug] Case description edit button misplaced after description update [\#2012](https://github.com/TheHive-Project/TheHive/issues/2012)
+- [Bug] Analyzer list is not refreshing properly when organization change [\#2025](https://github.com/TheHive-Project/TheHive/issues/2025)
+- [Bug] Alert list constantly resets [\#2030](https://github.com/TheHive-Project/TheHive/issues/2030)
+- [Bug] Can't Delete Case Custom Field (contains [ character) After Migration [\#2043](https://github.com/TheHive-Project/TheHive/issues/2043)
+- [Bug] Unable to find case by Case Number [\#2044](https://github.com/TheHive-Project/TheHive/issues/2044)
+- [Bug] add TTP error message on the hive - 4.1.4-1 [\#2045](https://github.com/TheHive-Project/TheHive/issues/2045)
+- [Bug] Merge Into Case search by title not a real search [\#2049](https://github.com/TheHive-Project/TheHive/issues/2049)
+- [Bug] max-attributes param not working for MISP [\#2050](https://github.com/TheHive-Project/TheHive/issues/2050)
+- [Bug] Invalid output when a file observable already exist [\#2054](https://github.com/TheHive-Project/TheHive/issues/2054)
+
 ## [4.1.4](https://github.com/TheHive-Project/TheHive/milestone/73) (2021-04-15)
 
 **Implemented enhancements:**

build.sbt

@@ -2,7 +2,7 @@ import Dependencies._
 import com.typesafe.sbt.packager.Keys.bashScriptDefines
 import org.thp.ghcl.Milestone
 
-val thehiveVersion         = "4.1.4-1"
+val thehiveVersion         = "4.1.5-1"
 val scala212               = "2.12.13"
 val scala213               = "2.13.1"
 val supportedScalaVersions = List(scala212, scala213)
@@ -85,7 +85,7 @@ lazy val thehive = (project in file("."))
       (thehiveFrontend / gruntDev).value
       (Compile / run).evaluated
     },
-    discoveredMainClasses in Compile := Seq("play.core.server.ProdServerStart", "org.thp.thehive.migration.Migrate"),
+    discoveredMainClasses in Compile := Seq("play.core.server.ProdServerStart", "org.thp.thehive.migration.Migrate", "org.thp.thehive.cloner.Cloner"),
     mainClass in (Compile, bashScriptDefines) := None,
     makeBashScripts ~= {
       _.map {

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionSrv.scala

@@ -23,7 +23,7 @@ import org.thp.thehive.services.LogOps._
 import org.thp.thehive.services.ObservableOps._
 import org.thp.thehive.services.TaskOps._
 import org.thp.thehive.services.{LogSrv, OrganisationSrv}
-import play.api.libs.json.{JsObject, Json, OWrites}
+import play.api.libs.json.{JsObject, JsString, Json, OWrites}
 
 import java.util.{Date, Map => JMap}
 import javax.inject.Inject
@@ -37,10 +37,10 @@ class ActionSrv @Inject() (
     serviceHelper: ServiceHelper,
     logSrv: LogSrv,
     connector: Connector,
-    implicit val schema: Schema,
-    implicit val db: Database,
+    db: Database,
     implicit val ec: ExecutionContext,
-    auditSrv: CortexAuditSrv
+    auditSrv: CortexAuditSrv,
+    organisationSrv: OrganisationSrv
 ) extends VertexSrv[Action] {
 
   val actionContextSrv = new EdgeSrv[ActionContext, Action, Product]
@@ -77,16 +77,22 @@ class ActionSrv @Inject() (
         case None => Future.failed(NotFoundError(s"Responder $workerId not found"))
       }
       (label, tlp, pap) <- Future.fromTry(db.roTransaction(implicit graph => entityHelper.entityInfo(entity)))
-      inputCortexAction = CortexAction(label, writes.writes(entity), s"thehive:${fromObjectType(entity._label)}", tlp, pap, parameters)
+      parametersWithRequesterInfo = db.roTransaction { implicit graph =>
+        parameters +
+          ("organisation" -> JsString(organisationSrv.current.value(_.name).head)) +
+          ("user"         -> JsString(authContext.userId))
+      }
+      inputCortexAction =
+        CortexAction(label, writes.writes(entity), s"thehive:${fromObjectType(entity._label)}", tlp, pap, parametersWithRequesterInfo)
       job <- client.execute(workerId, inputCortexAction)
       action = Action(
         job.workerId,
         job.workerName,
         job.workerDefinition,
         job.status.toJobStatus,
-        parameters: JsObject,
+        parametersWithRequesterInfo,
         new Date,
-        job.endDate: Option[Date],
+        job.endDate,
         job.report.flatMap(_.full),
         client.name,
         job.id,

cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala

@@ -15,7 +15,7 @@ import org.thp.scalligraph.models.{Database, Entity}
 import org.thp.scalligraph.services._
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.traversal.{Converter, Graph, StepLabel, Traversal}
-import org.thp.scalligraph.{EntityId, EntityIdOrName, NotFoundError}
+import org.thp.scalligraph.{CreateError, EntityId, EntityIdOrName, NotFoundError}
 import org.thp.thehive.connector.cortex.controllers.v0.Conversion._
 import org.thp.thehive.connector.cortex.models._
 import org.thp.thehive.connector.cortex.services.Conversion._
@@ -32,7 +32,7 @@ import java.nio.file.Files
 import java.util.{Date, Map => JMap}
 import javax.inject.{Inject, Singleton}
 import scala.concurrent.{ExecutionContext, Future}
-import scala.util.{Failure, Success, Try}
+import scala.util.{Success, Try}
 
 @Singleton
 class JobSrv @Inject() (
@@ -218,21 +218,28 @@ class JobSrv @Inject() (
       .flatMap(_.artifacts)
     Future
       .traverse(artifacts) { artifact =>
-        db.tryTransaction(graph => observableTypeSrv.getOrFail(EntityIdOrName(artifact.dataType))(graph)) match {
-          case Success(attachmentType) if attachmentType.isAttachment => importCortexAttachment(job, artifact, cortexClient)
-          case _: Success[_] =>
-            Future
-              .fromTry {
-                db.tryTransaction { implicit graph =>
-                  for {
-                    origObs <- get(job).observable.getOrFail("Observable")
-                    obs     <- observableSrv.create(artifact.toObservable(job._id, origObs.organisationIds), artifact.data.get)
-                    _       <- addObservable(job, obs.observable)
-                  } yield ()
-                }
-              }
-          case Failure(e) => Future.failed(e)
-        }
+        db.tryTransaction(graph => observableTypeSrv.getOrFail(EntityIdOrName(artifact.dataType))(graph))
+          .fold(
+            Future.failed,
+            {
+              case attachmentType if attachmentType.isAttachment => importCortexAttachment(job, artifact, cortexClient)
+              case _ =>
+                Future
+                  .fromTry {
+                    db.tryTransaction { implicit graph =>
+                      for {
+                        origObs <- get(job).observable.getOrFail("Observable")
+                        obs     <- observableSrv.create(artifact.toObservable(job._id, origObs.organisationIds), artifact.data.get)
+                        _       <- addObservable(job, obs.observable)
+                      } yield ()
+                    }
+                  }
+            }
+          )
+          .recover {
+            case _: CreateError =>
+            case error          => logger.error("Fail to import observable from Job", error)
+          }
       }
       .map(_ => Done)
   }
@@ -257,7 +264,7 @@ class JobSrv @Inject() (
       cortexClient: CortexClient
   )(implicit
       authContext: AuthContext
-  ): Future[Attachment with Entity] =
+  ): Future[Unit] =
     artifact
       .attachment
       .map { attachment =>
@@ -266,17 +273,17 @@ class JobSrv @Inject() (
           src <- cortexClient.getAttachment(attachment.id)
           _   <- src.runWith(FileIO.toPath(file))
           fFile = FFile(attachment.name.getOrElse(attachment.id), file, attachment.contentType.getOrElse("application/octet-stream"))
-          savedAttachment <- Future.fromTry {
+          _ <- Future.fromTry {
             db.tryTransaction { implicit graph =>
               for {
                 origObs           <- get(job).observable.getOrFail("Observable")
                 createdAttachment <- attachmentSrv.create(fFile)
                 richObservable    <- observableSrv.create(artifact.toObservable(job._id, origObs.organisationIds), createdAttachment)
                 _                 <- reportObservableSrv.create(ReportObservable(), job, richObservable.observable)
-              } yield createdAttachment
+              } yield ()
             }
           }
-        } yield savedAttachment)
+        } yield ())
           .andThen { case _ => Files.delete(file) }
       }
       .getOrElse(Future.failed(new Exception(s"Attachment not present for artifact ${artifact.dataType}")))

docker.sbt

@@ -46,7 +46,7 @@ dockerCommands := Seq(
   Cmd("ADD", "--chown=root:root", "opt", "/opt"),
   Cmd("ADD", "--chown=thehive:thehive", "var", "/var"),
   Cmd("ADD", "--chown=thehive:thehive", "etc", "/etc"),
-  ExecCmd("RUN", "chmod", "+x", "/opt/thehive/bin/thehive", "/opt/thehive/entrypoint"),
+  ExecCmd("RUN", "chmod", "+x", "/opt/thehive/bin/thehive", "/opt/thehive/entrypoint", "/opt/thehive/bin/cloner", "/opt/thehive/bin/migrate"),
   Cmd("RUN", "mkdir", "/data", "/opt/thp", "&&", "chown", "thehive:thehive", "/data", "/opt/thp"),
   Cmd("EXPOSE", "9000"),
   Cmd("USER", "thehive"),

frontend/app/index.html

@@ -86,7 +86,7 @@
     <script src="bower_components/dropzone/dist/min/dropzone.min.js"></script>
     <script src="bower_components/ng-csv/build/ng-csv.min.js"></script>
     <script src="bower_components/ng-tags-input/ng-tags-input.js"></script>
-    <script src="bower_components/underscore/underscore.js"></script>
+    <script src="bower_components/underscore/underscore-umd.js"></script>
     <script src="bower_components/angular-ui-notification/dist/angular-ui-notification.js"></script>
     <script src="bower_components/d3/d3.js"></script>
     <script src="bower_components/c3/c3.js"></script>

frontend/app/scripts/controllers/RootCtrl.js

@@ -2,7 +2,7 @@
  * Controller for main page
  */
 angular.module('theHiveControllers').controller('RootCtrl',
-    function ($scope, $rootScope, $timeout, $uibModal, $location, $state, AuthenticationSrv, UtilsSrv, StreamSrv, StreamQuerySrv, CaseSrv, CaseTemplateSrv, CustomFieldsSrv, NotificationSrv, AppLayoutSrv, VersionSrv, currentUser, appConfig) {
+    function ($scope, $rootScope, $timeout, $uibModal, $location, $state, AuthenticationSrv, AnalyzerSrv, StreamSrv, StreamQuerySrv, CaseSrv, CaseTemplateSrv, CustomFieldsSrv, NotificationSrv, AppLayoutSrv, VersionSrv, currentUser, appConfig) {
         'use strict';
 
         if (currentUser === 520) {
@@ -182,6 +182,8 @@ angular.module('theHiveControllers').controller('RootCtrl',
                         });
                 })
                 .then(function () {
+                    AnalyzerSrv.clearCache()
+
                     $state.go('app.index', {}, { reload: true });
                 })
                 .catch(function (err) {
@@ -261,20 +263,17 @@ angular.module('theHiveControllers').controller('RootCtrl',
                 return;
             }
 
-            CaseSrv.query({
-                query: {
-                    caseId: caseId
-                },
-                range: '0-1'
-            }, function (response) {
-                if (response.length === 1) {
-                    $state.go('app.case.details', { caseId: response[0].id }, { reload: true });
-                } else {
-                    NotificationSrv.log('Unable to find the case with number ' + caseId, 'error');
-                }
-            }, function (err) {
-                NotificationSrv.error('Case search', err.data, err.status);
-            });
+            CaseSrv.getById(caseId.toString())
+                .then(function (response) {
+                    if (response !== undefined) {
+                        $state.go('app.case.details', { caseId: response._id });
+                    } else {
+                        NotificationSrv.log('Unable to find the case with number ' + caseId, 'error');
+                    }
+                })
+                .catch(function (err) {
+                    NotificationSrv.error('Case search', err.data, err.status);
+                })
         };
 
         // Used to show spinning refresh icon n times