diff --git a/build.sbt b/build.sbt index 17936f1e83..f508df0111 100644 --- a/build.sbt +++ b/build.sbt @@ -57,7 +57,7 @@ lazy val circeDeps = Seq(circe, circeGeneric, circeParser) lazy val enumeratumDeps = Seq(enumeratum, enumeratumCirce) lazy val slickDeps = Seq(slick, slickHikariCP, postgres, h2) - +lazy val metaInfMatcher = """META-INF/.+\.(SF|DSA|RSA)""".r lazy val dockerSettings = Seq( dockerBuildCommand := { //force amd64 Architecture for k8s docker image compatability @@ -186,6 +186,7 @@ lazy val `hmda-platform` = (project in file("hmda")) case "META-INF/io.netty.versions.properties" => MergeStrategy.concat case "logback.xml" => MergeStrategy.concat case "META-INF/MANIFEST.MF" => MergeStrategy.discard + case metaInfMatcher(_) => MergeStrategy.discard case PathList("META-INF", xs@_*) => MergeStrategy.concat case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first case PathList("jakarta", xs@_*) => MergeStrategy.last @@ -227,6 +228,7 @@ lazy val `check-digit` = (project in file("check-digit")) case "application.conf" => MergeStrategy.concat case "META-INF/io.netty.versions.properties" => MergeStrategy.concat case "META-INF/MANIFEST.MF" => MergeStrategy.discard + case metaInfMatcher(_) => MergeStrategy.discard case PathList("META-INF", xs @ _*) => MergeStrategy.concat case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first case PathList("jakarta", xs @ _*) => MergeStrategy.last @@ -268,6 +270,7 @@ lazy val `check-digit` = (project in file("check-digit")) case "application.conf" => MergeStrategy.concat case "META-INF/io.netty.versions.properties" => MergeStrategy.concat case "META-INF/MANIFEST.MF" => MergeStrategy.discard + case metaInfMatcher(_) => MergeStrategy.discard case PathList("META-INF", xs@_*) => MergeStrategy.concat case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first case PathList("jakarta", xs@_*) => MergeStrategy.last @@ -304,6 +307,7 @@ lazy val `institutions-api` = (project in file("institutions-api")) case "application.conf" => MergeStrategy.concat case "META-INF/io.netty.versions.properties" => MergeStrategy.concat case "META-INF/MANIFEST.MF" => MergeStrategy.discard + case metaInfMatcher(_) => MergeStrategy.discard case PathList("META-INF", xs@_*) => MergeStrategy.concat case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first case PathList("jakarta", xs@_*) => MergeStrategy.last @@ -347,6 +351,7 @@ lazy val `hmda-data-publisher` = (project in file("hmda-data-publisher")) case "application.conf" => MergeStrategy.concat case "META-INF/io.netty.versions.properties" => MergeStrategy.concat case "META-INF/MANIFEST.MF" => MergeStrategy.discard + case metaInfMatcher(_) => MergeStrategy.discard case PathList("META-INF", xs@_*) => MergeStrategy.concat case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first case PathList("jakarta", xs@_*) => MergeStrategy.last @@ -385,6 +390,7 @@ lazy val `hmda-dashboard` = (project in file("hmda-dashboard")) case "application.conf" => MergeStrategy.concat case "META-INF/io.netty.versions.properties" => MergeStrategy.concat case "META-INF/MANIFEST.MF" => MergeStrategy.discard + case metaInfMatcher(_) => MergeStrategy.discard case PathList("META-INF", xs@_*) => MergeStrategy.concat case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first case PathList("jakarta", xs@_*) => MergeStrategy.last @@ -424,6 +430,7 @@ lazy val `ratespread-calculator` = (project in file("ratespread-calculator")) case "application.conf" => MergeStrategy.concat case "META-INF/io.netty.versions.properties" => MergeStrategy.concat case "META-INF/MANIFEST.MF" => MergeStrategy.discard + case metaInfMatcher(_) => MergeStrategy.discard case PathList("META-INF", xs@_*) => MergeStrategy.concat case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first case PathList("jakarta", xs@_*) => MergeStrategy.last @@ -463,6 +470,7 @@ lazy val `modified-lar` = (project in file("modified-lar")) case "application.conf" => MergeStrategy.concat case "META-INF/io.netty.versions.properties" => MergeStrategy.concat case "META-INF/MANIFEST.MF" => MergeStrategy.discard + case metaInfMatcher(_) => MergeStrategy.discard case PathList("META-INF", xs@_*) => MergeStrategy.concat case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first case PathList("jakarta", xs@_*) => MergeStrategy.last @@ -503,6 +511,7 @@ lazy val `irs-publisher` = (project in file("irs-publisher")) case "application.conf" => MergeStrategy.concat case "META-INF/io.netty.versions.properties" => MergeStrategy.concat case "META-INF/MANIFEST.MF" => MergeStrategy.discard + case metaInfMatcher(_) => MergeStrategy.discard case PathList("META-INF", xs@_*) => MergeStrategy.concat case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first case PathList("jakarta", xs@_*) => MergeStrategy.last @@ -543,6 +552,7 @@ lazy val `hmda-reporting` = (project in file("hmda-reporting")) case "application.conf" => MergeStrategy.concat case "META-INF/io.netty.versions.properties" => MergeStrategy.concat case "META-INF/MANIFEST.MF" => MergeStrategy.discard + case metaInfMatcher(_) => MergeStrategy.discard case PathList("META-INF", xs@_*) => MergeStrategy.concat case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first case PathList("jakarta", xs@_*) => MergeStrategy.last @@ -592,6 +602,7 @@ lazy val `hmda-analytics` = (project in file("hmda-analytics")) case "application.conf" => MergeStrategy.concat case "META-INF/io.netty.versions.properties" => MergeStrategy.concat case "META-INF/MANIFEST.MF" => MergeStrategy.discard + case metaInfMatcher(_) => MergeStrategy.discard case PathList("META-INF", xs@_*) => MergeStrategy.concat case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first case PathList("jakarta", xs@_*) => MergeStrategy.last @@ -634,6 +645,7 @@ lazy val `hmda-analytics` = (project in file("hmda-analytics")) case "application.conf" => MergeStrategy.concat case "META-INF/io.netty.versions.properties" => MergeStrategy.concat case "META-INF/MANIFEST.MF" => MergeStrategy.discard + case metaInfMatcher(_) => MergeStrategy.discard case PathList("META-INF", xs @ _*) => MergeStrategy.concat case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first case PathList("jakarta", xs @ _*) => MergeStrategy.last @@ -672,6 +684,7 @@ lazy val `rate-limit` = (project in file("rate-limit")) case "application.conf" => MergeStrategy.concat case "META-INF/io.netty.versions.properties" => MergeStrategy.concat case "META-INF/MANIFEST.MF" => MergeStrategy.discard + case metaInfMatcher(_) => MergeStrategy.discard case PathList("META-INF", xs@_*) => MergeStrategy.concat case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first case PathList("jakarta", xs@_*) => MergeStrategy.last @@ -712,6 +725,7 @@ lazy val `data-browser` = (project in file("data-browser")) case "application.conf" => MergeStrategy.concat case "META-INF/io.netty.versions.properties" => MergeStrategy.concat case "META-INF/MANIFEST.MF" => MergeStrategy.discard + case metaInfMatcher(_) => MergeStrategy.discard case PathList("META-INF", xs@_*) => MergeStrategy.concat case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first case PathList("jakarta", xs@_*) => MergeStrategy.last @@ -746,6 +760,7 @@ lazy val `submission-errors` = (project in file("submission-errors")) case "application.conf" => MergeStrategy.concat case "META-INF/io.netty.versions.properties" => MergeStrategy.concat case "META-INF/MANIFEST.MF" => MergeStrategy.discard + case metaInfMatcher(_) => MergeStrategy.discard case PathList("META-INF", xs@_*) => MergeStrategy.concat case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first case PathList("jakarta", xs@_*) => MergeStrategy.last @@ -777,6 +792,7 @@ lazy val `email-service` = (project in file("email-service")) case "application.conf" => MergeStrategy.concat case "META-INF/io.netty.versions.properties" => MergeStrategy.concat case "META-INF/MANIFEST.MF" => MergeStrategy.discard + case metaInfMatcher(_) => MergeStrategy.discard case PathList("META-INF", xs@_*) => MergeStrategy.concat case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first case PathList("jakarta", xs@_*) => MergeStrategy.last @@ -818,6 +834,7 @@ lazy val `hmda-quarterly-data-service` = (project in file ("hmda-quarterly-data- case "application.conf" => MergeStrategy.concat case "META-INF/io.netty.versions.properties" => MergeStrategy.concat case "META-INF/MANIFEST.MF" => MergeStrategy.discard + case metaInfMatcher(_) => MergeStrategy.discard case PathList("META-INF", xs@_*) => MergeStrategy.concat case PathList("org", "bouncycastle", xs @_*) => MergeStrategy.first case PathList("jakarta", xs@_*) => MergeStrategy.last diff --git a/common/src/main/scala/hmda/util/FieldNullifyUtility.scala b/common/src/main/scala/hmda/util/FieldNullifyUtility.scala new file mode 100644 index 0000000000..b420ce7412 --- /dev/null +++ b/common/src/main/scala/hmda/util/FieldNullifyUtility.scala @@ -0,0 +1,35 @@ +package hmda.util + +import hmda.model.institution.{Institution, InstitutionType, Parent, Respondent, TopHolder} + + +object FieldNullifyUtility { + def nullifyInstitutionFields(institution: Institution): Institution = { + //HMDA Ops will no longer provide updates to institution fields if they are not shared + //with the transmittal sheet model + if (institution.activityYear >= 2024) + Institution( + institution.activityYear, + institution.LEI, + institution.agency, + InstitutionType.valueOf(-1), + institution.institutionId_2017, + institution.taxId, + -1, + institution.emailDomains, + institution.respondent, + Parent(-1,Some("")), + -1, + -1, + TopHolder(-1,Some("")), + institution.hmdaFiler, + institution.quarterlyFiler, + institution.quarterlyFilerHasFiledQ1, + institution.quarterlyFilerHasFiledQ2, + institution.quarterlyFilerHasFiledQ3, + institution.notes + ) + else + institution + } +} diff --git a/hmda/src/main/scala/hmda/api/http/admin/InstitutionAdminHttpApi.scala b/hmda/src/main/scala/hmda/api/http/admin/InstitutionAdminHttpApi.scala index f21fe152c7..abeebc727d 100644 --- a/hmda/src/main/scala/hmda/api/http/admin/InstitutionAdminHttpApi.scala +++ b/hmda/src/main/scala/hmda/api/http/admin/InstitutionAdminHttpApi.scala @@ -1,8 +1,8 @@ package hmda.api.http.admin -import akka.cluster.sharding.typed.scaladsl.{ ClusterSharding, EntityRef } +import akka.cluster.sharding.typed.scaladsl.{ClusterSharding, EntityRef} import akka.http.scaladsl.model.headers.RawHeader -import akka.http.scaladsl.model.{ StatusCodes, Uri } +import akka.http.scaladsl.model.{StatusCodes, Uri} import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.Route import akka.util.Timeout @@ -15,14 +15,15 @@ import hmda.api.http.model.admin.InstitutionDeletedResponse import hmda.auth.OAuth2Authorization import hmda.messages.institution.InstitutionCommands._ import hmda.messages.institution.InstitutionEvents._ -import hmda.model.institution.{ Agency, Institution } +import hmda.model.institution.{Agency, Institution} import hmda.persistence.institution.InstitutionPersistence import hmda.persistence.institution.InstitutionPersistence.selectInstitution import hmda.util.http.FilingResponseUtils._ import hmda.api.http.EmailUtils._ +import hmda.util.FieldNullifyUtility.nullifyInstitutionFields -import scala.concurrent.{ ExecutionContext, Future } -import scala.util.{ Failure, Success } +import scala.concurrent.{ExecutionContext, Future} +import scala.util.{Failure, Success} object InstitutionAdminHttpApi { def create(config: Config, sharding: ClusterSharding)(implicit ec: ExecutionContext, t: Timeout): OAuth2Authorization => Route = @@ -180,7 +181,7 @@ private class InstitutionAdminHttpApi(config: Config, sharding: ClusterSharding) complete((StatusCodes.InternalServerError, errorResponse)) case Success(Some(i)) => - complete(i) + complete(nullifyInstitutionFields(i)) case Success(None) => complete(StatusCodes.NotFound) diff --git a/institutions-api/src/main/scala/hmda/institution/api/http/InstitutionConverter.scala b/institutions-api/src/main/scala/hmda/institution/api/http/InstitutionConverter.scala index 0f2315f28e..7df437460b 100644 --- a/institutions-api/src/main/scala/hmda/institution/api/http/InstitutionConverter.scala +++ b/institutions-api/src/main/scala/hmda/institution/api/http/InstitutionConverter.scala @@ -3,11 +3,13 @@ package hmda.institution.api.http import hmda.institution.query.{InstitutionEmailEntity, InstitutionEntity} import hmda.model.institution._ import hmda.util.CSVConsolidator.listDeDupeToList +import hmda.util.FieldNullifyUtility.nullifyInstitutionFields object InstitutionConverter { def convert(entity: InstitutionEntity, emails: Seq[String]): Institution = { - Institution( + + nullifyInstitutionFields(Institution( entity.activityYear, entity.lei, Agency.valueOf(entity.agency), @@ -38,11 +40,11 @@ object InstitutionConverter { entity.quarterlyFilerHasFiledQ2, entity.quarterlyFilerHasFiledQ3, entity.notes - ) + )) } def convert(institution: Institution): InstitutionEntity = - InstitutionEntity( + InstitutionEntity( institution.LEI, institution.activityYear, institution.agency.code, diff --git a/institutions-api/src/main/scala/hmda/institution/api/http/InstitutionQueryHttpApi.scala b/institutions-api/src/main/scala/hmda/institution/api/http/InstitutionQueryHttpApi.scala index 19c782e63c..0a2efdbcb3 100644 --- a/institutions-api/src/main/scala/hmda/institution/api/http/InstitutionQueryHttpApi.scala +++ b/institutions-api/src/main/scala/hmda/institution/api/http/InstitutionQueryHttpApi.scala @@ -132,7 +132,7 @@ private class InstitutionQueryHttpApi(config: Config)(implicit ec: ExecutionCont .map(_.sumLars(bankFilterList.toSeq) .recover({ case err: Throwable => log.debug("ts repo failure, most likely table not yet available for year, skipping...", err) - 0 + 0 }) .map(AnnualLarCount(yr, _))) .getOrElse(Future(AnnualLarCount(yr, 0)))