Merge pull request #5114 from kgudel/firstSignDate

PatrickGoRaft · web-flow · commit efcd6740dd12 · 2026-06-03T08:47:03.000-07:00
Update transmittal sheet to include first sign date
diff --git a/common/src/main/scala/hmda/query/ts/TransmittalSheetEntity.scala b/common/src/main/scala/hmda/query/ts/TransmittalSheetEntity.scala
@@ -26,7 +26,8 @@ case class TransmittalSheetEntity(
                                    submissionId: Option[String] = Some(""),
                                    createdAt: Option[java.sql.Timestamp] = Some(new java.sql.Timestamp(System.currentTimeMillis())),
                                    isQuarterly: Option[Boolean] = Some(false),
-                                   signDate: Option[Long] = Some(0L)
+                                   signDate: Option[Long] = Some(0L),
+                                   firstSignDate: Option[Long] = Some(0L)
                                  ) extends ColumnDataFormatter {
   def isEmpty: Boolean = lei == ""
 
@@ -35,7 +36,7 @@ case class TransmittalSheetEntity(
       s"$quarter|$name|$phone|" +
       s"$email|$street|$city|" +
       s"$state|$zipCode|$agency|" +
-      s"$totalLines|$taxId|$lei|${dateToString(signDate)}"
+      s"$totalLines|$taxId|$lei|${dateToString(signDate)}, ${dateToString(firstSignDate)}"
 
   def toPublicPSV: String =
     s"$year|$quarter|$lei|$taxId|$agency|" +
@@ -88,7 +89,7 @@ object TransmittalSheetEntity {
       s"$quarter|$name|$phone|" +
       s"$email|$street|$city|" +
       s"$state|$zipCode|$agency|" +
-      s"$totalLines|$taxId|$lei|${dateToString(signDate)}"
+      s"$totalLines|$taxId|$lei|${dateToString(signDate)}|${dateToString(firstSignDate)}"
    */
   object RegulatorParser extends PsvParsingCompanion[TransmittalSheetEntity] {
     override val psvReader: cormorant.Read[TransmittalSheetEntity] = { (a: CSV.Row) =>
@@ -108,9 +109,10 @@ object TransmittalSheetEntity {
         (rest, totalLines)      <- enforcePartialRead(readNext[Int], rest)
         (rest, taxId)           <- enforcePartialRead(readNext[String], rest)
         (rest, lei)             <- enforcePartialRead(readNext[String], rest)
-        signDateOrMore          <- readNext[String].readPartial(rest)
+        (rest, signDate)        <- enforcePartialRead(readNext[String], rest)
+        firstSignDateOrMore     <- readNext[String].readPartial(rest)
       } yield {
-        def create(signDate: String) = TransmittalSheetEntity(
+        def create(firstSignDate: String) = TransmittalSheetEntity(
           lei = lei,
           id = id,
           institutionName = institutionName,
@@ -126,12 +128,14 @@ object TransmittalSheetEntity {
           agency = agency,
           totalLines = totalLines,
           taxId = taxId,
-          signDate = dateFromString(signDate)
+          signDate = dateFromString(signDate),
+          firstSignDate = dateFromString(firstSignDate)
         )
 
-        signDateOrMore match {
-          case Left((more, signDate)) => Left(more -> create(signDate))
-          case Right(signDate)        => Right(create(signDate))
+        
+        firstSignDateOrMore match {
+          case Left((more, firstSignDate)) => Left(more -> create(firstSignDate))
+          case Right(firstSignDate)        => Right(create(firstSignDate))
         }
       }
     }
diff --git a/common/src/main/scala/hmda/query/ts/TransmittalSheetTable.scala b/common/src/main/scala/hmda/query/ts/TransmittalSheetTable.scala
@@ -25,6 +25,7 @@ class TransmittalSheetTable(tag: Tag, tableName: String) extends Table[Transmitt
   def createdAt = column[Option[Timestamp]]("created_at")
   def isQuarterly = column[Option[Boolean]]("is_quarterly")
   def signDate = column[Option[Long]]("sign_date")
+  def firstSignDate = column[Option[Long]]("sign_date")
 
   override def * =
     (
@@ -46,6 +47,7 @@ class TransmittalSheetTable(tag: Tag, tableName: String) extends Table[Transmitt
       submissionId,
       createdAt,
       isQuarterly,
-      signDate
+      signDate,
+      firstSignDate
     ) <> ((TransmittalSheetEntity.apply _).tupled, TransmittalSheetEntity.unapply)
 }
diff --git a/docs/spec/Agency_TS_Spec.csv b/docs/spec/Agency_TS_Spec.csv
@@ -14,4 +14,5 @@ agency, Federal Agency, Numeric, HMDA Filing Instructions Guide
 totalLines, Total Number of Entries Contained in Submission, Numeric, HMDA Filing Instructions Guide
 taxId, Federal Taxpayer Identification Number, Alphanumeric, HMDA Filing Instructions Guide
 lei, Legal Entity Identifier (LEI), Alphanumeric, HMDA Filing Instructions Guide
-signDate, HMDA Filing Sign Datetime, Alphanumeric, Collected by the HMDA Platform
+signDate, HMDA Filing Sign Datetime, Alphanumeric, Collected by the HMDA Platform
+firstSignDate, HMDA Filing First Sign Datetime, Alphanumeric, Collected by the HMDA Platform
diff --git a/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala b/hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala
@@ -140,6 +140,10 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo
         .map(l => l.submission.end)
         .runWith(Sink.lastOption)
 
+    def firstSignDateSubmissionHistory: Future[Seq[Long]] = {
+      submissionHistoryRepository.firstSignDate(submissionId)
+    }
+
     def deleteTsRow: Future[Done] =
       rawData.take(1)
         .map(s => TsCsvParser(s, fromCassandra = true))
@@ -189,16 +193,29 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo
           val enforceQuarterly = submissionId.period.quarter.isDefined
           for {
             signdate <- signDate
-            insertorupdate <- repo.insert(copyTs(ts, Some(signdate.getOrElse(0L)), enforceQuarterly))
+            firstsigndate <- firstSignDateSubmissionHistory
+            insertorupdate <- {
+              println("first sign date")
+              println(firstsigndate)
+              val resolvedSignDate = Some(signdate.getOrElse(0L))
+              val resolvedFirstSignDate = {
+                if (firstsigndate.head == 0) resolvedSignDate
+                else Some(firstsigndate.head)
+              }
+              repo.insert(copyTs(ts, resolvedSignDate, resolvedFirstSignDate, enforceQuarterly))
+            }
           } yield insertorupdate
         }
         .runWith(Sink.ignore)
 
-    def copyTs(ts: TransmittalSheetEntity, signdate: Option[Long], enforceQuarterly: Boolean): TransmittalSheetEntity =
+    def copyTs(ts: TransmittalSheetEntity, signdate: Option[Long], firstsigndate: Option[Long], enforceQuarterly: Boolean): TransmittalSheetEntity =
       if (enforceQuarterly) {
-        ts.copy(lei = ts.lei.toUpperCase, signDate = signdate, isQuarterly = Some(true))
+        ts.copy(lei = ts.lei.toUpperCase, signDate = signdate, firstSignDate = firstsigndate, isQuarterly = Some(true))
       } else {
-        ts.copy(lei = ts.lei.toUpperCase, signDate = signdate)
+        val newts = ts.copy(lei = ts.lei.toUpperCase, firstSignDate = firstsigndate, signDate = signdate)
+        println("new ts")
+        println(newts)
+        newts
       }
 
     def deleteLarRows: Future[Done] =
@@ -248,6 +265,9 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo
     def result =
       for {
 
+        firstSignDate <- firstSignDateSubmissionHistory
+        _ = log.info(s"First date signed $firstSignDate")
+
         _ <- deleteTsRow
         _ = if(tsDeletion)
           log.info(s"Attempt to remove data from TS table for  $submissionId  completed.")
@@ -268,7 +288,7 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo
         _ = log.info(s"Attempt to add data to LAR table for  $submissionId  completed.")
 
 
-        dateSigned   <- signDate
+        dateSigned <- signDate
         _ = log.info(s"Date signed $dateSigned")
 
         res <- insertSubmissionHistory
diff --git a/hmda-analytics/src/main/scala/hmda/analytics/query/SubmissionHistoryComponent.scala b/hmda-analytics/src/main/scala/hmda/analytics/query/SubmissionHistoryComponent.scala
@@ -28,6 +28,29 @@ trait SubmissionHistoryComponent {
            sign_date = ${signDate}
           """
       }
+    
+    def firstSignDate(submissionId: SubmissionId): Future[Seq[Long]] = {
+      val period = submissionId.period
+      val lei = submissionId.lei
+      val submissionIdLikeStatment = s"${lei}-${period}-%"
+      if (period.quarter.isDefined) {
+        config.db.run {
+          sql"""
+          SELECT MIN(sign_date) from #${tableName}
+          WHERE submission_id LIKE $submissionIdLikeStatment
+          """.as[Long]
+        }
+      }
+      else {
+        config.db.run {
+          sql"""
+          SELECT MIN(sign_date) from #${tableName}
+          WHERE submission_id LIKE $submissionIdLikeStatment
+          AND submission_id NOT LIKE '%-Q_-%'
+          """.as[Long]
+        }
+      }
+    }
   }
 
 }
diff --git a/hmda-analytics/src/main/scala/hmda/analytics/query/TransmittalSheetComponent.scala b/hmda-analytics/src/main/scala/hmda/analytics/query/TransmittalSheetComponent.scala
@@ -35,6 +35,7 @@ trait TransmittalSheetComponent {
     def createdAt       = column[Option[Timestamp]]("created_at")
     def isQuarterly     = column[Option[Boolean]]("is_quarterly")
     def signDate     = column[Option[Long]]("sign_date")
+    def firstSignDate   = column[Option[Long]]("first_sign_date")
 
     override def * =
       (
@@ -56,7 +57,8 @@ trait TransmittalSheetComponent {
         submissionId,
         createdAt,
         isQuarterly,
-        signDate
+        signDate,
+        firstSignDate
       ) <> ((TransmittalSheetEntity.apply _).tupled, TransmittalSheetEntity.unapply)
   }
 
@@ -84,6 +86,9 @@ trait TransmittalSheetComponent {
     def findByLei(lei: String): Future[Seq[TransmittalSheetEntity]] =
       db.run(table.filter(_.lei.toUpperCase === lei.toUpperCase).result)
 
+    def findByLeiAndQuarter(lei: String): Future[Seq[TransmittalSheetEntity]] =
+      db.run(table.filter(x => x.lei.toUpperCase === lei.toUpperCase && x.isQuarterly === true).result)
+
     def deleteByLei(lei: String): Future[Int] =
       db.run(table.filter(_.lei.toUpperCase === lei.toUpperCase).delete)
 
diff --git a/hmda-data-publisher/src/main/scala/hmda/publisher/query/component/SubmissionHistoryComponent.scala b/hmda-data-publisher/src/main/scala/hmda/publisher/query/component/SubmissionHistoryComponent.scala
@@ -0,0 +1,28 @@
+package hmda.publisher.query.component
+
+import hmda.publisher.helper.PGTableNameLoader
+import hmda.publisher.query.panel.InstitutionEmailEntity
+import hmda.query.DbConfiguration.dbConfig
+import hmda.query.repository.TableRepository
+import slick.basic.DatabaseConfig
+import slick.jdbc.JdbcProfile
+
+import scala.concurrent.Future
+
+trait SubmissionHistoryComponent extends PGTableNameLoader{
+
+
+  import dbConfig.profile.api._
+
+  class SubmissionHistoryTable(tag: Tag)
+    extends Table[SubmissionHistoryEntity](tag, submissionHistoryTableName) with SubmissionHistoryComponent {
+    def lei = column[String]("lei")
+    def submissionId= column[String]("submission_id")
+    def signDate = column[BigInt]("sign_date")
+
+    def * =
+      (lei, submissionId, signDate) <> (SubmissionHistoryEntity.tupled, SubmissionHistoryEntity.unapply)
+
+  }
+
+}
diff --git a/hmda-data-publisher/src/main/scala/hmda/publisher/query/component/TransmittalSheetTable.scala b/hmda-data-publisher/src/main/scala/hmda/publisher/query/component/TransmittalSheetTable.scala
@@ -27,6 +27,7 @@ abstract class AbstractTransmittalSheetTable[T](tag: Tag, tableName: String) ext
   def createdAt       = column[Option[Timestamp]]("created_at")
   def isQuarterly     = column[Option[Boolean]]("is_quarterly")
   def signDate        = column[Option[Long]]("sign_date")
+  def firstSignDate   = column[Option[Long]]("first_sign_date")
 
   def transmittalSheetEntityProjection =
     (
@@ -48,7 +49,8 @@ abstract class AbstractTransmittalSheetTable[T](tag: Tag, tableName: String) ext
       submissionId,
       createdAt,
       isQuarterly,
-      signDate
+      signDate,
+      firstSignDate
     ) <> ((TransmittalSheetEntity.apply _).tupled, TransmittalSheetEntity.unapply)
 }
 
