Not a PR, notes on seeding NFT

.run/local-nft-seeder.run.xml

@@ -18,7 +18,6 @@
     <method v="2">
       <option name="RunConfigurationTask" enabled="true" run_configuration_name="local_resources_up" run_configuration_type="docker-deploy" />
       <option name="Make" enabled="true" />
-      <option name="RunConfigurationTask" enabled="true" run_configuration_name="flywayClean-local-except-address" run_configuration_type="GradleRunConfiguration" />
     </method>
   </configuration>
-</component>
+</component>

src/main/kotlin/uk/gov/communities/prsdb/webapp/database/dao/NftDataSeederDao.kt

@@ -104,16 +104,6 @@ class NftDataSeederDao(
         return connection.prepareStatement(query)
     }
 
-    fun prepareCertificateUploadStatement(): PreparedStatement {
-        val query =
-            """
-            INSERT INTO certificate_upload 
-            (created_date, last_modified_date, category, property_ownership_id, file_upload_id) 
-            VALUES (?, ?, ?, ?, ?)
-            """
-        return connection.prepareStatement(query)
-    }
-
     fun prepareGasSafetyFileUploadsStatement(): PreparedStatement {
         val query =
             """

src/main/kotlin/uk/gov/communities/prsdb/webapp/services/NftDataSeeder.kt

@@ -1,11 +1,9 @@
 package uk.gov.communities.prsdb.webapp.services
 
-import kotlinx.datetime.LocalDate
 import org.hibernate.SessionFactory
 import org.springframework.beans.factory.annotation.Value
 import org.springframework.context.annotation.Profile
 import uk.gov.communities.prsdb.webapp.annotations.taskAnnotations.PrsdbTaskService
-import uk.gov.communities.prsdb.webapp.constants.GAS_SAFETY_CERT_VALIDITY_YEARS
 import uk.gov.communities.prsdb.webapp.constants.enums.CertificateType
 import uk.gov.communities.prsdb.webapp.constants.enums.RegistrationNumberType
 import uk.gov.communities.prsdb.webapp.database.dao.NftDataSeederDao
@@ -23,7 +21,6 @@ import uk.gov.communities.prsdb.webapp.helpers.extensions.PreparedStatementExten
 import uk.gov.communities.prsdb.webapp.helpers.extensions.PreparedStatementExtensions.Companion.setStringOrNull
 import uk.gov.communities.prsdb.webapp.models.dataModels.EpcDataModel
 import java.sql.Connection
-import java.sql.Date
 import java.sql.PreparedStatement
 import java.sql.Timestamp
 import java.time.LocalDateTime
@@ -43,7 +40,7 @@ class NftDataSeeder(
 
     private val registrationNumberGenerator = RegistrationNumberGenerator()
     private val landlordAddressGenerator = AddressGenerator()
-    private val propertyOwnershipAddressGenerator = AddressGenerator(restrictToAvailable = true)
+    private val propertyOwnershipAddressGenerator = AddressGenerator(restrictToAvailable = true, PREFERRED_PROPERTY_ADDRESS_IDS)
     private val incompletePropertyAddressGenerator = AddressGenerator(restrictToAvailable = true)
 
     fun seedDatabase() {
@@ -133,33 +130,39 @@ class NftDataSeeder(
         val licenceStmt = nftDataSeederDao.prepareLicenceStatement()
         val propertyOwnershipStmt = nftDataSeederDao.preparePropertyOwnershipStatement()
 
-        val fileUploadStmt = nftDataSeederDao.prepareFileUploadStatement()
-        val certificateUploadStmt = nftDataSeederDao.prepareCertificateUploadStatement()
-        val gasSafetyFileUploadsStmt = nftDataSeederDao.prepareGasSafetyFileUploadsStatement()
-        val electricalSafetyFileUploadsStmt = nftDataSeederDao.prepareElectricalSafetyFileUploadsStatement()
+        //  val fileUploadStmt = nftDataSeederDao.prepareFileUploadStatement()
+        //  val gasSafetyFileUploadsStmt = nftDataSeederDao.prepareGasSafetyFileUploadsStatement()
+        //  val electricalSafetyFileUploadsStmt = nftDataSeederDao.prepareElectricalSafetyFileUploadsStatement()
         val propertyComplianceStmt = nftDataSeederDao.preparePropertyComplianceStatement()
 
         val reminderEmailSentStmt = nftDataSeederDao.prepareReminderEmailSentStatement()
         val savedJourneyStateStmt = nftDataSeederDao.prepareSavedJourneyStateStatement()
         val incompletePropertyStmt = nftDataSeederDao.prepareLandlordIncompletePropertyStatement()
 
         try {
-            var registrationNumbersAdded = 0
+            var registrationNumbersAdded = REGISTRATION_NUMBERS_ALREADY_ADDED
 
-            var licencesAdded = 0
-            var propertyOwnershipsAdded = 0
+            var licencesAdded = LICENCES_ALREADY_ADDED
+            var propertyOwnershipsAdded = PROPERTY_OWNERSHIPS_ALREADY_ADDED
 
-            var fileUploadsAdded = 0
-            var complianceRecordsAdded = 0
+            var fileUploadsAdded = 0 // currently not adding to this
+            var complianceRecordsAdded = COMPLIANCE_RECORDS_ALREADY_ADDED
 
-            var reminderEmailsAdded = 0
-            var incompletePropertiesAdded = 0
+            var reminderEmailsAdded = REMINDER_EMAILS_ADDED
+            var incompletePropertiesAdded = INCOMPLETE_PROPERTIES_ADDED
 
-            fun propertyRegistrationsAdded() = propertyOwnershipsAdded + incompletePropertiesAdded
+            var newPropertyRegistrations = 0
 
             val numOfLandlordBatches = ceil(NUM_OF_LANDLORDS.toFloat() / BATCH_SIZE).toInt()
             for (landlordBatchNum in 1..numOfLandlordBatches) {
-                val landlordIdRange = ((landlordBatchNum - 1) * BATCH_SIZE + 1)..min(landlordBatchNum * BATCH_SIZE, NUM_OF_LANDLORDS)
+                val landlordIdRange =
+                    (LANDLORDS_ALREADY_ADDED + (landlordBatchNum - 1) * BATCH_SIZE + 1)..(
+                        LANDLORDS_ALREADY_ADDED +
+                            min(
+                                landlordBatchNum * BATCH_SIZE,
+                                NUM_OF_LANDLORDS,
+                            )
+                    )
                 val coreDetailsForLandlords = NftDataFaker.generateCoreDetailsForLandlords(landlordIdRange.toList())
 
                 coreDetailsForLandlords.forEach {
@@ -178,10 +181,10 @@ class NftDataSeeder(
                 registrationNumberGenerator.forgetUsedValues()
                 landlordAddressGenerator.forgetUsedValues()
 
-                log("Seeded ${landlordIdRange.last} landlords")
+                log("Seeded ${landlordIdRange.last - LANDLORDS_ALREADY_ADDED} / $NUM_OF_LANDLORDS landlords")
 
                 coreDetailsForLandlords.forEach { landlord ->
-                    val numOfPropertiesLeft = NUM_OF_PROPERTIES - propertyRegistrationsAdded()
+                    val numOfPropertiesLeft = NUM_OF_PROPERTIES - newPropertyRegistrations
                     val numOfPropertiesForLandlord = NftDataFaker.generateNumberOfPropertiesForLandlord().coerceAtMost(numOfPropertiesLeft)
 
                     repeat(numOfPropertiesForLandlord) {
@@ -203,22 +206,18 @@ class NftDataSeeder(
                                     landlord,
                                 )
 
-                            val probabilityOfComplianceRecord = if (isOccupied) 0.9 else 0.1
-                            if (NftDataFaker.generateBoolean(probabilityTrue = probabilityOfComplianceRecord)) {
-                                val complianceId = (++complianceRecordsAdded).toLong()
-                                fileUploadsAdded =
-                                    addPropertyComplianceToBatchReturningUpdatedFileUploadsAdded(
-                                        fileUploadStmt,
-                                        certificateUploadStmt,
-                                        gasSafetyFileUploadsStmt,
-                                        electricalSafetyFileUploadsStmt,
-                                        propertyComplianceStmt,
-                                        complianceId,
-                                        propertyOwnershipId,
-                                        propertyOwnershipCreatedDate,
-                                        fileUploadsAdded,
-                                    )
-                            }
+                            val complianceId = (++complianceRecordsAdded).toLong()
+                            fileUploadsAdded =
+                                addPropertyComplianceToBatchReturningUpdatedFileUploadsAdded(
+/*                                        fileUploadStmt,
+                                    gasSafetyFileUploadsStmt,
+                                    electricalSafetyFileUploadsStmt,*/
+                                    propertyComplianceStmt,
+                                    complianceId,
+                                    propertyOwnershipId,
+                                    propertyOwnershipCreatedDate,
+                                    fileUploadsAdded,
+                                )
                         } else {
                             val hasReminderEmailBeenSent = NftDataFaker.generateBoolean(probabilityTrue = 0.25)
                             addIncompletePropertyToBatch(
@@ -231,32 +230,47 @@ class NftDataSeeder(
                             )
                         }
 
-                        if (propertyOwnershipsAdded % BATCH_SIZE == 0 || propertyRegistrationsAdded() == NUM_OF_PROPERTIES) {
+                        newPropertyRegistrations++
+
+                        if ((propertyOwnershipsAdded - PROPERTY_OWNERSHIPS_ALREADY_ADDED) % BATCH_SIZE == 0 ||
+                            newPropertyRegistrations == NUM_OF_PROPERTIES
+                        ) {
                             registrationNumberStmt.executeBatch()
                             licenceStmt.executeBatch()
                             propertyOwnershipStmt.executeBatch()
                             registrationNumberGenerator.forgetUsedValues()
                             propertyOwnershipAddressGenerator.forgetUsedValues()
 
-                            fileUploadStmt.executeBatch()
-                            certificateUploadStmt.executeBatch()
+                            //                          fileUploadStmt.executeBatch()
                             propertyComplianceStmt.executeBatch()
-                            gasSafetyFileUploadsStmt.executeBatch()
-                            electricalSafetyFileUploadsStmt.executeBatch()
-                        }
-                        if (incompletePropertiesAdded % BATCH_SIZE == 0 || propertyRegistrationsAdded() == NUM_OF_PROPERTIES) {
+                            //                           gasSafetyFileUploadsStmt.executeBatch()
+                            //                          electricalSafetyFileUploadsStmt.executeBatch()
+
                             reminderEmailSentStmt.executeBatch()
                             savedJourneyStateStmt.executeBatch()
                             incompletePropertyStmt.executeBatch()
                             incompletePropertyAddressGenerator.forgetUsedValues()
                         }
 
-                        if (propertyRegistrationsAdded() % BATCH_SIZE == 0 || propertyRegistrationsAdded() == NUM_OF_PROPERTIES) {
-                            log("Seeded ${propertyRegistrationsAdded()} property ownerships/incomplete properties")
+                        if (newPropertyRegistrations % BATCH_SIZE == 0 || newPropertyRegistrations == NUM_OF_PROPERTIES) {
+                            log("Seeded $newPropertyRegistrations / $NUM_OF_PROPERTIES property registrations")
                         }
                     }
                 }
             }
+
+            // Flush any remaining batched items if we ran out of landlords before reaching NUM_OF_PROPERTIES
+            if (newPropertyRegistrations > 0 && newPropertyRegistrations % BATCH_SIZE != 0) {
+                registrationNumberStmt.executeBatch()
+                licenceStmt.executeBatch()
+                propertyOwnershipStmt.executeBatch()
+                propertyComplianceStmt.executeBatch()
+                reminderEmailSentStmt.executeBatch()
+                savedJourneyStateStmt.executeBatch()
+                incompletePropertyStmt.executeBatch()
+            }
+
+            log("Seeded $newPropertyRegistrations / $NUM_OF_PROPERTIES property registrations")
         } finally {
             prsdbUserStmt.close()
             registrationNumberStmt.close()
@@ -265,10 +279,9 @@ class NftDataSeeder(
             licenceStmt.close()
             propertyOwnershipStmt.close()
 
-            fileUploadStmt.close()
-            certificateUploadStmt.close()
+/*            fileUploadStmt.close()
             gasSafetyFileUploadsStmt.close()
-            electricalSafetyFileUploadsStmt.close()
+            electricalSafetyFileUploadsStmt.close()*/
             propertyComplianceStmt.close()
 
             reminderEmailSentStmt.close()
@@ -462,10 +475,9 @@ class NftDataSeeder(
     }
 
     private fun addPropertyComplianceToBatchReturningUpdatedFileUploadsAdded(
-        fileUploadStmt: PreparedStatement,
-        certificateUploadStmt: PreparedStatement,
+/*        fileUploadStmt: PreparedStatement,
         gasSafetyFileUploadsStmt: PreparedStatement,
-        electricalSafetyFileUploadsStmt: PreparedStatement,
+        electricalSafetyFileUploadsStmt: PreparedStatement,*/
         propertyComplianceStmt: PreparedStatement,
         complianceId: Long,
         propertyOwnershipId: Long,
@@ -477,7 +489,7 @@ class NftDataSeeder(
 
         var updatedFileUploadCount = currentFileUploadCount
 
-        if (complianceData.gasSafetyCertIssueDate?.after(
+/*        if (complianceData.gasSafetyCertIssueDate?.after(
                 Date.valueOf(
                     java.time.LocalDate
                         .now()
@@ -489,7 +501,6 @@ class NftDataSeeder(
             val gasSafetyUploadId = (++updatedFileUploadCount).toLong()
             addFileUploadToBatch(
                 fileUploadStmt,
-                certificateUploadStmt,
                 propertyOwnershipId,
                 createdDate,
                 CertificateType.GasSafetyCert,
@@ -503,7 +514,6 @@ class NftDataSeeder(
             val eicrUploadId = (++updatedFileUploadCount).toLong()
             addFileUploadToBatch(
                 fileUploadStmt,
-                certificateUploadStmt,
                 propertyOwnershipId,
                 createdDate,
                 complianceData.electricalCertType!!,
@@ -512,7 +522,7 @@ class NftDataSeeder(
             electricalSafetyFileUploadsStmt.setLong(1, complianceId)
             electricalSafetyFileUploadsStmt.setLong(2, eicrUploadId)
             electricalSafetyFileUploadsStmt.addBatch()
-        }
+        }*/
 
         propertyComplianceStmt.setLong(1, complianceId)
         propertyComplianceStmt.setTimestamp(2, createdDate)
@@ -541,7 +551,6 @@ class NftDataSeeder(
     // TODO PDJB-239: Upload files to S3
     private fun addFileUploadToBatch(
         fileUploadStmt: PreparedStatement,
-        certificateUploadStmt: PreparedStatement,
         propertyOwnershipId: Long,
         createdDate: Timestamp,
         certificateType: CertificateType,
@@ -558,25 +567,32 @@ class NftDataSeeder(
         fileUploadStmt.setString(5, NftDataFaker.generateETag())
         fileUploadStmt.setString(6, "fake-certificate.png")
         fileUploadStmt.addBatch()
-
-        certificateUploadStmt.setTimestamp(1, createdDate)
-        certificateUploadStmt.setTimestamp(2, NftDataFaker.generateLastModifiedDate(createdDate))
-        certificateUploadStmt.setInt(3, certificateType.ordinal)
-        certificateUploadStmt.setLong(4, propertyOwnershipId)
-        certificateUploadStmt.setLong(5, fileUploadId)
-        certificateUploadStmt.addBatch()
     }
 
     private fun log(message: String) {
         println("${LocalDateTime.now()} $message")
     }
 
     companion object {
-        const val NUM_OF_SYSTEM_OPERATORS = 150
-        const val NUM_OF_LC_USERS = 3000
-        const val NUM_OF_LANDLORDS = 2820000
-        const val NUM_OF_PROPERTIES = 4700000
-        const val BATCH_SIZE = 10000
+        const val NUM_OF_SYSTEM_OPERATORS = 0
+        const val NUM_OF_LC_USERS = 0
+        const val NUM_OF_LANDLORDS = 300
+        const val NUM_OF_PROPERTIES = 1000
+        const val BATCH_SIZE = 1000
+
+        const val REGISTRATION_NUMBERS_ALREADY_ADDED = 40
+        const val LICENCES_ALREADY_ADDED = 7
+        const val PROPERTY_OWNERSHIPS_ALREADY_ADDED = 20
+        const val COMPLIANCE_RECORDS_ALREADY_ADDED = 20
+        const val REMINDER_EMAILS_ADDED = 0
+        const val INCOMPLETE_PROPERTIES_ADDED = 1
+        const val LANDLORDS_ALREADY_ADDED = 20
+
+        val PREFERRED_PROPERTY_ADDRESS_IDS: List<Long> =
+            listOf(
+                33769665L,
+                10744667L,
+            )
     }
 
     private abstract class Generator<T> {
@@ -612,17 +628,24 @@ class NftDataSeeder(
 
     private inner class AddressGenerator(
         private val restrictToAvailable: Boolean = false,
+        preferredAddressIds: List<Long> = emptyList(),
     ) : Generator<Address>() {
+        private val preferredAddresses = ArrayDeque(preferredAddressIds)
         private val replenishmentSize = BATCH_SIZE * 5
 
         override fun replenishValues() {
             val valueSet = values.toSet()
             val newAddresses =
-                nftDataSeederDao.findAddresses(
-                    limit = replenishmentSize,
-                    offset = NftDataFaker.generateNumberLessThan(addressCount - replenishmentSize),
-                    restrictToAvailable,
-                )
+                if (preferredAddresses.isNotEmpty()) {
+                    val batch = (1..replenishmentSize).mapNotNull { preferredAddresses.removeFirstOrNull() }
+                    addressRepository.findAllById(batch)
+                } else {
+                    nftDataSeederDao.findAddresses(
+                        limit = replenishmentSize,
+                        offset = NftDataFaker.generateNumberLessThan(addressCount - replenishmentSize),
+                        restrictToAvailable,
+                    )
+                }
             values = (valueSet + newAddresses.shuffled()).toList()
         }
     }