diff --git a/.run/local-nft-seeder.run.xml b/.run/local-nft-seeder.run.xml index d16fdfa3da..73921eaa6e 100644 --- a/.run/local-nft-seeder.run.xml +++ b/.run/local-nft-seeder.run.xml @@ -18,7 +18,6 @@ - + \ No newline at end of file diff --git a/src/main/kotlin/uk/gov/communities/prsdb/webapp/database/dao/NftDataSeederDao.kt b/src/main/kotlin/uk/gov/communities/prsdb/webapp/database/dao/NftDataSeederDao.kt index a468ab8663..ee53fe7902 100644 --- a/src/main/kotlin/uk/gov/communities/prsdb/webapp/database/dao/NftDataSeederDao.kt +++ b/src/main/kotlin/uk/gov/communities/prsdb/webapp/database/dao/NftDataSeederDao.kt @@ -104,16 +104,6 @@ class NftDataSeederDao( return connection.prepareStatement(query) } - fun prepareCertificateUploadStatement(): PreparedStatement { - val query = - """ - INSERT INTO certificate_upload - (created_date, last_modified_date, category, property_ownership_id, file_upload_id) - VALUES (?, ?, ?, ?, ?) - """ - return connection.prepareStatement(query) - } - fun prepareGasSafetyFileUploadsStatement(): PreparedStatement { val query = """ diff --git a/src/main/kotlin/uk/gov/communities/prsdb/webapp/services/NftDataSeeder.kt b/src/main/kotlin/uk/gov/communities/prsdb/webapp/services/NftDataSeeder.kt index 6ff7d2419e..e155ea02b3 100644 --- a/src/main/kotlin/uk/gov/communities/prsdb/webapp/services/NftDataSeeder.kt +++ b/src/main/kotlin/uk/gov/communities/prsdb/webapp/services/NftDataSeeder.kt @@ -1,11 +1,9 @@ package uk.gov.communities.prsdb.webapp.services -import kotlinx.datetime.LocalDate import org.hibernate.SessionFactory import org.springframework.beans.factory.annotation.Value import org.springframework.context.annotation.Profile import uk.gov.communities.prsdb.webapp.annotations.taskAnnotations.PrsdbTaskService -import uk.gov.communities.prsdb.webapp.constants.GAS_SAFETY_CERT_VALIDITY_YEARS import uk.gov.communities.prsdb.webapp.constants.enums.CertificateType import uk.gov.communities.prsdb.webapp.constants.enums.RegistrationNumberType import uk.gov.communities.prsdb.webapp.database.dao.NftDataSeederDao @@ -23,7 +21,6 @@ import uk.gov.communities.prsdb.webapp.helpers.extensions.PreparedStatementExten import uk.gov.communities.prsdb.webapp.helpers.extensions.PreparedStatementExtensions.Companion.setStringOrNull import uk.gov.communities.prsdb.webapp.models.dataModels.EpcDataModel import java.sql.Connection -import java.sql.Date import java.sql.PreparedStatement import java.sql.Timestamp import java.time.LocalDateTime @@ -43,7 +40,7 @@ class NftDataSeeder( private val registrationNumberGenerator = RegistrationNumberGenerator() private val landlordAddressGenerator = AddressGenerator() - private val propertyOwnershipAddressGenerator = AddressGenerator(restrictToAvailable = true) + private val propertyOwnershipAddressGenerator = AddressGenerator(restrictToAvailable = true, PREFERRED_PROPERTY_ADDRESS_IDS) private val incompletePropertyAddressGenerator = AddressGenerator(restrictToAvailable = true) fun seedDatabase() { @@ -133,10 +130,9 @@ class NftDataSeeder( val licenceStmt = nftDataSeederDao.prepareLicenceStatement() val propertyOwnershipStmt = nftDataSeederDao.preparePropertyOwnershipStatement() - val fileUploadStmt = nftDataSeederDao.prepareFileUploadStatement() - val certificateUploadStmt = nftDataSeederDao.prepareCertificateUploadStatement() - val gasSafetyFileUploadsStmt = nftDataSeederDao.prepareGasSafetyFileUploadsStatement() - val electricalSafetyFileUploadsStmt = nftDataSeederDao.prepareElectricalSafetyFileUploadsStatement() + // val fileUploadStmt = nftDataSeederDao.prepareFileUploadStatement() + // val gasSafetyFileUploadsStmt = nftDataSeederDao.prepareGasSafetyFileUploadsStatement() + // val electricalSafetyFileUploadsStmt = nftDataSeederDao.prepareElectricalSafetyFileUploadsStatement() val propertyComplianceStmt = nftDataSeederDao.preparePropertyComplianceStatement() val reminderEmailSentStmt = nftDataSeederDao.prepareReminderEmailSentStatement() @@ -144,22 +140,29 @@ class NftDataSeeder( val incompletePropertyStmt = nftDataSeederDao.prepareLandlordIncompletePropertyStatement() try { - var registrationNumbersAdded = 0 + var registrationNumbersAdded = REGISTRATION_NUMBERS_ALREADY_ADDED - var licencesAdded = 0 - var propertyOwnershipsAdded = 0 + var licencesAdded = LICENCES_ALREADY_ADDED + var propertyOwnershipsAdded = PROPERTY_OWNERSHIPS_ALREADY_ADDED - var fileUploadsAdded = 0 - var complianceRecordsAdded = 0 + var fileUploadsAdded = 0 // currently not adding to this + var complianceRecordsAdded = COMPLIANCE_RECORDS_ALREADY_ADDED - var reminderEmailsAdded = 0 - var incompletePropertiesAdded = 0 + var reminderEmailsAdded = REMINDER_EMAILS_ADDED + var incompletePropertiesAdded = INCOMPLETE_PROPERTIES_ADDED - fun propertyRegistrationsAdded() = propertyOwnershipsAdded + incompletePropertiesAdded + var newPropertyRegistrations = 0 val numOfLandlordBatches = ceil(NUM_OF_LANDLORDS.toFloat() / BATCH_SIZE).toInt() for (landlordBatchNum in 1..numOfLandlordBatches) { - val landlordIdRange = ((landlordBatchNum - 1) * BATCH_SIZE + 1)..min(landlordBatchNum * BATCH_SIZE, NUM_OF_LANDLORDS) + val landlordIdRange = + (LANDLORDS_ALREADY_ADDED + (landlordBatchNum - 1) * BATCH_SIZE + 1)..( + LANDLORDS_ALREADY_ADDED + + min( + landlordBatchNum * BATCH_SIZE, + NUM_OF_LANDLORDS, + ) + ) val coreDetailsForLandlords = NftDataFaker.generateCoreDetailsForLandlords(landlordIdRange.toList()) coreDetailsForLandlords.forEach { @@ -178,10 +181,10 @@ class NftDataSeeder( registrationNumberGenerator.forgetUsedValues() landlordAddressGenerator.forgetUsedValues() - log("Seeded ${landlordIdRange.last} landlords") + log("Seeded ${landlordIdRange.last - LANDLORDS_ALREADY_ADDED} / $NUM_OF_LANDLORDS landlords") coreDetailsForLandlords.forEach { landlord -> - val numOfPropertiesLeft = NUM_OF_PROPERTIES - propertyRegistrationsAdded() + val numOfPropertiesLeft = NUM_OF_PROPERTIES - newPropertyRegistrations val numOfPropertiesForLandlord = NftDataFaker.generateNumberOfPropertiesForLandlord().coerceAtMost(numOfPropertiesLeft) repeat(numOfPropertiesForLandlord) { @@ -203,22 +206,18 @@ class NftDataSeeder( landlord, ) - val probabilityOfComplianceRecord = if (isOccupied) 0.9 else 0.1 - if (NftDataFaker.generateBoolean(probabilityTrue = probabilityOfComplianceRecord)) { - val complianceId = (++complianceRecordsAdded).toLong() - fileUploadsAdded = - addPropertyComplianceToBatchReturningUpdatedFileUploadsAdded( - fileUploadStmt, - certificateUploadStmt, - gasSafetyFileUploadsStmt, - electricalSafetyFileUploadsStmt, - propertyComplianceStmt, - complianceId, - propertyOwnershipId, - propertyOwnershipCreatedDate, - fileUploadsAdded, - ) - } + val complianceId = (++complianceRecordsAdded).toLong() + fileUploadsAdded = + addPropertyComplianceToBatchReturningUpdatedFileUploadsAdded( +/* fileUploadStmt, + gasSafetyFileUploadsStmt, + electricalSafetyFileUploadsStmt,*/ + propertyComplianceStmt, + complianceId, + propertyOwnershipId, + propertyOwnershipCreatedDate, + fileUploadsAdded, + ) } else { val hasReminderEmailBeenSent = NftDataFaker.generateBoolean(probabilityTrue = 0.25) addIncompletePropertyToBatch( @@ -231,32 +230,47 @@ class NftDataSeeder( ) } - if (propertyOwnershipsAdded % BATCH_SIZE == 0 || propertyRegistrationsAdded() == NUM_OF_PROPERTIES) { + newPropertyRegistrations++ + + if ((propertyOwnershipsAdded - PROPERTY_OWNERSHIPS_ALREADY_ADDED) % BATCH_SIZE == 0 || + newPropertyRegistrations == NUM_OF_PROPERTIES + ) { registrationNumberStmt.executeBatch() licenceStmt.executeBatch() propertyOwnershipStmt.executeBatch() registrationNumberGenerator.forgetUsedValues() propertyOwnershipAddressGenerator.forgetUsedValues() - fileUploadStmt.executeBatch() - certificateUploadStmt.executeBatch() + // fileUploadStmt.executeBatch() propertyComplianceStmt.executeBatch() - gasSafetyFileUploadsStmt.executeBatch() - electricalSafetyFileUploadsStmt.executeBatch() - } - if (incompletePropertiesAdded % BATCH_SIZE == 0 || propertyRegistrationsAdded() == NUM_OF_PROPERTIES) { + // gasSafetyFileUploadsStmt.executeBatch() + // electricalSafetyFileUploadsStmt.executeBatch() + reminderEmailSentStmt.executeBatch() savedJourneyStateStmt.executeBatch() incompletePropertyStmt.executeBatch() incompletePropertyAddressGenerator.forgetUsedValues() } - if (propertyRegistrationsAdded() % BATCH_SIZE == 0 || propertyRegistrationsAdded() == NUM_OF_PROPERTIES) { - log("Seeded ${propertyRegistrationsAdded()} property ownerships/incomplete properties") + if (newPropertyRegistrations % BATCH_SIZE == 0 || newPropertyRegistrations == NUM_OF_PROPERTIES) { + log("Seeded $newPropertyRegistrations / $NUM_OF_PROPERTIES property registrations") } } } } + + // Flush any remaining batched items if we ran out of landlords before reaching NUM_OF_PROPERTIES + if (newPropertyRegistrations > 0 && newPropertyRegistrations % BATCH_SIZE != 0) { + registrationNumberStmt.executeBatch() + licenceStmt.executeBatch() + propertyOwnershipStmt.executeBatch() + propertyComplianceStmt.executeBatch() + reminderEmailSentStmt.executeBatch() + savedJourneyStateStmt.executeBatch() + incompletePropertyStmt.executeBatch() + } + + log("Seeded $newPropertyRegistrations / $NUM_OF_PROPERTIES property registrations") } finally { prsdbUserStmt.close() registrationNumberStmt.close() @@ -265,10 +279,9 @@ class NftDataSeeder( licenceStmt.close() propertyOwnershipStmt.close() - fileUploadStmt.close() - certificateUploadStmt.close() +/* fileUploadStmt.close() gasSafetyFileUploadsStmt.close() - electricalSafetyFileUploadsStmt.close() + electricalSafetyFileUploadsStmt.close()*/ propertyComplianceStmt.close() reminderEmailSentStmt.close() @@ -462,10 +475,9 @@ class NftDataSeeder( } private fun addPropertyComplianceToBatchReturningUpdatedFileUploadsAdded( - fileUploadStmt: PreparedStatement, - certificateUploadStmt: PreparedStatement, +/* fileUploadStmt: PreparedStatement, gasSafetyFileUploadsStmt: PreparedStatement, - electricalSafetyFileUploadsStmt: PreparedStatement, + electricalSafetyFileUploadsStmt: PreparedStatement,*/ propertyComplianceStmt: PreparedStatement, complianceId: Long, propertyOwnershipId: Long, @@ -477,7 +489,7 @@ class NftDataSeeder( var updatedFileUploadCount = currentFileUploadCount - if (complianceData.gasSafetyCertIssueDate?.after( +/* if (complianceData.gasSafetyCertIssueDate?.after( Date.valueOf( java.time.LocalDate .now() @@ -489,7 +501,6 @@ class NftDataSeeder( val gasSafetyUploadId = (++updatedFileUploadCount).toLong() addFileUploadToBatch( fileUploadStmt, - certificateUploadStmt, propertyOwnershipId, createdDate, CertificateType.GasSafetyCert, @@ -503,7 +514,6 @@ class NftDataSeeder( val eicrUploadId = (++updatedFileUploadCount).toLong() addFileUploadToBatch( fileUploadStmt, - certificateUploadStmt, propertyOwnershipId, createdDate, complianceData.electricalCertType!!, @@ -512,7 +522,7 @@ class NftDataSeeder( electricalSafetyFileUploadsStmt.setLong(1, complianceId) electricalSafetyFileUploadsStmt.setLong(2, eicrUploadId) electricalSafetyFileUploadsStmt.addBatch() - } + }*/ propertyComplianceStmt.setLong(1, complianceId) propertyComplianceStmt.setTimestamp(2, createdDate) @@ -541,7 +551,6 @@ class NftDataSeeder( // TODO PDJB-239: Upload files to S3 private fun addFileUploadToBatch( fileUploadStmt: PreparedStatement, - certificateUploadStmt: PreparedStatement, propertyOwnershipId: Long, createdDate: Timestamp, certificateType: CertificateType, @@ -558,13 +567,6 @@ class NftDataSeeder( fileUploadStmt.setString(5, NftDataFaker.generateETag()) fileUploadStmt.setString(6, "fake-certificate.png") fileUploadStmt.addBatch() - - certificateUploadStmt.setTimestamp(1, createdDate) - certificateUploadStmt.setTimestamp(2, NftDataFaker.generateLastModifiedDate(createdDate)) - certificateUploadStmt.setInt(3, certificateType.ordinal) - certificateUploadStmt.setLong(4, propertyOwnershipId) - certificateUploadStmt.setLong(5, fileUploadId) - certificateUploadStmt.addBatch() } private fun log(message: String) { @@ -572,11 +574,25 @@ class NftDataSeeder( } companion object { - const val NUM_OF_SYSTEM_OPERATORS = 150 - const val NUM_OF_LC_USERS = 3000 - const val NUM_OF_LANDLORDS = 2820000 - const val NUM_OF_PROPERTIES = 4700000 - const val BATCH_SIZE = 10000 + const val NUM_OF_SYSTEM_OPERATORS = 0 + const val NUM_OF_LC_USERS = 0 + const val NUM_OF_LANDLORDS = 300 + const val NUM_OF_PROPERTIES = 1000 + const val BATCH_SIZE = 1000 + + const val REGISTRATION_NUMBERS_ALREADY_ADDED = 40 + const val LICENCES_ALREADY_ADDED = 7 + const val PROPERTY_OWNERSHIPS_ALREADY_ADDED = 20 + const val COMPLIANCE_RECORDS_ALREADY_ADDED = 20 + const val REMINDER_EMAILS_ADDED = 0 + const val INCOMPLETE_PROPERTIES_ADDED = 1 + const val LANDLORDS_ALREADY_ADDED = 20 + + val PREFERRED_PROPERTY_ADDRESS_IDS: List = + listOf( + 33769665L, + 10744667L, + ) } private abstract class Generator { @@ -612,17 +628,24 @@ class NftDataSeeder( private inner class AddressGenerator( private val restrictToAvailable: Boolean = false, + preferredAddressIds: List = emptyList(), ) : Generator
() { + private val preferredAddresses = ArrayDeque(preferredAddressIds) private val replenishmentSize = BATCH_SIZE * 5 override fun replenishValues() { val valueSet = values.toSet() val newAddresses = - nftDataSeederDao.findAddresses( - limit = replenishmentSize, - offset = NftDataFaker.generateNumberLessThan(addressCount - replenishmentSize), - restrictToAvailable, - ) + if (preferredAddresses.isNotEmpty()) { + val batch = (1..replenishmentSize).mapNotNull { preferredAddresses.removeFirstOrNull() } + addressRepository.findAllById(batch) + } else { + nftDataSeederDao.findAddresses( + limit = replenishmentSize, + offset = NftDataFaker.generateNumberLessThan(addressCount - replenishmentSize), + restrictToAvailable, + ) + } values = (valueSet + newAddresses.shuffled()).toList() } }