From 962bd1c7027c1c3c61c19a57ad6833f5bc111d89 Mon Sep 17 00:00:00 2001 From: Sebastian Schuberth Date: Tue, 14 Apr 2026 17:59:52 +0200 Subject: [PATCH 1/2] feat(scanner): Store detected and effective license in database Store detected licenses and effective license for packages and projects during the scanner phase to make them easily available via API routes without having to traverse all license findings from scan results. - Add detectedLicenses and effectiveLicense columns to packages and projects tables (Flyway V139 migration) - Add fields to Package and Project models - Add DAO support for new columns with comma-separated serialization - Create LicenseComputation.kt with functions to compute licenses - Integrate license computation in ScannerWorker - Expose new fields in API v1 Package and Project responses - Add unit tests for DAO and LicenseComputation Signed-off-by: Joel Pimentel Resolves #4607. --- .../src/commonMain/kotlin/ApiMappings.kt | 8 +- api/v1/model/src/commonMain/kotlin/Package.kt | 4 +- api/v1/model/src/commonMain/kotlin/Project.kt | 4 +- .../GetPackagesForAnalyzerRunQuery.kt | 8 +- .../GetProjectsForAnalyzerRunQuery.kt | 8 +- .../analyzerrun/DaoAnalyzerRunRepository.kt | 6 + .../repositories/analyzerrun/PackagesTable.kt | 12 +- .../repositories/analyzerrun/ProjectsTable.kt | 13 +- .../V145__storeLicensesInDatabase.sql | 7 + .../DaoAnalyzerRunRepositoryTest.kt | 32 ++++ dao/src/testFixtures/kotlin/Fixtures.kt | 13 +- model/src/commonMain/kotlin/runs/Package.kt | 4 +- model/src/commonMain/kotlin/runs/Project.kt | 4 +- .../src/main/kotlin/LicenseComputation.kt | 138 ++++++++++++++ .../scanner/src/main/kotlin/ScannerWorker.kt | 56 +++++- .../src/test/kotlin/LicenseComputationTest.kt | 170 ++++++++++++++++++ .../src/test/kotlin/ScannerWorkerTest.kt | 42 ++++- 17 files changed, 507 insertions(+), 22 deletions(-) create mode 100644 dao/src/main/resources/db/migration/V145__storeLicensesInDatabase.sql create mode 100644 workers/scanner/src/main/kotlin/LicenseComputation.kt create mode 100644 workers/scanner/src/test/kotlin/LicenseComputationTest.kt diff --git a/api/v1/mapping/src/commonMain/kotlin/ApiMappings.kt b/api/v1/mapping/src/commonMain/kotlin/ApiMappings.kt index a69b64b647..0c827ef0dc 100644 --- a/api/v1/mapping/src/commonMain/kotlin/ApiMappings.kt +++ b/api/v1/mapping/src/commonMain/kotlin/ApiMappings.kt @@ -813,7 +813,9 @@ fun Package.mapToApi( shortestDependencyPaths = shortestDependencyPaths.map { it.mapToApi() }, curations = curations, sourceCodeOrigins = sourceCodeOrigins?.map { it.mapToApi() }, - labels = labels + labels = labels, + detectedLicenses = detectedLicenses, + effectiveLicense = effectiveLicense ) fun PackageCurationData.mapToApi() = ApiPackageCurationData( @@ -864,7 +866,9 @@ fun Project.mapToApi() = ApiProject( vcsProcessed = vcsProcessed.mapToApi(), description = description, homepageUrl = homepageUrl, - scopeNames = scopeNames + scopeNames = scopeNames, + detectedLicenses = detectedLicenses, + effectiveLicense = effectiveLicense ) fun UserDisplayName.mapToApi() = ApiUserDisplayName(username = username, fullName = fullName) diff --git a/api/v1/model/src/commonMain/kotlin/Package.kt b/api/v1/model/src/commonMain/kotlin/Package.kt index 849513d29c..b3f186e5d0 100644 --- a/api/v1/model/src/commonMain/kotlin/Package.kt +++ b/api/v1/model/src/commonMain/kotlin/Package.kt @@ -40,7 +40,9 @@ data class Package( val shortestDependencyPaths: List, val curations: List, val sourceCodeOrigins: List? = null, - val labels: Map = emptyMap() + val labels: Map = emptyMap(), + val detectedLicenses: Set = emptySet(), + val effectiveLicense: String? = null ) /** diff --git a/api/v1/model/src/commonMain/kotlin/Project.kt b/api/v1/model/src/commonMain/kotlin/Project.kt index 9a43e05f62..20f02228a5 100644 --- a/api/v1/model/src/commonMain/kotlin/Project.kt +++ b/api/v1/model/src/commonMain/kotlin/Project.kt @@ -33,5 +33,7 @@ data class Project( val vcsProcessed: VcsInfo, val description: String, val homepageUrl: String, - val scopeNames: Set + val scopeNames: Set, + val detectedLicenses: Set = emptySet(), + val effectiveLicense: String? = null ) diff --git a/dao/src/main/kotlin/queries/analyzer/GetPackagesForAnalyzerRunQuery.kt b/dao/src/main/kotlin/queries/analyzer/GetPackagesForAnalyzerRunQuery.kt index 611adf7001..6f34f9f604 100644 --- a/dao/src/main/kotlin/queries/analyzer/GetPackagesForAnalyzerRunQuery.kt +++ b/dao/src/main/kotlin/queries/analyzer/GetPackagesForAnalyzerRunQuery.kt @@ -149,7 +149,13 @@ class GetPackagesForAnalyzerRunQuery( vcs = vcs, vcsProcessed = vcsProcessed, isMetadataOnly = resultRow[PackagesTable.isMetadataOnly], - isModified = resultRow[PackagesTable.isModified] + isModified = resultRow[PackagesTable.isModified], + detectedLicenses = resultRow[PackagesTable.detectedLicenses] + ?.split(',') + ?.filterNot { it.isEmpty() } + ?.toSet() + .orEmpty(), + effectiveLicense = resultRow[PackagesTable.effectiveLicense] ) } } diff --git a/dao/src/main/kotlin/queries/analyzer/GetProjectsForAnalyzerRunQuery.kt b/dao/src/main/kotlin/queries/analyzer/GetProjectsForAnalyzerRunQuery.kt index d480a9c694..2bec67679b 100644 --- a/dao/src/main/kotlin/queries/analyzer/GetProjectsForAnalyzerRunQuery.kt +++ b/dao/src/main/kotlin/queries/analyzer/GetProjectsForAnalyzerRunQuery.kt @@ -120,7 +120,13 @@ class GetProjectsForAnalyzerRunQuery( vcsProcessed = vcsProcessed, description = resultRow[ProjectsTable.description], homepageUrl = resultRow[ProjectsTable.homepageUrl], - scopeNames = scopeNamesByProjectId[projectId].orEmpty() + scopeNames = scopeNamesByProjectId[projectId].orEmpty(), + detectedLicenses = resultRow[ProjectsTable.detectedLicenses] + ?.split(',') + ?.filterNot { it.isEmpty() } + ?.toSet() + .orEmpty(), + effectiveLicense = resultRow[ProjectsTable.effectiveLicense] ) } } diff --git a/dao/src/main/kotlin/repositories/analyzerrun/DaoAnalyzerRunRepository.kt b/dao/src/main/kotlin/repositories/analyzerrun/DaoAnalyzerRunRepository.kt index 90217f8340..c1f6cc5532 100644 --- a/dao/src/main/kotlin/repositories/analyzerrun/DaoAnalyzerRunRepository.kt +++ b/dao/src/main/kotlin/repositories/analyzerrun/DaoAnalyzerRunRepository.kt @@ -211,6 +211,9 @@ private fun insertProject( createProcessedDeclaredLicense(project.processedDeclaredLicense, projectDao = projectDao) + projectDao.detectedLicenses = project.detectedLicenses.joinToString(",").takeIf { it.isNotEmpty() } + projectDao.effectiveLicense = project.effectiveLicense + return projectDao } @@ -284,6 +287,9 @@ private fun insertPackage( createProcessedDeclaredLicense(pkg.processedDeclaredLicense, pkgDao = pkgDao) + pkgDao.detectedLicenses = pkg.detectedLicenses.joinToString(",").takeIf { it.isNotEmpty() } + pkgDao.effectiveLicense = pkg.effectiveLicense + return pkgDao } diff --git a/dao/src/main/kotlin/repositories/analyzerrun/PackagesTable.kt b/dao/src/main/kotlin/repositories/analyzerrun/PackagesTable.kt index b74a5957f9..cdfba1afd2 100644 --- a/dao/src/main/kotlin/repositories/analyzerrun/PackagesTable.kt +++ b/dao/src/main/kotlin/repositories/analyzerrun/PackagesTable.kt @@ -62,6 +62,8 @@ object PackagesTable : SortableTable("packages") { val isMetadataOnly = bool("is_metadata_only").default(false) val isModified = bool("is_modified").default(false) val sourceCodeOrigins = text("source_code_origins").nullable() + val detectedLicenses = text("detected_licenses").nullable() + val effectiveLicense = text("effective_license").nullable() } class PackageDao(id: EntityID) : LongEntity(id) { @@ -185,6 +187,8 @@ class PackageDao(id: EntityID) : LongEntity(id) { ProcessedDeclaredLicensesTable.packageId var sourceCodeOrigins by PackagesTable.sourceCodeOrigins + var detectedLicenses by PackagesTable.detectedLicenses + var effectiveLicense by PackagesTable.effectiveLicense val labels by PackageLabelDao referrersOn PackageLabelsTable.packageId fun mapToModel() = Package( @@ -206,6 +210,12 @@ class PackageDao(id: EntityID) : LongEntity(id) { ?.split(',') ?.filterNot { it.isEmpty() } ?.map { SourceCodeOrigin.valueOf(it) }, - labels = labels.associate { it.key to it.value } + labels = labels.associate { it.key to it.value }, + detectedLicenses = detectedLicenses + ?.split(',') + ?.filterNot { it.isEmpty() } + ?.toSet() + .orEmpty(), + effectiveLicense = effectiveLicense ) } diff --git a/dao/src/main/kotlin/repositories/analyzerrun/ProjectsTable.kt b/dao/src/main/kotlin/repositories/analyzerrun/ProjectsTable.kt index 4b67638e36..b368ebbf9d 100644 --- a/dao/src/main/kotlin/repositories/analyzerrun/ProjectsTable.kt +++ b/dao/src/main/kotlin/repositories/analyzerrun/ProjectsTable.kt @@ -53,6 +53,8 @@ object ProjectsTable : SortableTable("projects") { val definitionFilePath = text("definition_file_path") val description = text("description") val homepageUrl = text("homepage_url") + val detectedLicenses = text("detected_licenses").nullable() + val effectiveLicense = text("effective_license").nullable() } class ProjectDao(id: EntityID) : LongEntity(id) { @@ -134,6 +136,9 @@ class ProjectDao(id: EntityID) : LongEntity(id) { val processedDeclaredLicense by ProcessedDeclaredLicenseDao backReferencedOn ProcessedDeclaredLicensesTable.projectId + var detectedLicenses by ProjectsTable.detectedLicenses + var effectiveLicense by ProjectsTable.effectiveLicense + fun mapToModel() = Project( identifier = identifier.mapToModel(), cpe = cpe, @@ -145,6 +150,12 @@ class ProjectDao(id: EntityID) : LongEntity(id) { vcsProcessed = vcsProcessed.mapToModel(), description = description, homepageUrl = homepageUrl, - scopeNames = scopeNames.mapTo(mutableSetOf(), ProjectScopeDao::name) + scopeNames = scopeNames.mapTo(mutableSetOf(), ProjectScopeDao::name), + detectedLicenses = detectedLicenses + ?.split(',') + ?.filterNot { it.isEmpty() } + ?.toSet() + .orEmpty(), + effectiveLicense = effectiveLicense ) } diff --git a/dao/src/main/resources/db/migration/V145__storeLicensesInDatabase.sql b/dao/src/main/resources/db/migration/V145__storeLicensesInDatabase.sql new file mode 100644 index 0000000000..a1982aa5e5 --- /dev/null +++ b/dao/src/main/resources/db/migration/V145__storeLicensesInDatabase.sql @@ -0,0 +1,7 @@ +ALTER TABLE packages + ADD COLUMN detected_licenses TEXT, + ADD COLUMN effective_license TEXT; + +ALTER TABLE projects + ADD COLUMN detected_licenses TEXT, + ADD COLUMN effective_license TEXT; \ No newline at end of file diff --git a/dao/src/test/kotlin/repositories/analyzerrun/DaoAnalyzerRunRepositoryTest.kt b/dao/src/test/kotlin/repositories/analyzerrun/DaoAnalyzerRunRepositoryTest.kt index ea3f78b7b7..abc3b90f80 100644 --- a/dao/src/test/kotlin/repositories/analyzerrun/DaoAnalyzerRunRepositoryTest.kt +++ b/dao/src/test/kotlin/repositories/analyzerrun/DaoAnalyzerRunRepositoryTest.kt @@ -143,6 +143,38 @@ class DaoAnalyzerRunRepositoryTest : StringSpec({ dbExtension.db.dbQuery { ProjectsTable.selectAll().count() } shouldBe 1 } + "create should deduplicate packages when license fields differ" { + val pkg1 = createPackage(1).copy( + detectedLicenses = setOf("MIT"), + effectiveLicense = "MIT" + ) + val pkg2 = createPackage(1).copy( + detectedLicenses = setOf("Apache-2.0"), + effectiveLicense = "Apache-2.0" + ) + + analyzerRunRepository.create(analyzerJobId, analyzerRun.copy(packages = setOf(pkg1))) + analyzerRunRepository.create(analyzerJobId, analyzerRun.copy(packages = setOf(pkg2))) + + dbExtension.db.dbQuery { PackagesTable.selectAll().count() } shouldBe 1 + } + + "create should deduplicate projects when license fields differ" { + val project1 = project.copy( + detectedLicenses = setOf("MIT"), + effectiveLicense = "MIT" + ) + val project2 = project.copy( + detectedLicenses = setOf("Apache-2.0"), + effectiveLicense = "Apache-2.0" + ) + + analyzerRunRepository.create(analyzerJobId, analyzerRun.copy(projects = setOf(project1))) + analyzerRunRepository.create(analyzerJobId, analyzerRun.copy(projects = setOf(project2))) + + dbExtension.db.dbQuery { ProjectsTable.selectAll().count() } shouldBe 1 + } + "create should handle unique constraint violations" { val txCount = 64 withContext(Dispatchers.IO) { diff --git a/dao/src/testFixtures/kotlin/Fixtures.kt b/dao/src/testFixtures/kotlin/Fixtures.kt index 2d725434ab..c415fd8bb7 100644 --- a/dao/src/testFixtures/kotlin/Fixtures.kt +++ b/dao/src/testFixtures/kotlin/Fixtures.kt @@ -262,7 +262,9 @@ class Fixtures(private val db: Database) { vcsProcessed = VcsInfo(RepositoryType.GIT, "https://example.com/git", "revision", ""), description = "", homepageUrl = "https://example.com", - scopeNames = setOf("compileClasspath", "runtimeClasspath") + scopeNames = setOf("compileClasspath", "runtimeClasspath"), + detectedLicenses = emptySet(), + effectiveLicense = null ) fun createAnalyzerRun( @@ -324,6 +326,7 @@ class Fixtures(private val db: Database) { results = results ) + @Suppress("LongParameterList") fun generatePackage( identifier: Identifier, authors: Set = emptySet(), @@ -356,7 +359,9 @@ class Fixtures(private val db: Database) { "https://example.com/git", "revision", "path" - ) + ), + detectedLicenses: Set = emptySet(), + effectiveLicense: String? = null ) = Package( identifier = identifier, purl = "pkg:${identifier.type}/${identifier.namespace}/${identifier.name}@${identifier.version}", @@ -371,6 +376,8 @@ class Fixtures(private val db: Database) { vcs = vcs, vcsProcessed = vcsProcessed, isMetadataOnly = false, - isModified = false + isModified = false, + detectedLicenses = detectedLicenses, + effectiveLicense = effectiveLicense ) } diff --git a/model/src/commonMain/kotlin/runs/Package.kt b/model/src/commonMain/kotlin/runs/Package.kt index 72a67a57c7..d0c8ab476c 100644 --- a/model/src/commonMain/kotlin/runs/Package.kt +++ b/model/src/commonMain/kotlin/runs/Package.kt @@ -38,7 +38,9 @@ data class Package( val isMetadataOnly: Boolean = false, val isModified: Boolean = false, val sourceCodeOrigins: List? = null, - val labels: Map = emptyMap() + val labels: Map = emptyMap(), + val detectedLicenses: Set = emptySet(), + val effectiveLicense: String? = null ) /** diff --git a/model/src/commonMain/kotlin/runs/Project.kt b/model/src/commonMain/kotlin/runs/Project.kt index 8135155fc6..a87965de74 100644 --- a/model/src/commonMain/kotlin/runs/Project.kt +++ b/model/src/commonMain/kotlin/runs/Project.kt @@ -30,5 +30,7 @@ data class Project( val vcsProcessed: VcsInfo, val description: String, val homepageUrl: String, - val scopeNames: Set + val scopeNames: Set, + val detectedLicenses: Set = emptySet(), + val effectiveLicense: String? = null ) diff --git a/workers/scanner/src/main/kotlin/LicenseComputation.kt b/workers/scanner/src/main/kotlin/LicenseComputation.kt new file mode 100644 index 0000000000..2a8a7d4339 --- /dev/null +++ b/workers/scanner/src/main/kotlin/LicenseComputation.kt @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2026 The ORT Server Authors (See ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.eclipse.apoapsis.ortserver.workers.scanner + +import org.eclipse.apoapsis.ortserver.dao.repositories.analyzerrun.PackageDao +import org.eclipse.apoapsis.ortserver.dao.repositories.analyzerrun.PackagesTable +import org.eclipse.apoapsis.ortserver.dao.repositories.analyzerrun.ProjectDao +import org.eclipse.apoapsis.ortserver.dao.repositories.analyzerrun.ProjectsTable +import org.eclipse.apoapsis.ortserver.dao.tables.shared.IdentifierDao +import org.eclipse.apoapsis.ortserver.model.runs.Identifier +import org.eclipse.apoapsis.ortserver.services.ortrun.mapToModel + +import org.jetbrains.exposed.v1.core.eq + +import org.ossreviewtoolkit.model.Identifier as OrtIdentifier +import org.ossreviewtoolkit.model.OrtResult +import org.ossreviewtoolkit.model.ScannerRun +import org.ossreviewtoolkit.model.licenses.LicenseInfoResolver +import org.ossreviewtoolkit.model.licenses.LicenseView + +/** + * Compute detected licenses for a package or project by aggregating license findings across all scan results + * for the given [id]. Returns the union of all license strings found in all provenances. + */ +fun computeDetectedLicenses( + scannerRun: ScannerRun, + id: OrtIdentifier +): Set = + scannerRun.getAllScanResults()[id] + .orEmpty() + .flatMap { scanResult -> scanResult.summary.licenseFindings } + .map { licenseFinding -> licenseFinding.license.toString() } + .toSet() + +/** + * Compute the effective license for a package or project using the [licenseInfoResolver]. + * + * The effective license follows the precedence: concluded → declared → detected (via LicenseView). + * Returns null if none of these are available. + */ +fun computeEffectiveLicense( + licenseInfoResolver: LicenseInfoResolver, + id: OrtIdentifier +): String? { + val resolvedLicenseInfo = licenseInfoResolver.resolveLicenseInfo(id) + + return resolvedLicenseInfo.effectiveLicense(LicenseView.CONCLUDED_OR_DECLARED_AND_DETECTED)?.toString() +} + +/** + * Compute detected and effective licenses for all packages and projects in the [ortResult] and store them in the + * database. Uses the [scannerRun] to aggregate detected licenses across all provenances and the + * [licenseInfoResolver] to compute effective licenses. + */ +fun computeAndStoreLicenses( + ortResult: OrtResult, + scannerRun: ScannerRun, + licenseInfoResolver: LicenseInfoResolver +) { + for (curatedPackage in ortResult.getPackages()) { + val id = curatedPackage.metadata.id + val modelId = id.mapToModel() + + val detectedLicenses = computeDetectedLicenses(scannerRun, id) + val effectiveLicense = computeEffectiveLicense(licenseInfoResolver, id) + + if (detectedLicenses.isNotEmpty() || effectiveLicense != null) { + updatePackageLicenses(modelId, detectedLicenses, effectiveLicense) + } + } + + for (project in ortResult.getProjects()) { + val id = project.id + val modelId = id.mapToModel() + + val detectedLicenses = computeDetectedLicenses(scannerRun, id) + val effectiveLicense = computeEffectiveLicense(licenseInfoResolver, id) + + if (detectedLicenses.isNotEmpty() || effectiveLicense != null) { + updateProjectLicenses(modelId, detectedLicenses, effectiveLicense) + } + } +} + +/** + * Update the license fields for the package identified by [modelId] in the database. + */ +fun updatePackageLicenses( + modelId: Identifier, + detectedLicenses: Set, + effectiveLicense: String? +) { + val identifierDao = IdentifierDao.findByIdentifier(modelId) + ?: return + + val packageDao = PackageDao.find { + PackagesTable.identifierId eq identifierDao.id + }.firstOrNull() ?: return + + packageDao.detectedLicenses = detectedLicenses.joinToString(",").takeIf { it.isNotEmpty() } + packageDao.effectiveLicense = effectiveLicense +} + +/** + * Update the license fields for the project identified by [modelId] in the database. + */ +fun updateProjectLicenses( + modelId: Identifier, + detectedLicenses: Set, + effectiveLicense: String? +) { + val identifierDao = IdentifierDao.findByIdentifier(modelId) + ?: return + + val projectDao = ProjectDao.find { + ProjectsTable.identifierId eq identifierDao.id + }.firstOrNull() ?: return + + projectDao.detectedLicenses = detectedLicenses.joinToString(",").takeIf { it.isNotEmpty() } + projectDao.effectiveLicense = effectiveLicense +} diff --git a/workers/scanner/src/main/kotlin/ScannerWorker.kt b/workers/scanner/src/main/kotlin/ScannerWorker.kt index 80f1ca0b19..e75aed7032 100644 --- a/workers/scanner/src/main/kotlin/ScannerWorker.kt +++ b/workers/scanner/src/main/kotlin/ScannerWorker.kt @@ -31,15 +31,27 @@ import org.eclipse.apoapsis.ortserver.services.ortrun.mapToOrt import org.eclipse.apoapsis.ortserver.transport.EndpointComponent import org.eclipse.apoapsis.ortserver.workers.common.JobIgnoredException import org.eclipse.apoapsis.ortserver.workers.common.RunResult +import org.eclipse.apoapsis.ortserver.workers.common.context.WorkerContext import org.eclipse.apoapsis.ortserver.workers.common.context.WorkerContextFactory import org.eclipse.apoapsis.ortserver.workers.common.env.EnvironmentService +import org.eclipse.apoapsis.ortserver.workers.common.readConfigFileValueWithDefault import org.eclipse.apoapsis.ortserver.workers.common.resolutions.OrtServerResolutionProvider +import org.eclipse.apoapsis.ortserver.workers.common.resolvedConfigurationContext import org.eclipse.apoapsis.ortserver.workers.common.validateForProcessing import org.jetbrains.exposed.v1.jdbc.Database +import org.ossreviewtoolkit.model.OrtResult import org.ossreviewtoolkit.model.Provenance import org.ossreviewtoolkit.model.Severity +import org.ossreviewtoolkit.model.config.CopyrightGarbage +import org.ossreviewtoolkit.model.config.LicenseFilePatterns +import org.ossreviewtoolkit.model.licenses.DefaultLicenseInfoProvider +import org.ossreviewtoolkit.model.licenses.LicenseClassifications +import org.ossreviewtoolkit.model.licenses.LicenseInfoResolver +import org.ossreviewtoolkit.model.utils.FileArchiver +import org.ossreviewtoolkit.utils.ort.ORT_COPYRIGHT_GARBAGE_FILENAME +import org.ossreviewtoolkit.utils.ort.ORT_LICENSE_CLASSIFICATIONS_FILENAME import org.ossreviewtoolkit.utils.ort.ORT_VERSION import org.slf4j.LoggerFactory @@ -53,7 +65,8 @@ class ScannerWorker( private val contextFactory: WorkerContextFactory, private val environmentService: EnvironmentService, private val adminConfigService: AdminConfigService, - private val issueResolutionService: IssueResolutionService + private val issueResolutionService: IssueResolutionService, + private val fileArchiver: FileArchiver ) { suspend fun run(jobId: Long, traceId: String): RunResult = runCatching { val (scannerJob, ortRun, ortResult) = db.dbQuery { @@ -93,6 +106,16 @@ class ScannerWorker( val scannerRunResult = runner.run(context, ortResult, scannerJob.configuration, scannerRunId) + val licenseInfoResolver = buildLicenseInfoResolver(ortResult, context, fileArchiver) + + db.dbQuery { + computeAndStoreLicenses( + ortResult = ortResult, + scannerRun = scannerRunResult.scannerRun, + licenseInfoResolver = licenseInfoResolver + ) + } + val issues = scannerRunResult.extractIssues() val allIssues = issues + scannerRunResult.scannerRun.issues.values.flatten().map { it.mapToModel() } @@ -172,3 +195,34 @@ private fun OrtScannerResult.extractIssues(): Set { } } } + +/** + * Build a [LicenseInfoResolver] using the same pattern as [EvaluatorRunner] and [ReporterRunner]. + */ +internal fun buildLicenseInfoResolver( + ortResult: OrtResult, + context: WorkerContext, + fileArchiver: FileArchiver +): LicenseInfoResolver { + val copyrightGarbage = context.configManager.readConfigFileValueWithDefault( + path = null, + defaultPath = ORT_COPYRIGHT_GARBAGE_FILENAME, + fallbackValue = CopyrightGarbage(), + context = context.resolvedConfigurationContext + ) + + context.configManager.readConfigFileValueWithDefault( + path = null, + defaultPath = ORT_LICENSE_CLASSIFICATIONS_FILENAME, + fallbackValue = LicenseClassifications(), + context = context.resolvedConfigurationContext + ) + + return LicenseInfoResolver( + provider = DefaultLicenseInfoProvider(ortResult), + copyrightGarbage = copyrightGarbage, + addAuthorsToCopyrights = true, + archiver = fileArchiver, + licenseFilePatterns = LicenseFilePatterns.DEFAULT + ) +} diff --git a/workers/scanner/src/test/kotlin/LicenseComputationTest.kt b/workers/scanner/src/test/kotlin/LicenseComputationTest.kt new file mode 100644 index 0000000000..8f68d93136 --- /dev/null +++ b/workers/scanner/src/test/kotlin/LicenseComputationTest.kt @@ -0,0 +1,170 @@ +/* + * Copyright (C) 2026 The ORT Server Authors (See ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.eclipse.apoapsis.ortserver.workers.scanner + +import io.kotest.core.spec.style.StringSpec +import io.kotest.matchers.collections.shouldBeEmpty +import io.kotest.matchers.collections.shouldContainExactlyInAnyOrder +import io.kotest.matchers.nulls.shouldBeNull +import io.kotest.matchers.shouldBe + +import io.mockk.every +import io.mockk.mockk + +import java.time.Instant + +import org.ossreviewtoolkit.model.ArtifactProvenance +import org.ossreviewtoolkit.model.Hash +import org.ossreviewtoolkit.model.Identifier +import org.ossreviewtoolkit.model.LicenseFinding +import org.ossreviewtoolkit.model.OrtResult +import org.ossreviewtoolkit.model.RemoteArtifact +import org.ossreviewtoolkit.model.ScanResult +import org.ossreviewtoolkit.model.ScanSummary +import org.ossreviewtoolkit.model.ScannerDetails +import org.ossreviewtoolkit.model.ScannerRun +import org.ossreviewtoolkit.model.TextLocation +import org.ossreviewtoolkit.model.licenses.DefaultLicenseInfoProvider +import org.ossreviewtoolkit.model.licenses.LicenseInfoResolver + +class LicenseComputationTest : StringSpec({ + "computeDetectedLicenses should aggregate licenses across multiple provenances" { + val id = Identifier("Maven", "com.example", "test-pkg", "1.0") + + val provenance1 = ArtifactProvenance( + sourceArtifact = RemoteArtifact( + url = "https://example.com/pkg1.jar", + hash = Hash.NONE + ) + ) + val provenance2 = ArtifactProvenance( + sourceArtifact = RemoteArtifact( + url = "https://example.com/pkg2.jar", + hash = Hash.NONE + ) + ) + + val scanResult1 = ScanResult( + provenance = provenance1, + scanner = ScannerDetails("scanner1", "1.0", ""), + summary = ScanSummary( + startTime = Instant.now(), + endTime = Instant.now(), + licenseFindings = setOf( + LicenseFinding("MIT", TextLocation("file1.txt", 1)), + LicenseFinding("Apache-2.0", TextLocation("file2.txt", 1)) + ) + ) + ) + + val scanResult2 = ScanResult( + provenance = provenance2, + scanner = ScannerDetails("scanner2", "1.0", ""), + summary = ScanSummary( + startTime = Instant.now(), + endTime = Instant.now(), + licenseFindings = setOf( + LicenseFinding("Apache-2.0", TextLocation("file3.txt", 1)), + LicenseFinding("GPL-3.0", TextLocation("file4.txt", 1)) + ) + ) + ) + + val scannerRun = mockk(relaxed = true) { + every { getAllScanResults() } returns mapOf(id to listOf(scanResult1, scanResult2)) + } + + val result = computeDetectedLicenses(scannerRun, id) + + result shouldContainExactlyInAnyOrder setOf("MIT", "Apache-2.0", "GPL-3.0") + } + + "computeDetectedLicenses should return empty set when no scan results exist" { + val id = Identifier("Maven", "com.example", "no-scan-pkg", "1.0") + + val scannerRun = mockk(relaxed = true) { + every { getAllScanResults() } returns emptyMap() + } + + val result = computeDetectedLicenses(scannerRun, id) + + result.shouldBeEmpty() + } + + "computeDetectedLicenses should deduplicate licenses across provenances" { + val id = Identifier("NPM", "@example", "lib", "2.0") + + val provenance = ArtifactProvenance( + sourceArtifact = RemoteArtifact( + url = "https://example.com/lib.tgz", + hash = Hash.NONE + ) + ) + + val scanResult1 = ScanResult( + provenance = provenance, + scanner = ScannerDetails("scanner1", "1.0", ""), + summary = ScanSummary( + startTime = Instant.now(), + endTime = Instant.now(), + licenseFindings = setOf( + LicenseFinding("MIT", TextLocation("a.txt", 1)) + ) + ) + ) + + val scanResult2 = ScanResult( + provenance = provenance, + scanner = ScannerDetails("scanner2", "1.0", ""), + summary = ScanSummary( + startTime = Instant.now(), + endTime = Instant.now(), + licenseFindings = setOf( + LicenseFinding("MIT", TextLocation("b.txt", 1)) + ) + ) + ) + + val scannerRun = mockk(relaxed = true) { + every { getAllScanResults() } returns mapOf(id to listOf(scanResult1, scanResult2)) + } + + val result = computeDetectedLicenses(scannerRun, id) + + result shouldBe setOf("MIT") + } + + "computeEffectiveLicense should return null when no license info is available" { + val id = Identifier("Maven", "com.example", "no-license-pkg", "1.0") + + val ortResult = OrtResult.EMPTY + val resolver = LicenseInfoResolver( + provider = DefaultLicenseInfoProvider(ortResult), + copyrightGarbage = org.ossreviewtoolkit.model.config.CopyrightGarbage(), + addAuthorsToCopyrights = false, + archiver = mockk(relaxed = true), + licenseFilePatterns = org.ossreviewtoolkit.model.config.LicenseFilePatterns.DEFAULT + ) + + val result = computeEffectiveLicense(resolver, id) + + result.shouldBeNull() + } +}) diff --git a/workers/scanner/src/test/kotlin/ScannerWorkerTest.kt b/workers/scanner/src/test/kotlin/ScannerWorkerTest.kt index 606f51eba0..f3c9d39a19 100644 --- a/workers/scanner/src/test/kotlin/ScannerWorkerTest.kt +++ b/workers/scanner/src/test/kotlin/ScannerWorkerTest.kt @@ -119,6 +119,12 @@ class ScannerWorkerTest : StringSpec({ every { analyzerRun.mapToOrt() } returns mockk() every { ortRun.mapToOrt(any(), any(), any(), any(), any(), any()) } returns OrtResult.EMPTY + mockkStatic("org.eclipse.apoapsis.ortserver.workers.scanner.ScannerWorkerKt") + every { buildLicenseInfoResolver(any(), any(), any()) } returns mockk(relaxed = true) + + mockkStatic("org.eclipse.apoapsis.ortserver.workers.scanner.LicenseComputationKt") + every { computeAndStoreLicenses(any(), any(), any()) } just runs + val ortRunService = mockk { every { createScannerRun(any()) } returns mockk { every { id } returns scannerJob.id @@ -191,7 +197,8 @@ class ScannerWorkerTest : StringSpec({ contextFactory, environmentService, mockk(relaxed = true), - mockIssueResolutionService() + mockIssueResolutionService(), + mockk(relaxed = true) ) mockkTransaction { @@ -223,6 +230,12 @@ class ScannerWorkerTest : StringSpec({ every { analyzerRun.mapToOrt() } returns mockk() every { ortRun.mapToOrt(any(), any(), any(), any(), any(), any()) } returns OrtResult.EMPTY + mockkStatic("org.eclipse.apoapsis.ortserver.workers.scanner.ScannerWorkerKt") + every { buildLicenseInfoResolver(any(), any(), any()) } returns mockk(relaxed = true) + + mockkStatic("org.eclipse.apoapsis.ortserver.workers.scanner.LicenseComputationKt") + every { computeAndStoreLicenses(any(), any(), any()) } just runs + val ortRunService = mockk { every { createScannerRun(any()) } returns mockk { every { id } returns scannerJob.id @@ -337,7 +350,8 @@ class ScannerWorkerTest : StringSpec({ contextFactory, environmentService, mockk(relaxed = true), - mockIssueResolutionService() + mockIssueResolutionService(), + mockk(relaxed = true) ) mockkTransaction { @@ -389,7 +403,8 @@ class ScannerWorkerTest : StringSpec({ mockk(), mockk(), mockk(relaxed = true), - mockIssueResolutionService() + mockIssueResolutionService(), + mockk(relaxed = true) ) mockkTransaction { @@ -416,6 +431,12 @@ class ScannerWorkerTest : StringSpec({ every { analyzerRun.mapToOrt() } returns mockk() every { ortRun.mapToOrt(any(), any(), any(), any(), any(), any()) } returns OrtResult.EMPTY + mockkStatic("org.eclipse.apoapsis.ortserver.workers.scanner.ScannerWorkerKt") + every { buildLicenseInfoResolver(any(), any(), any()) } returns mockk(relaxed = true) + + mockkStatic("org.eclipse.apoapsis.ortserver.workers.scanner.LicenseComputationKt") + every { computeAndStoreLicenses(any(), any(), any()) } just runs + val ortRunService = mockk { every { createScannerRun(any()) } returns mockk { every { id } returns scannerJob.id @@ -463,7 +484,8 @@ class ScannerWorkerTest : StringSpec({ contextFactory, environmentService, mockk(relaxed = true), - mockIssueResolutionService() + mockIssueResolutionService(), + mockk(relaxed = true) ) mockkTransaction { @@ -531,7 +553,8 @@ class ScannerWorkerTest : StringSpec({ contextFactory, environmentService, mockk(relaxed = true), - mockIssueResolutionService() + mockIssueResolutionService(), + mockk(relaxed = true) ) mockkTransaction { @@ -641,7 +664,8 @@ class ScannerWorkerTest : StringSpec({ contextFactory, environmentService, mockk(relaxed = true), - mockIssueResolutionService() + mockIssueResolutionService(), + mockk(relaxed = true) ) mockkTransaction { @@ -757,7 +781,8 @@ class ScannerWorkerTest : StringSpec({ contextFactory, environmentService, mockk(relaxed = true), - mockIssueResolutionService() + mockIssueResolutionService(), + mockk(relaxed = true) ) mockkTransaction { @@ -781,7 +806,8 @@ class ScannerWorkerTest : StringSpec({ mockk(), mockk(), mockk(relaxed = true), - mockIssueResolutionService() + mockIssueResolutionService(), + mockk(relaxed = true) ) mockkTransaction { From ed6934cbb5c9de1876aa466cfd584f2fa6db9c56 Mon Sep 17 00:00:00 2001 From: Joelp03 Date: Mon, 20 Apr 2026 14:40:42 -0400 Subject: [PATCH 2/2] fix(scanner): Apply curations before storing detected licenses Detected licenses must have PackageCuration and PackageConfiguration applied before storing, otherwise wrong data could be shown. - Change computeDetectedLicenses to use LicenseInfoResolver which applies LicenseFindingCuration and PathExclude from PackageConfiguration - Remove scannerRun parameter from computeAndStoreLicenses since it's no longer needed after using LicenseInfoResolver Signed-off-by: Joelp03 --- ....sql => V146__storeLicensesInDatabase.sql} | 0 .../src/main/kotlin/LicenseComputation.kt | 29 +-- .../scanner/src/main/kotlin/ScannerWorker.kt | 1 - .../src/test/kotlin/LicenseComputationTest.kt | 165 ++++++++---------- .../src/test/kotlin/ScannerWorkerTest.kt | 6 +- 5 files changed, 91 insertions(+), 110 deletions(-) rename dao/src/main/resources/db/migration/{V145__storeLicensesInDatabase.sql => V146__storeLicensesInDatabase.sql} (100%) diff --git a/dao/src/main/resources/db/migration/V145__storeLicensesInDatabase.sql b/dao/src/main/resources/db/migration/V146__storeLicensesInDatabase.sql similarity index 100% rename from dao/src/main/resources/db/migration/V145__storeLicensesInDatabase.sql rename to dao/src/main/resources/db/migration/V146__storeLicensesInDatabase.sql diff --git a/workers/scanner/src/main/kotlin/LicenseComputation.kt b/workers/scanner/src/main/kotlin/LicenseComputation.kt index 2a8a7d4339..52cdd469e2 100644 --- a/workers/scanner/src/main/kotlin/LicenseComputation.kt +++ b/workers/scanner/src/main/kotlin/LicenseComputation.kt @@ -30,24 +30,27 @@ import org.eclipse.apoapsis.ortserver.services.ortrun.mapToModel import org.jetbrains.exposed.v1.core.eq import org.ossreviewtoolkit.model.Identifier as OrtIdentifier +import org.ossreviewtoolkit.model.LicenseSource import org.ossreviewtoolkit.model.OrtResult -import org.ossreviewtoolkit.model.ScannerRun import org.ossreviewtoolkit.model.licenses.LicenseInfoResolver import org.ossreviewtoolkit.model.licenses.LicenseView /** - * Compute detected licenses for a package or project by aggregating license findings across all scan results - * for the given [id]. Returns the union of all license strings found in all provenances. + * Compute detected licenses for a package or project using the [licenseInfoResolver]. + * + * Returns the set of licenses with [LicenseSource.DETECTED], which respects LicenseFindingCuration and PathExclude + * from PackageConfiguration. */ fun computeDetectedLicenses( - scannerRun: ScannerRun, + licenseInfoResolver: LicenseInfoResolver, id: OrtIdentifier -): Set = - scannerRun.getAllScanResults()[id] - .orEmpty() - .flatMap { scanResult -> scanResult.summary.licenseFindings } - .map { licenseFinding -> licenseFinding.license.toString() } +): Set { + val resolvedLicenseInfo = licenseInfoResolver.resolveLicenseInfo(id) + return resolvedLicenseInfo.licenses + .filter { it.sources.any { source -> source == LicenseSource.DETECTED } } + .map { it.license.toString() } .toSet() +} /** * Compute the effective license for a package or project using the [licenseInfoResolver]. @@ -66,19 +69,17 @@ fun computeEffectiveLicense( /** * Compute detected and effective licenses for all packages and projects in the [ortResult] and store them in the - * database. Uses the [scannerRun] to aggregate detected licenses across all provenances and the - * [licenseInfoResolver] to compute effective licenses. + * database. Uses the [licenseInfoResolver] to compute both detected and effective licenses with curations applied. */ fun computeAndStoreLicenses( ortResult: OrtResult, - scannerRun: ScannerRun, licenseInfoResolver: LicenseInfoResolver ) { for (curatedPackage in ortResult.getPackages()) { val id = curatedPackage.metadata.id val modelId = id.mapToModel() - val detectedLicenses = computeDetectedLicenses(scannerRun, id) + val detectedLicenses = computeDetectedLicenses(licenseInfoResolver, id) val effectiveLicense = computeEffectiveLicense(licenseInfoResolver, id) if (detectedLicenses.isNotEmpty() || effectiveLicense != null) { @@ -90,7 +91,7 @@ fun computeAndStoreLicenses( val id = project.id val modelId = id.mapToModel() - val detectedLicenses = computeDetectedLicenses(scannerRun, id) + val detectedLicenses = computeDetectedLicenses(licenseInfoResolver, id) val effectiveLicense = computeEffectiveLicense(licenseInfoResolver, id) if (detectedLicenses.isNotEmpty() || effectiveLicense != null) { diff --git a/workers/scanner/src/main/kotlin/ScannerWorker.kt b/workers/scanner/src/main/kotlin/ScannerWorker.kt index e75aed7032..66ccfd5a50 100644 --- a/workers/scanner/src/main/kotlin/ScannerWorker.kt +++ b/workers/scanner/src/main/kotlin/ScannerWorker.kt @@ -111,7 +111,6 @@ class ScannerWorker( db.dbQuery { computeAndStoreLicenses( ortResult = ortResult, - scannerRun = scannerRunResult.scannerRun, licenseInfoResolver = licenseInfoResolver ) } diff --git a/workers/scanner/src/test/kotlin/LicenseComputationTest.kt b/workers/scanner/src/test/kotlin/LicenseComputationTest.kt index 8f68d93136..db0270fbbf 100644 --- a/workers/scanner/src/test/kotlin/LicenseComputationTest.kt +++ b/workers/scanner/src/test/kotlin/LicenseComputationTest.kt @@ -21,134 +21,115 @@ package org.eclipse.apoapsis.ortserver.workers.scanner import io.kotest.core.spec.style.StringSpec import io.kotest.matchers.collections.shouldBeEmpty -import io.kotest.matchers.collections.shouldContainExactlyInAnyOrder +import io.kotest.matchers.collections.shouldContain import io.kotest.matchers.nulls.shouldBeNull import io.kotest.matchers.shouldBe -import io.mockk.every import io.mockk.mockk -import java.time.Instant +import org.eclipse.apoapsis.ortserver.shared.orttestdata.OrtTestData -import org.ossreviewtoolkit.model.ArtifactProvenance -import org.ossreviewtoolkit.model.Hash import org.ossreviewtoolkit.model.Identifier -import org.ossreviewtoolkit.model.LicenseFinding import org.ossreviewtoolkit.model.OrtResult -import org.ossreviewtoolkit.model.RemoteArtifact -import org.ossreviewtoolkit.model.ScanResult -import org.ossreviewtoolkit.model.ScanSummary -import org.ossreviewtoolkit.model.ScannerDetails -import org.ossreviewtoolkit.model.ScannerRun -import org.ossreviewtoolkit.model.TextLocation +import org.ossreviewtoolkit.model.config.CopyrightGarbage +import org.ossreviewtoolkit.model.config.LicenseFilePatterns import org.ossreviewtoolkit.model.licenses.DefaultLicenseInfoProvider import org.ossreviewtoolkit.model.licenses.LicenseInfoResolver class LicenseComputationTest : StringSpec({ - "computeDetectedLicenses should aggregate licenses across multiple provenances" { - val id = Identifier("Maven", "com.example", "test-pkg", "1.0") + "computeDetectedLicenses returns curated licenses using OrtTestData" { + val id = OrtTestData.pkgIdentifier - val provenance1 = ArtifactProvenance( - sourceArtifact = RemoteArtifact( - url = "https://example.com/pkg1.jar", - hash = Hash.NONE - ) - ) - val provenance2 = ArtifactProvenance( - sourceArtifact = RemoteArtifact( - url = "https://example.com/pkg2.jar", - hash = Hash.NONE - ) - ) + val ortResult = OrtTestData.result - val scanResult1 = ScanResult( - provenance = provenance1, - scanner = ScannerDetails("scanner1", "1.0", ""), - summary = ScanSummary( - startTime = Instant.now(), - endTime = Instant.now(), - licenseFindings = setOf( - LicenseFinding("MIT", TextLocation("file1.txt", 1)), - LicenseFinding("Apache-2.0", TextLocation("file2.txt", 1)) - ) - ) + val resolver = LicenseInfoResolver( + provider = DefaultLicenseInfoProvider(ortResult), + copyrightGarbage = CopyrightGarbage(), + addAuthorsToCopyrights = false, + archiver = mockk(relaxed = true), + licenseFilePatterns = LicenseFilePatterns.DEFAULT ) - val scanResult2 = ScanResult( - provenance = provenance2, - scanner = ScannerDetails("scanner2", "1.0", ""), - summary = ScanSummary( - startTime = Instant.now(), - endTime = Instant.now(), - licenseFindings = setOf( - LicenseFinding("Apache-2.0", TextLocation("file3.txt", 1)), - LicenseFinding("GPL-3.0", TextLocation("file4.txt", 1)) + val result = computeDetectedLicenses(resolver, id) + + result shouldContain "LicenseRef-detected1-concluded" + result shouldContain "LicenseRef-detected2" + result shouldContain "LicenseRef-detected3" + } + + "computeDetectedLicenses returns all detected licenses when no curations exist" { + val id = OrtTestData.pkgIdentifier + + val ortResult = OrtTestData.result.copy( + repository = OrtTestData.repository.copy( + config = OrtTestData.repository.config.copy( + packageConfigurations = emptyList() ) + ), + resolvedConfiguration = OrtTestData.resolvedConfiguration.copy( + packageConfigurations = emptyList() ) ) - val scannerRun = mockk(relaxed = true) { - every { getAllScanResults() } returns mapOf(id to listOf(scanResult1, scanResult2)) - } + val resolver = LicenseInfoResolver( + provider = DefaultLicenseInfoProvider(ortResult), + copyrightGarbage = CopyrightGarbage(), + addAuthorsToCopyrights = false, + archiver = mockk(relaxed = true), + licenseFilePatterns = LicenseFilePatterns.DEFAULT + ) - val result = computeDetectedLicenses(scannerRun, id) + val result = computeDetectedLicenses(resolver, id) - result shouldContainExactlyInAnyOrder setOf("MIT", "Apache-2.0", "GPL-3.0") + result shouldContain "LicenseRef-detected1" + result shouldContain "LicenseRef-detected2" + result shouldContain "LicenseRef-detected3" + result shouldContain "LicenseRef-detected-excluded" } - "computeDetectedLicenses should return empty set when no scan results exist" { + "computeDetectedLicenses returns empty set when no scan results for id" { val id = Identifier("Maven", "com.example", "no-scan-pkg", "1.0") - val scannerRun = mockk(relaxed = true) { - every { getAllScanResults() } returns emptyMap() - } + val ortResult = OrtResult.EMPTY - val result = computeDetectedLicenses(scannerRun, id) + val resolver = LicenseInfoResolver( + provider = DefaultLicenseInfoProvider(ortResult), + copyrightGarbage = CopyrightGarbage(), + addAuthorsToCopyrights = false, + archiver = mockk(relaxed = true), + licenseFilePatterns = LicenseFilePatterns.DEFAULT + ) + + val result = computeDetectedLicenses(resolver, id) result.shouldBeEmpty() } - "computeDetectedLicenses should deduplicate licenses across provenances" { - val id = Identifier("NPM", "@example", "lib", "2.0") - - val provenance = ArtifactProvenance( - sourceArtifact = RemoteArtifact( - url = "https://example.com/lib.tgz", - hash = Hash.NONE - ) - ) - - val scanResult1 = ScanResult( - provenance = provenance, - scanner = ScannerDetails("scanner1", "1.0", ""), - summary = ScanSummary( - startTime = Instant.now(), - endTime = Instant.now(), - licenseFindings = setOf( - LicenseFinding("MIT", TextLocation("a.txt", 1)) + "computeDetectedLicenses deduplicates licenses across provenances" { + val ortResult = OrtTestData.result.copy( + repository = OrtTestData.repository.copy( + config = OrtTestData.repository.config.copy( + packageConfigurations = emptyList() ) + ), + resolvedConfiguration = OrtTestData.resolvedConfiguration.copy( + packageConfigurations = emptyList() ) ) + val id = OrtTestData.pkgIdentifier - val scanResult2 = ScanResult( - provenance = provenance, - scanner = ScannerDetails("scanner2", "1.0", ""), - summary = ScanSummary( - startTime = Instant.now(), - endTime = Instant.now(), - licenseFindings = setOf( - LicenseFinding("MIT", TextLocation("b.txt", 1)) - ) - ) + val resolver = LicenseInfoResolver( + provider = DefaultLicenseInfoProvider(ortResult), + copyrightGarbage = CopyrightGarbage(), + addAuthorsToCopyrights = false, + archiver = mockk(relaxed = true), + licenseFilePatterns = LicenseFilePatterns.DEFAULT ) - val scannerRun = mockk(relaxed = true) { - every { getAllScanResults() } returns mapOf(id to listOf(scanResult1, scanResult2)) - } - - val result = computeDetectedLicenses(scannerRun, id) + val result = computeDetectedLicenses(resolver, id) - result shouldBe setOf("MIT") + val licenseCounts = result.groupingBy { it }.eachCount() + licenseCounts.values.all { it == 1 } shouldBe true } "computeEffectiveLicense should return null when no license info is available" { @@ -157,10 +138,10 @@ class LicenseComputationTest : StringSpec({ val ortResult = OrtResult.EMPTY val resolver = LicenseInfoResolver( provider = DefaultLicenseInfoProvider(ortResult), - copyrightGarbage = org.ossreviewtoolkit.model.config.CopyrightGarbage(), + copyrightGarbage = CopyrightGarbage(), addAuthorsToCopyrights = false, archiver = mockk(relaxed = true), - licenseFilePatterns = org.ossreviewtoolkit.model.config.LicenseFilePatterns.DEFAULT + licenseFilePatterns = LicenseFilePatterns.DEFAULT ) val result = computeEffectiveLicense(resolver, id) diff --git a/workers/scanner/src/test/kotlin/ScannerWorkerTest.kt b/workers/scanner/src/test/kotlin/ScannerWorkerTest.kt index f3c9d39a19..69ae558831 100644 --- a/workers/scanner/src/test/kotlin/ScannerWorkerTest.kt +++ b/workers/scanner/src/test/kotlin/ScannerWorkerTest.kt @@ -123,7 +123,7 @@ class ScannerWorkerTest : StringSpec({ every { buildLicenseInfoResolver(any(), any(), any()) } returns mockk(relaxed = true) mockkStatic("org.eclipse.apoapsis.ortserver.workers.scanner.LicenseComputationKt") - every { computeAndStoreLicenses(any(), any(), any()) } just runs + every { computeAndStoreLicenses(any(), any()) } just runs val ortRunService = mockk { every { createScannerRun(any()) } returns mockk { @@ -234,7 +234,7 @@ class ScannerWorkerTest : StringSpec({ every { buildLicenseInfoResolver(any(), any(), any()) } returns mockk(relaxed = true) mockkStatic("org.eclipse.apoapsis.ortserver.workers.scanner.LicenseComputationKt") - every { computeAndStoreLicenses(any(), any(), any()) } just runs + every { computeAndStoreLicenses(any(), any()) } just runs val ortRunService = mockk { every { createScannerRun(any()) } returns mockk { @@ -435,7 +435,7 @@ class ScannerWorkerTest : StringSpec({ every { buildLicenseInfoResolver(any(), any(), any()) } returns mockk(relaxed = true) mockkStatic("org.eclipse.apoapsis.ortserver.workers.scanner.LicenseComputationKt") - every { computeAndStoreLicenses(any(), any(), any()) } just runs + every { computeAndStoreLicenses(any(), any()) } just runs val ortRunService = mockk { every { createScannerRun(any()) } returns mockk {