Skip to content

Commit ed6934c

Browse files
Joelp03Joelp03
committed
fix(scanner): Apply curations before storing detected licenses
Detected licenses must have PackageCuration and PackageConfiguration applied before storing, otherwise wrong data could be shown. - Change computeDetectedLicenses to use LicenseInfoResolver which applies LicenseFindingCuration and PathExclude from PackageConfiguration - Remove scannerRun parameter from computeAndStoreLicenses since it's no longer needed after using LicenseInfoResolver Signed-off-by: Joelp03 <joelpimentel1995@gmail.com>
1 parent 962bd1c commit ed6934c

5 files changed

Lines changed: 91 additions & 110 deletions

File tree

dao/src/main/resources/db/migration/V145__storeLicensesInDatabase.sql renamed to dao/src/main/resources/db/migration/V146__storeLicensesInDatabase.sql

File renamed without changes.

‎workers/scanner/src/main/kotlin/LicenseComputation.kt‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -30,24 +30,27 @@ import org.eclipse.apoapsis.ortserver.services.ortrun.mapToModel
3030
import org.jetbrains.exposed.v1.core.eq
3131

3232
import org.ossreviewtoolkit.model.Identifier as OrtIdentifier
33+
import org.ossreviewtoolkit.model.LicenseSource
3334
import org.ossreviewtoolkit.model.OrtResult
34-
import org.ossreviewtoolkit.model.ScannerRun
3535
import org.ossreviewtoolkit.model.licenses.LicenseInfoResolver
3636
import org.ossreviewtoolkit.model.licenses.LicenseView
3737

3838
/**
39-
* Compute detected licenses for a package or project by aggregating license findings across all scan results
40-
* for the given [id]. Returns the union of all license strings found in all provenances.
39+
* Compute detected licenses for a package or project using the [licenseInfoResolver].
40+
*
41+
* Returns the set of licenses with [LicenseSource.DETECTED], which respects LicenseFindingCuration and PathExclude
42+
* from PackageConfiguration.
4143
*/
4244
fun computeDetectedLicenses(
43-
scannerRun: ScannerRun,
45+
licenseInfoResolver: LicenseInfoResolver,
4446
id: OrtIdentifier
45-
): Set<String> =
46-
scannerRun.getAllScanResults()[id]
47-
.orEmpty()
48-
.flatMap { scanResult -> scanResult.summary.licenseFindings }
49-
.map { licenseFinding -> licenseFinding.license.toString() }
47+
): Set<String> {
48+
val resolvedLicenseInfo = licenseInfoResolver.resolveLicenseInfo(id)
49+
return resolvedLicenseInfo.licenses
50+
.filter { it.sources.any { source -> source == LicenseSource.DETECTED } }
51+
.map { it.license.toString() }
5052
.toSet()
53+
}
5154

5255
/**
5356
* Compute the effective license for a package or project using the [licenseInfoResolver].
@@ -66,19 +69,17 @@ fun computeEffectiveLicense(
6669

6770
/**
6871
* Compute detected and effective licenses for all packages and projects in the [ortResult] and store them in the
69-
* database. Uses the [scannerRun] to aggregate detected licenses across all provenances and the
70-
* [licenseInfoResolver] to compute effective licenses.
72+
* database. Uses the [licenseInfoResolver] to compute both detected and effective licenses with curations applied.
7173
*/
7274
fun computeAndStoreLicenses(
7375
ortResult: OrtResult,
74-
scannerRun: ScannerRun,
7576
licenseInfoResolver: LicenseInfoResolver
7677
) {
7778
for (curatedPackage in ortResult.getPackages()) {
7879
val id = curatedPackage.metadata.id
7980
val modelId = id.mapToModel()
8081

81-
val detectedLicenses = computeDetectedLicenses(scannerRun, id)
82+
val detectedLicenses = computeDetectedLicenses(licenseInfoResolver, id)
8283
val effectiveLicense = computeEffectiveLicense(licenseInfoResolver, id)
8384

8485
if (detectedLicenses.isNotEmpty() || effectiveLicense != null) {
@@ -90,7 +91,7 @@ fun computeAndStoreLicenses(
9091
val id = project.id
9192
val modelId = id.mapToModel()
9293

93-
val detectedLicenses = computeDetectedLicenses(scannerRun, id)
94+
val detectedLicenses = computeDetectedLicenses(licenseInfoResolver, id)
9495
val effectiveLicense = computeEffectiveLicense(licenseInfoResolver, id)
9596

9697
if (detectedLicenses.isNotEmpty() || effectiveLicense != null) {

‎workers/scanner/src/main/kotlin/ScannerWorker.kt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,6 @@ class ScannerWorker(
111111
db.dbQuery {
112112
computeAndStoreLicenses(
113113
ortResult = ortResult,
114-
scannerRun = scannerRunResult.scannerRun,
115114
licenseInfoResolver = licenseInfoResolver
116115
)
117116
}

‎workers/scanner/src/test/kotlin/LicenseComputationTest.kt‎

Lines changed: 73 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -21,134 +21,115 @@ package org.eclipse.apoapsis.ortserver.workers.scanner
2121

2222
import io.kotest.core.spec.style.StringSpec
2323
import io.kotest.matchers.collections.shouldBeEmpty
24-
import io.kotest.matchers.collections.shouldContainExactlyInAnyOrder
24+
import io.kotest.matchers.collections.shouldContain
2525
import io.kotest.matchers.nulls.shouldBeNull
2626
import io.kotest.matchers.shouldBe
2727

28-
import io.mockk.every
2928
import io.mockk.mockk
3029

31-
import java.time.Instant
30+
import org.eclipse.apoapsis.ortserver.shared.orttestdata.OrtTestData
3231

33-
import org.ossreviewtoolkit.model.ArtifactProvenance
34-
import org.ossreviewtoolkit.model.Hash
3532
import org.ossreviewtoolkit.model.Identifier
36-
import org.ossreviewtoolkit.model.LicenseFinding
3733
import org.ossreviewtoolkit.model.OrtResult
38-
import org.ossreviewtoolkit.model.RemoteArtifact
39-
import org.ossreviewtoolkit.model.ScanResult
40-
import org.ossreviewtoolkit.model.ScanSummary
41-
import org.ossreviewtoolkit.model.ScannerDetails
42-
import org.ossreviewtoolkit.model.ScannerRun
43-
import org.ossreviewtoolkit.model.TextLocation
34+
import org.ossreviewtoolkit.model.config.CopyrightGarbage
35+
import org.ossreviewtoolkit.model.config.LicenseFilePatterns
4436
import org.ossreviewtoolkit.model.licenses.DefaultLicenseInfoProvider
4537
import org.ossreviewtoolkit.model.licenses.LicenseInfoResolver
4638

4739
class LicenseComputationTest : StringSpec({
48-
"computeDetectedLicenses should aggregate licenses across multiple provenances" {
49-
val id = Identifier("Maven", "com.example", "test-pkg", "1.0")
40+
"computeDetectedLicenses returns curated licenses using OrtTestData" {
41+
val id = OrtTestData.pkgIdentifier
5042

51-
val provenance1 = ArtifactProvenance(
52-
sourceArtifact = RemoteArtifact(
53-
url = "https://example.com/pkg1.jar",
54-
hash = Hash.NONE
55-
)
56-
)
57-
val provenance2 = ArtifactProvenance(
58-
sourceArtifact = RemoteArtifact(
59-
url = "https://example.com/pkg2.jar",
60-
hash = Hash.NONE
61-
)
62-
)
43+
val ortResult = OrtTestData.result
6344

64-
val scanResult1 = ScanResult(
65-
provenance = provenance1,
66-
scanner = ScannerDetails("scanner1", "1.0", ""),
67-
summary = ScanSummary(
68-
startTime = Instant.now(),
69-
endTime = Instant.now(),
70-
licenseFindings = setOf(
71-
LicenseFinding("MIT", TextLocation("file1.txt", 1)),
72-
LicenseFinding("Apache-2.0", TextLocation("file2.txt", 1))
73-
)
74-
)
45+
val resolver = LicenseInfoResolver(
46+
provider = DefaultLicenseInfoProvider(ortResult),
47+
copyrightGarbage = CopyrightGarbage(),
48+
addAuthorsToCopyrights = false,
49+
archiver = mockk(relaxed = true),
50+
licenseFilePatterns = LicenseFilePatterns.DEFAULT
7551
)
7652

77-
val scanResult2 = ScanResult(
78-
provenance = provenance2,
79-
scanner = ScannerDetails("scanner2", "1.0", ""),
80-
summary = ScanSummary(
81-
startTime = Instant.now(),
82-
endTime = Instant.now(),
83-
licenseFindings = setOf(
84-
LicenseFinding("Apache-2.0", TextLocation("file3.txt", 1)),
85-
LicenseFinding("GPL-3.0", TextLocation("file4.txt", 1))
53+
val result = computeDetectedLicenses(resolver, id)
54+
55+
result shouldContain "LicenseRef-detected1-concluded"
56+
result shouldContain "LicenseRef-detected2"
57+
result shouldContain "LicenseRef-detected3"
58+
}
59+
60+
"computeDetectedLicenses returns all detected licenses when no curations exist" {
61+
val id = OrtTestData.pkgIdentifier
62+
63+
val ortResult = OrtTestData.result.copy(
64+
repository = OrtTestData.repository.copy(
65+
config = OrtTestData.repository.config.copy(
66+
packageConfigurations = emptyList()
8667
)
68+
),
69+
resolvedConfiguration = OrtTestData.resolvedConfiguration.copy(
70+
packageConfigurations = emptyList()
8771
)
8872
)
8973

90-
val scannerRun = mockk<ScannerRun>(relaxed = true) {
91-
every { getAllScanResults() } returns mapOf(id to listOf(scanResult1, scanResult2))
92-
}
74+
val resolver = LicenseInfoResolver(
75+
provider = DefaultLicenseInfoProvider(ortResult),
76+
copyrightGarbage = CopyrightGarbage(),
77+
addAuthorsToCopyrights = false,
78+
archiver = mockk(relaxed = true),
79+
licenseFilePatterns = LicenseFilePatterns.DEFAULT
80+
)
9381

94-
val result = computeDetectedLicenses(scannerRun, id)
82+
val result = computeDetectedLicenses(resolver, id)
9583

96-
result shouldContainExactlyInAnyOrder setOf("MIT", "Apache-2.0", "GPL-3.0")
84+
result shouldContain "LicenseRef-detected1"
85+
result shouldContain "LicenseRef-detected2"
86+
result shouldContain "LicenseRef-detected3"
87+
result shouldContain "LicenseRef-detected-excluded"
9788
}
9889

99-
"computeDetectedLicenses should return empty set when no scan results exist" {
90+
"computeDetectedLicenses returns empty set when no scan results for id" {
10091
val id = Identifier("Maven", "com.example", "no-scan-pkg", "1.0")
10192

102-
val scannerRun = mockk<ScannerRun>(relaxed = true) {
103-
every { getAllScanResults() } returns emptyMap()
104-
}
93+
val ortResult = OrtResult.EMPTY
10594

106-
val result = computeDetectedLicenses(scannerRun, id)
95+
val resolver = LicenseInfoResolver(
96+
provider = DefaultLicenseInfoProvider(ortResult),
97+
copyrightGarbage = CopyrightGarbage(),
98+
addAuthorsToCopyrights = false,
99+
archiver = mockk(relaxed = true),
100+
licenseFilePatterns = LicenseFilePatterns.DEFAULT
101+
)
102+
103+
val result = computeDetectedLicenses(resolver, id)
107104

108105
result.shouldBeEmpty()
109106
}
110107

111-
"computeDetectedLicenses should deduplicate licenses across provenances" {
112-
val id = Identifier("NPM", "@example", "lib", "2.0")
113-
114-
val provenance = ArtifactProvenance(
115-
sourceArtifact = RemoteArtifact(
116-
url = "https://example.com/lib.tgz",
117-
hash = Hash.NONE
118-
)
119-
)
120-
121-
val scanResult1 = ScanResult(
122-
provenance = provenance,
123-
scanner = ScannerDetails("scanner1", "1.0", ""),
124-
summary = ScanSummary(
125-
startTime = Instant.now(),
126-
endTime = Instant.now(),
127-
licenseFindings = setOf(
128-
LicenseFinding("MIT", TextLocation("a.txt", 1))
108+
"computeDetectedLicenses deduplicates licenses across provenances" {
109+
val ortResult = OrtTestData.result.copy(
110+
repository = OrtTestData.repository.copy(
111+
config = OrtTestData.repository.config.copy(
112+
packageConfigurations = emptyList()
129113
)
114+
),
115+
resolvedConfiguration = OrtTestData.resolvedConfiguration.copy(
116+
packageConfigurations = emptyList()
130117
)
131118
)
119+
val id = OrtTestData.pkgIdentifier
132120

133-
val scanResult2 = ScanResult(
134-
provenance = provenance,
135-
scanner = ScannerDetails("scanner2", "1.0", ""),
136-
summary = ScanSummary(
137-
startTime = Instant.now(),
138-
endTime = Instant.now(),
139-
licenseFindings = setOf(
140-
LicenseFinding("MIT", TextLocation("b.txt", 1))
141-
)
142-
)
121+
val resolver = LicenseInfoResolver(
122+
provider = DefaultLicenseInfoProvider(ortResult),
123+
copyrightGarbage = CopyrightGarbage(),
124+
addAuthorsToCopyrights = false,
125+
archiver = mockk(relaxed = true),
126+
licenseFilePatterns = LicenseFilePatterns.DEFAULT
143127
)
144128

145-
val scannerRun = mockk<ScannerRun>(relaxed = true) {
146-
every { getAllScanResults() } returns mapOf(id to listOf(scanResult1, scanResult2))
147-
}
148-
149-
val result = computeDetectedLicenses(scannerRun, id)
129+
val result = computeDetectedLicenses(resolver, id)
150130

151-
result shouldBe setOf("MIT")
131+
val licenseCounts = result.groupingBy { it }.eachCount()
132+
licenseCounts.values.all { it == 1 } shouldBe true
152133
}
153134

154135
"computeEffectiveLicense should return null when no license info is available" {
@@ -157,10 +138,10 @@ class LicenseComputationTest : StringSpec({
157138
val ortResult = OrtResult.EMPTY
158139
val resolver = LicenseInfoResolver(
159140
provider = DefaultLicenseInfoProvider(ortResult),
160-
copyrightGarbage = org.ossreviewtoolkit.model.config.CopyrightGarbage(),
141+
copyrightGarbage = CopyrightGarbage(),
161142
addAuthorsToCopyrights = false,
162143
archiver = mockk(relaxed = true),
163-
licenseFilePatterns = org.ossreviewtoolkit.model.config.LicenseFilePatterns.DEFAULT
144+
licenseFilePatterns = LicenseFilePatterns.DEFAULT
164145
)
165146

166147
val result = computeEffectiveLicense(resolver, id)

‎workers/scanner/src/test/kotlin/ScannerWorkerTest.kt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ class ScannerWorkerTest : StringSpec({
123123
every { buildLicenseInfoResolver(any(), any(), any()) } returns mockk(relaxed = true)
124124

125125
mockkStatic("org.eclipse.apoapsis.ortserver.workers.scanner.LicenseComputationKt")
126-
every { computeAndStoreLicenses(any(), any(), any()) } just runs
126+
every { computeAndStoreLicenses(any(), any()) } just runs
127127

128128
val ortRunService = mockk<OrtRunService> {
129129
every { createScannerRun(any()) } returns mockk {
@@ -234,7 +234,7 @@ class ScannerWorkerTest : StringSpec({
234234
every { buildLicenseInfoResolver(any(), any(), any()) } returns mockk(relaxed = true)
235235

236236
mockkStatic("org.eclipse.apoapsis.ortserver.workers.scanner.LicenseComputationKt")
237-
every { computeAndStoreLicenses(any(), any(), any()) } just runs
237+
every { computeAndStoreLicenses(any(), any()) } just runs
238238

239239
val ortRunService = mockk<OrtRunService> {
240240
every { createScannerRun(any()) } returns mockk {
@@ -435,7 +435,7 @@ class ScannerWorkerTest : StringSpec({
435435
every { buildLicenseInfoResolver(any(), any(), any()) } returns mockk(relaxed = true)
436436

437437
mockkStatic("org.eclipse.apoapsis.ortserver.workers.scanner.LicenseComputationKt")
438-
every { computeAndStoreLicenses(any(), any(), any()) } just runs
438+
every { computeAndStoreLicenses(any(), any()) } just runs
439439

440440
val ortRunService = mockk<OrtRunService> {
441441
every { createScannerRun(any()) } returns mockk {

0 commit comments

Comments
 (0)