From 5c6f6d59167c48b50326a61e710307754e0a5afa Mon Sep 17 00:00:00 2001 From: Thomas Schafer Date: Thu, 6 Feb 2025 10:50:15 +0000 Subject: [PATCH 1/8] fix: fall back to license from package if not present in versioned package --- internal/utils/spdx.go | 12 +++++-- internal/utils/spdx_test.go | 41 ++++++++++++++++----- lib/ecosystems/enrich_cyclonedx.go | 8 ++--- lib/ecosystems/enrich_cyclonedx_test.go | 47 ++++++++++++++++++++++--- lib/ecosystems/enrich_spdx.go | 8 ++--- 5 files changed, 92 insertions(+), 24 deletions(-) diff --git a/internal/utils/spdx.go b/internal/utils/spdx.go index 6e2059c..962c47c 100644 --- a/internal/utils/spdx.go +++ b/internal/utils/spdx.go @@ -32,9 +32,15 @@ func GetPurlFromSPDXPackage(pkg *spdx_2_3.Package) (*packageurl.PackageURL, erro return &purl, nil } -func GetSPDXLicenseExpressionFromEcosystemsLicense(data *packages.VersionWithDependencies) string { - if data == nil || data.Licenses == nil || *data.Licenses == "" { +func GetSPDXLicenseExpressionFromEcosystemsLicense(pkgVersionData *packages.VersionWithDependencies, pkgData *packages.Package) string { + licenses := "" + if pkgVersionData != nil && pkgVersionData.Licenses != nil && *pkgVersionData.Licenses != "" { + licenses = *pkgVersionData.Licenses + } else if pkgData != nil && pkgData.Licenses != nil && *pkgData.Licenses != "" { + licenses = *pkgData.Licenses + } + if licenses == "" { return "" } - return fmt.Sprintf("(%s)", strings.Join(strings.Split(*data.Licenses, ","), " OR ")) + return fmt.Sprintf("(%s)", strings.Join(strings.Split(licenses, ","), " OR ")) } diff --git a/internal/utils/spdx_test.go b/internal/utils/spdx_test.go index 245ee29..58c3bac 100644 --- a/internal/utils/spdx_test.go +++ b/internal/utils/spdx_test.go @@ -11,29 +11,52 @@ import ( func TestGetSPDXLicenseExpressionFromEcosystemsLicense(t *testing.T) { assert := assert.New(t) - licenses := "GPLv2,MIT" - data := packages.VersionWithDependencies{Licenses: &licenses} - expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(&data) + versionedLicenses := "GPLv2,MIT" + pkgVersionData := packages.VersionWithDependencies{Licenses: &versionedLicenses} + latestLicenses := "Apache-2.0" + pkgData := packages.Package{Licenses: &latestLicenses} + expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) assert.Equal("(GPLv2 OR MIT)", expression) } func TestGetSPDXLicenseExpressionFromEcosystemsLicense_NoData(t *testing.T) { assert := assert.New(t) - expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(nil) + expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(nil, nil) assert.Equal("", expression) } +func TestGetSPDXLicenseExpressionFromEcosystemsLicense_NoVersionedData(t *testing.T) { + assert := assert.New(t) + pkgVersionData := packages.VersionWithDependencies{} + latestLicenses := "Apache-2.0" + pkgData := packages.Package{Licenses: &latestLicenses} + expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) + assert.Equal("(Apache-2.0)", expression) +} + +func TestGetSPDXLicenseExpressionFromEcosystemsLicense_NoLatestData(t *testing.T) { + assert := assert.New(t) + versionedLicenses := "GPLv2,MIT" + pkgVersionData := packages.VersionWithDependencies{Licenses: &versionedLicenses} + pkgData := packages.Package{} + expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) + assert.Equal("(GPLv2 OR MIT)", expression) +} + func TestGetSPDXLicenseExpressionFromEcosystemsLicense_NoLicenses(t *testing.T) { assert := assert.New(t) - data := packages.VersionWithDependencies{} - expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(&data) + pkgVersionData := packages.VersionWithDependencies{} + pkgData := packages.Package{} + expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) assert.Equal("", expression) } func TestGetSPDXLicenseExpressionFromEcosystemsLicense_EmptyLicenses(t *testing.T) { assert := assert.New(t) - licenses := "" - data := packages.VersionWithDependencies{Licenses: &licenses} - expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(&data) + versionedLicenses := "" + pkgVersionData := packages.VersionWithDependencies{Licenses: &versionedLicenses} + latestLicenses := "" + pkgData := packages.Package{Licenses: &latestLicenses} + expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) assert.Equal("", expression) } diff --git a/lib/ecosystems/enrich_cyclonedx.go b/lib/ecosystems/enrich_cyclonedx.go index 2f73e5f..4061f1f 100644 --- a/lib/ecosystems/enrich_cyclonedx.go +++ b/lib/ecosystems/enrich_cyclonedx.go @@ -31,7 +31,7 @@ import ( ) type cdxPackageEnricher = func(*cdx.Component, *packages.Package) -type cdxPackageVersionEnricher = func(*cdx.Component, *packages.VersionWithDependencies) +type cdxPackageVersionEnricher = func(*cdx.Component, *packages.VersionWithDependencies, *packages.Package) var cdxPackageEnrichers = []cdxPackageEnricher{ enrichCDXDescription, @@ -58,8 +58,8 @@ func enrichCDXDescription(comp *cdx.Component, data *packages.Package) { } } -func enrichCDXLicense(comp *cdx.Component, data *packages.VersionWithDependencies) { - expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(data) +func enrichCDXLicense(comp *cdx.Component, pkgVersionData *packages.VersionWithDependencies, pkgData *packages.Package) { + expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(pkgVersionData, pkgData) if expression != "" { licenses := cdx.LicenseChoice{Expression: expression} comp.Licenses = &cdx.Licenses{licenses} @@ -248,7 +248,7 @@ func enrichCDX(bom *cdx.BOM, logger *zerolog.Logger) { } for _, enrichFunc := range cdxPackageVersionEnrichers { - enrichFunc(comp, packageVersionResp.JSON200) + enrichFunc(comp, packageVersionResp.JSON200, packageResp.JSON200) } }(comps[i]) diff --git a/lib/ecosystems/enrich_cyclonedx_test.go b/lib/ecosystems/enrich_cyclonedx_test.go index 9eaf553..3d33a44 100644 --- a/lib/ecosystems/enrich_cyclonedx_test.go +++ b/lib/ecosystems/enrich_cyclonedx_test.go @@ -190,15 +190,54 @@ func TestEnrichLicense(t *testing.T) { Name: "cyclonedx-go", Version: "v0.3.0", } - lic := "BSD-3-Clause" - pack := &packages.VersionWithDependencies{ - Licenses: &lic, + versionedLicenses := "BSD-3-Clause" + pkgVersionData := &packages.VersionWithDependencies{Licenses: &versionedLicenses} + latestLicenses := "Apache-2.0" + pkgData := &packages.Package{Licenses: &latestLicenses} + + enrichCDXLicense(component, pkgVersionData, pkgData) + + licenses := *component.Licenses + comp := cdx.LicenseChoice(cdx.LicenseChoice{Expression: "(BSD-3-Clause)"}) + assert.Equal(t, 1, len(licenses)) + assert.Equal(t, comp, licenses[0]) +} + +func TestEnrichLicenseNoVersionedLicense(t *testing.T) { + component := &cdx.Component{ + Type: cdx.ComponentTypeLibrary, + Name: "cyclonedx-go", + Version: "v0.3.0", + } + versionedLicenses := "" + pkgVersionData := &packages.VersionWithDependencies{Licenses: &versionedLicenses} + latestLicenses := "Apache-2.0" + pkgData := &packages.Package{Licenses: &latestLicenses} + + enrichCDXLicense(component, pkgVersionData, pkgData) + + licenses := *component.Licenses + comp := cdx.LicenseChoice(cdx.LicenseChoice{Expression: "(Apache-2.0)"}) + assert.Equal(t, 1, len(licenses)) + assert.Equal(t, comp, licenses[0]) +} + +func TestEnrichLicenseNoLatestLicense(t *testing.T) { + component := &cdx.Component{ + Type: cdx.ComponentTypeLibrary, + Name: "cyclonedx-go", + Version: "v0.3.0", } + versionedLicenses := "BSD-3-Clause" + pkgVersionData := &packages.VersionWithDependencies{Licenses: &versionedLicenses} + latestLicenses := "" + pkgData := &packages.Package{Licenses: &latestLicenses} - enrichCDXLicense(component, pack) + enrichCDXLicense(component, pkgVersionData, pkgData) licenses := *component.Licenses comp := cdx.LicenseChoice(cdx.LicenseChoice{Expression: "(BSD-3-Clause)"}) + assert.Equal(t, 1, len(licenses)) assert.Equal(t, comp, licenses[0]) } diff --git a/lib/ecosystems/enrich_spdx.go b/lib/ecosystems/enrich_spdx.go index 860ccc1..1013c46 100644 --- a/lib/ecosystems/enrich_spdx.go +++ b/lib/ecosystems/enrich_spdx.go @@ -64,7 +64,7 @@ func enrichSPDX(bom *spdx.Document, logger *zerolog.Logger) { continue } - enrichSPDXLicense(pkg, pkgVersionData) + enrichSPDXLicense(pkg, pkgVersionData, pkgData) } } @@ -96,10 +96,10 @@ func enrichSPDXSupplier(pkg *v2_3.Package, data *packages.Package) { } } -func enrichSPDXLicense(pkg *v2_3.Package, data *packages.VersionWithDependencies) { - expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(data) +func enrichSPDXLicense(pkg *v2_3.Package, pkgVersionData *packages.VersionWithDependencies, pkgData *packages.Package) { + expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(pkgVersionData, pkgData) if expression != "" { - pkg.PackageLicenseConcluded = *data.Licenses + pkg.PackageLicenseConcluded = *pkgVersionData.Licenses } } From c6eb75ff742987143ff26873045ed3f0d7280f7a Mon Sep 17 00:00:00 2001 From: Thomas Schafer Date: Thu, 6 Feb 2025 16:04:38 +0000 Subject: [PATCH 2/8] chore: use normalized licenses from versioned package --- internal/utils/spdx.go | 12 ++++++------ internal/utils/spdx_test.go | 12 ++++++------ lib/ecosystems/enrich_cyclonedx_test.go | 12 ++++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/internal/utils/spdx.go b/internal/utils/spdx.go index 962c47c..658768e 100644 --- a/internal/utils/spdx.go +++ b/internal/utils/spdx.go @@ -33,14 +33,14 @@ func GetPurlFromSPDXPackage(pkg *spdx_2_3.Package) (*packageurl.PackageURL, erro } func GetSPDXLicenseExpressionFromEcosystemsLicense(pkgVersionData *packages.VersionWithDependencies, pkgData *packages.Package) string { - licenses := "" + licenses := []string{} if pkgVersionData != nil && pkgVersionData.Licenses != nil && *pkgVersionData.Licenses != "" { - licenses = *pkgVersionData.Licenses - } else if pkgData != nil && pkgData.Licenses != nil && *pkgData.Licenses != "" { - licenses = *pkgData.Licenses + licenses = strings.Split(*pkgVersionData.Licenses, ",") + } else if pkgData != nil && len(pkgData.NormalizedLicenses) > 0 { + licenses = pkgData.NormalizedLicenses } - if licenses == "" { + if len(licenses) == 0 { return "" } - return fmt.Sprintf("(%s)", strings.Join(strings.Split(licenses, ","), " OR ")) + return fmt.Sprintf("(%s)", strings.Join(licenses, " OR ")) } diff --git a/internal/utils/spdx_test.go b/internal/utils/spdx_test.go index 58c3bac..65eb3b7 100644 --- a/internal/utils/spdx_test.go +++ b/internal/utils/spdx_test.go @@ -13,8 +13,8 @@ func TestGetSPDXLicenseExpressionFromEcosystemsLicense(t *testing.T) { assert := assert.New(t) versionedLicenses := "GPLv2,MIT" pkgVersionData := packages.VersionWithDependencies{Licenses: &versionedLicenses} - latestLicenses := "Apache-2.0" - pkgData := packages.Package{Licenses: &latestLicenses} + latestLicenses := []string{"Apache-2.0"} + pkgData := packages.Package{NormalizedLicenses: latestLicenses} expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) assert.Equal("(GPLv2 OR MIT)", expression) } @@ -28,8 +28,8 @@ func TestGetSPDXLicenseExpressionFromEcosystemsLicense_NoData(t *testing.T) { func TestGetSPDXLicenseExpressionFromEcosystemsLicense_NoVersionedData(t *testing.T) { assert := assert.New(t) pkgVersionData := packages.VersionWithDependencies{} - latestLicenses := "Apache-2.0" - pkgData := packages.Package{Licenses: &latestLicenses} + latestLicenses := []string{"Apache-2.0"} + pkgData := packages.Package{NormalizedLicenses: latestLicenses} expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) assert.Equal("(Apache-2.0)", expression) } @@ -55,8 +55,8 @@ func TestGetSPDXLicenseExpressionFromEcosystemsLicense_EmptyLicenses(t *testing. assert := assert.New(t) versionedLicenses := "" pkgVersionData := packages.VersionWithDependencies{Licenses: &versionedLicenses} - latestLicenses := "" - pkgData := packages.Package{Licenses: &latestLicenses} + latestLicenses := []string{""} + pkgData := packages.Package{NormalizedLicenses: latestLicenses} expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) assert.Equal("", expression) } diff --git a/lib/ecosystems/enrich_cyclonedx_test.go b/lib/ecosystems/enrich_cyclonedx_test.go index 3d33a44..ef9c6c3 100644 --- a/lib/ecosystems/enrich_cyclonedx_test.go +++ b/lib/ecosystems/enrich_cyclonedx_test.go @@ -192,8 +192,8 @@ func TestEnrichLicense(t *testing.T) { } versionedLicenses := "BSD-3-Clause" pkgVersionData := &packages.VersionWithDependencies{Licenses: &versionedLicenses} - latestLicenses := "Apache-2.0" - pkgData := &packages.Package{Licenses: &latestLicenses} + latestLicenses := []string{"Apache-2.0"} + pkgData := &packages.Package{NormalizedLicenses: latestLicenses} enrichCDXLicense(component, pkgVersionData, pkgData) @@ -211,8 +211,8 @@ func TestEnrichLicenseNoVersionedLicense(t *testing.T) { } versionedLicenses := "" pkgVersionData := &packages.VersionWithDependencies{Licenses: &versionedLicenses} - latestLicenses := "Apache-2.0" - pkgData := &packages.Package{Licenses: &latestLicenses} + latestLicenses := []string{"Apache-2.0"} + pkgData := &packages.Package{NormalizedLicenses: latestLicenses} enrichCDXLicense(component, pkgVersionData, pkgData) @@ -230,8 +230,8 @@ func TestEnrichLicenseNoLatestLicense(t *testing.T) { } versionedLicenses := "BSD-3-Clause" pkgVersionData := &packages.VersionWithDependencies{Licenses: &versionedLicenses} - latestLicenses := "" - pkgData := &packages.Package{Licenses: &latestLicenses} + latestLicenses := []string{""} + pkgData := &packages.Package{NormalizedLicenses: latestLicenses} enrichCDXLicense(component, pkgVersionData, pkgData) From b88f63db04f1b6ded96d9a86e15de3a920ed3861 Mon Sep 17 00:00:00 2001 From: Thomas Schafer Date: Thu, 6 Feb 2025 16:34:41 +0000 Subject: [PATCH 3/8] fix: retain previous behaviour of using unformatted licenses --- internal/utils/spdx.go | 7 ++++++- internal/utils/spdx_test.go | 14 +++++++------- lib/ecosystems/enrich_cyclonedx.go | 2 +- lib/ecosystems/enrich_spdx.go | 7 ++++--- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/internal/utils/spdx.go b/internal/utils/spdx.go index 658768e..d6ed141 100644 --- a/internal/utils/spdx.go +++ b/internal/utils/spdx.go @@ -32,13 +32,18 @@ func GetPurlFromSPDXPackage(pkg *spdx_2_3.Package) (*packageurl.PackageURL, erro return &purl, nil } -func GetSPDXLicenseExpressionFromEcosystemsLicense(pkgVersionData *packages.VersionWithDependencies, pkgData *packages.Package) string { +func GetLicensesFromEcosystemsLicense(pkgVersionData *packages.VersionWithDependencies, pkgData *packages.Package) []string { licenses := []string{} if pkgVersionData != nil && pkgVersionData.Licenses != nil && *pkgVersionData.Licenses != "" { licenses = strings.Split(*pkgVersionData.Licenses, ",") } else if pkgData != nil && len(pkgData.NormalizedLicenses) > 0 { licenses = pkgData.NormalizedLicenses } + return licenses +} + +func GetLicenseExpressionFromEcosystemsLicense(pkgVersionData *packages.VersionWithDependencies, pkgData *packages.Package) string { + licenses := GetLicensesFromEcosystemsLicense(pkgVersionData, pkgData) if len(licenses) == 0 { return "" } diff --git a/internal/utils/spdx_test.go b/internal/utils/spdx_test.go index 65eb3b7..3882d2f 100644 --- a/internal/utils/spdx_test.go +++ b/internal/utils/spdx_test.go @@ -15,13 +15,13 @@ func TestGetSPDXLicenseExpressionFromEcosystemsLicense(t *testing.T) { pkgVersionData := packages.VersionWithDependencies{Licenses: &versionedLicenses} latestLicenses := []string{"Apache-2.0"} pkgData := packages.Package{NormalizedLicenses: latestLicenses} - expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) + expression := utils.GetLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) assert.Equal("(GPLv2 OR MIT)", expression) } func TestGetSPDXLicenseExpressionFromEcosystemsLicense_NoData(t *testing.T) { assert := assert.New(t) - expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(nil, nil) + expression := utils.GetLicenseExpressionFromEcosystemsLicense(nil, nil) assert.Equal("", expression) } @@ -30,7 +30,7 @@ func TestGetSPDXLicenseExpressionFromEcosystemsLicense_NoVersionedData(t *testin pkgVersionData := packages.VersionWithDependencies{} latestLicenses := []string{"Apache-2.0"} pkgData := packages.Package{NormalizedLicenses: latestLicenses} - expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) + expression := utils.GetLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) assert.Equal("(Apache-2.0)", expression) } @@ -39,7 +39,7 @@ func TestGetSPDXLicenseExpressionFromEcosystemsLicense_NoLatestData(t *testing.T versionedLicenses := "GPLv2,MIT" pkgVersionData := packages.VersionWithDependencies{Licenses: &versionedLicenses} pkgData := packages.Package{} - expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) + expression := utils.GetLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) assert.Equal("(GPLv2 OR MIT)", expression) } @@ -47,7 +47,7 @@ func TestGetSPDXLicenseExpressionFromEcosystemsLicense_NoLicenses(t *testing.T) assert := assert.New(t) pkgVersionData := packages.VersionWithDependencies{} pkgData := packages.Package{} - expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) + expression := utils.GetLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) assert.Equal("", expression) } @@ -55,8 +55,8 @@ func TestGetSPDXLicenseExpressionFromEcosystemsLicense_EmptyLicenses(t *testing. assert := assert.New(t) versionedLicenses := "" pkgVersionData := packages.VersionWithDependencies{Licenses: &versionedLicenses} - latestLicenses := []string{""} + latestLicenses := []string{} pkgData := packages.Package{NormalizedLicenses: latestLicenses} - expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) + expression := utils.GetLicenseExpressionFromEcosystemsLicense(&pkgVersionData, &pkgData) assert.Equal("", expression) } diff --git a/lib/ecosystems/enrich_cyclonedx.go b/lib/ecosystems/enrich_cyclonedx.go index 4061f1f..c5358ee 100644 --- a/lib/ecosystems/enrich_cyclonedx.go +++ b/lib/ecosystems/enrich_cyclonedx.go @@ -59,7 +59,7 @@ func enrichCDXDescription(comp *cdx.Component, data *packages.Package) { } func enrichCDXLicense(comp *cdx.Component, pkgVersionData *packages.VersionWithDependencies, pkgData *packages.Package) { - expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(pkgVersionData, pkgData) + expression := utils.GetLicenseExpressionFromEcosystemsLicense(pkgVersionData, pkgData) if expression != "" { licenses := cdx.LicenseChoice{Expression: expression} comp.Licenses = &cdx.Licenses{licenses} diff --git a/lib/ecosystems/enrich_spdx.go b/lib/ecosystems/enrich_spdx.go index 1013c46..2c9f863 100644 --- a/lib/ecosystems/enrich_spdx.go +++ b/lib/ecosystems/enrich_spdx.go @@ -18,6 +18,7 @@ package ecosystems import ( "errors" + "strings" "github.com/package-url/packageurl-go" "github.com/rs/zerolog" @@ -97,9 +98,9 @@ func enrichSPDXSupplier(pkg *v2_3.Package, data *packages.Package) { } func enrichSPDXLicense(pkg *v2_3.Package, pkgVersionData *packages.VersionWithDependencies, pkgData *packages.Package) { - expression := utils.GetSPDXLicenseExpressionFromEcosystemsLicense(pkgVersionData, pkgData) - if expression != "" { - pkg.PackageLicenseConcluded = *pkgVersionData.Licenses + licenses := utils.GetLicensesFromEcosystemsLicense(pkgVersionData, pkgData) + if len(licenses) > 0 { + pkg.PackageLicenseConcluded = strings.Join(licenses, ",") } } From c7e91fa1d92f3c37c9933bfaf74ac497bb6dcde5 Mon Sep 17 00:00:00 2001 From: Thomas Schafer Date: Thu, 6 Feb 2025 17:12:00 +0000 Subject: [PATCH 4/8] test: add test to verify spdx behaviour is retained --- lib/ecosystems/enrich_spdx_test.go | 84 +++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 24 deletions(-) diff --git a/lib/ecosystems/enrich_spdx_test.go b/lib/ecosystems/enrich_spdx_test.go index a161bc6..815ec72 100644 --- a/lib/ecosystems/enrich_spdx_test.go +++ b/lib/ecosystems/enrich_spdx_test.go @@ -31,34 +31,20 @@ import ( "github.com/snyk/parlay/lib/sbom" ) -func TestEnrichSBOM_SPDX(t *testing.T) { +func testEnrichSBOM(t *testing.T, ecosysteMsPackageResponse map[string]interface{}, ecosysteMsRegistryResponse map[string]interface{}, assertions func(bom *v2_3.Document)) { httpmock.Activate() defer httpmock.DeactivateAndReset() httpmock.RegisterResponder("GET", `=~^https://packages.ecosyste.ms/api/v1/registries/.*/packages/.*/versions`, func(r *http.Request) (*http.Response, error) { - return httpmock.NewJsonResponse(200, map[string]interface{}{ - // This is the license we expect to see for the specific package version - "licenses": "MIT", - }) + return httpmock.NewJsonResponse(200, ecosysteMsPackageResponse) }, ) httpmock.RegisterResponder("GET", `=~^https://packages.ecosyste.ms/api/v1/registries`, func(req *http.Request) (*http.Response, error) { - return httpmock.NewJsonResponse(200, map[string]interface{}{ - "description": "description", - "normalized_licenses": []string{ - // This license should be ignored as it corresponds to the latest version of the package - "BSD-3-Clause", - }, - "homepage": "https://github.com/spdx/tools-golang", - "repo_metadata": map[string]interface{}{ - "owner_record": map[string]interface{}{ - "name": "Acme Corp", - }, - }, - }) - }) + return httpmock.NewJsonResponse(200, ecosysteMsRegistryResponse) + }, + ) doc, err := sbom.DecodeSBOMDocument([]byte(`{"spdxVersion":"SPDX-2.3","SPDXID":"SPDXRef-DOCUMENT"}`)) require.NoError(t, err) @@ -86,11 +72,7 @@ func TestEnrichSBOM_SPDX(t *testing.T) { pkgs := bom.Packages - assert.Equal(t, "description", pkgs[0].PackageDescription) - assert.Equal(t, "MIT", pkgs[0].PackageLicenseConcluded) - assert.Equal(t, "https://github.com/spdx/tools-golang", pkgs[0].PackageHomePage) - assert.Equal(t, "Organization", pkgs[0].PackageSupplier.SupplierType) - assert.Equal(t, "Acme Corp", pkgs[0].PackageSupplier.Supplier) + assertions(bom) httpmock.GetTotalCallCount() calls := httpmock.GetCallCountInfo() @@ -100,6 +82,60 @@ func TestEnrichSBOM_SPDX(t *testing.T) { require.NoError(t, doc.Encode(buf)) } +func TestEnrichSBOM_SPDX(t *testing.T) { + testEnrichSBOM( + t, + map[string]interface{}{ + "licenses": "MIT", + }, + map[string]interface{}{ + "description": "description", + "normalized_licenses": []string{"BSD-3-Clause"}, + "homepage": "https://github.com/spdx/tools-golang", + "repo_metadata": map[string]interface{}{ + "owner_record": map[string]interface{}{ + "name": "Acme Corp", + }, + }, + }, + func(bom *v2_3.Document) { + pkgs := bom.Packages + assert.Equal(t, "description", pkgs[0].PackageDescription) + assert.Equal(t, "MIT", pkgs[0].PackageLicenseConcluded) + assert.Equal(t, "https://github.com/spdx/tools-golang", pkgs[0].PackageHomePage) + assert.Equal(t, "Organization", pkgs[0].PackageSupplier.SupplierType) + assert.Equal(t, "Acme Corp", pkgs[0].PackageSupplier.Supplier) + }, + ) +} + +func TestEnrichSBOM_MissingVersionedLicense(t *testing.T) { + testEnrichSBOM( + t, + map[string]interface{}{ + "licenses": "", + }, + map[string]interface{}{ + "description": "description", + "normalized_licenses": []string{"BSD-3-Clause", "Apache-2.0"}, + "homepage": "https://github.com/spdx/tools-golang", + "repo_metadata": map[string]interface{}{ + "owner_record": map[string]interface{}{ + "name": "Acme Corp", + }, + }, + }, + func(bom *v2_3.Document) { + pkgs := bom.Packages + assert.Equal(t, "description", pkgs[0].PackageDescription) + assert.Equal(t, "BSD-3-Clause,Apache-2.0", pkgs[0].PackageLicenseConcluded) + assert.Equal(t, "https://github.com/spdx/tools-golang", pkgs[0].PackageHomePage) + assert.Equal(t, "Organization", pkgs[0].PackageSupplier.SupplierType) + assert.Equal(t, "Acme Corp", pkgs[0].PackageSupplier.Supplier) + }, + ) +} + func TestEnrichSBOM_SPDX_NoSupplierName(t *testing.T) { httpmock.Activate() defer httpmock.DeactivateAndReset() From cae78c1cd695fa384f96203d389410b49ce333db Mon Sep 17 00:00:00 2001 From: Thomas Schafer Date: Fri, 7 Feb 2025 09:09:29 +0000 Subject: [PATCH 5/8] docs: update readme with license data info --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 0bed157..d27b1ad 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,10 @@ You can also return raw JSON information about a specific repository: parlay ecosystems repo https://github.com/open-policy-agent/conftest ``` +### License data + +parlay enriches components and packages with their license information from ecosyste.ms on a best-effort basis. It prefers the license data of the package version at hand; however, it may not always be possible to retrieve the license for a specific version (see [ecosyste.ms issue here](https://github.com/ecosyste-ms/packages/issues/1027) for more info). In this case, parlay will fall back to enriching with the license data of the package's latest release. In rare cases — where the licensing model of a package changed over time — this may result in license data inaccuracies. + ## Enriching with Snyk From cc45f98148e8970a850a15dd226706d4ed6946bf Mon Sep 17 00:00:00 2001 From: Thomas Schafer Date: Fri, 7 Feb 2025 09:37:20 +0000 Subject: [PATCH 6/8] chore: don't allocate string slice unnecessarily --- internal/utils/spdx.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/utils/spdx.go b/internal/utils/spdx.go index d6ed141..2942996 100644 --- a/internal/utils/spdx.go +++ b/internal/utils/spdx.go @@ -33,13 +33,12 @@ func GetPurlFromSPDXPackage(pkg *spdx_2_3.Package) (*packageurl.PackageURL, erro } func GetLicensesFromEcosystemsLicense(pkgVersionData *packages.VersionWithDependencies, pkgData *packages.Package) []string { - licenses := []string{} if pkgVersionData != nil && pkgVersionData.Licenses != nil && *pkgVersionData.Licenses != "" { - licenses = strings.Split(*pkgVersionData.Licenses, ",") + return strings.Split(*pkgVersionData.Licenses, ",") } else if pkgData != nil && len(pkgData.NormalizedLicenses) > 0 { - licenses = pkgData.NormalizedLicenses + return pkgData.NormalizedLicenses } - return licenses + return nil } func GetLicenseExpressionFromEcosystemsLicense(pkgVersionData *packages.VersionWithDependencies, pkgData *packages.Package) string { From 85eb8ad1ed20b09859bb403485f7527fef28e3eb Mon Sep 17 00:00:00 2001 From: Thomas Schafer Date: Fri, 7 Feb 2025 13:19:25 +0000 Subject: [PATCH 7/8] style: simplify testing abstraction and pass in raw json string --- lib/ecosystems/enrich_spdx_test.go | 191 +++++++++++++++++------------ 1 file changed, 113 insertions(+), 78 deletions(-) diff --git a/lib/ecosystems/enrich_spdx_test.go b/lib/ecosystems/enrich_spdx_test.go index 815ec72..af245f0 100644 --- a/lib/ecosystems/enrich_spdx_test.go +++ b/lib/ecosystems/enrich_spdx_test.go @@ -18,6 +18,8 @@ package ecosystems import ( "bytes" + "encoding/json" + "fmt" "net/http" "testing" @@ -31,20 +33,52 @@ import ( "github.com/snyk/parlay/lib/sbom" ) -func testEnrichSBOM(t *testing.T, ecosysteMsPackageResponse map[string]interface{}, ecosysteMsRegistryResponse map[string]interface{}, assertions func(bom *v2_3.Document)) { +func parseJson(jsonStr string) map[string]any { + var result map[string]any + + err := json.Unmarshal([]byte(jsonStr), &result) + if err != nil { + panic(fmt.Errorf("failed to parse JSON: %w", err)) + } + + return result +} + +func setupHttpmock(packageVersionsResponse, packageResponse *string) { httpmock.Activate() - defer httpmock.DeactivateAndReset() - httpmock.RegisterResponder("GET", `=~^https://packages.ecosyste.ms/api/v1/registries/.*/packages/.*/versions`, - func(r *http.Request) (*http.Response, error) { - return httpmock.NewJsonResponse(200, ecosysteMsPackageResponse) - }, - ) - httpmock.RegisterResponder("GET", `=~^https://packages.ecosyste.ms/api/v1/registries`, - func(req *http.Request) (*http.Response, error) { - return httpmock.NewJsonResponse(200, ecosysteMsRegistryResponse) - }, - ) + if packageVersionsResponse != nil { + httpmock.RegisterResponder("GET", `=~^https://packages.ecosyste.ms/api/v1/registries/.*/packages/.*/versions`, + func(r *http.Request) (*http.Response, error) { + return httpmock.NewJsonResponse(200, parseJson(*packageVersionsResponse)) + }, + ) + } + + if packageResponse != nil { + httpmock.RegisterResponder("GET", `=~^https://packages.ecosyste.ms/api/v1/registries`, + func(req *http.Request) (*http.Response, error) { + return httpmock.NewJsonResponse(200, parseJson(*packageResponse)) + }) + } +} + +func TestEnrichSBOM_SPDX(t *testing.T) { + packageVersionResponse := `{ + "licenses": "MIT" + }` + packageResponse := `{ + "description": "description", + "normalized_licenses": ["BSD-3-Clause"], + "homepage": "https://github.com/spdx/tools-golang", + "repo_metadata": { + "owner_record": { + "name": "Acme Corp" + } + } + }` + setupHttpmock(&packageVersionResponse, &packageResponse) + defer httpmock.DeactivateAndReset() doc, err := sbom.DecodeSBOMDocument([]byte(`{"spdxVersion":"SPDX-2.3","SPDXID":"SPDXRef-DOCUMENT"}`)) require.NoError(t, err) @@ -72,7 +106,11 @@ func testEnrichSBOM(t *testing.T, ecosysteMsPackageResponse map[string]interface pkgs := bom.Packages - assertions(bom) + assert.Equal(t, "description", pkgs[0].PackageDescription) + assert.Equal(t, "MIT", pkgs[0].PackageLicenseConcluded) + assert.Equal(t, "https://github.com/spdx/tools-golang", pkgs[0].PackageHomePage) + assert.Equal(t, "Organization", pkgs[0].PackageSupplier.SupplierType) + assert.Equal(t, "Acme Corp", pkgs[0].PackageSupplier.Supplier) httpmock.GetTotalCallCount() calls := httpmock.GetCallCountInfo() @@ -82,80 +120,77 @@ func testEnrichSBOM(t *testing.T, ecosysteMsPackageResponse map[string]interface require.NoError(t, doc.Encode(buf)) } -func TestEnrichSBOM_SPDX(t *testing.T) { - testEnrichSBOM( - t, - map[string]interface{}{ - "licenses": "MIT", - }, - map[string]interface{}{ - "description": "description", - "normalized_licenses": []string{"BSD-3-Clause"}, - "homepage": "https://github.com/spdx/tools-golang", - "repo_metadata": map[string]interface{}{ - "owner_record": map[string]interface{}{ - "name": "Acme Corp", - }, - }, - }, - func(bom *v2_3.Document) { - pkgs := bom.Packages - assert.Equal(t, "description", pkgs[0].PackageDescription) - assert.Equal(t, "MIT", pkgs[0].PackageLicenseConcluded) - assert.Equal(t, "https://github.com/spdx/tools-golang", pkgs[0].PackageHomePage) - assert.Equal(t, "Organization", pkgs[0].PackageSupplier.SupplierType) - assert.Equal(t, "Acme Corp", pkgs[0].PackageSupplier.Supplier) - }, - ) -} - func TestEnrichSBOM_MissingVersionedLicense(t *testing.T) { - testEnrichSBOM( - t, - map[string]interface{}{ - "licenses": "", - }, - map[string]interface{}{ - "description": "description", - "normalized_licenses": []string{"BSD-3-Clause", "Apache-2.0"}, - "homepage": "https://github.com/spdx/tools-golang", - "repo_metadata": map[string]interface{}{ - "owner_record": map[string]interface{}{ - "name": "Acme Corp", + packageVersionResponse := `{ + "licenses": "" + }` + packageResponse := `{ + "description": "description", + "normalized_licenses": ["BSD-3-Clause", "Apache-2.0"], + "homepage": "https://github.com/spdx/tools-golang", + "repo_metadata": { + "owner_record": { + "name": "Acme Corp" + } + } + }` + setupHttpmock(&packageVersionResponse, &packageResponse) + defer httpmock.DeactivateAndReset() + + doc, err := sbom.DecodeSBOMDocument([]byte(`{"spdxVersion":"SPDX-2.3","SPDXID":"SPDXRef-DOCUMENT"}`)) + require.NoError(t, err) + + bom, ok := doc.BOM.(*v2_3.Document) + require.True(t, ok) + + bom.Packages = []*v2_3.Package{ + { + PackageSPDXIdentifier: "pkg:golang/github.com/spdx/tools-golang@v0.5.2", + PackageName: "github.com/spdx/tools-golang", + PackageVersion: "v0.5.2", + PackageExternalReferences: []*v2_3.PackageExternalReference{ + { + Category: common.CategoryPackageManager, + RefType: "purl", + Locator: "pkg:golang/github.com/spdx/tools-golang@v0.5.2", }, }, }, - func(bom *v2_3.Document) { - pkgs := bom.Packages - assert.Equal(t, "description", pkgs[0].PackageDescription) - assert.Equal(t, "BSD-3-Clause,Apache-2.0", pkgs[0].PackageLicenseConcluded) - assert.Equal(t, "https://github.com/spdx/tools-golang", pkgs[0].PackageHomePage) - assert.Equal(t, "Organization", pkgs[0].PackageSupplier.SupplierType) - assert.Equal(t, "Acme Corp", pkgs[0].PackageSupplier.Supplier) - }, - ) + } + logger := zerolog.Nop() + + EnrichSBOM(doc, &logger) + + pkgs := bom.Packages + + assert.Equal(t, "description", pkgs[0].PackageDescription) + assert.Equal(t, "BSD-3-Clause,Apache-2.0", pkgs[0].PackageLicenseConcluded) + assert.Equal(t, "https://github.com/spdx/tools-golang", pkgs[0].PackageHomePage) + assert.Equal(t, "Organization", pkgs[0].PackageSupplier.SupplierType) + assert.Equal(t, "Acme Corp", pkgs[0].PackageSupplier.Supplier) + + httpmock.GetTotalCallCount() + calls := httpmock.GetCallCountInfo() + assert.Equal(t, len(pkgs), calls[`GET =~^https://packages.ecosyste.ms/api/v1/registries`]) + + buf := bytes.NewBuffer(nil) + require.NoError(t, doc.Encode(buf)) } func TestEnrichSBOM_SPDX_NoSupplierName(t *testing.T) { - httpmock.Activate() + packageResponse := `{ + "description": "description", + "normalized_licenses": ["BSD-3-Clause"], + "homepage": "https://github.com/spdx/tools-golang", + "repo_metadata": { + "owner_record": { + "name": "" + } + } + }` + setupHttpmock(nil, &packageResponse) defer httpmock.DeactivateAndReset() - httpmock.RegisterResponder("GET", `=~^https://packages.ecosyste.ms/api/v1/registries`, - func(req *http.Request) (*http.Response, error) { - return httpmock.NewJsonResponse(200, map[string]interface{}{ - "description": "description", - "normalized_licenses": []string{ - "BSD-3-Clause", - }, - "homepage": "https://github.com/spdx/tools-golang", - "repo_metadata": map[string]interface{}{ - "owner_record": map[string]interface{}{ - "name": "", - }, - }, - }) - }) - doc, err := sbom.DecodeSBOMDocument([]byte(`{"spdxVersion":"SPDX-2.3","SPDXID":"SPDXRef-DOCUMENT"}`)) require.NoError(t, err) From a0e16d0c666c374d01decefc6f240d16aae640a8 Mon Sep 17 00:00:00 2001 From: Thomas Schafer Date: Fri, 7 Feb 2025 15:12:34 +0000 Subject: [PATCH 8/8] chore: call t.Helper in helper methods, and use require.NoError instead of panicking --- lib/ecosystems/enrich_spdx_test.go | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/lib/ecosystems/enrich_spdx_test.go b/lib/ecosystems/enrich_spdx_test.go index af245f0..e6defb1 100644 --- a/lib/ecosystems/enrich_spdx_test.go +++ b/lib/ecosystems/enrich_spdx_test.go @@ -19,7 +19,6 @@ package ecosystems import ( "bytes" "encoding/json" - "fmt" "net/http" "testing" @@ -33,24 +32,21 @@ import ( "github.com/snyk/parlay/lib/sbom" ) -func parseJson(jsonStr string) map[string]any { +func parseJson(t *testing.T, jsonStr string) map[string]any { + t.Helper() var result map[string]any - - err := json.Unmarshal([]byte(jsonStr), &result) - if err != nil { - panic(fmt.Errorf("failed to parse JSON: %w", err)) - } - + require.NoError(t, json.Unmarshal([]byte(jsonStr), &result)) return result } -func setupHttpmock(packageVersionsResponse, packageResponse *string) { +func setupHttpmock(t *testing.T, packageVersionsResponse, packageResponse *string) { + t.Helper() httpmock.Activate() if packageVersionsResponse != nil { httpmock.RegisterResponder("GET", `=~^https://packages.ecosyste.ms/api/v1/registries/.*/packages/.*/versions`, func(r *http.Request) (*http.Response, error) { - return httpmock.NewJsonResponse(200, parseJson(*packageVersionsResponse)) + return httpmock.NewJsonResponse(200, parseJson(t, *packageVersionsResponse)) }, ) } @@ -58,7 +54,7 @@ func setupHttpmock(packageVersionsResponse, packageResponse *string) { if packageResponse != nil { httpmock.RegisterResponder("GET", `=~^https://packages.ecosyste.ms/api/v1/registries`, func(req *http.Request) (*http.Response, error) { - return httpmock.NewJsonResponse(200, parseJson(*packageResponse)) + return httpmock.NewJsonResponse(200, parseJson(t, *packageResponse)) }) } } @@ -77,7 +73,7 @@ func TestEnrichSBOM_SPDX(t *testing.T) { } } }` - setupHttpmock(&packageVersionResponse, &packageResponse) + setupHttpmock(t, &packageVersionResponse, &packageResponse) defer httpmock.DeactivateAndReset() doc, err := sbom.DecodeSBOMDocument([]byte(`{"spdxVersion":"SPDX-2.3","SPDXID":"SPDXRef-DOCUMENT"}`)) @@ -134,7 +130,7 @@ func TestEnrichSBOM_MissingVersionedLicense(t *testing.T) { } } }` - setupHttpmock(&packageVersionResponse, &packageResponse) + setupHttpmock(t, &packageVersionResponse, &packageResponse) defer httpmock.DeactivateAndReset() doc, err := sbom.DecodeSBOMDocument([]byte(`{"spdxVersion":"SPDX-2.3","SPDXID":"SPDXRef-DOCUMENT"}`)) @@ -188,7 +184,7 @@ func TestEnrichSBOM_SPDX_NoSupplierName(t *testing.T) { } } }` - setupHttpmock(nil, &packageResponse) + setupHttpmock(t, nil, &packageResponse) defer httpmock.DeactivateAndReset() doc, err := sbom.DecodeSBOMDocument([]byte(`{"spdxVersion":"SPDX-2.3","SPDXID":"SPDXRef-DOCUMENT"}`))