Skip to content

Commit

Permalink
feat(RELEASE-1214): add support for spdx sbom format (#307)
Browse files Browse the repository at this point in the history
Build team is moving from cyclonedx to spdx sbom format.
To support both formats until the transition is over:
* The original upload_rpm_data was copied as
  upload_rpm_data_cyclonedx to preserve the original
  behavior for now
* upload_rpm_data was modified to work with spdx format

Signed-off-by: Martin Malina <[email protected]>
  • Loading branch information
mmalina authored Nov 12, 2024
1 parent 61ecbd2 commit 47a7016
Show file tree
Hide file tree
Showing 5 changed files with 892 additions and 139 deletions.
162 changes: 90 additions & 72 deletions pyxis/test_upload_rpm_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,7 @@
get_image_rpm_data,
create_image_rpm_manifest,
update_container_content_sets,
load_sbom_components,
check_bom_ref_duplicates,
load_sbom_packages,
construct_rpm_items_and_content_sets,
)

Expand All @@ -18,38 +17,90 @@
SBOM_PATH = "mypath"
RPM_MANIFEST_ID = "abcd1234"
CONTENT_SETS = ["myrepo1", "myrepo2"]
COMPONENTS = [
PACKAGES = [
{ # all fields
"purl": "pkg:rpm/rhel/[email protected]?arch=x86_64&"
+ "upstream=pkg1-1-2.el8.src.rpm&distro=rhel-8.0&repository_id=myrepo1",
"externalRefs": [
{
"referenceType": "purl",
"referenceLocator": "pkg:rpm/rhel/[email protected]?arch=x86_64&"
+ "upstream=pkg1-1-2.el8.src.rpm&distro=rhel-8.0&repository_id=myrepo1",
}
]
},
{ # no version, same repository_id
"purl": "pkg:rpm/rhel/pkg2?arch=noarch&upstream=pkg2-1-2.el8.src.rpm&distro=rhel-8.0"
+ "&repository_id=myrepo1",
"externalRefs": [
{
"referenceType": "purl",
"referenceLocator": "pkg:rpm/rhel/pkg2?arch=noarch"
+ "&upstream=pkg2-1-2.el8.src.rpm&distro=rhel-8.0&repository_id=myrepo1",
}
]
},
{ # no architecture, different repository_id
"purl": "pkg:rpm/rhel/[email protected]?upstream=pkg3-9-8.el8.src.rpm&distro=rhel-8.0"
+ "&repository_id=myrepo2",
"externalRefs": [
{
"referenceType": "purl",
"referenceLocator": "pkg:rpm/rhel/[email protected]?upstream=pkg3-9-8.el8.src.rpm"
+ "&distro=rhel-8.0&repository_id=myrepo2",
}
]
},
{ # no upstream
"purl": "pkg:rpm/rhel/[email protected]?arch=x86_64&distro=rhel-8.0",
"externalRefs": [
{
"referenceType": "purl",
"referenceLocator": "pkg:rpm/rhel/[email protected]?arch=x86_64&distro=rhel-8.0",
}
]
},
{ # with RH publisher
"purl": "pkg:rpm/rhel/pkg5?arch=noarch&upstream=pkg5-1-2.el8.src.rpm&distro=rhel-8.0",
"publisher": "Red Hat, Inc.",
"externalRefs": [
{
"referenceType": "purl",
"referenceLocator": "pkg:rpm/rhel/pkg5?arch=noarch"
+ "&upstream=pkg5-1-2.el8.src.rpm&distro=rhel-8.0",
}
],
"supplier": "Organization: Red Hat, Inc.",
},
{ # with other publisher
"purl": "pkg:rpm/rhel/pkg6?arch=noarch&upstream=pkg6-1-2.el8.src.rpm&distro=rhel-8.0",
"publisher": "Blue Shoe, inc.",
"externalRefs": [
{
"referenceType": "purl",
"referenceLocator": "pkg:rpm/rhel/pkg6?arch=noarch"
+ "&upstream=pkg6-1-2.el8.src.rpm&distro=rhel-8.0",
}
],
"supplier": "Organization: Blue Shoe, inc.",
},
{ # not an rpm
"purl": "pkg:golang/./staging/src@(devel)#k8s.io/api",
"externalRefs": [
{
"referenceType": "purl",
"referenceLocator": "pkg:golang/./staging/src@(devel)#k8s.io/api",
}
]
},
{}, # no externalRefs
{ # no purl ref
"externalRefs": [
{
"referenceType": "cpe23Type",
"referenceLocator": "cpe:2.3:a:alpine_baselayout:alpine-baselayout:"
+ "3.2.0-r18:*:*:*:*:*:*:*",
},
]
},
{ # no purl
"bom_ref": "ref",
{ # non-rpm purl
"externalRefs": [{"referenceLocator": "pkg:pypi/[email protected]", "referenceType": "purl"}]
},
{ # with redhat namespace, but no publisher
"purl": "pkg:rpm/redhat/[email protected]?arch=noarch",
"externalRefs": [
{
"referenceType": "purl",
"referenceLocator": "pkg:rpm/redhat/[email protected]?arch=noarch",
}
]
},
]

Expand Down Expand Up @@ -135,11 +186,11 @@ def test_upload_container_rpm_data_with_retry__fails_http_other(
@patch("upload_rpm_data.update_container_content_sets")
@patch("upload_rpm_data.create_image_rpm_manifest")
@patch("upload_rpm_data.construct_rpm_items_and_content_sets")
@patch("upload_rpm_data.load_sbom_components")
@patch("upload_rpm_data.load_sbom_packages")
@patch("upload_rpm_data.get_image_rpm_data")
def test_upload_container_rpm_data__success(
mock_get_image_rpm_data,
mock_load_sbom_components,
mock_load_sbom_packages,
mock_construct_rpm_items_and_content_sets,
mock_create_image_rpm_manifest,
mock_update_container_content_sets,
Expand All @@ -154,12 +205,12 @@ def test_upload_container_rpm_data__success(
"content_sets": None,
"rpm_manifest": None,
}
mock_load_sbom_components.return_value = COMPONENTS
mock_load_sbom_packages.return_value = PACKAGES
mock_construct_rpm_items_and_content_sets.return_value = ([{"name": "pkg"}], ["myrepo1"])

upload_container_rpm_data(GRAPHQL_API, IMAGE_ID, SBOM_PATH)

mock_construct_rpm_items_and_content_sets.assert_called_once_with(COMPONENTS)
mock_construct_rpm_items_and_content_sets.assert_called_once_with(PACKAGES)
mock_create_image_rpm_manifest.assert_called_once_with(
GRAPHQL_API,
IMAGE_ID,
Expand All @@ -175,11 +226,11 @@ def test_upload_container_rpm_data__success(
@patch("upload_rpm_data.update_container_content_sets")
@patch("upload_rpm_data.create_image_rpm_manifest")
@patch("upload_rpm_data.construct_rpm_items_and_content_sets")
@patch("upload_rpm_data.load_sbom_components")
@patch("upload_rpm_data.load_sbom_packages")
@patch("upload_rpm_data.get_image_rpm_data")
def test_upload_container_rpm_data__data_already_exists(
mock_get_image_rpm_data,
mock_load_sbom_components,
mock_load_sbom_packages,
mock_construct_rpm_items_and_content_sets,
mock_create_image_rpm_manifest,
mock_update_container_content_sets,
Expand All @@ -193,13 +244,13 @@ def test_upload_container_rpm_data__data_already_exists(
"content_sets": CONTENT_SETS,
"rpm_manifest": {"_id": RPM_MANIFEST_ID},
}
mock_load_sbom_components.return_value = COMPONENTS
mock_load_sbom_packages.return_value = PACKAGES
mock_construct_rpm_items_and_content_sets.return_value = ([{"name": "pkg"}], CONTENT_SETS)

upload_container_rpm_data(GRAPHQL_API, IMAGE_ID, SBOM_PATH)

mock_load_sbom_components.assert_called_once()
mock_construct_rpm_items_and_content_sets.assert_called_once_with(COMPONENTS)
mock_load_sbom_packages.assert_called_once()
mock_construct_rpm_items_and_content_sets.assert_called_once_with(PACKAGES)
mock_create_image_rpm_manifest.assert_not_called()
mock_update_container_content_sets.assert_not_called()

Expand Down Expand Up @@ -293,78 +344,45 @@ def test_update_container_content_sets__error(mock_post):


@patch("json.load")
@patch("upload_rpm_data.check_bom_ref_duplicates")
@patch("builtins.open")
def test_load_sbom_components__success(mock_open, mock_check_bom_ref_duplicates, mock_load):
fake_components = [1, 2, 3, 4]
mock_load.return_value = {"components": fake_components}
def test_load_sbom_packages__success(mock_open, mock_load):
fake_packages = [1, 2, 3, 4]
mock_load.return_value = {"packages": fake_packages}

loaded_components = load_sbom_components(SBOM_PATH)
loaded_packages = load_sbom_packages(SBOM_PATH)

mock_load.assert_called_once_with(mock_open.return_value.__enter__.return_value)
mock_check_bom_ref_duplicates.assert_called_once_with(loaded_components)
assert fake_components == loaded_components
assert fake_packages == loaded_packages


@patch("json.load")
@patch("upload_rpm_data.check_bom_ref_duplicates")
@patch("builtins.open")
def test_load_sbom_components__no_components_key(
mock_open, mock_check_bom_ref_duplicates, mock_load
):
def test_load_sbom_packages__no_components_key(mock_open, mock_load):
mock_load.return_value = {}

loaded_components = load_sbom_components(SBOM_PATH)
loaded_components = load_sbom_packages(SBOM_PATH)

mock_load.assert_called_once_with(mock_open.return_value.__enter__.return_value)
mock_check_bom_ref_duplicates.assert_called_once_with(loaded_components)
assert loaded_components == []


@patch("json.load")
@patch("upload_rpm_data.check_bom_ref_duplicates")
@patch("builtins.open")
def test_load_sbom_components__json_load_fails(
mock_open, mock_check_bom_ref_duplicates, mock_load
):
def test_load_sbom_packages__json_load_fails(mock_open, mock_load):
mock_load.side_effect = ValueError

with pytest.raises(ValueError):
load_sbom_components(SBOM_PATH)
load_sbom_packages(SBOM_PATH)

mock_load.assert_called_once_with(mock_open.return_value.__enter__.return_value)
mock_check_bom_ref_duplicates.assert_not_called()


def test_check_bom_ref_duplicates__no_duplicates():
components = [
{"bom-ref": "a"},
{"bom-ref": "b"},
{},
{"bom-ref": "c"},
]

check_bom_ref_duplicates(components)


def test_check_bom_ref_duplicates__duplicates_found():
components = [
{"bom-ref": "a"},
{"bom-ref": "b"},
{},
{"bom-ref": "a"},
]

with pytest.raises(ValueError):
check_bom_ref_duplicates(components)


def test_construct_rpm_items_and_content_sets__success():
"""Only rpm purls are added, the version, release,
and architecture fields are added if present.
All unique repository_id values are returned."""

rpms, content_sets = construct_rpm_items_and_content_sets(COMPONENTS)
rpms, content_sets = construct_rpm_items_and_content_sets(PACKAGES)

assert rpms == [
{
Expand Down Expand Up @@ -426,8 +444,8 @@ def test_construct_rpm_items_and_content_sets__success():
assert content_sets == ["myrepo1", "myrepo2"]


def test_construct_rpm_items_and_content_sets__no_components_result_in_empty_list():
"""An empty list of components results in an empty list of rpms and content_sets"""
def test_construct_rpm_items_and_content_sets__no_packages_result_in_empty_list():
"""An empty list of packages results in an empty list of rpms and content_sets"""
rpms, content_sets = construct_rpm_items_and_content_sets([])

assert rpms == []
Expand Down
Loading

0 comments on commit 47a7016

Please sign in to comment.