diff --git a/docs/generic.md b/docs/generic.md index 6c9377bb4..0227119b1 100644 --- a/docs/generic.md +++ b/docs/generic.md @@ -1,6 +1,7 @@ # Generic fetcher - [Specifying artifacts to fetch](#specifying-artifacts-to-fetch) +- [Authentication](#authentication) - [Using fetched dependencies](#using-fetched-dependencies) - [Full example walkthrough](#example) @@ -24,12 +25,13 @@ Below are sections for each type of supported artifact. Several artifacts of different types can be specified in a single lockfile. The lockfile must always contain a `metadata` header and a list of `artifacts`. -Currently, the only supported version is 1.0: +Supported versions are 1.0 and 2.0. Version 2.0 is required when using +[authentication](#authentication): ```yaml --- metadata: - version: "1.0" + version: "1.0" # or "2.0" for auth support artifacts: [] ``` @@ -129,6 +131,96 @@ These files will be reported with `pkg:maven` purl in the output SBOM, because the URL is fully assembled from the provided attributes and therefore the file can be assumed to be a maven artifact. +## Authentication + +The generic fetcher supports per-artifact authentication for downloading from +private repositories and registries. Authentication requires lockfile version +`"2.0"`. Artifacts without `auth` do not require authentication and will still +use `.netrc` credentials if available. + +### Auth types + +Each artifact can specify an `auth` block with exactly one auth type. + +#### Bearer token + +Header-based token authentication. Supports most platforms (GitHub, GitLab, +Gitea, JFrog Artifactory, etc.). + +| Field | Required | Description | +|----------|----------|-----------------------------------------------------------------------------| +| `header` | No | HTTP header name. Defaults to `Authorization` | +| `value` | Yes | Header value, supports `$VAR` / `${VAR}` environment variable interpolation | + +#### HTTP Basic + +Username and password authentication encoded as a Base64 `Authorization` header. + +| Field | Required | Description | +|------------|----------|----------------------------------------------------| +| `username` | Yes | Username, supports `$VAR` / `${VAR}` interpolation | +| `password` | Yes | Password, supports `$VAR` / `${VAR}` interpolation | + +### Environment variable interpolation + +Secret values should be provided via environment variables using `$VAR` or +`${VAR}` syntax. Hermeto will fail with a clear error if any referenced variable +is not set. Use `\$` for a literal dollar sign. + +### Examples + +**GitLab** (custom `PRIVATE-TOKEN` header): + +```yaml +metadata: + version: "2.0" +artifacts: + - download_url: "https://gitlab.example.com/api/v4/projects/123/repository/archive.tar.gz" + checksum: "sha256:abc123..." + auth: + bearer: + header: PRIVATE-TOKEN + value: "$GITLAB_TOKEN" +``` + +**GitHub** (standard Bearer token): + +```yaml +metadata: + version: "2.0" +artifacts: + - download_url: "https://api.github.com/repos/owner/repo/tarball/v1.0.0" + checksum: "sha256:abc123..." + auth: + bearer: + value: "Bearer $GITHUB_TOKEN" +``` + +**Mixed** (authenticated and public artifacts): + +```yaml +metadata: + version: "2.0" +artifacts: + - download_url: "https://gitlab.example.com/api/v4/projects/123/repository/archive.tar.gz" + checksum: "sha256:..." + auth: + bearer: + header: PRIVATE-TOKEN + value: "$GITLAB_TOKEN" + + - download_url: "https://example.com/public-file.zip" + checksum: "sha256:..." +``` + +Then run hermeto with the required environment variables set: + +```shell +export GITLAB_TOKEN="glpat-xxxxxxxxxxxxxxxxxxxx" +export GITHUB_TOKEN="github_pat_xxxxxxxxxxxxxxxxxxxxx" +hermeto fetch-deps generic +``` + ## Using fetched dependencies Hermeto downloads the files into the `deps/generic/` subpath of the output diff --git a/hermeto/core/package_managers/general.py b/hermeto/core/package_managers/general.py index dc57c04ba..f5301c6bc 100644 --- a/hermeto/core/package_managers/general.py +++ b/hermeto/core/package_managers/general.py @@ -72,7 +72,7 @@ async def _async_download_binary_file( session: aiohttp_retry.RetryClient, url: str, download_path: StrPath, - auth: aiohttp.BasicAuth | None = None, + headers: dict[str, str] | None = None, ssl_context: ssl.SSLContext | None = None, chunk_size: int = 8192, ) -> None: @@ -95,7 +95,7 @@ async def _async_download_binary_file( async with session.get( url, timeout=timeout, - auth=auth, + headers=headers, raise_for_status=True, ssl=ssl_context, ) as resp: @@ -120,14 +120,14 @@ async def async_download_files( files_to_download: Mapping[str, StrPath], concurrency_limit: int, ssl_context: ssl.SSLContext | None = None, - auth: aiohttp.BasicAuth | None = None, + headers: Mapping[str, dict[str, str]] | None = None, ) -> None: """Asynchronous function to download files. - :param files_to_download: Mapping of URLs and file paths to download. + :param files_to_download: Mapping of URLs to file paths to download. :param concurrency_limit: Max number of concurrent tasks (downloads). :param ssl_context: Optional SSL context for the requests. - :param auth: Optional authorization data for proxies. + :param headers: Optional per-URL headers mapping (URL -> headers dict). """ trace_config = aiohttp.TraceConfig() num_attempts: int = int(DEFAULT_RETRY_OPTIONS["total"]) @@ -171,7 +171,7 @@ async def async_download_files( url, download_path, ssl_context=ssl_context, - auth=auth, + headers=headers.get(url) if headers else None, ) ) ) diff --git a/hermeto/core/package_managers/generic/main.py b/hermeto/core/package_managers/generic/main.py index e023231d2..f85a78bd6 100644 --- a/hermeto/core/package_managers/generic/main.py +++ b/hermeto/core/package_managers/generic/main.py @@ -14,13 +14,16 @@ from hermeto.core.models.output import RequestOutput from hermeto.core.models.sbom import Component, create_backend_annotation from hermeto.core.package_managers.general import async_download_files -from hermeto.core.package_managers.generic.models import GenericLockfileV1 +from hermeto.core.package_managers.generic.models import GenericLockfile, GenericLockfileAdapter from hermeto.core.rooted_path import RootedPath log = logging.getLogger(__name__) DEFAULT_LOCKFILE_NAME = "artifacts.lock.yaml" DEFAULT_DEPS_DIR = "deps/generic" +Url = str +AuthHeaders = dict[str, str] + def fetch_generic_source(request: Request) -> RequestOutput: """ @@ -82,14 +85,25 @@ def _resolve_generic_lockfile(lockfile_path: Path, output_dir: RootedPath) -> li log.info(f"Reading generic lockfile: {lockfile_path}") lockfile = _load_lockfile(lockfile_path, output_dir) - to_download: dict[str, str | os.PathLike[str]] = {} + to_download: dict[Url, str | os.PathLike[str]] = {} + auth_headers: dict[Url, AuthHeaders] = {} for artifact in lockfile.artifacts: # create the parent directory for the artifact Path.mkdir(Path(artifact.filename).parent, parents=True, exist_ok=True) - to_download[str(artifact.download_url)] = artifact.filename - - asyncio.run(async_download_files(to_download, get_config().runtime.concurrency_limit)) + url = str(artifact.download_url) + to_download[url] = artifact.filename + auth = getattr(artifact, "auth", None) + if auth: + auth_headers[url] = auth.get_headers() + + asyncio.run( + async_download_files( + to_download, + get_config().runtime.concurrency_limit, + headers=auth_headers or None, + ) + ) # verify checksums for artifact in lockfile.artifacts: @@ -97,7 +111,7 @@ def _resolve_generic_lockfile(lockfile_path: Path, output_dir: RootedPath) -> li return [artifact.get_sbom_component() for artifact in lockfile.artifacts] -def _load_lockfile(lockfile_path: Path, output_dir: RootedPath) -> GenericLockfileV1: +def _load_lockfile(lockfile_path: Path, output_dir: RootedPath) -> GenericLockfile: """ Load the generic lockfile from the given path. @@ -115,7 +129,7 @@ def _load_lockfile(lockfile_path: Path, output_dir: RootedPath) -> GenericLockfi ) try: - lockfile = GenericLockfileV1.model_validate( + lockfile = GenericLockfileAdapter.validate_python( lockfile_data, context={"output_dir": output_dir} ) except ValidationError as e: diff --git a/hermeto/core/package_managers/generic/models.py b/hermeto/core/package_managers/generic/models.py index f8d8b8548..fc020c6dc 100644 --- a/hermeto/core/package_managers/generic/models.py +++ b/hermeto/core/package_managers/generic/models.py @@ -1,10 +1,14 @@ # SPDX-License-Identifier: GPL-3.0-only +import os import re from abc import ABC, abstractmethod +from base64 import b64encode from collections import Counter +from collections.abc import Sequence from functools import cached_property from pathlib import Path -from typing import Annotated, Literal +from string import Template +from typing import Annotated, Any, Literal, Optional, Union from urllib.parse import urljoin, urlparse from packageurl import PackageURL @@ -12,11 +16,15 @@ AnyUrl, BaseModel, ConfigDict, + Discriminator, PlainSerializer, + Tag, + TypeAdapter, field_validator, model_validator, ) from pydantic_core.core_schema import ValidationInfo +from typing_extensions import override from hermeto.core.checksum import ChecksumInfo from hermeto.core.errors import PackageManagerError @@ -26,6 +34,78 @@ CHECKSUM_FORMAT = re.compile(r"^[a-zA-Z0-9]+:[a-zA-Z0-9]+$") +class AbstractAuth(BaseModel, ABC): + """Abstract base class for artifact authentication.""" + + model_config = ConfigDict(extra="forbid") + + @abstractmethod + def to_headers(self) -> dict[str, str]: + """Return the authentication headers.""" + + @model_validator(mode="after") + def validate_headers(self) -> "AbstractAuth": + """Validate that the headers are not empty.""" + if not self.to_headers(): + raise ValueError("Headers cannot be empty") + return self + + @field_validator("*") + @classmethod + def _expand_env_vars_in_fields(cls, value: Any) -> Any: + """Expand environment variables in the fields.""" + + if not isinstance(value, str): + return value + + return Template(value).substitute(os.environ) + + +class BasicAuth(AbstractAuth): + """Defines format of the basic auth section in the lockfile.""" + + username: str + password: str + + @override + def to_headers(self) -> dict[str, str]: + encoded = b64encode(f"{self.username}:{self.password}".encode()).decode() + return {"Authorization": f"Basic {encoded}"} + + +class BearerAuth(AbstractAuth): + """Defines format of the bearer auth section in the lockfile.""" + + header: Optional[str] = None + value: str + + @override + def to_headers(self) -> dict[str, str]: + return {self.header or "Authorization": self.value} + + +class LockfileArtifactAuth(BaseModel): + """Defines format of the auth section in the lockfile.""" + + basic: Optional[BasicAuth] = None + bearer: Optional[BearerAuth] = None + model_config = ConfigDict(extra="forbid") + + @model_validator(mode="before") + @classmethod + def _check_mutually_exclusive(cls, values: dict) -> dict: + if ("basic" not in values and "bearer" not in values) or ( + "basic" in values and "bearer" in values + ): + raise ValueError("Exactly one of the auth types must be set") + return values + + def get_headers(self) -> dict[str, str]: + """Return the headers for the artifact.""" + auth = self.basic or self.bearer + return auth.to_headers() if auth else {} + + class LockfileMetadata(BaseModel): """Defines format of the metadata section in the lockfile.""" @@ -33,6 +113,13 @@ class LockfileMetadata(BaseModel): model_config = ConfigDict(extra="forbid") +class LockfileMetadataV2(BaseModel): + """Defines format of the metadata section in the lockfile, version 2.0.""" + + version: Literal["2.0"] + model_config = ConfigDict(extra="forbid") + + class LockfileArtifactBase(BaseModel, ABC): """ Base class for artifacts in the lockfile. @@ -125,6 +212,12 @@ def get_sbom_component(self) -> Component: return component +class LockfileArtifactUrlV2(LockfileArtifactUrl): + """V2 URL artifact that supports optional authentication.""" + + auth: Optional[LockfileArtifactAuth] = None + + class LockfileArtifactMavenAttributes(BaseModel): """Attributes for a Maven artifact in the lockfile.""" @@ -217,6 +310,27 @@ def get_sbom_component(self) -> Component: ) +class LockfileArtifactMavenV2(LockfileArtifactMaven): + """V2 Maven artifact that supports optional authentication.""" + + auth: Optional[LockfileArtifactAuth] = None + + +def _validate_no_artifact_conflicts( + artifacts: Sequence[LockfileArtifactUrl | LockfileArtifactMaven], +) -> None: + """Validate that all artifacts have unique filenames and download_urls.""" + urls = Counter(a.download_url for a in artifacts) + filenames = Counter(a.filename for a in artifacts) + duplicate_urls = [str(u) for u, count in urls.most_common() if count > 1] + duplicate_filenames = [t for t, count in filenames.most_common() if count > 1] + if duplicate_urls or duplicate_filenames: + raise ValueError( + (f"Duplicate download_urls: {duplicate_urls}\n" if duplicate_urls else "") + + (f"Duplicate filenames: {duplicate_filenames}" if duplicate_filenames else "") + ) + + class GenericLockfileV1(BaseModel): """Defines format of our generic lockfile, version 1.0.""" @@ -227,14 +341,31 @@ class GenericLockfileV1(BaseModel): @model_validator(mode="after") def no_artifact_conflicts(self) -> "GenericLockfileV1": """Validate that all artifacts have unique filenames and download_urls.""" - urls = Counter(a.download_url for a in self.artifacts) - filenames = Counter(a.filename for a in self.artifacts) - duplicate_urls = [str(u) for u, count in urls.most_common() if count > 1] - duplicate_filenames = [t for t, count in filenames.most_common() if count > 1] - if duplicate_urls or duplicate_filenames: - raise ValueError( - (f"Duplicate download_urls: {duplicate_urls}\n" if duplicate_urls else "") - + (f"Duplicate filenames: {duplicate_filenames}" if duplicate_filenames else "") - ) + _validate_no_artifact_conflicts(self.artifacts) + return self + +class GenericLockfileV2(BaseModel): + """Defines format of our generic lockfile, version 2.0.""" + + metadata: LockfileMetadataV2 + artifacts: list[LockfileArtifactUrlV2 | LockfileArtifactMavenV2] + model_config = ConfigDict(extra="forbid") + + @model_validator(mode="after") + def no_artifact_conflicts(self) -> "GenericLockfileV2": + """Validate that all artifacts have unique filenames and download_urls.""" + _validate_no_artifact_conflicts(self.artifacts) return self + + +GenericLockfile = Annotated[ + Union[ + Annotated[GenericLockfileV1, Tag("1.0")], + Annotated[GenericLockfileV2, Tag("2.0")], + ], + Discriminator(lambda x: x.get("metadata", {}).get("version")), +] +GenericLockfileAdapter: TypeAdapter[GenericLockfileV1 | GenericLockfileV2] = TypeAdapter( + GenericLockfile +) diff --git a/hermeto/core/package_managers/npm/resolver.py b/hermeto/core/package_managers/npm/resolver.py index c73b1e04f..10231dd95 100644 --- a/hermeto/core/package_managers/npm/resolver.py +++ b/hermeto/core/package_managers/npm/resolver.py @@ -78,13 +78,19 @@ async def _async_download_tar(files_to_download_list: list[dict[str, dict[str, A ftdl = [e for e in files_to_download_list if e] if not ftdl: return - # NOTE: when present proxy auth is the same for all packages accessible - # through a proxy. - auth = lambda ftd: next(iter(ftd.values()))["proxy_auth"] + + def headers(ftd: dict[str, dict[str, Any]]) -> dict[str, dict[str, str]] | None: + # NOTE: when present proxy auth is the same for all packages accessible + # through a proxy. + auth = next(iter(ftd.values()))["proxy_auth"] + if not auth: + return None + return {it["fetch_url"]: {"Authorization": auth.encode()} for it in ftd.values()} + ftd = lambda ftd: {it["fetch_url"]: it["download_path"] for it in ftd.values()} adf = partial(async_download_files, concurrency_limit=get_config().runtime.concurrency_limit) - await asyncio.gather(*[adf(files_to_download=ftd(f), auth=auth(f)) for f in ftdl]) + await asyncio.gather(*[adf(files_to_download=ftd(f), headers=headers(f)) for f in ftdl]) def _get_npm_dependencies( diff --git a/tests/integration/test_data/generic_e2e_basic_auth/.build-config.yaml b/tests/integration/test_data/generic_e2e_basic_auth/.build-config.yaml new file mode 100644 index 000000000..ecfb4f6a7 --- /dev/null +++ b/tests/integration/test_data/generic_e2e_basic_auth/.build-config.yaml @@ -0,0 +1,2 @@ +environment_variables: [] +project_files: [] diff --git a/tests/integration/test_data/generic_e2e_basic_auth/bom.json b/tests/integration/test_data/generic_e2e_basic_auth/bom.json new file mode 100644 index 000000000..47dbcf709 --- /dev/null +++ b/tests/integration/test_data/generic_e2e_basic_auth/bom.json @@ -0,0 +1,47 @@ +{ + "annotations": [ + { + "annotator": { + "organization": { + "name": "red hat" + } + }, + "subjects": [ + "pkg:generic/six-1.17.0-py2.py3-none-any.whl?checksum=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274&download_url=https://127.0.0.1:8443/repository/pypi-proxy/packages/six/1.17.0/six-1.17.0-py2.py3-none-any.whl" + ], + "text": "hermeto:backend:generic", + "timestamp": "2025-01-01T00:00:00Z" + } + ], + "bomFormat": "CycloneDX", + "components": [ + { + "bom-ref": "pkg:generic/six-1.17.0-py2.py3-none-any.whl?checksum=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274&download_url=https://127.0.0.1:8443/repository/pypi-proxy/packages/six/1.17.0/six-1.17.0-py2.py3-none-any.whl", + "externalReferences": [ + { + "type": "distribution", + "url": "https://127.0.0.1:8443/repository/pypi-proxy/packages/six/1.17.0/six-1.17.0-py2.py3-none-any.whl" + } + ], + "name": "six-1.17.0-py2.py3-none-any.whl", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:generic/six-1.17.0-py2.py3-none-any.whl?checksum=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274&download_url=https://127.0.0.1:8443/repository/pypi-proxy/packages/six/1.17.0/six-1.17.0-py2.py3-none-any.whl", + "type": "file" + } + ], + "metadata": { + "tools": [ + { + "name": "hermeto", + "vendor": "red hat" + } + ] + }, + "specVersion": "1.6", + "version": 1 +} diff --git a/tests/integration/test_data/generic_e2e_bearer_auth/.build-config.yaml b/tests/integration/test_data/generic_e2e_bearer_auth/.build-config.yaml new file mode 100644 index 000000000..ecfb4f6a7 --- /dev/null +++ b/tests/integration/test_data/generic_e2e_bearer_auth/.build-config.yaml @@ -0,0 +1,2 @@ +environment_variables: [] +project_files: [] diff --git a/tests/integration/test_data/generic_e2e_bearer_auth/bom.json b/tests/integration/test_data/generic_e2e_bearer_auth/bom.json new file mode 100644 index 000000000..da98e8961 --- /dev/null +++ b/tests/integration/test_data/generic_e2e_bearer_auth/bom.json @@ -0,0 +1,47 @@ +{ + "annotations": [ + { + "annotator": { + "organization": { + "name": "red hat" + } + }, + "subjects": [ + "pkg:generic/six-1.17.0-py2.py3-none-any.whl?checksum=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274&download_url=https://127.0.0.1:8445/repository/pypi-proxy/packages/six/1.17.0/six-1.17.0-py2.py3-none-any.whl" + ], + "text": "hermeto:backend:generic", + "timestamp": "2025-01-01T00:00:00Z" + } + ], + "bomFormat": "CycloneDX", + "components": [ + { + "bom-ref": "pkg:generic/six-1.17.0-py2.py3-none-any.whl?checksum=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274&download_url=https://127.0.0.1:8445/repository/pypi-proxy/packages/six/1.17.0/six-1.17.0-py2.py3-none-any.whl", + "externalReferences": [ + { + "type": "distribution", + "url": "https://127.0.0.1:8445/repository/pypi-proxy/packages/six/1.17.0/six-1.17.0-py2.py3-none-any.whl" + } + ], + "name": "six-1.17.0-py2.py3-none-any.whl", + "properties": [ + { + "name": "hermeto:found_by", + "value": "hermeto" + } + ], + "purl": "pkg:generic/six-1.17.0-py2.py3-none-any.whl?checksum=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274&download_url=https://127.0.0.1:8445/repository/pypi-proxy/packages/six/1.17.0/six-1.17.0-py2.py3-none-any.whl", + "type": "file" + } + ], + "metadata": { + "tools": [ + { + "name": "hermeto", + "vendor": "red hat" + } + ] + }, + "specVersion": "1.6", + "version": 1 +} diff --git a/tests/integration/test_generic.py b/tests/integration/test_generic.py index 25c0aede3..d8109a77f 100644 --- a/tests/integration/test_generic.py +++ b/tests/integration/test_generic.py @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-3.0-only +import os from pathlib import Path import pytest @@ -100,3 +101,69 @@ def test_e2e_generic( expected_cmd_output, hermeto_image, ) + + +@pytest.mark.parametrize( + "test_params", + [ + pytest.param( + utils.TestParameters( + branch="generic/e2e-basic-auth", + packages=({"path": ".", "type": "generic"},), + check_output=True, + expected_output="All dependencies fetched successfully", + ), + id="generic_e2e_basic_auth", + marks=pytest.mark.skipif( + os.getenv("HERMETO_TEST_LOCAL_NEXUS") != "1", + reason="HERMETO_TEST_LOCAL_NEXUS!=1", + ), + ), + pytest.param( + utils.TestParameters( + branch="generic/e2e-bearer-auth", + packages=({"path": ".", "type": "generic"},), + check_output=True, + expected_output="All dependencies fetched successfully", + ), + id="generic_e2e_bearer_auth", + marks=pytest.mark.skipif( + os.getenv("HERMETO_TEST_LOCAL_NEXUS") != "1", + reason="HERMETO_TEST_LOCAL_NEXUS!=1", + ), + ), + pytest.param( + utils.TestParameters( + branch="generic/e2e-auth-wrong-creds", + packages=({"path": ".", "type": "generic"},), + check_output=False, + expected_error=ExitError.ERR_FETCH, + expected_output="401", + ), + id="generic_e2e_auth_wrong_creds", + marks=pytest.mark.skipif( + os.getenv("HERMETO_TEST_LOCAL_NEXUS") != "1", + reason="HERMETO_TEST_LOCAL_NEXUS!=1", + ), + ), + ], +) +def test_generic_auth( + test_params: utils.TestParameters, + hermeto_image: utils.HermetoImage, + tmp_path: Path, + test_repo_dir: Path, + test_data_dir: Path, + request: pytest.FixtureRequest, +) -> None: + """ + Test generic fetcher with authentication from the lockfile. + + :param test_params: Test case arguments + :param tmp_path: Temp directory for pytest + """ + test_case = request.node.callspec.id + + utils.fetch_deps_and_check_output( + tmp_path, test_case, test_params, test_repo_dir, test_data_dir, hermeto_image + ) diff --git a/tests/nexusserver/bearer_token b/tests/nexusserver/bearer_token new file mode 100644 index 000000000..b2de42b22 --- /dev/null +++ b/tests/nexusserver/bearer_token @@ -0,0 +1 @@ +set $bearer_token "SecretToken"; diff --git a/tests/nexusserver/docker-compose.yml b/tests/nexusserver/docker-compose.yml index fa827bce5..b9cfeb216 100644 --- a/tests/nexusserver/docker-compose.yml +++ b/tests/nexusserver/docker-compose.yml @@ -19,9 +19,11 @@ services: ports: - "8443:443" - "8444:444" + - "8445:445" volumes: - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro,z - ./htpasswd:/etc/nginx/auth/htpasswd:ro,z + - ./bearer_token:/etc/nginx/auth/bearer_token:ro,z - ../certificates/server.crt:/etc/nginx/certs/server.crt:ro,z - ../certificates/server.key:/etc/nginx/certs/server.key:ro,z - ../certificates/CA.crt:/etc/nginx/certs/CA.crt:ro,z diff --git a/tests/nexusserver/nginx.conf b/tests/nexusserver/nginx.conf index b07d0a974..f79945708 100644 --- a/tests/nexusserver/nginx.conf +++ b/tests/nexusserver/nginx.conf @@ -36,3 +36,26 @@ server { proxy_set_header X-Forwarded-Proto $scheme; } } + +server { + listen 445 ssl; + server_name 127.0.0.1; + + ssl_certificate /etc/nginx/certs/server.crt; + ssl_certificate_key /etc/nginx/certs/server.key; + + location / { + include /etc/nginx/auth/bearer_token; + + if ($http_authorization != "Bearer $bearer_token") { + return 401; + } + + proxy_pass http://nexus:8081; + proxy_set_header Authorization ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} diff --git a/tests/unit/package_managers/npm/test_resolver.py b/tests/unit/package_managers/npm/test_resolver.py index 05b2eee84..42643b29d 100644 --- a/tests/unit/package_managers/npm/test_resolver.py +++ b/tests/unit/package_managers/npm/test_resolver.py @@ -1024,7 +1024,7 @@ def test_npm_proxy_credentials_do_not_propagate_to_nonregistry_hosts( _get_npm_dependencies(rooted_tmp_path, deps_to_download) for call in mock_async_download_files.mock_calls: - assert call.kwargs["auth"] is None, "Found credentials where they should not be!" + assert call.kwargs["headers"] is None, "Found credentials where they should not be!" @pytest.mark.parametrize( @@ -1083,7 +1083,7 @@ def test_npm_proxy_credentials_propagate_to_registry_hosts( msg = "Not found credentials where they should be!" for call in mock_async_download_files.mock_calls: - assert call.kwargs["auth"] is not None, msg + assert call.kwargs["headers"] is not None, msg @pytest.mark.parametrize( diff --git a/tests/unit/package_managers/test_general.py b/tests/unit/package_managers/test_general.py index 163c20d79..6b2947728 100644 --- a/tests/unit/package_managers/test_general.py +++ b/tests/unit/package_managers/test_general.py @@ -180,7 +180,7 @@ async def mock_aenter() -> MagicMock: assert session.get.call_args == mock.call( url, timeout=aiohttp.ClientTimeout(total=None, connect=30, sock_read=300), - auth=None, + headers=None, raise_for_status=True, ssl=None, ) diff --git a/tests/unit/package_managers/test_generic.py b/tests/unit/package_managers/test_generic.py index 9a371ef41..2ea0114cb 100644 --- a/tests/unit/package_managers/test_generic.py +++ b/tests/unit/package_managers/test_generic.py @@ -4,6 +4,7 @@ from unittest import mock import pytest +from pydantic import ValidationError from hermeto import APP_NAME from hermeto.core.errors import ( @@ -23,6 +24,11 @@ _resolve_lockfile_path, fetch_generic_source, ) +from hermeto.core.package_managers.generic.models import ( + BasicAuth, + BearerAuth, + LockfileArtifactAuth, +) from hermeto.core.rooted_path import RootedPath LOCKFILE_WRONG_VERSION = """ @@ -125,6 +131,83 @@ checksum: md5:32112bed1914cfe3799600f962750b1d """ +LOCKFILE_WITH_BASIC_AUTH = """ +metadata: + version: '2.0' +artifacts: + - download_url: https://example.com/artifact + filename: archive.zip + checksum: md5:3a18656e1cea70504b905836dee14db0 + auth: + basic: + username: user + password: passwd +""" + +LOCKFILE_WITH_BEARER_AUTH = """ +metadata: + version: '2.0' +artifacts: + - download_url: https://example.com/artifact + filename: archive.zip + checksum: md5:3a18656e1cea70504b905836dee14db0 + auth: + bearer: + value: "Bearer secret123" +""" + +LOCKFILE_WITH_CUSTOM_HEADER_BEARER_AUTH = """ +metadata: + version: '2.0' +artifacts: + - download_url: https://example.com/artifact + filename: archive.zip + checksum: md5:3a18656e1cea70504b905836dee14db0 + auth: + bearer: + header: X-Custom-Auth + value: secret123 +""" + +LOCKFILE_WITH_MIXED_AUTH = """ +metadata: + version: '2.0' +artifacts: + - download_url: https://example.com/authed + filename: authed.zip + checksum: md5:3a18656e1cea70504b905836dee14db0 + auth: + basic: + username: user + password: passwd + - download_url: https://example.com/public + filename: public.zip + checksum: md5:32112bed1914cfe3799600f962750b1d +""" + +LOCKFILE_V2_VALID = """ +metadata: + version: '2.0' +artifacts: + - download_url: https://example.com/artifact + filename: archive.zip + checksum: md5:3a18656e1cea70504b905836dee14db0 + - download_url: https://example.com/more/complex/path/file.tar.gz?foo=bar#fragment + checksum: md5:32112bed1914cfe3799600f962750b1d +""" + +LOCKFILE_V1_WITH_AUTH = """ +metadata: + version: '1.0' +artifacts: + - download_url: https://example.com/artifact + filename: archive.zip + checksum: md5:3a18656e1cea70504b905836dee14db0 + auth: + bearer: + value: my-token +""" + @pytest.mark.parametrize( ["model_input", "components"], @@ -251,6 +334,11 @@ def test_resolve_generic_no_lockfile(mock_load: mock.Mock, rooted_tmp_path: Root InvalidLockfileFormat, id="wrong_checksum_format", ), + pytest.param( + LOCKFILE_V1_WITH_AUTH, + InvalidLockfileFormat, + id="v1_rejects_auth", + ), ], ) @mock.patch("hermeto.core.package_managers.generic.main.async_download_files") @@ -387,3 +475,259 @@ def test_load_generic_lockfile_valid(rooted_tmp_path: RootedPath) -> None: f.write(LOCKFILE_VALID) assert _load_lockfile(lockfile_path.path, rooted_tmp_path).model_dump() == expected_lockfile + + +def test_load_generic_lockfile_v2_valid(rooted_tmp_path: RootedPath) -> None: + lockfile_path = rooted_tmp_path.join_within_root(DEFAULT_LOCKFILE_NAME) + with open(lockfile_path, "w") as f: + f.write(LOCKFILE_WITH_BEARER_AUTH) + + lockfile = _load_lockfile(lockfile_path.path, rooted_tmp_path) + assert lockfile.metadata.version == "2.0" + assert lockfile.artifacts[0].auth is not None + assert lockfile.artifacts[0].auth.bearer is not None + assert lockfile.artifacts[0].auth.bearer.value == "Bearer secret123" + + +class TestEnvVarExpansion: + """Tests for environment variable expansion in auth fields.""" + + @pytest.mark.parametrize( + ["env_vars", "value", "expected"], + [ + pytest.param( + {"MY_TOKEN": "secret123"}, + "Bearer $MY_TOKEN", + "Bearer secret123", + id="dollar_var_syntax", + ), + pytest.param( + {"MY_TOKEN": "secret123"}, + "Bearer ${MY_TOKEN}", + "Bearer secret123", + id="braced_var_syntax", + ), + pytest.param( + {}, + "no-expansion", + "no-expansion", + id="literal_no_expansion", + ), + pytest.param( + {"A": "first", "B": "second"}, + "$A and ${B}", + "first and second", + id="multiple_vars_mixed_syntax", + ), + pytest.param( + {"TOKEN": "secret123"}, + "Token:$TOKEN", + "Token:secret123", + id="no_space_before_var", + ), + pytest.param( + {"EMPTY_VAR": ""}, + "Bearer $EMPTY_VAR", + "Bearer ", + id="empty_env_var", + ), + ], + ) + def test_expand_env_vars( + self, + monkeypatch: pytest.MonkeyPatch, + env_vars: dict[str, str], + value: str, + expected: str, + ) -> None: + for name, val in env_vars.items(): + monkeypatch.setenv(name, val) + auth = BearerAuth(value=value) + assert auth.value == expected + + @pytest.mark.parametrize( + ["value", "missing_var"], + [ + pytest.param("Bearer $NONEXISTENT_VAR", "NONEXISTENT_VAR", id="dollar_syntax"), + pytest.param("Bearer ${NONEXISTENT_VAR}", "NONEXISTENT_VAR", id="braced_syntax"), + ], + ) + def test_unset_var_raises(self, value: str, missing_var: str) -> None: + with pytest.raises(KeyError): + BearerAuth(value=value) + + def test_expansion_in_all_fields(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("USER", "admin") + monkeypatch.setenv("PASS", "s3cret") + auth = BasicAuth(username="$USER", password="$PASS") # noqa: S106 + assert auth.username == "admin" + assert auth.password == "s3cret" # noqa: S105 + + +class TestLockfileArtifactAuth: + """Tests for LockfileArtifactAuth model.""" + + @pytest.mark.parametrize( + ["kwargs", "has_basic", "has_bearer"], + [ + pytest.param( + {"basic": BasicAuth(username="user", password="passwd")}, # noqa: S106 + True, + False, + id="basic_only", + ), + pytest.param( + {"bearer": BearerAuth(value="secret123")}, + False, + True, + id="bearer_only", + ), + ], + ) + def test_valid_combinations( + self, kwargs: dict[str, Any], has_basic: bool, has_bearer: bool + ) -> None: + auth = LockfileArtifactAuth(**kwargs) + assert (auth.basic is not None) == has_basic + assert (auth.bearer is not None) == has_bearer + + def test_both_raises(self) -> None: + with pytest.raises(ValidationError, match="Exactly one of the auth types must be set"): + LockfileArtifactAuth( + basic=BasicAuth(username="user", password="passwd"), # noqa: S106 + bearer=BearerAuth(value="secret123"), + ) + + @pytest.mark.parametrize( + ["kwargs", "expected"], + [ + pytest.param( + {"basic": BasicAuth(username="user", password="passwd")}, # noqa: S106 + {"Authorization": "Basic dXNlcjpwYXNzd2Q="}, + id="basic", + ), + pytest.param( + {"basic": BasicAuth(username="user@domain", password="p@ss:word!")}, # noqa: S106 + {"Authorization": "Basic dXNlckBkb21haW46cEBzczp3b3JkIQ=="}, + id="basic_special_chars", + ), + pytest.param( + {"bearer": BearerAuth(value="Bearer secret123")}, + {"Authorization": "Bearer secret123"}, + id="bearer_default_header", + ), + pytest.param( + {"bearer": BearerAuth(header="X-Token", value="secret123")}, + {"X-Token": "secret123"}, + id="bearer_custom_header", + ), + ], + ) + def test_get_headers(self, kwargs: dict[str, Any], expected: dict[str, str]) -> None: + auth = LockfileArtifactAuth(**kwargs) + assert auth.get_headers() == expected + + def test_raise_error_when_no_auth_type_set(self) -> None: + with pytest.raises(ValidationError, match="Exactly one of the auth types must be set"): + LockfileArtifactAuth() + + +@pytest.mark.parametrize( + ["lockfile_content", "expected_url", "expected_headers"], + [ + pytest.param( + LOCKFILE_WITH_BASIC_AUTH, + "https://example.com/artifact", + {"Authorization": "Basic dXNlcjpwYXNzd2Q="}, + id="basic_auth", + ), + pytest.param( + LOCKFILE_WITH_BEARER_AUTH, + "https://example.com/artifact", + {"Authorization": "Bearer secret123"}, + id="bearer_auth_default_header", + ), + pytest.param( + LOCKFILE_WITH_CUSTOM_HEADER_BEARER_AUTH, + "https://example.com/artifact", + {"X-Custom-Auth": "secret123"}, + id="bearer_auth_custom_header", + ), + ], +) +@mock.patch("hermeto.core.package_managers.generic.main.asyncio.run") +@mock.patch("hermeto.core.package_managers.generic.main.async_download_files") +@mock.patch("hermeto.core.package_managers.generic.main.must_match_any_checksum") +def test_resolve_generic_lockfile_with_auth( + mock_checksums: mock.Mock, + mock_download: mock.Mock, + mock_asyncio_run: mock.Mock, + lockfile_content: str, + expected_url: str, + expected_headers: dict[str, str], + rooted_tmp_path: RootedPath, +) -> None: + lockfile_path = rooted_tmp_path.join_within_root(DEFAULT_LOCKFILE_NAME) + with open(lockfile_path, "w") as f: + f.write(lockfile_content) + + _resolve_generic_lockfile(lockfile_path.path, rooted_tmp_path) + + assert mock_download.call_args.kwargs["headers"] == {expected_url: expected_headers} + + +@mock.patch("hermeto.core.package_managers.generic.main.asyncio.run") +@mock.patch("hermeto.core.package_managers.generic.main.async_download_files") +@mock.patch("hermeto.core.package_managers.generic.main.must_match_any_checksum") +def test_resolve_generic_lockfile_mixed_auth( + mock_checksums: mock.Mock, + mock_download: mock.Mock, + mock_asyncio_run: mock.Mock, + rooted_tmp_path: RootedPath, +) -> None: + lockfile_path = rooted_tmp_path.join_within_root(DEFAULT_LOCKFILE_NAME) + with open(lockfile_path, "w") as f: + f.write(LOCKFILE_WITH_MIXED_AUTH) + + _resolve_generic_lockfile(lockfile_path.path, rooted_tmp_path) + + headers = mock_download.call_args.kwargs["headers"] + # Only the authed artifact should have headers + assert "https://example.com/authed" in headers + assert "https://example.com/public" not in headers + + +@mock.patch("hermeto.core.package_managers.generic.main.asyncio.run") +@mock.patch("hermeto.core.package_managers.generic.main.async_download_files") +@mock.patch("hermeto.core.package_managers.generic.main.must_match_any_checksum") +def test_resolve_generic_lockfile_no_auth( + mock_checksums: mock.Mock, + mock_download: mock.Mock, + mock_asyncio_run: mock.Mock, + rooted_tmp_path: RootedPath, +) -> None: + lockfile_path = rooted_tmp_path.join_within_root(DEFAULT_LOCKFILE_NAME) + with open(lockfile_path, "w") as f: + f.write(LOCKFILE_VALID) + + _resolve_generic_lockfile(lockfile_path.path, rooted_tmp_path) + + assert mock_download.call_args.kwargs["headers"] is None + + +@mock.patch("hermeto.core.package_managers.generic.main.asyncio.run") +@mock.patch("hermeto.core.package_managers.generic.main.async_download_files") +@mock.patch("hermeto.core.package_managers.generic.main.must_match_any_checksum") +def test_resolve_generic_lockfile_v2_no_auth( + mock_checksums: mock.Mock, + mock_download: mock.Mock, + mock_asyncio_run: mock.Mock, + rooted_tmp_path: RootedPath, +) -> None: + lockfile_path = rooted_tmp_path.join_within_root(DEFAULT_LOCKFILE_NAME) + with open(lockfile_path, "w") as f: + f.write(LOCKFILE_V2_VALID) + + _resolve_generic_lockfile(lockfile_path.path, rooted_tmp_path) + + assert mock_download.call_args.kwargs["headers"] is None