diff --git a/cmd/helm/dependency.go b/cmd/helm/dependency.go index 03874742cba..1703b715775 100644 --- a/cmd/helm/dependency.go +++ b/cmd/helm/dependency.go @@ -71,6 +71,34 @@ the dependency charts stored locally. The path should start with a prefix of If the dependency chart is retrieved locally, it is not required to have the repository added to helm by "helm add repo". Version matching is also supported for this case. + +A repository can be defined as a git URL. The path must start with a prefix of +"git+" followed by a valid git repository URL. + + # Chart.yaml + dependencies: + - name: helm-chart + version: "main" + repository: "git+https://github.com/helm/helm-chart.git" + +The 'repository' can be the https or ssh URL that you would use to clone a git +repo or add as a git remote, prefixed with 'git:'. +For example 'git+ssh://github.com:helm/helm-chart.git' or +'git+https://github.com/helm/helm-chart.git' + +When using a 'git[+subprotocol]>://' repository, the 'version' must be a valid +tag or branch name for the git repo, for example 'main'. + +Limitations when working with git repositories: +* Helm will use the 'git' executable on your system to retrieve information +about the repo. The 'git' command must be properly configured and available +on the PATH. +* When specifying a private repo, if git tries to query the user for +username/password for an HTTPS URL, or for a certificate password for an SSH +URL, it may cause Helm to hang. Input is not forwarded to the child git +process, so it will not be able to receive user input. Authentication can be +configured by using a git credentials helper which can read the credentials +from environment variables, from operating system keychain, etc. ` const dependencyListDesc = ` diff --git a/go.mod b/go.mod index e200d4fcb11..fe05ff6f051 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 + github.com/whilp/git-urls v1.0.0 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/crypto v0.17.0 golang.org/x/term v0.15.0 diff --git a/go.sum b/go.sum index 2799262df39..ee441d7d6ab 100644 --- a/go.sum +++ b/go.sum @@ -386,6 +386,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU= +github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= diff --git a/internal/fileutil/testdata/testdir/testfile b/internal/fileutil/testdata/testdir/testfile new file mode 100644 index 00000000000..5a8e18c53ed --- /dev/null +++ b/internal/fileutil/testdata/testdir/testfile @@ -0,0 +1 @@ +helm \ No newline at end of file diff --git a/internal/gitutil/gitutil.go b/internal/gitutil/gitutil.go new file mode 100644 index 00000000000..c8d1a2fa1c1 --- /dev/null +++ b/internal/gitutil/gitutil.go @@ -0,0 +1,93 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitutil + +import ( + "net/url" + "os" + "regexp" + "strings" + + "github.com/pkg/errors" + giturls "github.com/whilp/git-urls" + + "github.com/Masterminds/vcs" +) + +var gitRepositoryURLRe = regexp.MustCompile(`^git(\+\w+)?://`) + +type GitRepositoryURL struct { + RepositoryURL string + GitRemoteURL *url.URL + PathUnderGitRepository string +} + +// HasGitReference returns true if a git repository contains a specified ref (branch/tag) +func HasGitReference(gitRepo, ref string) (bool, error) { + local, err := os.MkdirTemp("", "helm-git-") + if err != nil { + return false, err + } + repo, err := vcs.NewRepo(gitRepo, local) + + if err != nil { + return false, err + } + + if err := repo.Get(); err != nil { + return false, err + } + defer os.RemoveAll(local) + return repo.IsReference(ref), nil +} + +// IsGitRepository determines whether a URL is to be treated as a git repository URL +func IsGitRepository(url string) bool { + return gitRepositoryURLRe.MatchString(url) +} + +// ParseGitRepositoryURL creates a new GitRepositoryURL from a string +func ParseGitRepositoryURL(repositoryURL string) (*GitRepositoryURL, error) { + gitRemoteURL, err := giturls.Parse(strings.TrimPrefix(repositoryURL, "git+")) + + if err != nil { + return nil, err + } + + if gitRemoteURL.User != nil { + return nil, errors.Errorf("git repository URL should not contain credentials - please use git credential helpers") + } + + path := "" + + if gitRemoteURL.Fragment != "" { + query, err := url.ParseQuery(gitRemoteURL.Fragment) + if err != nil { + return nil, err + } + + path = query.Get("subdirectory") + } + + gitRemoteURL.Fragment = "" + + return &GitRepositoryURL{ + RepositoryURL: repositoryURL, + GitRemoteURL: gitRemoteURL, + PathUnderGitRepository: path, + }, err +} diff --git a/internal/gitutil/gitutil_test.go b/internal/gitutil/gitutil_test.go new file mode 100644 index 00000000000..db2ec6e8289 --- /dev/null +++ b/internal/gitutil/gitutil_test.go @@ -0,0 +1,79 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitutil + +import ( + "testing" +) + +func TestIsGitUrl(t *testing.T) { + // Test table: Given url, IsGitRepository should return expect. + tests := []struct { + url string + expect bool + }{ + {"oci://example.com/example/chart", false}, + {"git://example.com/example/chart", true}, + {"git+https://example.com/example/chart", true}, + } + + for _, test := range tests { + if IsGitRepository(test.url) != test.expect { + t.Errorf("Expected %t for %s", test.expect, test.url) + } + } +} + +func TestParseGitRepositoryURL(t *testing.T) { + // Test table: Given url, ParseGitRepositoryURL should return expect. + tests := []struct { + url string + expectRepositoryURL string + expectGitRemoteURL string + expectedPathUnderGitRepository string + }{ + { + url: "git://example.com/example/chart", + expectRepositoryURL: "git://example.com/example/chart", + expectGitRemoteURL: "git://example.com/example/chart", + }, + { + url: "git+https://example.com/example/chart", + expectRepositoryURL: "git+https://example.com/example/chart", + expectGitRemoteURL: "https://example.com/example/chart", + }, + { + url: "git+https://example.com/example/chart#subdirectory=charts/some-chart", + expectRepositoryURL: "git+https://example.com/example/chart#subdirectory=charts/some-chart", + expectGitRemoteURL: "https://example.com/example/chart", + expectedPathUnderGitRepository: "charts/some-chart", + }, + } + + for _, test := range tests { + parsed, _ := ParseGitRepositoryURL(test.url) + if parsed.RepositoryURL != test.expectRepositoryURL { + t.Errorf("Expected RepositoryURL %s for %s, but got %s", test.expectRepositoryURL, test.url, parsed.RepositoryURL) + } + if parsed.GitRemoteURL.String() != test.expectGitRemoteURL { + t.Errorf("Expected GitRemoteURL %s for %s, but got %s", test.expectGitRemoteURL, test.url, parsed.GitRemoteURL) + } + if parsed.PathUnderGitRepository != test.expectedPathUnderGitRepository { + t.Errorf("Expected PathUnderGitRepository %s for %s, but got %s", test.expectGitRemoteURL, test.url, parsed.PathUnderGitRepository) + } + } +} diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go index 5e8921f960e..07dcee93bde 100644 --- a/internal/resolver/resolver.go +++ b/internal/resolver/resolver.go @@ -27,6 +27,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/pkg/errors" + "helm.sh/helm/v3/internal/gitutil" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/helmpath" @@ -35,6 +36,8 @@ import ( "helm.sh/helm/v3/pkg/repo" ) +var hasGitReference = gitutil.HasGitReference + // Resolver resolves dependencies from semantic version ranges to a particular version. type Resolver struct { chartpath string @@ -58,9 +61,13 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string locked := make([]*chart.Dependency, len(reqs)) missing := []string{} for i, d := range reqs { - constraint, err := semver.NewConstraint(d.Version) - if err != nil { - return nil, errors.Wrapf(err, "dependency %q has an invalid version/constraint format", d.Name) + var constraint *semver.Constraints + var err error + if !gitutil.IsGitRepository(d.Repository) { + constraint, err = semver.NewConstraint(d.Version) + if err != nil { + return nil, errors.Wrapf(err, "dependency %q has an invalid version/constraint format", d.Name) + } } if d.Repository == "" { @@ -76,6 +83,7 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string } continue } + if strings.HasPrefix(d.Repository, "file://") { chartpath, err := GetLocalPath(d.Repository, r.chartpath) @@ -107,6 +115,31 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string continue } + if gitutil.IsGitRepository(d.Repository) { + + gitURL, err := gitutil.ParseGitRepositoryURL(d.Repository) + if err != nil { + return nil, err + } + + found, err := hasGitReference(gitURL.GitRemoteURL.String(), d.Version) + if err != nil { + return nil, err + } + + if !found { + return nil, fmt.Errorf(`dependency %q is missing git branch or tag: %s. + When using a "git[+subprotocol]://" type repository, the "version" should be a valid branch or tag name`, d.Name, d.Version) + } + + locked[i] = &chart.Dependency{ + Name: d.Name, + Repository: d.Repository, + Version: d.Version, + } + continue + } + repoName := repoNames[d.Name] // if the repository was not defined, but the dependency defines a repository url, bypass the cache if repoName == "" && d.Repository != "" { diff --git a/internal/resolver/resolver_test.go b/internal/resolver/resolver_test.go index a798521751f..ecbc1e86873 100644 --- a/internal/resolver/resolver_test.go +++ b/internal/resolver/resolver_test.go @@ -16,6 +16,7 @@ limitations under the License. package resolver import ( + "fmt" "runtime" "testing" @@ -23,12 +24,22 @@ import ( "helm.sh/helm/v3/pkg/registry" ) +func fakeGitReference(_, ref string) (bool, error) { + gitRefs := map[string]string{ + "1.0.0": "", + "main": "", + } + + _, found := gitRefs[ref] + return found, nil +} func TestResolve(t *testing.T) { + hasGitReference = fakeGitReference tests := []struct { - name string - req []*chart.Dependency - expect *chart.Lock - err bool + name string + req []*chart.Dependency + expect *chart.Lock + expectError string }{ { name: "repo from invalid version", @@ -40,14 +51,14 @@ func TestResolve(t *testing.T) { {Name: "base", Repository: "file://base", Version: "0.1.0"}, }, }, - err: true, + expectError: "can't get a valid version for repositories base. Try changing the version constraint in Chart.yaml", }, { name: "version failure", req: []*chart.Dependency{ {Name: "oedipus-rex", Repository: "http://example.com", Version: ">a1"}, }, - err: true, + expectError: `dependency "oedipus-rex" has an invalid version/constraint format: improper constraint: >a1`, }, { name: "cache index failure", @@ -65,14 +76,14 @@ func TestResolve(t *testing.T) { req: []*chart.Dependency{ {Name: "redis", Repository: "http://example.com", Version: "1.0.0"}, }, - err: true, + expectError: "redis chart not found in repo http://example.com", }, { name: "constraint not satisfied failure", req: []*chart.Dependency{ {Name: "alpine", Repository: "http://example.com", Version: ">=1.0.0"}, }, - err: true, + expectError: "can't get a valid version for repositories alpine. Try changing the version constraint in Chart.yaml", }, { name: "valid lock", @@ -112,7 +123,7 @@ func TestResolve(t *testing.T) { req: []*chart.Dependency{ {Name: "nonexistent", Repository: "file://testdata/nonexistent", Version: "0.1.0"}, }, - err: true, + expectError: "directory testdata/chartpath/testdata/nonexistent not found", }, { name: "repo from valid path under charts path", @@ -135,7 +146,66 @@ func TestResolve(t *testing.T) { {Name: "nonexistentlocaldependency", Repository: "", Version: "0.1.0"}, }, }, - err: true, + expectError: "directory testdata/chartpath/charts/nonexistentdependency not found", + }, + { + name: "repo from git https url", + req: []*chart.Dependency{ + {Name: "gitdependencyok", Repository: "git+https://github.com/helm/helmchart.git", Version: "1.0.0"}, + }, + expect: &chart.Lock{ + Dependencies: []*chart.Dependency{ + {Name: "gitdependencyok", Repository: "git+https://github.com/helm/helmchart.git", Version: "1.0.0"}, + }, + }, + }, + { + name: "repo from git https url", + req: []*chart.Dependency{ + {Name: "gitdependencyerror", Repository: "git+https://github.com/helm/helmchart.git", Version: "2.0.0"}, + }, + expect: &chart.Lock{ + Dependencies: []*chart.Dependency{ + {Name: "gitdependencyerror", Repository: "git+https://github.com/helm/helmchart.git", Version: "2.0.0"}, + }, + }, + expectError: `dependency "gitdependencyerror" is missing git branch or tag: 2.0.0. + When using a "git[+subprotocol]://" type repository, the "version" should be a valid branch or tag name`, + }, + { + name: "repo from git ssh url", + req: []*chart.Dependency{ + {Name: "gitdependency", Repository: "git://github.com:helm/helmchart.git", Version: "1.0.0"}, + }, + expect: &chart.Lock{ + Dependencies: []*chart.Dependency{ + {Name: "gitdependency", Repository: "git://github.com:helm/helmchart.git", Version: "1.0.0"}, + }, + }, + }, + { + name: "repo from git ssh url", + req: []*chart.Dependency{ + {Name: "gitdependencyerror", Repository: "git://github.com:helm/helmchart.git", Version: "2.0.0"}, + }, + expect: &chart.Lock{ + Dependencies: []*chart.Dependency{ + {Name: "gitdependencyerror", Repository: "git://github.com:helm/helmchart.git", Version: "2.0.0"}, + }, + }, + expectError: `dependency "gitdependencyerror" is missing git branch or tag: 2.0.0. + When using a "git[+subprotocol]://" type repository, the "version" should be a valid branch or tag name`, + }, + { + name: "repo from git with non-semver version", + req: []*chart.Dependency{ + {Name: "gitdependencyok", Repository: "git+https://github.com/helm/helmchart.git", Version: "main"}, + }, + expect: &chart.Lock{ + Dependencies: []*chart.Dependency{ + {Name: "gitdependencyok", Repository: "git+https://github.com/helm/helmchart.git", Version: "main"}, + }, + }, }, } @@ -145,15 +215,16 @@ func TestResolve(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { l, err := r.Resolve(tt.req, repoNames) - if err != nil { - if tt.err { - return + + if tt.expectError != "" { + if tt.expectError != fmt.Sprint(err) { + t.Errorf("%s: expected error %q, got %q", tt.name, tt.expectError, err) } - t.Fatal(err) + return } - if tt.err { - t.Fatalf("Expected error in test %q", tt.name) + if err != nil { + t.Fatal(err) } if h, err := HashReq(tt.req, tt.expect.Dependencies); err != nil { diff --git a/pkg/downloader/chart_downloader.go b/pkg/downloader/chart_downloader.go index a95894e00e8..0e5160f9d07 100644 --- a/pkg/downloader/chart_downloader.go +++ b/pkg/downloader/chart_downloader.go @@ -27,6 +27,7 @@ import ( "github.com/pkg/errors" "helm.sh/helm/v3/internal/fileutil" + "helm.sh/helm/v3/internal/gitutil" "helm.sh/helm/v3/internal/urlutil" "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/helmpath" @@ -107,6 +108,13 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven idx := strings.LastIndexByte(name, ':') name = fmt.Sprintf("%s-%s.tgz", name[:idx], name[idx+1:]) } + if gitutil.IsGitRepository(ref) { + gitGetter, ok := g.(*getter.GitGetter) + if !ok { + return "", nil, fmt.Errorf("can't convert to GITGetter") + } + name = fmt.Sprintf("%s-%s.tgz", gitGetter.ChartName(), version) + } destfile := filepath.Join(dest, name) if err := fileutil.AtomicWriteFile(destfile, data, 0644); err != nil { @@ -180,7 +188,7 @@ func (c *ChartDownloader) getOciURI(ref, version string, u *url.URL) (*url.URL, // the URL using the appropriate Getter. // // A reference may be an HTTP URL, an oci reference URL, a 'reponame/chartname' -// reference, or a local path. +// reference, git URL, or a local path. // // A version is a SemVer string (1.2.3-beta.1+f334a6789). // @@ -199,6 +207,14 @@ func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, er return c.getOciURI(ref, version, u) } + if gitutil.IsGitRepository(ref) { + _, err := gitutil.ParseGitRepositoryURL(ref) + if err != nil { + return nil, errors.Wrapf(err, "invalid git URL format: %s", ref) + } + return u, err + } + rf, err := loadRepoConfig(c.RepositoryConfig) if err != nil { return u, err diff --git a/pkg/downloader/chart_downloader_test.go b/pkg/downloader/chart_downloader_test.go index 131e2130612..8509d4a5e2b 100644 --- a/pkg/downloader/chart_downloader_test.go +++ b/pkg/downloader/chart_downloader_test.go @@ -16,6 +16,7 @@ limitations under the License. package downloader import ( + "fmt" "os" "path/filepath" "testing" @@ -35,11 +36,14 @@ const ( func TestResolveChartRef(t *testing.T) { tests := []struct { name, ref, expect, version string - fail bool + expectError string }{ {name: "full URL", ref: "http://example.com/foo-1.2.3.tgz", expect: "http://example.com/foo-1.2.3.tgz"}, {name: "full URL, HTTPS", ref: "https://example.com/foo-1.2.3.tgz", expect: "https://example.com/foo-1.2.3.tgz"}, {name: "full URL, with authentication", ref: "http://username:password@example.com/foo-1.2.3.tgz", expect: "http://username:password@example.com/foo-1.2.3.tgz"}, + {name: "helmchart", ref: "git+https://github.com/helmchart/helmchart.git", expect: "git+https://github.com/helmchart/helmchart.git"}, + {name: "helmchart", ref: "git://github.com/helmchart/helmchart.git", expect: "git://github.com/helmchart/helmchart.git"}, + {name: "helmchart", ref: "git+https://username:password@github.com/helmchart/helmchart.git", expectError: "invalid git URL format: git+https://username:password@github.com/helmchart/helmchart.git: git repository URL should not contain credentials - please use git credential helpers"}, {name: "reference, testing repo", ref: "testing/alpine", expect: "http://example.com/alpine-1.2.3.tgz"}, {name: "reference, version, testing repo", ref: "testing/alpine", version: "0.2.0", expect: "http://example.com/alpine-0.2.0.tgz"}, {name: "reference, version, malformed repo", ref: "malformed/alpine", version: "1.2.3", expect: "http://dl.example.com/alpine-1.2.3.tgz"}, @@ -49,10 +53,10 @@ func TestResolveChartRef(t *testing.T) { {name: "reference, testing-relative-trailing-slash repo", ref: "testing-relative-trailing-slash/foo", expect: "http://example.com/helm/charts/foo-1.2.3.tgz"}, {name: "reference, testing-relative-trailing-slash repo", ref: "testing-relative-trailing-slash/bar", expect: "http://example.com/helm/bar-1.2.3.tgz"}, {name: "encoded URL", ref: "encoded-url/foobar", expect: "http://example.com/with%2Fslash/charts/foobar-4.2.1.tgz"}, - {name: "full URL, HTTPS, irrelevant version", ref: "https://example.com/foo-1.2.3.tgz", version: "0.1.0", expect: "https://example.com/foo-1.2.3.tgz", fail: true}, - {name: "full URL, file", ref: "file:///foo-1.2.3.tgz", fail: true}, - {name: "invalid", ref: "invalid-1.2.3", fail: true}, - {name: "not found", ref: "nosuchthing/invalid-1.2.3", fail: true}, + {name: "full URL, HTTPS, irrelevant version", ref: "https://example.com/foo-1.2.3.tgz", version: "0.1.0", expect: "https://example.com/foo-1.2.3.tgz"}, + {name: "full URL, file", ref: "file:///foo-1.2.3.tgz", expectError: "repo not found"}, + {name: "invalid", ref: "invalid-1.2.3", expectError: "non-absolute URLs should be in form of repo_name/path_to_chart, got: invalid-1.2.3"}, + {name: "not found", ref: "nosuchthing/invalid-1.2.3", expectError: "repo nosuchthing not found"}, } c := ChartDownloader{ @@ -67,10 +71,13 @@ func TestResolveChartRef(t *testing.T) { for _, tt := range tests { u, err := c.ResolveChartVersion(tt.ref, tt.version) - if err != nil { - if tt.fail { - continue + if tt.expectError != "" { + if tt.expectError != fmt.Sprint(err) { + t.Errorf("%s: expected error %q, got %q", tt.name, tt.expectError, err) } + continue + } + if err != nil { t.Errorf("%s: failed with error %q", tt.name, err) continue } diff --git a/pkg/downloader/manager.go b/pkg/downloader/manager.go index 68c9c6e006d..b6369782b9e 100644 --- a/pkg/downloader/manager.go +++ b/pkg/downloader/manager.go @@ -33,6 +33,7 @@ import ( "github.com/pkg/errors" "sigs.k8s.io/yaml" + "helm.sh/helm/v3/internal/gitutil" "helm.sh/helm/v3/internal/resolver" "helm.sh/helm/v3/internal/third_party/dep/fs" "helm.sh/helm/v3/internal/urlutil" @@ -342,6 +343,7 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { } version := "" + if registry.IsOCI(churl) { churl, version, err = parseOCIRef(churl) if err != nil { @@ -352,6 +354,17 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { getter.WithTagName(version)) } + if gitutil.IsGitRepository(churl) { + version = dep.Version + + dl.Options = append(dl.Options, getter.WithTagName(version)) + dl.Options = append(dl.Options, getter.WithChartName(dep.Name)) + + if m.Debug { + fmt.Fprintf(m.Out, "Downloading %s from git repo %s\n", dep.Name, churl) + } + } + if _, _, err = dl.DownloadTo(churl, version, tmpPath); err != nil { saveError = errors.Wrapf(err, "could not download %s", churl) break @@ -471,8 +484,8 @@ func (m *Manager) hasAllRepos(deps []*chart.Dependency) error { missing := []string{} Loop: for _, dd := range deps { - // If repo is from local path or OCI, continue - if strings.HasPrefix(dd.Repository, "file://") || registry.IsOCI(dd.Repository) { + // If repo is from local path, OCI or a git url, continue - there is no local repo cache to check + if strings.HasPrefix(dd.Repository, "file://") || registry.IsOCI(dd.Repository) || gitutil.IsGitRepository(dd.Repository) { continue } @@ -593,6 +606,13 @@ func (m *Manager) resolveRepoNames(deps []*chart.Dependency) (map[string]string, reposMap[dd.Name] = dd.Repository continue } + // if dep chart is from a git url, assume it is valid for now. + // if the repo does not exist then it will later error when we try to fetch branches and tags. + // we could check for the repo existence here, but trying to avoid another git request. + if gitutil.IsGitRepository(dd.Repository) { + reposMap[dd.Name] = dd.Repository + continue + } found := false @@ -704,6 +724,10 @@ func (m *Manager) parallelRepoUpdate(repos []*repo.Entry) error { // // If it finds a URL that is "relative", it will prepend the repoURL. func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (url, username, password string, insecureskiptlsverify, passcredentialsall bool, caFile, certFile, keyFile string, err error) { + if gitutil.IsGitRepository(repoURL) { + return repoURL, "", "", false, false, "", "", "", nil + } + if registry.IsOCI(repoURL) { return fmt.Sprintf("%s/%s:%s", repoURL, name, version), "", "", false, false, "", "", "", nil } diff --git a/pkg/downloader/manager_test.go b/pkg/downloader/manager_test.go index f7ab1a5682d..48fcc3a2d10 100644 --- a/pkg/downloader/manager_test.go +++ b/pkg/downloader/manager_test.go @@ -80,55 +80,36 @@ func TestFindChartURL(t *testing.T) { t.Fatal(err) } - name := "alpine" - version := "0.1.0" - repoURL := "http://example.com/charts" - - churl, username, password, insecureSkipTLSVerify, passcredentialsall, _, _, _, err := m.findChartURL(name, version, repoURL, repos) - if err != nil { - t.Fatal(err) - } - - if churl != "https://charts.helm.sh/stable/alpine-0.1.0.tgz" { - t.Errorf("Unexpected URL %q", churl) - } - if username != "" { - t.Errorf("Unexpected username %q", username) - } - if password != "" { - t.Errorf("Unexpected password %q", password) - } - if passcredentialsall != false { - t.Errorf("Unexpected passcredentialsall %t", passcredentialsall) - } - if insecureSkipTLSVerify { - t.Errorf("Unexpected insecureSkipTLSVerify %t", insecureSkipTLSVerify) + tests := []struct { + name, version, repoURL, expectChurl, expectUserName, expectPassword string + expectInsecureSkipTLSVerify, expectPasscredentialsall bool + }{ + {name: "alpine", version: "0.1.0", repoURL: "http://example.com/charts", expectChurl: "https://charts.helm.sh/stable/alpine-0.1.0.tgz", expectUserName: "", expectPassword: "", expectInsecureSkipTLSVerify: false, expectPasscredentialsall: false}, + {name: "tlsfoo", version: "1.2.3", repoURL: "https://example-https-insecureskiptlsverify.com", expectChurl: "https://example.com/tlsfoo-1.2.3.tgz", expectUserName: "", expectPassword: "", expectInsecureSkipTLSVerify: true, expectPasscredentialsall: false}, + {name: "helm-test", version: "master", repoURL: "git+https://github.com/rally25rs/helm-test-chart.git", expectChurl: "git+https://github.com/rally25rs/helm-test-chart.git", expectUserName: "", expectPassword: "", expectInsecureSkipTLSVerify: false, expectPasscredentialsall: false}, } - - name = "tlsfoo" - version = "1.2.3" - repoURL = "https://example-https-insecureskiptlsverify.com" - - churl, username, password, insecureSkipTLSVerify, passcredentialsall, _, _, _, err = m.findChartURL(name, version, repoURL, repos) - if err != nil { - t.Fatal(err) + for _, tt := range tests { + churl, username, password, insecureSkipTLSVerify, passcredentialsall, _, _, _, err := m.findChartURL(tt.name, tt.version, tt.repoURL, repos) + if err != nil { + t.Fatal(err) + } + if churl != tt.expectChurl { + t.Errorf("Unexpected URL %q", churl) + } + if username != tt.expectUserName { + t.Errorf("Unexpected username %q", username) + } + if password != tt.expectPassword { + t.Errorf("Unexpected password %q", password) + } + if insecureSkipTLSVerify != tt.expectInsecureSkipTLSVerify { + t.Errorf("Unexpected insecureSkipTLSVerify %t", insecureSkipTLSVerify) + } + if passcredentialsall != tt.expectPasscredentialsall { + t.Errorf("Unexpected passcredentialsall %t", passcredentialsall) + } } - if !insecureSkipTLSVerify { - t.Errorf("Unexpected insecureSkipTLSVerify %t", insecureSkipTLSVerify) - } - if churl != "https://example.com/tlsfoo-1.2.3.tgz" { - t.Errorf("Unexpected URL %q", churl) - } - if username != "" { - t.Errorf("Unexpected username %q", username) - } - if password != "" { - t.Errorf("Unexpected password %q", password) - } - if passcredentialsall != false { - t.Errorf("Unexpected passcredentialsall %t", passcredentialsall) - } } func TestGetRepoNames(t *testing.T) { @@ -193,6 +174,13 @@ func TestGetRepoNames(t *testing.T) { }, expect: map[string]string{}, }, + { + name: "repo from git url", + req: []*chart.Dependency{ + {Name: "local-dep", Repository: "git+https://github.com/git/git"}, + }, + expect: map[string]string{"local-dep": "git+https://github.com/git/git"}, + }, } for _, tt := range tests { diff --git a/pkg/getter/getter.go b/pkg/getter/getter.go index 45ab4da7e4e..e28ba80024e 100644 --- a/pkg/getter/getter.go +++ b/pkg/getter/getter.go @@ -43,6 +43,7 @@ type options struct { passCredentialsAll bool userAgent string version string + chartName string registryClient *registry.Client timeout time.Duration transport *http.Transport @@ -67,6 +68,11 @@ func WithBasicAuth(username, password string) Option { opts.password = password } } +func WithChartName(chartName string) Option { + return func(opts *options) { + opts.chartName = chartName + } +} func WithPassCredentialsAll(pass bool) Option { return func(opts *options) { @@ -201,11 +207,16 @@ var ociProvider = Provider{ New: NewOCIGetter, } +var gitProvider = Provider{ + Schemes: []string{"git", "git+http", "git+https", "git+rsync", "git+ftp", "git+file", "git+ssh"}, + New: NewGitGetter, +} + // All finds all of the registered getters as a list of Provider instances. // Currently, the built-in getters and the discovered plugins with downloader // notations are collected. func All(settings *cli.EnvSettings) Providers { - result := Providers{httpProvider, ociProvider} + result := Providers{httpProvider, ociProvider, gitProvider} pluginDownloaders, _ := collectPlugins(settings) result = append(result, pluginDownloaders...) return result diff --git a/pkg/getter/getter_test.go b/pkg/getter/getter_test.go index ab14784abfb..c331f2c0fdc 100644 --- a/pkg/getter/getter_test.go +++ b/pkg/getter/getter_test.go @@ -57,8 +57,8 @@ func TestAll(t *testing.T) { env.PluginsDirectory = pluginDir all := All(env) - if len(all) != 4 { - t.Errorf("expected 4 providers (default plus three plugins), got %d", len(all)) + if len(all) != 5 { + t.Errorf("expected 5 providers (default plus three plugins), got %d", len(all)) } if _, err := all.ByScheme("test2"); err != nil { diff --git a/pkg/getter/gitgetter.go b/pkg/getter/gitgetter.go new file mode 100644 index 00000000000..e23343f0113 --- /dev/null +++ b/pkg/getter/gitgetter.go @@ -0,0 +1,114 @@ +/* +Copyright The Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package getter + +import ( + "bytes" + "fmt" + "os" + + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" + + "github.com/Masterminds/vcs" + securejoin "github.com/cyphar/filepath-securejoin" + + "helm.sh/helm/v3/internal/gitutil" +) + +// GitGetter is the default git backend handler +type GitGetter struct { + opts options +} + +func (g *GitGetter) ChartName() string { + return g.opts.chartName +} + +// Get performs a Get from repo.Getter and returns the body. +func (g *GitGetter) Get(href string, options ...Option) (*bytes.Buffer, error) { + for _, opt := range options { + opt(&g.opts) + } + return g.get(href) +} + +func (g *GitGetter) get(href string) (*bytes.Buffer, error) { + gitURL, err := gitutil.ParseGitRepositoryURL(href) + if err != nil { + return nil, err + } + version := g.opts.version + chartName := g.opts.chartName + if version == "" { + return nil, fmt.Errorf("the version must be a valid tag or branch name for the git repo, not nil") + } + tmpDir, err := os.MkdirTemp("", "helm-git-") + if err != nil { + return nil, err + } + + gitTmpDir, err := securejoin.SecureJoin(tmpDir, chartName) + if err != nil { + return nil, err + } + + if err := os.MkdirAll(gitTmpDir, 0755); err != nil { + return nil, err + } + defer os.RemoveAll(tmpDir) + + repo, err := vcs.NewRepo(gitURL.GitRemoteURL.String(), gitTmpDir) + if err != nil { + return nil, err + } + if err := repo.Get(); err != nil { + return nil, err + } + if err := repo.UpdateVersion(version); err != nil { + return nil, err + } + + chartDir, err := securejoin.SecureJoin(gitTmpDir, gitURL.PathUnderGitRepository) + if err != nil { + return nil, err + } + + ch, err := loader.LoadDir(chartDir) + if err != nil { + return nil, err + } + + tarballPath, err := chartutil.Save(ch, tmpDir) + if err != nil { + return nil, err + } + + buf, err := os.ReadFile(tarballPath) + return bytes.NewBuffer(buf), err +} + +// NewGitGetter constructs a valid git client as a Getter +func NewGitGetter(ops ...Option) (Getter, error) { + + client := GitGetter{} + + for _, opt := range ops { + opt(&client.opts) + } + + return &client, nil +} diff --git a/pkg/getter/gitgetter_test.go b/pkg/getter/gitgetter_test.go new file mode 100644 index 00000000000..94bbb0d62b7 --- /dev/null +++ b/pkg/getter/gitgetter_test.go @@ -0,0 +1,31 @@ +/* +Copyright The Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package getter + +import ( + "testing" +) + +func TestNewGITGetter(t *testing.T) { + g, err := NewGitGetter() + if err != nil { + t.Fatal(err) + } + + if _, ok := g.(*GitGetter); !ok { + t.Fatal("Expected NewGITGetter to produce an *GITGetter") + } +} diff --git a/pkg/getter/ocigetter.go b/pkg/getter/ocigetter.go index 209786bd7d6..68ecb63c1a2 100644 --- a/pkg/getter/ocigetter.go +++ b/pkg/getter/ocigetter.go @@ -29,7 +29,7 @@ import ( "helm.sh/helm/v3/pkg/registry" ) -// OCIGetter is the default HTTP(/S) backend handler +// OCIGetter is the default OCI backend handler type OCIGetter struct { opts options transport *http.Transport