diff --git a/enricher/enricherlist/list.go b/enricher/enricherlist/list.go index 8c75f5e4c..a4fb57fbb 100644 --- a/enricher/enricherlist/list.go +++ b/enricher/enricherlist/list.go @@ -30,6 +30,7 @@ import ( "github.com/google/osv-scalibr/enricher/reachability/java" "github.com/google/osv-scalibr/enricher/secrets/convert" "github.com/google/osv-scalibr/enricher/secrets/hashicorp" + "github.com/google/osv-scalibr/enricher/transitivedependency/pomxml" "github.com/google/osv-scalibr/enricher/transitivedependency/requirements" "github.com/google/osv-scalibr/enricher/vex/filter" "github.com/google/osv-scalibr/enricher/vulnmatch/osvdev" @@ -138,6 +139,7 @@ var ( // TransitiveDependency enrichers. TransitiveDependency = InitMap{ requirements.Name: {requirements.New}, + pomxml.Name: {pomxml.New}, } // PackageDeprecation enricher. diff --git a/enricher/transitivedependency/internal/grouping.go b/enricher/transitivedependency/internal/grouping.go new file mode 100644 index 000000000..9d981c344 --- /dev/null +++ b/enricher/transitivedependency/internal/grouping.go @@ -0,0 +1,68 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package internal contains miscellaneous functions and objects useful within transitive dependency enrichers +package internal + +import ( + "slices" + + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/inventory" + "github.com/google/osv-scalibr/log" +) + +// PackageWithIndex holds the package with its index in inv.Packages +type PackageWithIndex struct { + Pkg *extractor.Package + Index int +} + +// GroupPackagesFromPlugin groups packages that were added by a particular plugin by the first location +// that they are found and returns a map of location -> package name -> package with index. +func GroupPackagesFromPlugin(pkgs []*extractor.Package, pluginName string) map[string]map[string]PackageWithIndex { + result := make(map[string]map[string]PackageWithIndex) + for i, pkg := range pkgs { + if !slices.Contains(pkg.Plugins, pluginName) { + continue + } + if len(pkg.Locations) == 0 { + log.Warnf("package %s has no locations", pkg.Name) + continue + } + // Use the path where this package is first found. + path := pkg.Locations[0] + if _, ok := result[path]; !ok { + result[path] = make(map[string]PackageWithIndex) + } + result[path][pkg.Name] = PackageWithIndex{pkg, i} + } + return result +} + +// Add handles supplementing an inventory with enriched packages +func Add(enrichedPkgs []*extractor.Package, inv *inventory.Inventory, pluginName string, existingPackages map[string]PackageWithIndex) { + for _, pkg := range enrichedPkgs { + indexPkg, ok := existingPackages[pkg.Name] + if ok { + // This dependency is in manifest, update the version and plugins. + i := indexPkg.Index + inv.Packages[i].Version = pkg.Version + inv.Packages[i].Plugins = append(inv.Packages[i].Plugins, pluginName) + } else { + // This dependency is not found in manifest, so it's a transitive dependency. + inv.Packages = append(inv.Packages, pkg) + } + } +} diff --git a/enricher/transitivedependency/pomxml/pomxml.go b/enricher/transitivedependency/pomxml/pomxml.go new file mode 100644 index 000000000..34de4b697 --- /dev/null +++ b/enricher/transitivedependency/pomxml/pomxml.go @@ -0,0 +1,281 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package pomxml implements an enricher to perform dependency resolution for Java pom.xml files. +package pomxml + +import ( + "context" + "fmt" + "maps" + "slices" + "strings" + + "deps.dev/util/maven" + "deps.dev/util/resolve" + mavenresolve "deps.dev/util/resolve/maven" + cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/clients/datasource" + "github.com/google/osv-scalibr/clients/resolution" + "github.com/google/osv-scalibr/enricher" + "github.com/google/osv-scalibr/enricher/transitivedependency/internal" + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scalibr/extractor/filesystem/language/java/javalockfile" + "github.com/google/osv-scalibr/extractor/filesystem/language/java/pomxml" + "github.com/google/osv-scalibr/internal/mavenutil" + "github.com/google/osv-scalibr/inventory" + "github.com/google/osv-scalibr/plugin" + "github.com/google/osv-scalibr/purl" +) + +const ( + // Name is the unique name of this enricher. + Name = "transitivedependency/pomxml" +) + +// Enricher performs dependency resolution for pom.xml. +type Enricher struct { + DepClient resolve.Client + MavenClient *datasource.MavenRegistryAPIClient +} + +// Name returns the name of the enricher. +func (Enricher) Name() string { + return Name +} + +// Version returns the version of the enricher. +func (Enricher) Version() int { + return 0 +} + +// Requirements returns the requirements of the enricher. +func (Enricher) Requirements() *plugin.Capabilities { + return &plugin.Capabilities{ + Network: plugin.NetworkOnline, + DirectFS: true, + } +} + +// RequiredPlugins returns the names of the plugins required by the enricher. +func (Enricher) RequiredPlugins() []string { + return []string{pomxml.Name} +} + +// Config is the configuration for the pomxmlnet Extractor. +type Config struct { + *datasource.MavenRegistryAPIClient + + DependencyClient resolve.Client +} + +// New makes a new pom.xml transitive enricher with the given config. +func New(cfg *cpb.PluginConfig) enricher.Enricher { + upstreamRegistry := "" + specific := plugin.FindConfig(cfg, func(c *cpb.PluginSpecificConfig) *cpb.POMXMLNetConfig { return c.GetPomXmlNet() }) + if specific != nil { + upstreamRegistry = specific.UpstreamRegistry + } + + // No need to check errors since we are using the default Maven Central URL. + mavenClient, _ := datasource.NewMavenRegistryAPIClient(context.Background(), datasource.MavenRegistry{ + URL: upstreamRegistry, + ReleasesEnabled: true, + }, cfg.LocalRegistry, cfg.DisableGoogleAuth) + depClient := resolution.NewMavenRegistryClientWithAPI(mavenClient) + + return &Enricher{ + DepClient: depClient, + MavenClient: mavenClient, + } +} + +// Enrich enriches the inventory in pom.xml files with transitive dependencies. +func (e Enricher) Enrich(ctx context.Context, input *enricher.ScanInput, inv *inventory.Inventory) error { + pkgGroups := internal.GroupPackagesFromPlugin(inv.Packages, pomxml.Name) + + for path, pkgMap := range pkgGroups { + f, err := input.ScanRoot.FS.Open(path) + + if err != nil { + return err + } + + enrichedInv, err := e.extract(ctx, &filesystem.ScanInput{ + Path: path, + Reader: f, + Info: nil, + FS: input.ScanRoot.FS, + Root: input.ScanRoot.Path, + }) + + if err != nil { + return err + } + + internal.Add(enrichedInv.Packages, inv, Name, pkgMap) + } + + return nil +} + +func (e Enricher) extract(ctx context.Context, input *filesystem.ScanInput) (inventory.Inventory, error) { + var project maven.Project + if err := datasource.NewMavenDecoder(input.Reader).Decode(&project); err != nil { + return inventory.Inventory{}, fmt.Errorf("could not extract: %w", err) + } + // Empty JDK and ActivationOS indicates merging the default profiles. + if err := project.MergeProfiles("", maven.ActivationOS{}); err != nil { + return inventory.Inventory{}, fmt.Errorf("failed to merge profiles: %w", err) + } + // Interpolate the repositories so that properties are resolved. + if err := project.InterpolateRepositories(); err != nil { + return inventory.Inventory{}, fmt.Errorf("failed to interpolate project: %w", err) + } + // Clear the registries that may be from other extraction. + e.MavenClient = e.MavenClient.WithoutRegistries() + for _, repo := range project.Repositories { + if repo.URL.ContainsProperty() { + continue + } + if err := e.MavenClient.AddRegistry(ctx, datasource.MavenRegistry{ + URL: string(repo.URL), + ID: string(repo.ID), + ReleasesEnabled: repo.Releases.Enabled.Boolean(), + SnapshotsEnabled: repo.Snapshots.Enabled.Boolean(), + }); err != nil { + return inventory.Inventory{}, fmt.Errorf("failed to add registry %s: %w", repo.URL, err) + } + } + // Merging parents data by parsing local parent pom.xml or fetching from upstream. + if err := mavenutil.MergeParents(ctx, project.Parent, &project, mavenutil.Options{ + Input: input, + Client: e.MavenClient, + AddRegistry: true, + AllowLocal: true, + InitialParentIndex: 1, + }); err != nil { + return inventory.Inventory{}, fmt.Errorf("failed to merge parents: %w", err) + } + // Process the dependencies: + // - dedupe dependencies and dependency management + // - import dependency management + // - fill in missing dependency version requirement + project.ProcessDependencies(func(groupID, artifactID, version maven.String) (maven.DependencyManagement, error) { + return mavenutil.GetDependencyManagement(ctx, e.MavenClient, groupID, artifactID, version) + }) + + if registries := e.MavenClient.GetRegistries(); len(registries) > 0 { + clientRegs := make([]resolution.Registry, len(registries)) + for i, reg := range registries { + clientRegs[i] = reg + } + if cl, ok := e.DepClient.(resolution.ClientWithRegistries); ok { + if err := cl.AddRegistries(ctx, clientRegs); err != nil { + return inventory.Inventory{}, err + } + } + } + + overrideClient := resolution.NewOverrideClient(e.DepClient) + resolver := mavenresolve.NewResolver(overrideClient) + + // Resolve the dependencies. + root := resolve.Version{ + VersionKey: resolve.VersionKey{ + PackageKey: resolve.PackageKey{ + System: resolve.Maven, + Name: project.ProjectKey.Name(), + }, + VersionType: resolve.Concrete, + Version: string(project.Version), + }} + reqs := make([]resolve.RequirementVersion, len(project.Dependencies)+len(project.DependencyManagement.Dependencies)) + for i, d := range project.Dependencies { + reqs[i] = resolve.RequirementVersion{ + VersionKey: resolve.VersionKey{ + PackageKey: resolve.PackageKey{ + System: resolve.Maven, + Name: d.Name(), + }, + VersionType: resolve.Requirement, + Version: string(d.Version), + }, + Type: resolve.MavenDepType(d, ""), + } + } + for i, d := range project.DependencyManagement.Dependencies { + reqs[len(project.Dependencies)+i] = resolve.RequirementVersion{ + VersionKey: resolve.VersionKey{ + PackageKey: resolve.PackageKey{ + System: resolve.Maven, + Name: d.Name(), + }, + VersionType: resolve.Requirement, + Version: string(d.Version), + }, + Type: resolve.MavenDepType(d, mavenutil.OriginManagement), + } + } + overrideClient.AddVersion(root, reqs) + + g, err := resolver.Resolve(ctx, root.VersionKey) + if err != nil { + return inventory.Inventory{}, fmt.Errorf("failed resolving %v: %w", root, err) + } + if len(g.Nodes) <= 1 && g.Error != "" { + // Multi-registry error may be appended to the resolved graph so only return error when the graph is empty. + return inventory.Inventory{}, fmt.Errorf("failed resolving %v: %s", root, g.Error) + } + + details := map[string]*extractor.Package{} + for i := 1; i < len(g.Nodes); i++ { + // Ignore the first node which is the root. + node := g.Nodes[i] + depGroups := []string{} + groupID, artifactID, _ := strings.Cut(node.Version.Name, ":") + // We are only able to know dependency groups of direct dependencies but + // not transitive dependencies because the nodes in the resolve graph does + // not have the scope information. + isDirect := false + for _, dep := range project.Dependencies { + if dep.Name() != node.Version.Name { + continue + } + isDirect = true + if dep.Scope != "" && dep.Scope != "compile" { + depGroups = append(depGroups, string(dep.Scope)) + } + break + } + pkg := extractor.Package{ + Name: node.Version.Name, + Version: node.Version.Version, + PURLType: purl.TypeMaven, + Metadata: &javalockfile.Metadata{ + ArtifactID: artifactID, + GroupID: groupID, + DepGroupVals: depGroups, + IsTransitive: !isDirect, + }, + // TODO(#408): Add merged paths in here as well + Locations: []string{input.Path}, + Plugins: []string{Name}, + } + details[pkg.Name] = &pkg + } + + return inventory.Inventory{Packages: slices.Collect(maps.Values(details))}, nil +} diff --git a/enricher/transitivedependency/pomxml/pomxml_test.go b/enricher/transitivedependency/pomxml/pomxml_test.go new file mode 100644 index 000000000..aa1ee7e12 --- /dev/null +++ b/enricher/transitivedependency/pomxml/pomxml_test.go @@ -0,0 +1,268 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pomxml_test + +import ( + "sort" + "testing" + + "github.com/google/go-cmp/cmp" + cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" + "github.com/google/osv-scalibr/clients/clienttest" + "github.com/google/osv-scalibr/clients/datasource" + "github.com/google/osv-scalibr/enricher" + "github.com/google/osv-scalibr/enricher/transitivedependency/pomxml" + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem/language/java/javalockfile" + scalibrfs "github.com/google/osv-scalibr/fs" + "github.com/google/osv-scalibr/inventory" + "github.com/google/osv-scalibr/purl" +) + +func TestEnricher_Enrich(t *testing.T) { + input := enricher.ScanInput{ + ScanRoot: &scalibrfs.ScanRoot{ + Path: "testdata", + FS: scalibrfs.DirFS("."), + }, + } + inv := inventory.Inventory{ + Packages: []*extractor.Package{ + { + // Not a Java package. + Name: "abc", + Version: "1.0.0", + PURLType: purl.TypePyPi, + Locations: []string{"testdata/poetry/poetry.lock"}, + Plugins: []string{"python/poetrylock"}, + }, + { + // Not extracted from a pom.xml + Name: "abc", + Version: "1.0.0", + PURLType: purl.TypeMaven, + Locations: []string{"testdata/java/gradle.lockfile"}, + Plugins: []string{"java/gradlelockfile"}, + }, + { + Name: "org.direct:alice", + Version: "1.0.0", + PURLType: purl.TypeMaven, + Locations: []string{"testdata/transitive.xml"}, + Plugins: []string{"java/pomxml"}, + Metadata: &javalockfile.Metadata{ + ArtifactID: "alice", + GroupID: "org.direct", + IsTransitive: false, + DepGroupVals: []string{}, + }, + }, + { + Name: "org.direct:bob", + Version: "2.0.0", + PURLType: purl.TypeMaven, + Locations: []string{"testdata/transitive.xml"}, + Plugins: []string{"java/pomxml"}, + Metadata: &javalockfile.Metadata{ + ArtifactID: "bob", + GroupID: "org.direct", + IsTransitive: false, + DepGroupVals: []string{}, + }, + }, + { + Name: "org.direct:chris", + Version: "3.0.0", + PURLType: purl.TypeMaven, + Locations: []string{"testdata/transitive.xml"}, + Plugins: []string{"java/pomxml"}, + Metadata: &javalockfile.Metadata{ + ArtifactID: "chris", + GroupID: "org.direct", + IsTransitive: false, + DepGroupVals: []string{}, + }, + }, + }, + } + + srv := clienttest.NewMockHTTPServer(t) + srv.SetResponse(t, "org/upstream/parent-pom/1.0/parent-pom-1.0.pom", []byte(` + + org.upstream + parent-pom + 1.0 + pom + + + org.eve + eve + 5.0.0 + + + + `)) + srv.SetResponse(t, "org/import/import/1.2.3/import-1.2.3.pom", []byte(` + + org.import + import + 1.2.3 + pom + + + + org.frank + frank + 6.0.0 + + + + + `)) + + apiClient, err := datasource.NewDefaultMavenRegistryAPIClient(t.Context(), srv.URL) + if err != nil { + t.Fatalf("%v", err) + } + + resolutionClient := clienttest.NewMockResolutionClient(t, "testdata/universe/basic-universe.yaml") + + enrichy := pomxml.New(&cpb.PluginConfig{}) + enrichy.(*pomxml.Enricher).DepClient = resolutionClient + enrichy.(*pomxml.Enricher).MavenClient = apiClient + + err = enrichy.Enrich(t.Context(), &input, &inv) + if err != nil { + t.Fatalf("failed to enrich: %v", err) + } + + wantInventory := inventory.Inventory{ + Packages: []*extractor.Package{ + { + // Not a Java package. + Name: "abc", + Version: "1.0.0", + PURLType: purl.TypePyPi, + Locations: []string{"testdata/poetry/poetry.lock"}, + Plugins: []string{"python/poetrylock"}, + }, + { + // Not extracted from a pom.xml + Name: "abc", + Version: "1.0.0", + PURLType: purl.TypeMaven, + Locations: []string{"testdata/java/gradle.lockfile"}, + Plugins: []string{"java/gradlelockfile"}, + }, + { + Name: "org.direct:alice", + Version: "1.0.0", + PURLType: purl.TypeMaven, + Locations: []string{"testdata/transitive.xml"}, + Plugins: []string{"java/pomxml", "transitivedependency/pomxml"}, + Metadata: &javalockfile.Metadata{ + ArtifactID: "alice", + GroupID: "org.direct", + IsTransitive: false, + DepGroupVals: []string{}, + }, + }, + { + Name: "org.direct:bob", + Version: "2.0.0", + PURLType: purl.TypeMaven, + Locations: []string{"testdata/transitive.xml"}, + Plugins: []string{"java/pomxml", "transitivedependency/pomxml"}, + Metadata: &javalockfile.Metadata{ + ArtifactID: "bob", + GroupID: "org.direct", + IsTransitive: false, + DepGroupVals: []string{}, + }, + }, + { + Name: "org.direct:chris", + Version: "3.0.0", + PURLType: purl.TypeMaven, + Locations: []string{"testdata/transitive.xml"}, + Plugins: []string{"java/pomxml", "transitivedependency/pomxml"}, + Metadata: &javalockfile.Metadata{ + ArtifactID: "chris", + GroupID: "org.direct", + IsTransitive: false, + DepGroupVals: []string{}, + }, + }, + { + Name: "org.transitive:chuck", + Version: "1.1.1", + PURLType: purl.TypeMaven, + Locations: []string{"testdata/transitive.xml"}, + Plugins: []string{"transitivedependency/pomxml"}, + Metadata: &javalockfile.Metadata{ + ArtifactID: "chuck", + GroupID: "org.transitive", + IsTransitive: true, + DepGroupVals: []string{}, + }, + }, + { + Name: "org.transitive:dave", + Version: "2.2.2", + PURLType: purl.TypeMaven, + Locations: []string{"testdata/transitive.xml"}, + Plugins: []string{"transitivedependency/pomxml"}, + Metadata: &javalockfile.Metadata{ + ArtifactID: "dave", + GroupID: "org.transitive", + IsTransitive: true, + DepGroupVals: []string{}, + }, + }, + { + Name: "org.transitive:eve", + Version: "3.3.3", + PURLType: purl.TypeMaven, + Locations: []string{"testdata/transitive.xml"}, + Plugins: []string{"transitivedependency/pomxml"}, + Metadata: &javalockfile.Metadata{ + ArtifactID: "eve", + GroupID: "org.transitive", + IsTransitive: true, + DepGroupVals: []string{}, + }, + }, + { + Name: "org.transitive:frank", + Version: "4.4.4", + PURLType: purl.TypeMaven, + Locations: []string{"testdata/transitive.xml"}, + Plugins: []string{"transitivedependency/pomxml"}, + Metadata: &javalockfile.Metadata{ + ArtifactID: "frank", + GroupID: "org.transitive", + IsTransitive: true, + DepGroupVals: []string{}, + }, + }, + }, + } + sort.Slice(inv.Packages, func(i, j int) bool { + return inv.Packages[i].Name < inv.Packages[j].Name + }) + if diff := cmp.Diff(wantInventory, inv); diff != "" { + t.Errorf("%s.Enrich() diff (-want +got):\n%s", enrichy.Name(), diff) + } +} diff --git a/enricher/transitivedependency/pomxml/testdata/transitive.xml b/enricher/transitivedependency/pomxml/testdata/transitive.xml new file mode 100644 index 000000000..52e416a0b --- /dev/null +++ b/enricher/transitivedependency/pomxml/testdata/transitive.xml @@ -0,0 +1,33 @@ + + com.mycompany.app + my-app + 1.0 + + + + + org.transitive + frank + 4.4.4 + + + + + + + org.direct + alice + 1.0.0 + + + org.direct + bob + 2.0.0 + + + org.direct + chris + 3.0.0 + + + diff --git a/enricher/transitivedependency/pomxml/testdata/universe/basic-universe.yaml b/enricher/transitivedependency/pomxml/testdata/universe/basic-universe.yaml new file mode 100644 index 000000000..2bf2b3272 --- /dev/null +++ b/enricher/transitivedependency/pomxml/testdata/universe/basic-universe.yaml @@ -0,0 +1,60 @@ +system: maven +schema: | + com.google.code.findbugs:jsr305 + 3.0.2 + io.netty:netty-all + 4.1.9 + 4.1.42.Final + junit:junit + 4.12 + org.alice:alice + 1.0.0 + org.apache.maven:maven-artifact + 1.0.0 + org.bob:bob + 2.0.0 + org.chuck:chuck + 3.0.0 + org.dave:dave + 4.0.0 + org.direct:alice + 1.0.0 + org.transitive:chuck@1.1.1 + org.transitive:dave@2.2.2 + org.direct:bob + 2.0.0 + org.transitive:eve@3.3.3 + org.direct:chris + 3.0.0 + org.transitive:frank@3.3.3 + org.eve:eve + 5.0.0 + org.frank:frank + 6.0.0 + org.mine:my.package + 2.3.4 + org.mine:mypackage + 1.0.0 + org.mine:ranged-package + 9.4.35 + 9.4.36 + 9.4.37 + 9.5 + org.slf4j:slf4j-log4j12 + 1.7.25 + org.transitive:chuck + 1.1.1 + 2.2.2 + org.transitive:eve@2.2.2 + 3.3.3 + org.transitive:dave + 1.1.1 + 2.2.2 + 3.3.3 + org.transitive:eve + 1.1.1 + 2.2.2 + 3.3.3 + org.transitive:frank + 3.3.3 + 4.4.4 diff --git a/enricher/transitivedependency/requirements/requirements.go b/enricher/transitivedependency/requirements/requirements.go index 814af8c4c..ce8880197 100644 --- a/enricher/transitivedependency/requirements/requirements.go +++ b/enricher/transitivedependency/requirements/requirements.go @@ -27,6 +27,7 @@ import ( cpb "github.com/google/osv-scalibr/binary/proto/config_go_proto" "github.com/google/osv-scalibr/clients/resolution" "github.com/google/osv-scalibr/enricher" + "github.com/google/osv-scalibr/enricher/transitivedependency/internal" "github.com/google/osv-scalibr/extractor" "github.com/google/osv-scalibr/extractor/filesystem/language/python/requirements" "github.com/google/osv-scalibr/inventory" @@ -76,19 +77,19 @@ func New(cfg *cpb.PluginConfig) enricher.Enricher { // Enrich enriches the inventory in requirements.txt with transitive dependencies. func (e Enricher) Enrich(ctx context.Context, input *enricher.ScanInput, inv *inventory.Inventory) error { - pkgGroups := groupPackages(inv.Packages) + pkgGroups := internal.GroupPackagesFromPlugin(inv.Packages, requirements.Name) for path, pkgMap := range pkgGroups { - packages := make([]packageWithIndex, 0, len(pkgMap)) + packages := make([]internal.PackageWithIndex, 0, len(pkgMap)) for _, indexPkg := range pkgMap { packages = append(packages, indexPkg) } - slices.SortFunc(packages, func(a, b packageWithIndex) int { - return a.index - b.index + slices.SortFunc(packages, func(a, b internal.PackageWithIndex) int { + return a.Index - b.Index }) list := make([]*extractor.Package, 0, len(packages)) for _, indexPkg := range packages { - list = append(list, indexPkg.pkg) + list = append(list, indexPkg.Pkg) } if len(list) == 0 || len(list[0].Metadata.(*requirements.Metadata).HashCheckingModeValues) > 0 { // Do not perform transitive extraction with hash-checking mode. @@ -105,50 +106,11 @@ func (e Enricher) Enrich(ctx context.Context, input *enricher.ScanInput, inv *in continue } - for _, pkg := range pkgs { - indexPkg, ok := pkgMap[pkg.Name] - if ok { - // This dependency is in manifest, update the version and plugins. - i := indexPkg.index - inv.Packages[i].Version = pkg.Version - inv.Packages[i].Plugins = append(inv.Packages[i].Plugins, Name) - } else { - // This dependency is not found in manifest, so it's a transitive dependency. - inv.Packages = append(inv.Packages, pkg) - } - } + internal.Add(pkgs, inv, Name, pkgMap) } return nil } -// packageWithIndex holds the package with its index in inv.Packages -type packageWithIndex struct { - pkg *extractor.Package - index int -} - -// groupPackages groups packages found in requirements.txt by the first location that they are found -// and returns a map of location -> package name -> package with index. -func groupPackages(pkgs []*extractor.Package) map[string]map[string]packageWithIndex { - result := make(map[string]map[string]packageWithIndex) - for i, pkg := range pkgs { - if !slices.Contains(pkg.Plugins, requirements.Name) { - continue - } - if len(pkg.Locations) == 0 { - log.Warnf("package %s has no locations", pkg.Name) - continue - } - // Use the path where this package is first found. - path := pkg.Locations[0] - if _, ok := result[path]; !ok { - result[path] = make(map[string]packageWithIndex) - } - result[path][pkg.Name] = packageWithIndex{pkg, i} - } - return result -} - // resolve performs dependency resolution for packages found in a single requirements.txt. func (e Enricher) resolve(ctx context.Context, path string, list []*extractor.Package) ([]*extractor.Package, error) { overrideClient := resolution.NewOverrideClient(e.Client) diff --git a/extractor/filesystem/language/java/pomxml/pomxml_test.go b/extractor/filesystem/language/java/pomxml/pomxml_test.go index e4710dc39..5d6d3ad26 100644 --- a/extractor/filesystem/language/java/pomxml/pomxml_test.go +++ b/extractor/filesystem/language/java/pomxml/pomxml_test.go @@ -336,6 +336,50 @@ func TestExtractor_Extract(t *testing.T) { }, }, }, + { + Name: "transitive dependencies", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/transitive.xml", + }, + WantPackages: []*extractor.Package{ + { + Name: "org.direct:alice", + Version: "1.0.0", + PURLType: purl.TypeMaven, + Locations: []string{"testdata/transitive.xml"}, + Metadata: &javalockfile.Metadata{ + ArtifactID: "alice", + GroupID: "org.direct", + IsTransitive: false, + DepGroupVals: []string{}, + }, + }, + { + Name: "org.direct:bob", + Version: "2.0.0", + PURLType: purl.TypeMaven, + Locations: []string{"testdata/transitive.xml"}, + Metadata: &javalockfile.Metadata{ + ArtifactID: "bob", + GroupID: "org.direct", + IsTransitive: false, + DepGroupVals: []string{}, + }, + }, + { + Name: "org.direct:chris", + Version: "3.0.0", + PURLType: purl.TypeMaven, + Locations: []string{"testdata/transitive.xml"}, + Metadata: &javalockfile.Metadata{ + ArtifactID: "chris", + GroupID: "org.direct", + IsTransitive: false, + DepGroupVals: []string{}, + }, + }, + }, + }, } for _, tt := range tests { diff --git a/extractor/filesystem/language/java/pomxml/testdata/transitive.xml b/extractor/filesystem/language/java/pomxml/testdata/transitive.xml new file mode 100644 index 000000000..52e416a0b --- /dev/null +++ b/extractor/filesystem/language/java/pomxml/testdata/transitive.xml @@ -0,0 +1,33 @@ + + com.mycompany.app + my-app + 1.0 + + + + + org.transitive + frank + 4.4.4 + + + + + + + org.direct + alice + 1.0.0 + + + org.direct + bob + 2.0.0 + + + org.direct + chris + 3.0.0 + + +