Veles custom extractor for aws access key

extractor/filesystem/list/list.go

@@ -96,14 +96,14 @@ import (
 	"github.com/google/osv-scalibr/extractor/filesystem/runtime/nodejs/nvm"
 	"github.com/google/osv-scalibr/extractor/filesystem/sbom/cdx"
 	"github.com/google/osv-scalibr/extractor/filesystem/sbom/spdx"
+	"github.com/google/osv-scalibr/extractor/filesystem/secrets/awsaccesskey"
 	"github.com/google/osv-scalibr/extractor/filesystem/secrets/convert"
 	"github.com/google/osv-scalibr/extractor/filesystem/secrets/mariadb"
 	"github.com/google/osv-scalibr/extractor/filesystem/secrets/mysqlmylogin"
 	"github.com/google/osv-scalibr/extractor/filesystem/secrets/onepasswordconnecttoken"
 	"github.com/google/osv-scalibr/extractor/filesystem/secrets/pgpass"
 	"github.com/google/osv-scalibr/veles"
 	"github.com/google/osv-scalibr/veles/secrets/anthropicapikey"
-	"github.com/google/osv-scalibr/veles/secrets/awsaccesskey"
 	"github.com/google/osv-scalibr/veles/secrets/azurestorageaccountaccesskey"
 	"github.com/google/osv-scalibr/veles/secrets/azuretoken"
 	"github.com/google/osv-scalibr/veles/secrets/cratesioapitoken"
@@ -283,6 +283,7 @@ var (
 		pgpass.Name:                  {noCFG(pgpass.New)},
 		onepasswordconnecttoken.Name: {noCFG(onepasswordconnecttoken.New)},
 		mariadb.Name:                 {noCFG(mariadb.NewDefault)},
+		awsaccesskey.Name:            {noCFG(awsaccesskey.New)},
 	}
 
 	// SecretDetectors for Detector interface.
@@ -331,7 +332,6 @@ var (
 		{onepasswordkeys.NewRecoveryTokenDetector(), "secrets/onepasswordrecoverycode", 0},
 		{gcshmackey.NewDetector(), "secrets/gcshmackey", 0},
 		{vapid.NewDetector(), "secrets/vapidkey", 0},
-		{awsaccesskey.NewDetector(), "secrets/awsaccesskey", 0},
 		{recaptchakey.NewDetector(), "secrets/recaptchakey", 0},
 	})
 

extractor/filesystem/secrets/awsaccesskey/awsaccesskey.go

@@ -0,0 +1,46 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package awsaccesskey extends the veles awsaccesskey.Detector to search inside the `~/.aws/credentials` file
+package awsaccesskey
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/google/osv-scalibr/extractor/filesystem"
+	"github.com/google/osv-scalibr/veles/secrets/awsaccesskey"
+
+	"github.com/google/osv-scalibr/extractor/filesystem/secrets/convert"
+)
+
+const (
+	// Name is the name of the extractor
+	Name = "secrets/awsaccesskey"
+	// Version is the version of the extractor
+	Version = 0
+)
+
+// New returns a filesystem.Extractor which extracts AWS Access Keys using the awsaccesskey.Detector
+func New() filesystem.Extractor {
+	return convert.FromVelesDetectorWithRequire(
+		awsaccesskey.NewDetector(), Name, Version, FileRequired,
+	)
+}
+
+// FileRequired returns true if a file contains aws credentials.
+func FileRequired(api filesystem.FileAPI) bool {
+	path := filepath.ToSlash(api.Path())
+	return strings.HasSuffix(path, ".aws/credentials")
+}

extractor/filesystem/secrets/awsaccesskey/awsaccesskey_test.go

@@ -0,0 +1,119 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awsaccesskey_test
+
+import (
+	"runtime"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/google/osv-scalibr/extractor/filesystem/secrets/awsaccesskey"
+	"github.com/google/osv-scalibr/extractor/filesystem/simplefileapi"
+	"github.com/google/osv-scalibr/inventory"
+	"github.com/google/osv-scalibr/testing/extracttest"
+	awsaccesskeydetector "github.com/google/osv-scalibr/veles/secrets/awsaccesskey"
+)
+
+func TestExtractor_FileRequired(t *testing.T) {
+	tests := []struct {
+		inputPath string
+		want      bool
+		isWindows bool
+	}{
+		{inputPath: "", want: false},
+
+		// linux
+		{inputPath: `/Users/example-user/.aws/credentials`, want: true},
+		{inputPath: `/Users/example-user/bad/path`, want: false},
+
+		// windows
+		{inputPath: `C:\Users\USERNAME\.aws\credentials`, isWindows: true, want: true},
+		{inputPath: `C:\Users\USERNAME\another\bad\path`, isWindows: true, want: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.inputPath, func(t *testing.T) {
+			if tt.isWindows && runtime.GOOS != "windows" {
+				t.Skipf("Skipping test %q for %q", t.Name(), runtime.GOOS)
+			}
+			e := awsaccesskey.New()
+			got := e.FileRequired(simplefileapi.New(tt.inputPath, nil))
+			if got != tt.want {
+				t.Errorf("FileRequired(%s) got = %v, want %v", tt.inputPath, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestExtractor_Extract(t *testing.T) {
+	tests := []*struct {
+		Name        string
+		Path        string
+		WantSecrets []*inventory.Secret
+		WantErr     error
+	}{
+		{
+			Name:        "empty",
+			Path:        "empty",
+			WantSecrets: nil,
+		},
+		{
+			Name: "aws_credentials",
+			Path: "aws_credentials",
+			WantSecrets: []*inventory.Secret{
+				{
+					Secret: awsaccesskeydetector.Credentials{
+						AccessID: "AIKA1984R439T439HTH4",
+						Secret:   "32r923jr023rk320rk2a3rkB34tj340r32Ckt433",
+					},
+					Location: "aws_credentials",
+				},
+			},
+		},
+		{
+			Name:        "random_content",
+			Path:        "random_content",
+			WantSecrets: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.Name, func(t *testing.T) {
+			extr := awsaccesskey.New()
+
+			inputCfg := extracttest.ScanInputMockConfig{
+				Path:         tt.Path,
+				FakeScanRoot: "testdata",
+			}
+
+			scanInput := extracttest.GenerateScanInputMock(t, inputCfg)
+			defer extracttest.CloseTestScanInput(t, scanInput)
+
+			got, err := extr.Extract(t.Context(), &scanInput)
+
+			if diff := cmp.Diff(tt.WantErr, err, cmpopts.EquateErrors()); diff != "" {
+				t.Errorf("%s.Extract(%q) error diff (-want +got):\n%s", extr.Name(), tt.Path, diff)
+				return
+			}
+
+			wantInv := inventory.Inventory{Secrets: tt.WantSecrets}
+			opts := []cmp.Option{cmpopts.SortSlices(extracttest.PackageCmpLess), cmpopts.EquateEmpty()}
+			if diff := cmp.Diff(wantInv, got, opts...); diff != "" {
+				t.Errorf("%s.Extract(%q) diff (-want +got):\n%s", extr.Name(), tt.Path, diff)
+			}
+		})
+	}
+}

extractor/filesystem/secrets/awsaccesskey/testdata/aws_credentials

@@ -0,0 +1,3 @@
+[default]
+aws_access_key_id = AIKA1984R439T439HTH4
+aws_secret_access_key = 32r923jr023rk320rk2a3rkB34tj340r32Ckt433

extractor/filesystem/secrets/awsaccesskey/testdata/random_content

@@ -0,0 +1,10 @@
+localhost:5432:mydb:myuser:mypassword
+hostname:1234:testdb:testuser:testpass123
+hostname:1234:testdb:testuser:passw*ord
+# space inside one group (except password)
+hostname:1234:testdb:testuser:passw ord
+hostname:1234:db name:testuser:password
+# this is a comment and should be ignored
+*:*:db:admin:supersecret
+# valid with escaped :
+prod.example.com:5432:db:admin:pass\:word

extractor/filesystem/secrets/convert/convert.go

@@ -95,9 +95,13 @@ var _ filesystem.Extractor = &detectorWrapper{}
 func SetupVelesExtractors(extractors []filesystem.Extractor) ([]filesystem.Extractor, error) {
 	result := make([]filesystem.Extractor, 0, len(extractors))
 	detectors := []veles.Detector{}
+
 	for _, e := range extractors {
-		if d, ok := e.(veles.Detector); ok {
+		if d, isDetector := e.(veles.Detector); isDetector {
 			detectors = append(detectors, d)
+			if _, keepExtractor := e.(extractorKeeper); keepExtractor {
+				result = append(result, e)
+			}
 		} else {
 			result = append(result, e)
 		}

extractor/filesystem/secrets/convert/specific.go

@@ -0,0 +1,100 @@
+package convert
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/google/osv-scalibr/extractor/filesystem"
+	"github.com/google/osv-scalibr/inventory"
+	"github.com/google/osv-scalibr/plugin"
+	"github.com/google/osv-scalibr/veles"
+)
+
+// FromVelesDetectorWithRequire works similar to FromVelesDetector but allows specifying additional files to look at on top of the default ones.
+func FromVelesDetectorWithRequire(velesDetector veles.Detector, name string, version int, fileRequired func(filesystem.FileAPI) bool) filesystem.Extractor {
+	return &detectorWithRequire{
+		velesDetector: velesDetector,
+		name:          name,
+		version:       version,
+		fileRequired:  fileRequired,
+	}
+}
+
+// extractorKeeper signals that a Detector also functions as a standalone filesystem.Extractor.
+type extractorKeeper interface {
+	KeepExtractor() bool
+}
+
+// Assert that detectorWithRequire implements the required interfaces.
+var _ veles.Detector = &detectorWithRequire{}
+var _ filesystem.Extractor = &detectorWithRequire{}
+var _ extractorKeeper = &detectorWithRequire{}
+
+// detectorWithRequire is a wrapper around the veles.Detector interface that
+// implements the additional functions of the filesystem Extractor interface.
+type detectorWithRequire struct {
+	velesDetector veles.Detector
+	name          string
+	version       int
+	fileRequired  func(filesystem.FileAPI) bool
+	e             *veles.DetectionEngine
+}
+
+// KeepExtractor signals that this detector must also be registered as a standalone
+// filesystem.Extractor to handle the additional files specified in the fileRequired callback.
+func (d *detectorWithRequire) KeepExtractor() bool { return true }
+
+// MaxSecretLen returns the maximum length a secret from this Detector can have.
+func (d *detectorWithRequire) MaxSecretLen() uint32 {
+	return d.velesDetector.MaxSecretLen()
+}
+
+// Detect finds candidate secrets in the data and returns them alongside their
+// starting positions.
+func (d *detectorWithRequire) Detect(data []byte) ([]veles.Secret, []int) {
+	return d.velesDetector.Detect(data)
+}
+
+// Name of the secret extractor.
+func (d *detectorWithRequire) Name() string {
+	return d.name
+}
+
+// Version of the secret extractor.
+func (d *detectorWithRequire) Version() int {
+	return d.version
+}
+
+// Requirements of the secret extractor.
+func (d *detectorWithRequire) Requirements() *plugin.Capabilities {
+	// Veles plugins don't have any special requirements.
+	return &plugin.Capabilities{}
+}
+
+// FileRequired returns the provided file required callback.
+func (d *detectorWithRequire) FileRequired(api filesystem.FileAPI) bool {
+	return d.fileRequired(api)
+}
+
+// Extract extracts secret from the filesystem using the provided detector.
+func (d *detectorWithRequire) Extract(ctx context.Context, input *filesystem.ScanInput) (inventory.Inventory, error) {
+	if d.e == nil {
+		var err error
+		d.e, err = veles.NewDetectionEngine([]veles.Detector{d.velesDetector})
+		if err != nil {
+			return inventory.Inventory{}, err
+		}
+	}
+	secrets, err := d.e.Detect(ctx, input.Reader)
+	if err != nil {
+		return inventory.Inventory{}, fmt.Errorf("unable to scan for secrets: %w", err)
+	}
+	i := inventory.Inventory{}
+	for _, s := range secrets {
+		i.Secrets = append(i.Secrets, &inventory.Secret{
+			Secret:   s,
+			Location: input.Path,
+		})
+	}
+	return i, nil
+}

scalibr_test.go

@@ -21,6 +21,7 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -78,6 +79,7 @@ func TestScan(t *testing.T) {
 	fs := scalibrfs.DirFS(tmp)
 	tmpRoot := []*scalibrfs.ScanRoot{{FS: fs, Path: tmp}}
 	_ = os.WriteFile(filepath.Join(tmp, "file.txt"), []byte("Content"), 0644)
+	_ = os.WriteFile(filepath.Join(tmp, "config"), []byte("Content"), 0644)
 
 	pkgName := "software"
 	fakeExtractor := fe.New(
@@ -366,6 +368,35 @@ func TestScan(t *testing.T) {
 				},
 			},
 		},
+		{
+			desc: "Veles_secret_detector_with_extractor",
+			cfg: &scalibr.ScanConfig{
+				Plugins: []plugin.Plugin{
+					// use the fakeSecretDetector1 also on config files
+					cf.FromVelesDetectorWithRequire(
+						fakeSecretDetector1, "secret-detector", 1,
+						func(fa filesystem.FileAPI) bool {
+							return strings.HasSuffix(fa.Path(), "config")
+						},
+					),
+				},
+				ScanRoots: tmpRoot,
+			},
+			want: &scalibr.ScanResult{
+				Version: version.ScannerVersion,
+				Status:  success,
+				PluginStatus: []*plugin.Status{
+					{Name: "secret-detector", Version: 1, Status: success},
+					{Name: "secrets/veles", Version: 1, Status: success},
+				},
+				Inventory: inventory.Inventory{
+					Secrets: []*inventory.Secret{
+						{Secret: velestest.NewFakeStringSecret("Con"), Location: "file.txt"},
+						{Secret: velestest.NewFakeStringSecret("Con"), Location: "config"},
+					},
+				},
+			},
+		},
 	}
 
 	for _, tc := range testCases {