diff --git a/extractor/filesystem/list/list.go b/extractor/filesystem/list/list.go index cf7c76733..451e6ea4b 100644 --- a/extractor/filesystem/list/list.go +++ b/extractor/filesystem/list/list.go @@ -96,6 +96,7 @@ import ( "github.com/google/osv-scalibr/extractor/filesystem/runtime/nodejs/nvm" "github.com/google/osv-scalibr/extractor/filesystem/sbom/cdx" "github.com/google/osv-scalibr/extractor/filesystem/sbom/spdx" + "github.com/google/osv-scalibr/extractor/filesystem/secrets/awsaccesskey" "github.com/google/osv-scalibr/extractor/filesystem/secrets/convert" "github.com/google/osv-scalibr/extractor/filesystem/secrets/mariadb" "github.com/google/osv-scalibr/extractor/filesystem/secrets/mysqlmylogin" @@ -103,7 +104,6 @@ import ( "github.com/google/osv-scalibr/extractor/filesystem/secrets/pgpass" "github.com/google/osv-scalibr/veles" "github.com/google/osv-scalibr/veles/secrets/anthropicapikey" - "github.com/google/osv-scalibr/veles/secrets/awsaccesskey" "github.com/google/osv-scalibr/veles/secrets/azurestorageaccountaccesskey" "github.com/google/osv-scalibr/veles/secrets/azuretoken" "github.com/google/osv-scalibr/veles/secrets/cratesioapitoken" @@ -283,6 +283,7 @@ var ( pgpass.Name: {noCFG(pgpass.New)}, onepasswordconnecttoken.Name: {noCFG(onepasswordconnecttoken.New)}, mariadb.Name: {noCFG(mariadb.NewDefault)}, + awsaccesskey.Name: {noCFG(awsaccesskey.New)}, } // SecretDetectors for Detector interface. @@ -331,7 +332,6 @@ var ( {onepasswordkeys.NewRecoveryTokenDetector(), "secrets/onepasswordrecoverycode", 0}, {gcshmackey.NewDetector(), "secrets/gcshmackey", 0}, {vapid.NewDetector(), "secrets/vapidkey", 0}, - {awsaccesskey.NewDetector(), "secrets/awsaccesskey", 0}, {recaptchakey.NewDetector(), "secrets/recaptchakey", 0}, }) diff --git a/extractor/filesystem/secrets/awsaccesskey/awsaccesskey.go b/extractor/filesystem/secrets/awsaccesskey/awsaccesskey.go new file mode 100644 index 000000000..304c8f38a --- /dev/null +++ b/extractor/filesystem/secrets/awsaccesskey/awsaccesskey.go @@ -0,0 +1,46 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package awsaccesskey extends the veles awsaccesskey.Detector to search inside the `~/.aws/credentials` file +package awsaccesskey + +import ( + "path/filepath" + "strings" + + "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scalibr/veles/secrets/awsaccesskey" + + "github.com/google/osv-scalibr/extractor/filesystem/secrets/convert" +) + +const ( + // Name is the name of the extractor + Name = "secrets/awsaccesskey" + // Version is the version of the extractor + Version = 0 +) + +// New returns a filesystem.Extractor which extracts AWS Access Keys using the awsaccesskey.Detector +func New() filesystem.Extractor { + return convert.FromVelesDetectorWithRequire( + awsaccesskey.NewDetector(), Name, Version, FileRequired, + ) +} + +// FileRequired returns true if a file contains aws credentials. +func FileRequired(api filesystem.FileAPI) bool { + path := filepath.ToSlash(api.Path()) + return strings.HasSuffix(path, ".aws/credentials") +} diff --git a/extractor/filesystem/secrets/awsaccesskey/awsaccesskey_test.go b/extractor/filesystem/secrets/awsaccesskey/awsaccesskey_test.go new file mode 100644 index 000000000..e28199099 --- /dev/null +++ b/extractor/filesystem/secrets/awsaccesskey/awsaccesskey_test.go @@ -0,0 +1,119 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package awsaccesskey_test + +import ( + "runtime" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/osv-scalibr/extractor/filesystem/secrets/awsaccesskey" + "github.com/google/osv-scalibr/extractor/filesystem/simplefileapi" + "github.com/google/osv-scalibr/inventory" + "github.com/google/osv-scalibr/testing/extracttest" + awsaccesskeydetector "github.com/google/osv-scalibr/veles/secrets/awsaccesskey" +) + +func TestExtractor_FileRequired(t *testing.T) { + tests := []struct { + inputPath string + want bool + isWindows bool + }{ + {inputPath: "", want: false}, + + // linux + {inputPath: `/Users/example-user/.aws/credentials`, want: true}, + {inputPath: `/Users/example-user/bad/path`, want: false}, + + // windows + {inputPath: `C:\Users\USERNAME\.aws\credentials`, isWindows: true, want: true}, + {inputPath: `C:\Users\USERNAME\another\bad\path`, isWindows: true, want: false}, + } + + for _, tt := range tests { + t.Run(tt.inputPath, func(t *testing.T) { + if tt.isWindows && runtime.GOOS != "windows" { + t.Skipf("Skipping test %q for %q", t.Name(), runtime.GOOS) + } + e := awsaccesskey.New() + got := e.FileRequired(simplefileapi.New(tt.inputPath, nil)) + if got != tt.want { + t.Errorf("FileRequired(%s) got = %v, want %v", tt.inputPath, got, tt.want) + } + }) + } +} + +func TestExtractor_Extract(t *testing.T) { + tests := []*struct { + Name string + Path string + WantSecrets []*inventory.Secret + WantErr error + }{ + { + Name: "empty", + Path: "empty", + WantSecrets: nil, + }, + { + Name: "aws_credentials", + Path: "aws_credentials", + WantSecrets: []*inventory.Secret{ + { + Secret: awsaccesskeydetector.Credentials{ + AccessID: "AIKA1984R439T439HTH4", + Secret: "32r923jr023rk320rk2a3rkB34tj340r32Ckt433", + }, + Location: "aws_credentials", + }, + }, + }, + { + Name: "random_content", + Path: "random_content", + WantSecrets: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.Name, func(t *testing.T) { + extr := awsaccesskey.New() + + inputCfg := extracttest.ScanInputMockConfig{ + Path: tt.Path, + FakeScanRoot: "testdata", + } + + scanInput := extracttest.GenerateScanInputMock(t, inputCfg) + defer extracttest.CloseTestScanInput(t, scanInput) + + got, err := extr.Extract(t.Context(), &scanInput) + + if diff := cmp.Diff(tt.WantErr, err, cmpopts.EquateErrors()); diff != "" { + t.Errorf("%s.Extract(%q) error diff (-want +got):\n%s", extr.Name(), tt.Path, diff) + return + } + + wantInv := inventory.Inventory{Secrets: tt.WantSecrets} + opts := []cmp.Option{cmpopts.SortSlices(extracttest.PackageCmpLess), cmpopts.EquateEmpty()} + if diff := cmp.Diff(wantInv, got, opts...); diff != "" { + t.Errorf("%s.Extract(%q) diff (-want +got):\n%s", extr.Name(), tt.Path, diff) + } + }) + } +} diff --git a/extractor/filesystem/secrets/awsaccesskey/testdata/aws_credentials b/extractor/filesystem/secrets/awsaccesskey/testdata/aws_credentials new file mode 100644 index 000000000..7ea6ade7a --- /dev/null +++ b/extractor/filesystem/secrets/awsaccesskey/testdata/aws_credentials @@ -0,0 +1,3 @@ +[default] +aws_access_key_id = AIKA1984R439T439HTH4 +aws_secret_access_key = 32r923jr023rk320rk2a3rkB34tj340r32Ckt433 diff --git a/extractor/filesystem/secrets/awsaccesskey/testdata/empty b/extractor/filesystem/secrets/awsaccesskey/testdata/empty new file mode 100644 index 000000000..e69de29bb diff --git a/extractor/filesystem/secrets/awsaccesskey/testdata/random_content b/extractor/filesystem/secrets/awsaccesskey/testdata/random_content new file mode 100644 index 000000000..e62a6bb7a --- /dev/null +++ b/extractor/filesystem/secrets/awsaccesskey/testdata/random_content @@ -0,0 +1,10 @@ +localhost:5432:mydb:myuser:mypassword +hostname:1234:testdb:testuser:testpass123 +hostname:1234:testdb:testuser:passw*ord +# space inside one group (except password) +hostname:1234:testdb:testuser:passw ord +hostname:1234:db name:testuser:password +# this is a comment and should be ignored +*:*:db:admin:supersecret +# valid with escaped : +prod.example.com:5432:db:admin:pass\:word diff --git a/extractor/filesystem/secrets/convert/convert.go b/extractor/filesystem/secrets/convert/convert.go index 3572b1610..e5f90978a 100644 --- a/extractor/filesystem/secrets/convert/convert.go +++ b/extractor/filesystem/secrets/convert/convert.go @@ -95,9 +95,13 @@ var _ filesystem.Extractor = &detectorWrapper{} func SetupVelesExtractors(extractors []filesystem.Extractor) ([]filesystem.Extractor, error) { result := make([]filesystem.Extractor, 0, len(extractors)) detectors := []veles.Detector{} + for _, e := range extractors { - if d, ok := e.(veles.Detector); ok { + if d, isDetector := e.(veles.Detector); isDetector { detectors = append(detectors, d) + if _, keepExtractor := e.(extractorKeeper); keepExtractor { + result = append(result, e) + } } else { result = append(result, e) } diff --git a/extractor/filesystem/secrets/convert/specific.go b/extractor/filesystem/secrets/convert/specific.go new file mode 100644 index 000000000..e1c179958 --- /dev/null +++ b/extractor/filesystem/secrets/convert/specific.go @@ -0,0 +1,100 @@ +package convert + +import ( + "context" + "fmt" + + "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scalibr/inventory" + "github.com/google/osv-scalibr/plugin" + "github.com/google/osv-scalibr/veles" +) + +// FromVelesDetectorWithRequire works similar to FromVelesDetector but allows specifying additional files to look at on top of the default ones. +func FromVelesDetectorWithRequire(velesDetector veles.Detector, name string, version int, fileRequired func(filesystem.FileAPI) bool) filesystem.Extractor { + return &detectorWithRequire{ + velesDetector: velesDetector, + name: name, + version: version, + fileRequired: fileRequired, + } +} + +// extractorKeeper signals that a Detector also functions as a standalone filesystem.Extractor. +type extractorKeeper interface { + KeepExtractor() bool +} + +// Assert that detectorWithRequire implements the required interfaces. +var _ veles.Detector = &detectorWithRequire{} +var _ filesystem.Extractor = &detectorWithRequire{} +var _ extractorKeeper = &detectorWithRequire{} + +// detectorWithRequire is a wrapper around the veles.Detector interface that +// implements the additional functions of the filesystem Extractor interface. +type detectorWithRequire struct { + velesDetector veles.Detector + name string + version int + fileRequired func(filesystem.FileAPI) bool + e *veles.DetectionEngine +} + +// KeepExtractor signals that this detector must also be registered as a standalone +// filesystem.Extractor to handle the additional files specified in the fileRequired callback. +func (d *detectorWithRequire) KeepExtractor() bool { return true } + +// MaxSecretLen returns the maximum length a secret from this Detector can have. +func (d *detectorWithRequire) MaxSecretLen() uint32 { + return d.velesDetector.MaxSecretLen() +} + +// Detect finds candidate secrets in the data and returns them alongside their +// starting positions. +func (d *detectorWithRequire) Detect(data []byte) ([]veles.Secret, []int) { + return d.velesDetector.Detect(data) +} + +// Name of the secret extractor. +func (d *detectorWithRequire) Name() string { + return d.name +} + +// Version of the secret extractor. +func (d *detectorWithRequire) Version() int { + return d.version +} + +// Requirements of the secret extractor. +func (d *detectorWithRequire) Requirements() *plugin.Capabilities { + // Veles plugins don't have any special requirements. + return &plugin.Capabilities{} +} + +// FileRequired returns the provided file required callback. +func (d *detectorWithRequire) FileRequired(api filesystem.FileAPI) bool { + return d.fileRequired(api) +} + +// Extract extracts secret from the filesystem using the provided detector. +func (d *detectorWithRequire) Extract(ctx context.Context, input *filesystem.ScanInput) (inventory.Inventory, error) { + if d.e == nil { + var err error + d.e, err = veles.NewDetectionEngine([]veles.Detector{d.velesDetector}) + if err != nil { + return inventory.Inventory{}, err + } + } + secrets, err := d.e.Detect(ctx, input.Reader) + if err != nil { + return inventory.Inventory{}, fmt.Errorf("unable to scan for secrets: %w", err) + } + i := inventory.Inventory{} + for _, s := range secrets { + i.Secrets = append(i.Secrets, &inventory.Secret{ + Secret: s, + Location: input.Path, + }) + } + return i, nil +} diff --git a/scalibr_test.go b/scalibr_test.go index 29d5352e3..2a9893296 100644 --- a/scalibr_test.go +++ b/scalibr_test.go @@ -21,6 +21,7 @@ import ( "io/fs" "os" "path/filepath" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -78,6 +79,7 @@ func TestScan(t *testing.T) { fs := scalibrfs.DirFS(tmp) tmpRoot := []*scalibrfs.ScanRoot{{FS: fs, Path: tmp}} _ = os.WriteFile(filepath.Join(tmp, "file.txt"), []byte("Content"), 0644) + _ = os.WriteFile(filepath.Join(tmp, "config"), []byte("Content"), 0644) pkgName := "software" fakeExtractor := fe.New( @@ -366,6 +368,35 @@ func TestScan(t *testing.T) { }, }, }, + { + desc: "Veles_secret_detector_with_extractor", + cfg: &scalibr.ScanConfig{ + Plugins: []plugin.Plugin{ + // use the fakeSecretDetector1 also on config files + cf.FromVelesDetectorWithRequire( + fakeSecretDetector1, "secret-detector", 1, + func(fa filesystem.FileAPI) bool { + return strings.HasSuffix(fa.Path(), "config") + }, + ), + }, + ScanRoots: tmpRoot, + }, + want: &scalibr.ScanResult{ + Version: version.ScannerVersion, + Status: success, + PluginStatus: []*plugin.Status{ + {Name: "secret-detector", Version: 1, Status: success}, + {Name: "secrets/veles", Version: 1, Status: success}, + }, + Inventory: inventory.Inventory{ + Secrets: []*inventory.Secret{ + {Secret: velestest.NewFakeStringSecret("Con"), Location: "file.txt"}, + {Secret: velestest.NewFakeStringSecret("Con"), Location: "config"}, + }, + }, + }, + }, } for _, tc := range testCases { @@ -376,10 +407,15 @@ func TestScan(t *testing.T) { tc.want.StartTime = got.StartTime tc.want.EndTime = got.EndTime - // Ignore timestamps. - ignoreFields := cmpopts.IgnoreFields(inventory.SecretValidationResult{}, "At") + opts := []cmp.Option{ + fe.AllowUnexported, + // Ignore timestamps. + cmpopts.IgnoreFields(inventory.SecretValidationResult{}, "At"), + // Sort secrets. + cmpopts.SortSlices(func(a, b *inventory.Secret) bool { return a.Location < b.Location }), + } - if diff := cmp.Diff(tc.want, got, fe.AllowUnexported, ignoreFields); diff != "" { + if diff := cmp.Diff(tc.want, got, opts...); diff != "" { t.Errorf("scalibr.New().Scan(%v): unexpected diff (-want +got):\n%s", tc.cfg, diff) } })