Skip to content

Commit 410845e

Browse files
copybara-githubcopybara-github
committed
Merge pull request #1568 from doyensec:veles-custom-extractor-for-aws-access-key
PiperOrigin-RevId: 838793730
2 parents 901a475 + ae71ff2 commit 410845e

File tree

9 files changed

+335
-4
lines changed

9 files changed

+335
-4
lines changed

‎extractor/filesystem/list/list.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,14 +96,14 @@ import (
9696
"github.com/google/osv-scalibr/extractor/filesystem/runtime/nodejs/nvm"
9797
"github.com/google/osv-scalibr/extractor/filesystem/sbom/cdx"
9898
"github.com/google/osv-scalibr/extractor/filesystem/sbom/spdx"
99+
"github.com/google/osv-scalibr/extractor/filesystem/secrets/awsaccesskey"
99100
"github.com/google/osv-scalibr/extractor/filesystem/secrets/convert"
100101
"github.com/google/osv-scalibr/extractor/filesystem/secrets/mariadb"
101102
"github.com/google/osv-scalibr/extractor/filesystem/secrets/mysqlmylogin"
102103
"github.com/google/osv-scalibr/extractor/filesystem/secrets/onepasswordconnecttoken"
103104
"github.com/google/osv-scalibr/extractor/filesystem/secrets/pgpass"
104105
"github.com/google/osv-scalibr/veles"
105106
"github.com/google/osv-scalibr/veles/secrets/anthropicapikey"
106-
"github.com/google/osv-scalibr/veles/secrets/awsaccesskey"
107107
"github.com/google/osv-scalibr/veles/secrets/azurestorageaccountaccesskey"
108108
"github.com/google/osv-scalibr/veles/secrets/azuretoken"
109109
"github.com/google/osv-scalibr/veles/secrets/cratesioapitoken"
@@ -283,6 +283,7 @@ var (
283283
pgpass.Name: {noCFG(pgpass.New)},
284284
onepasswordconnecttoken.Name: {noCFG(onepasswordconnecttoken.New)},
285285
mariadb.Name: {noCFG(mariadb.NewDefault)},
286+
awsaccesskey.Name: {noCFG(awsaccesskey.New)},
286287
}
287288

288289
// SecretDetectors for Detector interface.
@@ -331,7 +332,6 @@ var (
331332
{onepasswordkeys.NewRecoveryTokenDetector(), "secrets/onepasswordrecoverycode", 0},
332333
{gcshmackey.NewDetector(), "secrets/gcshmackey", 0},
333334
{vapid.NewDetector(), "secrets/vapidkey", 0},
334-
{awsaccesskey.NewDetector(), "secrets/awsaccesskey", 0},
335335
{recaptchakey.NewDetector(), "secrets/recaptchakey", 0},
336336
})
337337

‎extractor/filesystem/secrets/awsaccesskey/awsaccesskey.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// Package awsaccesskey extends the veles awsaccesskey.Detector to search inside the `~/.aws/credentials` file
16+
package awsaccesskey
17+
18+
import (
19+
"path/filepath"
20+
"strings"
21+
22+
"github.com/google/osv-scalibr/extractor/filesystem"
23+
"github.com/google/osv-scalibr/veles/secrets/awsaccesskey"
24+
25+
"github.com/google/osv-scalibr/extractor/filesystem/secrets/convert"
26+
)
27+
28+
const (
29+
// Name is the name of the extractor
30+
Name = "secrets/awsaccesskey"
31+
// Version is the version of the extractor
32+
Version = 0
33+
)
34+
35+
// New returns a filesystem.Extractor which extracts AWS Access Keys using the awsaccesskey.Detector
36+
func New() filesystem.Extractor {
37+
return convert.FromVelesDetectorWithRequire(
38+
awsaccesskey.NewDetector(), Name, Version, FileRequired,
39+
)
40+
}
41+
42+
// FileRequired returns true if a file contains aws credentials.
43+
func FileRequired(api filesystem.FileAPI) bool {
44+
path := filepath.ToSlash(api.Path())
45+
return strings.HasSuffix(path, ".aws/credentials")
46+
}

‎extractor/filesystem/secrets/awsaccesskey/awsaccesskey_test.go‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package awsaccesskey_test
16+
17+
import (
18+
"runtime"
19+
"testing"
20+
21+
"github.com/google/go-cmp/cmp"
22+
"github.com/google/go-cmp/cmp/cmpopts"
23+
"github.com/google/osv-scalibr/extractor/filesystem/secrets/awsaccesskey"
24+
"github.com/google/osv-scalibr/extractor/filesystem/simplefileapi"
25+
"github.com/google/osv-scalibr/inventory"
26+
"github.com/google/osv-scalibr/testing/extracttest"
27+
awsaccesskeydetector "github.com/google/osv-scalibr/veles/secrets/awsaccesskey"
28+
)
29+
30+
func TestExtractor_FileRequired(t *testing.T) {
31+
tests := []struct {
32+
inputPath string
33+
want bool
34+
isWindows bool
35+
}{
36+
{inputPath: "", want: false},
37+
38+
// linux
39+
{inputPath: `/Users/example-user/.aws/credentials`, want: true},
40+
{inputPath: `/Users/example-user/bad/path`, want: false},
41+
42+
// windows
43+
{inputPath: `C:\Users\USERNAME\.aws\credentials`, isWindows: true, want: true},
44+
{inputPath: `C:\Users\USERNAME\another\bad\path`, isWindows: true, want: false},
45+
}
46+
47+
for _, tt := range tests {
48+
t.Run(tt.inputPath, func(t *testing.T) {
49+
if tt.isWindows && runtime.GOOS != "windows" {
50+
t.Skipf("Skipping test %q for %q", t.Name(), runtime.GOOS)
51+
}
52+
e := awsaccesskey.New()
53+
got := e.FileRequired(simplefileapi.New(tt.inputPath, nil))
54+
if got != tt.want {
55+
t.Errorf("FileRequired(%s) got = %v, want %v", tt.inputPath, got, tt.want)
56+
}
57+
})
58+
}
59+
}
60+
61+
func TestExtractor_Extract(t *testing.T) {
62+
tests := []*struct {
63+
Name string
64+
Path string
65+
WantSecrets []*inventory.Secret
66+
WantErr error
67+
}{
68+
{
69+
Name: "empty",
70+
Path: "empty",
71+
WantSecrets: nil,
72+
},
73+
{
74+
Name: "aws_credentials",
75+
Path: "aws_credentials",
76+
WantSecrets: []*inventory.Secret{
77+
{
78+
Secret: awsaccesskeydetector.Credentials{
79+
AccessID: "AIKA1984R439T439HTH4",
80+
Secret: "32r923jr023rk320rk2a3rkB34tj340r32Ckt433",
81+
},
82+
Location: "aws_credentials",
83+
},
84+
},
85+
},
86+
{
87+
Name: "random_content",
88+
Path: "random_content",
89+
WantSecrets: nil,
90+
},
91+
}
92+
93+
for _, tt := range tests {
94+
t.Run(tt.Name, func(t *testing.T) {
95+
extr := awsaccesskey.New()
96+
97+
inputCfg := extracttest.ScanInputMockConfig{
98+
Path: tt.Path,
99+
FakeScanRoot: "testdata",
100+
}
101+
102+
scanInput := extracttest.GenerateScanInputMock(t, inputCfg)
103+
defer extracttest.CloseTestScanInput(t, scanInput)
104+
105+
got, err := extr.Extract(t.Context(), &scanInput)
106+
107+
if diff := cmp.Diff(tt.WantErr, err, cmpopts.EquateErrors()); diff != "" {
108+
t.Errorf("%s.Extract(%q) error diff (-want +got):\n%s", extr.Name(), tt.Path, diff)
109+
return
110+
}
111+
112+
wantInv := inventory.Inventory{Secrets: tt.WantSecrets}
113+
opts := []cmp.Option{cmpopts.SortSlices(extracttest.PackageCmpLess), cmpopts.EquateEmpty()}
114+
if diff := cmp.Diff(wantInv, got, opts...); diff != "" {
115+
t.Errorf("%s.Extract(%q) diff (-want +got):\n%s", extr.Name(), tt.Path, diff)
116+
}
117+
})
118+
}
119+
}

‎extractor/filesystem/secrets/awsaccesskey/testdata/aws_credentials‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[default]
2+
aws_access_key_id = AIKA1984R439T439HTH4
3+
aws_secret_access_key = 32r923jr023rk320rk2a3rkB34tj340r32Ckt433

‎extractor/filesystem/secrets/awsaccesskey/testdata/empty‎

Whitespace-only changes.

‎extractor/filesystem/secrets/awsaccesskey/testdata/random_content‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
localhost:5432:mydb:myuser:mypassword
2+
hostname:1234:testdb:testuser:testpass123
3+
hostname:1234:testdb:testuser:passw*ord
4+
# space inside one group (except password)
5+
hostname:1234:testdb:testuser:passw ord
6+
hostname:1234:db name:testuser:password
7+
# this is a comment and should be ignored
8+
*:*:db:admin:supersecret
9+
# valid with escaped :
10+
prod.example.com:5432:db:admin:pass\:word

‎extractor/filesystem/secrets/convert/convert.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,9 +95,13 @@ var _ filesystem.Extractor = &detectorWrapper{}
9595
func SetupVelesExtractors(extractors []filesystem.Extractor) ([]filesystem.Extractor, error) {
9696
result := make([]filesystem.Extractor, 0, len(extractors))
9797
detectors := []veles.Detector{}
98+
9899
for _, e := range extractors {
99-
if d, ok := e.(veles.Detector); ok {
100+
if d, isDetector := e.(veles.Detector); isDetector {
100101
detectors = append(detectors, d)
102+
if _, keepExtractor := e.(extractorKeeper); keepExtractor {
103+
result = append(result, e)
104+
}
101105
} else {
102106
result = append(result, e)
103107
}

‎extractor/filesystem/secrets/convert/specific.go‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
// Copyright 2025 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package convert
16+
17+
import (
18+
"context"
19+
"fmt"
20+
21+
"github.com/google/osv-scalibr/extractor/filesystem"
22+
"github.com/google/osv-scalibr/inventory"
23+
"github.com/google/osv-scalibr/plugin"
24+
"github.com/google/osv-scalibr/veles"
25+
)
26+
27+
// FromVelesDetectorWithRequire works similar to FromVelesDetector but allows specifying additional files to look at on top of the default ones.
28+
func FromVelesDetectorWithRequire(velesDetector veles.Detector, name string, version int, fileRequired func(filesystem.FileAPI) bool) filesystem.Extractor {
29+
return &detectorWithRequire{
30+
velesDetector: velesDetector,
31+
name: name,
32+
version: version,
33+
fileRequired: fileRequired,
34+
}
35+
}
36+
37+
// extractorKeeper signals that a Detector also functions as a standalone filesystem.Extractor.
38+
type extractorKeeper interface {
39+
KeepExtractor() bool
40+
}
41+
42+
// Assert that detectorWithRequire implements the required interfaces.
43+
var _ veles.Detector = &detectorWithRequire{}
44+
var _ filesystem.Extractor = &detectorWithRequire{}
45+
var _ extractorKeeper = &detectorWithRequire{}
46+
47+
// detectorWithRequire is a wrapper around the veles.Detector interface that
48+
// implements the additional functions of the filesystem Extractor interface.
49+
type detectorWithRequire struct {
50+
velesDetector veles.Detector
51+
name string
52+
version int
53+
fileRequired func(filesystem.FileAPI) bool
54+
e *veles.DetectionEngine
55+
}
56+
57+
// KeepExtractor signals that this detector must also be registered as a standalone
58+
// filesystem.Extractor to handle the additional files specified in the fileRequired callback.
59+
func (d *detectorWithRequire) KeepExtractor() bool { return true }
60+
61+
// MaxSecretLen returns the maximum length a secret from this Detector can have.
62+
func (d *detectorWithRequire) MaxSecretLen() uint32 {
63+
return d.velesDetector.MaxSecretLen()
64+
}
65+
66+
// Detect finds candidate secrets in the data and returns them alongside their
67+
// starting positions.
68+
func (d *detectorWithRequire) Detect(data []byte) ([]veles.Secret, []int) {
69+
return d.velesDetector.Detect(data)
70+
}
71+
72+
// Name of the secret extractor.
73+
func (d *detectorWithRequire) Name() string {
74+
return d.name
75+
}
76+
77+
// Version of the secret extractor.
78+
func (d *detectorWithRequire) Version() int {
79+
return d.version
80+
}
81+
82+
// Requirements of the secret extractor.
83+
func (d *detectorWithRequire) Requirements() *plugin.Capabilities {
84+
// Veles plugins don't have any special requirements.
85+
return &plugin.Capabilities{}
86+
}
87+
88+
// FileRequired returns the provided file required callback.
89+
func (d *detectorWithRequire) FileRequired(api filesystem.FileAPI) bool {
90+
return d.fileRequired(api)
91+
}
92+
93+
// Extract extracts secret from the filesystem using the provided detector.
94+
func (d *detectorWithRequire) Extract(ctx context.Context, input *filesystem.ScanInput) (inventory.Inventory, error) {
95+
if d.e == nil {
96+
var err error
97+
d.e, err = veles.NewDetectionEngine([]veles.Detector{d.velesDetector})
98+
if err != nil {
99+
return inventory.Inventory{}, err
100+
}
101+
}
102+
secrets, err := d.e.Detect(ctx, input.Reader)
103+
if err != nil {
104+
return inventory.Inventory{}, fmt.Errorf("unable to scan for secrets: %w", err)
105+
}
106+
i := inventory.Inventory{}
107+
for _, s := range secrets {
108+
i.Secrets = append(i.Secrets, &inventory.Secret{
109+
Secret: s,
110+
Location: input.Path,
111+
})
112+
}
113+
return i, nil
114+
}

0 commit comments

Comments
 (0)