refactor the OSV scanner from the ground up (#1124)

Author: ptzianos

components/scanners/osv-scanner/Dockerfile

@@ -0,0 +1,40 @@
+# Used only to build Go binaries.
+FROM golang:1.24.6 AS builder
+
+ARG SDK_VERSION=unset
+ARG COMPONENT_PATH=.
+ARG COMPONENT_BINARY_SOURCE_PATH=cmd/main.go
+
+WORKDIR /wrk
+
+# Copy only go related files.
+COPY ${COMPONENT_PATH} ./
+
+# Security hardening and building flags for minimal binaries.
+#
+# These CGO_CPPFLAGS help preventing overflows.
+# Add a small overhead at compile time.
+RUN CGO_CPPFLAGS="-D_FORTIFY_SOURCE=2 -fstack-protector-all" \
+    # Makes memory exploitation harder.
+    # Add a small overhead at compile time.
+    GOFLAGS="-buildmode=pie" \
+      go build \
+        -ldflags "-X github.com/smithy-security/smithy/sdk.Version=${SDK_VERSION} -s -w" \
+        -trimpath \
+        -o \
+        app \
+        ${COMPONENT_BINARY_SOURCE_PATH}
+
+# Create a workspace to clone repos to.
+RUN mkdir -p /workspace
+
+# Used to actually run the binary in minimal image.
+FROM gcr.io/distroless/base-debian12
+
+COPY --from=builder /wrk/app /bin/app
+
+# Setting the workdir where we'll clone repositories.
+WORKDIR /workspace
+
+# Set the binary as the entry point
+ENTRYPOINT ["/bin/app"]

…ts/scanners/osv-scanner/scanner/Makefile components/scanners/osv-scanner/Makefilecomponents/scanners/osv-scanner/scanner/Makefile renamed to components/scanners/osv-scanner/Makefile components/scanners/osv-scanner/scanner/Makefile renamed to components/scanners/osv-scanner/Makefile

@@ -1,7 +1,24 @@
 # osv-scanner
 
 This component implements a [scanner](https://github.com/smithy-security/smithy/blob/main/sdk/component/component.go)
-that parses json reports output by [osv-scan](https://google.github.io/osv-scanner/) into [ocsf](https://github.com/ocsf) format.
+that invokes [osv-scan](https://google.github.io/osv-scanner/) for all modules in the repository and adds information
+in the output related to the lines where each vulnerable dependency discovered is located to assist the reviewing
+process. The results generated will be in [ocsf](https://github.com/ocsf) format.
+
+The module discovery is not left to the OSV scanner itself but it is implemented by us so that we can add more
+intelligence. When a repository is scanned, if there is no git diff available, we will search for modules of supported
+languages and pass them to the scanner. If a module is identified the scanner will not look further down the filesystem
+tree. If a git diff is available, it will be loaded and we will look in it for modifications to dependency files of
+supported languages.
+
+Supporting a language means that we can get the results from the OSV scanner and add information about which line of
+the dependency file they are declared.
+
+## Supported Languages
+
+1. Golang
+2. JavaScript/TypeScript
+3. Elixir
 
 ## Environment variables
 
@@ -13,9 +30,10 @@ as the following:
 
 | Environment Variable     | Type   | Required | Default    | Description                                             |
 |--------------------------|--------|----------|------------|---------------------------------------------------------|
-| RAW\_OUT\_FILE\_PATH  | string | yes      | -          | The path where to find the osv-scan report                 |
-| TARGET\_TYPE         | string | false    | repository | The type of target that was used to generate the report |
+| WORKSPACE\_PATH          | string | yes      | .          | The path where root of the repository is located        |
+| GIT\_RAW\_DIFF\_PATH     | string | no       | -          | The path of the git diff file                           |
 
 ## Test data
 
-The `results.json` file used in tests was generated with the following steps:
+The `raw-results.json` files used in the internal/transformer tests were generated by running the OSV scanner for well
+known vulnerable repositories.

components/scanners/osv-scanner/README.md

@@ -3,34 +3,38 @@ package main
 import (
 	"context"
 	"log"
-	"time"
 
 	"github.com/go-errors/errors"
-
 	"github.com/smithy-security/smithy/sdk/component"
 
-	"github.com/smithy-security/smithy/components/scanners/osv-scanner/internal/transformer"
+	"github.com/smithy-security/smithy/components/scanners/osv-scanner/pkg/config"
+	"github.com/smithy-security/smithy/components/scanners/osv-scanner/pkg/transformer"
 )
 
 func main() {
-	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
 	if err := Main(ctx); err != nil {
 		log.Fatalf("unexpected error: %v", err)
 	}
 }
 
-// Main is the main entrypoint of this component
+// Main is the entrypoint to the scanner
 func Main(ctx context.Context, opts ...component.RunnerOption) error {
-	opts = append(opts, component.RunnerWithComponentName("bandit"))
+	opts = append(opts, component.RunnerWithComponentName("osv-scanner"))
+
+	cfg, err := config.New()
+	if err != nil {
+		return errors.Errorf("could not extract configuration values from environment: %w", err)
+	}
 
-	ocsfTransformer, err := transformer.New()
+	osvScannerTranformer, err := transformer.New(cfg)
 	if err != nil {
 		return errors.Errorf("could not create transformer: %w", err)
 	}
 
-	if err := component.RunScanner(ctx, ocsfTransformer, opts...); err != nil {
+	if err := component.RunScanner(ctx, osvScannerTranformer, opts...); err != nil {
 		return errors.Errorf("could not run scanner: %w", err)
 	}
 

components/scanners/osv-scanner/cmd/main.go

@@ -3,16 +3,8 @@ description: "Scans third party dependencies of multiple languages."
 type: scanner
 steps:
   - name: scanner
-    image: components/scanners/osv-scanner/scanner
-    executable: /bin/bash
-    env_vars: 
-      RAW_OUT_FILE: "{{ scratchWorkspace }}/output.json"
-    args:
-      - -c
-      - /entrypoint.sh scan source -r --format=sarif --call-analysis=true {{ sourceCodeWorkspace }}
-  - name: parser
     image: components/scanners/osv-scanner
     env_vars:
-      RAW_OUT_FILE: "{{ scratchWorkspace }}/output.json"
+      GIT_RAW_DIFF_PATH: "{{ targetMetadataWorkspace }}/raw.diff"
       WORKSPACE_PATH: "{{ sourceCodeWorkspace }}"
     executable: /bin/app

components/scanners/osv-scanner/component.yaml

@@ -1,67 +1,226 @@
 module github.com/smithy-security/smithy/components/scanners/osv-scanner
 
-go 1.23.4
+go 1.24.6
+
+toolchain go1.24.10
 
 require (
+	github.com/bluekeyes/go-gitdiff v0.8.1
 	github.com/go-errors/errors v1.5.1
-	github.com/jonboulle/clockwork v0.5.0
+	github.com/google/osv-scanner/v2 v2.2.4
+	github.com/ossf/osv-schema/bindings/go v0.0.0-20251012234424-434020c6442f
+	github.com/pandatix/go-cvss v0.6.2
 	github.com/smithy-security/pkg/env v0.0.3
-	github.com/smithy-security/pkg/sarif v0.0.14
-	github.com/smithy-security/smithy/sdk v0.0.19-alpha
-	github.com/stretchr/testify v1.10.0
-	google.golang.org/protobuf v1.36.6
+	github.com/smithy-security/pkg/languages v0.0.2-0.20251114203348-2ddb8b3c56f8
+	github.com/smithy-security/pkg/utils v0.0.2
+	github.com/smithy-security/smithy/sdk v0.0.23-alpha
+	github.com/stretchr/testify v1.11.1
+	google.golang.org/protobuf v1.36.10
 )
 
 require (
 	ariga.io/atlas v0.29.0 // indirect
+	bitbucket.org/creachadair/stringset v0.0.14 // indirect
+	cloud.google.com/go/compute/metadata v0.8.4 // indirect
+	dario.cat/mergo v1.0.2 // indirect
+	deps.dev/api/v3 v3.0.0-20250917073939-6ff3dd7d2eea // indirect
+	deps.dev/api/v3alpha v0.0.0-20250903005441-604c45d5b44b // indirect
+	deps.dev/util/maven v0.0.0-20250917073939-6ff3dd7d2eea // indirect
+	deps.dev/util/pypi v0.0.0-20250903005441-604c45d5b44b // indirect
+	deps.dev/util/resolve v0.0.0-20250917073939-6ff3dd7d2eea // indirect
+	deps.dev/util/semver v0.0.0-20250917073939-6ff3dd7d2eea // indirect
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect
+	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20250520111509-a70c2aa677fa // indirect
+	github.com/BurntSushi/toml v1.5.0 // indirect
+	github.com/CycloneDX/cyclonedx-go v0.9.3 // indirect
+	github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.2.0 // indirect
 	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/Microsoft/hcsshim v0.13.0 // indirect
+	github.com/ProtonMail/go-crypto v1.3.0 // indirect
 	github.com/abice/go-enum v0.6.0 // indirect
 	github.com/agext/levenshtein v1.2.3 // indirect
+	github.com/anchore/go-lzo v0.1.0 // indirect
+	github.com/anchore/go-struct-converter v0.0.0-20250211213226-cce56d595160 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/bmatcuk/doublestar v1.3.4 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/cloudflare/circl v1.6.1 // indirect
+	github.com/compose-spec/compose-go/v2 v2.8.1 // indirect
+	github.com/containerd/cgroups/v3 v3.0.5 // indirect
+	github.com/containerd/containerd v1.7.27 // indirect
+	github.com/containerd/containerd/api v1.9.0 // indirect
+	github.com/containerd/continuity v0.4.5 // indirect
+	github.com/containerd/errdefs v1.0.0 // indirect
+	github.com/containerd/errdefs/pkg v0.3.0 // indirect
+	github.com/containerd/fifo v1.1.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/platforms v1.0.0-rc.1 // indirect
+	github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect
+	github.com/containerd/ttrpc v1.2.7 // indirect
+	github.com/containerd/typeurl/v2 v2.2.3 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
+	github.com/cyphar/filepath-securejoin v0.4.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/deitch/magic v0.0.0-20240306090643-c67ab88f10cb // indirect
+	github.com/diskfs/go-diskfs v1.7.0 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/djherbis/times v1.6.0 // indirect
+	github.com/docker/cli v28.3.3+incompatible // indirect
+	github.com/docker/distribution v2.8.3+incompatible // indirect
+	github.com/docker/docker v28.3.3+incompatible // indirect
+	github.com/docker/docker-credential-helpers v0.9.3 // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-events v0.0.0-20250114142523-c867878c5e32 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/dsoprea/go-exfat v0.0.0-20190906070738-5e932fbdb589 // indirect
+	github.com/dsoprea/go-logging v0.0.0-20200710184922-b02d349568dd // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/edsrzf/mmap-go v1.2.0 // indirect
+	github.com/elliotwutingfeng/asciiset v0.0.0-20230602022725-51bbb787efab // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/erikvarga/go-rpmdb v0.0.0-20250523120114-a15a62cd4593 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
+	github.com/go-git/go-billy/v5 v5.6.2 // indirect
+	github.com/go-git/go-git/v5 v5.16.3 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/inflect v0.19.0 // indirect
+	github.com/go-restruct/restruct v1.2.0-alpha // indirect
+	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/golang/mock v1.6.0 // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/go-containerregistry v0.20.6 // indirect
+	github.com/google/osv-scalibr v0.3.7-0.20251023161426-90e9ac9cc1b3 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/hcl/v2 v2.18.1 // indirect
 	github.com/huandu/xstrings v1.3.3 // indirect
+	github.com/ianlancetaylor/demangle v0.0.0-20250628045327-2d64ad6b7ec5 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/pgx/v5 v5.6.0 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/jedib0t/go-pretty/v6 v6.6.8 // indirect
+	github.com/jonboulle/clockwork v0.5.0 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/labstack/gommon v0.4.1 // indirect
+	github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40 // indirect
+	github.com/masahiro331/go-ext4-filesystem v0.0.0-20240620024024-ca14e6327bbd // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.16 // indirect
+	github.com/mattn/go-shellwords v1.0.12 // indirect
 	github.com/mattn/go-sqlite3 v1.14.24 // indirect
 	github.com/mattn/goveralls v0.0.12 // indirect
+	github.com/micromdm/plist v0.2.1 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/moby/buildkit v0.23.2 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/locker v1.0.1 // indirect
+	github.com/moby/sys/mountinfo v0.7.2 // indirect
+	github.com/moby/sys/sequential v0.6.0 // indirect
+	github.com/moby/sys/signal v0.7.1 // indirect
+	github.com/moby/sys/user v0.4.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.1 // indirect
+	github.com/opencontainers/runtime-spec v1.2.1 // indirect
+	github.com/opencontainers/selinux v1.12.0 // indirect
+	github.com/owenrumney/go-sarif/v3 v3.2.3 // indirect
 	github.com/package-url/packageurl-go v0.1.3 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
+	github.com/pierrec/lz4/v4 v4.1.17 // indirect
+	github.com/pjbgf/sha1cd v0.4.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pkg/xattr v0.4.9 // indirect
+	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
+	github.com/rust-secure-code/go-rustaudit v0.0.0-20250226111315-e20ec32e963c // indirect
+	github.com/saferwall/pe v1.5.7 // indirect
+	github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 // indirect
+	github.com/secDre4mer/pkcs7 v0.0.0-20240322103146-665324a4461d // indirect
+	github.com/sergi/go-diff v1.4.0 // indirect
+	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
-	github.com/smithy-security/pkg/utils v0.0.2 // indirect
+	github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af // indirect
+	github.com/skeema/knownhosts v1.3.1 // indirect
+	github.com/spdx/gordf v0.0.0-20250128162952-000978ccd6fb // indirect
+	github.com/spdx/tools-golang v0.5.5 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
 	github.com/sqlc-dev/sqlc v1.27.0 // indirect
+	github.com/thoas/go-funk v0.9.3 // indirect
+	github.com/tidwall/gjson v1.18.0 // indirect
+	github.com/tidwall/jsonc v0.3.2 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	github.com/tink-crypto/tink-go/v2 v2.4.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.15 // indirect
+	github.com/tklauser/numcpus v0.10.0 // indirect
+	github.com/tonistiigi/go-csvvalue v0.0.0-20240814133006-030d3b2625d0 // indirect
+	github.com/ulikunitz/xz v0.5.15 // indirect
 	github.com/urfave/cli/v2 v2.26.0 // indirect
+	github.com/vbatts/tar-split v0.12.1 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zclconf/go-cty v1.14.4 // indirect
+	go.etcd.io/bbolt v1.4.2 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect
+	go.opentelemetry.io/otel v1.37.0 // indirect
+	go.opentelemetry.io/otel/metric v1.37.0 // indirect
+	go.opentelemetry.io/otel/trace v1.37.0 // indirect
 	go.uber.org/mock v0.5.0 // indirect
-	golang.org/x/crypto v0.24.0 // indirect
-	golang.org/x/mod v0.18.0 // indirect
-	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/sys v0.22.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
-	golang.org/x/tools v0.22.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.26.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/crypto v0.43.0 // indirect
+	golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc // indirect
+	golang.org/x/mod v0.28.0 // indirect
+	golang.org/x/net v0.46.0 // indirect
+	golang.org/x/oauth2 v0.30.0 // indirect
+	golang.org/x/sync v0.17.0 // indirect
+	golang.org/x/sys v0.37.0 // indirect
+	golang.org/x/telemetry v0.0.0-20250908211612-aef8a434d053 // indirect
+	golang.org/x/text v0.30.0 // indirect
+	golang.org/x/tools v0.37.0 // indirect
 	golang.org/x/tools/cmd/cover v0.1.0-deprecated // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect
-	google.golang.org/grpc v1.65.0 // indirect
+	golang.org/x/vuln v1.1.4 // indirect
+	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
+	google.golang.org/genproto v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b // indirect
+	google.golang.org/grpc v1.76.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	modernc.org/libc v1.66.3 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
+	modernc.org/sqlite v1.38.0 // indirect
+	osv.dev/bindings/go v0.0.0-20251013010847-b847e93bd9b0 // indirect
+	sigs.k8s.io/yaml v1.5.0 // indirect
+	www.velocidex.com/golang/go-ntfs v0.2.0 // indirect
+	www.velocidex.com/golang/regparser v0.0.0-20250203141505-31e704a67ef7 // indirect
 )