From 07c8d8f317b25d155ca0f9550533f5d35dbee864 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliv=C3=A9r=20Falvai?= Date: Fri, 6 Sep 2024 15:05:20 +0200 Subject: [PATCH] Execute binary steplib steps --- .../steplib_step_executable/step.yml | 29 ++++++ go.mod | 2 +- go.sum | 6 +- tools/tools.go | 17 ---- tools/tools_test.go | 17 ---- .../stepman/activator/steplib/activate.go | 99 ++++++------------- .../activator/steplib/activate_executable.go | 84 ++++++++++++++++ .../activator/steplib/activate_source.go | 77 +++++++++++++++ .../bitrise-io/stepman/models/models.go | 20 +++- vendor/modules.txt | 2 +- 10 files changed, 241 insertions(+), 112 deletions(-) create mode 100644 _tests/integration/steplib_step_executable/step.yml create mode 100644 vendor/github.com/bitrise-io/stepman/activator/steplib/activate_executable.go create mode 100644 vendor/github.com/bitrise-io/stepman/activator/steplib/activate_source.go diff --git a/_tests/integration/steplib_step_executable/step.yml b/_tests/integration/steplib_step_executable/step.yml new file mode 100644 index 000000000..90ee23411 --- /dev/null +++ b/_tests/integration/steplib_step_executable/step.yml @@ -0,0 +1,29 @@ +title: Dummy step +summary: Dummy step +description: Dummy step +website: https://github.com/... +source_code_url: https://github.com/... +support_url: https://github.com/.../issues +project_type_tags: +- ios +type_tags: +- script +is_always_run: false +is_skippable: false + +toolkit: + go: + package_name: github.com/bitrise-io/steps-go-toolkit-test + +source: + git: https://github.com/bitrise-steplib/steps-deploy-to-bitrise-io.git + commit: 45842c8d910ffd036a494cfb7456a730f6c7a0b5 + +executables: + darwin-arm64: + url: + hash: + darwin-amd64: + linux-amd64: + +inputs: [] diff --git a/go.mod b/go.mod index 284b47311..089290b2d 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/bitrise-io/go-utils v1.0.13 github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.22 github.com/bitrise-io/goinp v0.0.0-20240103152431-054ed78518ef - github.com/bitrise-io/stepman v0.0.0-20240828074035-6ae1a5f5efde + github.com/bitrise-io/stepman v0.0.0-20240827073840-3b3418afd742 github.com/go-git/go-git/v5 v5.12.0 github.com/gofrs/uuid v4.3.1+incompatible github.com/hashicorp/go-version v1.4.0 diff --git a/go.sum b/go.sum index a8c047ae1..ad848d752 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,10 @@ github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.22 h1:/SD9xE4LlX/Ju9YZ+n/yW/uDs7h github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.22/go.mod h1:Laih4ji980SQkRgdnMCH0g4u2GZI/5nnbqmYT9UfKFQ= github.com/bitrise-io/goinp v0.0.0-20240103152431-054ed78518ef h1:R5FOa8RHjqZwMN9g1FQ8W7nXxQAG7iwq1Cw+mUk5S9A= github.com/bitrise-io/goinp v0.0.0-20240103152431-054ed78518ef/go.mod h1:27ldH2bkCdYN5CEJ6x92EK+gkd5EcDBkA7dMrSKQFYU= -github.com/bitrise-io/stepman v0.0.0-20240828074035-6ae1a5f5efde h1:LinFhZG5OdayDh1T1JO8QANsNwQWzqORZ0A9EGHQ0ps= -github.com/bitrise-io/stepman v0.0.0-20240828074035-6ae1a5f5efde/go.mod h1:Lq9nEqKerBD35w3eSU8lf83F7uZPkXfmRSZEUDJN40w= +github.com/bitrise-io/stepman v0.0.0-20240827070535-210db1eae58e h1:bTy1G+zKuDZOckY7e9F1tKKS3pvi8Yxwj8hwTFumPX4= +github.com/bitrise-io/stepman v0.0.0-20240827070535-210db1eae58e/go.mod h1:Lq9nEqKerBD35w3eSU8lf83F7uZPkXfmRSZEUDJN40w= +github.com/bitrise-io/stepman v0.0.0-20240827073840-3b3418afd742 h1:n9T1IXoYrv53dgA6ok55qpATMOHO1GO1a1XEzFgdJ+E= +github.com/bitrise-io/stepman v0.0.0-20240827073840-3b3418afd742/go.mod h1:Lq9nEqKerBD35w3eSU8lf83F7uZPkXfmRSZEUDJN40w= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= diff --git a/tools/tools.go b/tools/tools.go index 04a28820d..d29aec945 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -15,8 +15,6 @@ import ( envmanModels "github.com/bitrise-io/envman/models" "github.com/bitrise-io/go-utils/command" "github.com/bitrise-io/go-utils/pathutil" - stepman "github.com/bitrise-io/stepman/cli" - stepmanModels "github.com/bitrise-io/stepman/models" "golang.org/x/sys/unix" ) @@ -125,21 +123,6 @@ func InstallFromURL(toolBinName, downloadURL string) error { return nil } -// ------------------ -// --- Stepman - -// StepmanSetup ... -func StepmanSetup(collection string) error { - log := log.NewLogger(log.GetGlobalLoggerOpts()) - return stepman.Setup(collection, "", log) -} - -// StepmanStepInfo ... -func StepmanStepInfo(collection, stepID, stepVersion string) (stepmanModels.StepInfoModel, error) { - log := log.NewLogger(log.GetGlobalLoggerOpts()) - return stepman.QueryStepInfo(collection, stepID, stepVersion, log) -} - // // Share diff --git a/tools/tools_test.go b/tools/tools_test.go index b7b56a077..5276eb0f4 100644 --- a/tools/tools_test.go +++ b/tools/tools_test.go @@ -9,7 +9,6 @@ import ( "github.com/bitrise-io/envman/models" - "github.com/bitrise-io/bitrise/configs" "github.com/bitrise-io/go-utils/command" "github.com/bitrise-io/go-utils/pathutil" "github.com/stretchr/testify/require" @@ -71,22 +70,6 @@ func TestMoveFileDifferentDevices(t *testing.T) { } } -func TestStepmanJSONStepLibStepInfo(t *testing.T) { - // setup - require.NoError(t, configs.InitPaths()) - - // Valid params -- Err should empty, output filled - require.Equal(t, nil, StepmanSetup("https://github.com/bitrise-io/bitrise-steplib")) - - info, err := StepmanStepInfo("https://github.com/bitrise-io/bitrise-steplib", "script", "0.9.0") - require.NoError(t, err) - require.NotEqual(t, "", info.JSON()) - - // Invalid params -- Err returned, output is invalid - info, err = StepmanStepInfo("https://github.com/bitrise-io/bitrise-steplib", "script", "2.x") - require.Error(t, err) -} - func TestEnvmanJSONPrint(t *testing.T) { // Initialized envstore -- Err should empty, output filled testDirPth, err := pathutil.NormalizedOSTempDirPath("test_env_store") diff --git a/vendor/github.com/bitrise-io/stepman/activator/steplib/activate.go b/vendor/github.com/bitrise-io/stepman/activator/steplib/activate.go index 0f192a5dd..037ce9406 100644 --- a/vendor/github.com/bitrise-io/stepman/activator/steplib/activate.go +++ b/vendor/github.com/bitrise-io/stepman/activator/steplib/activate.go @@ -4,7 +4,9 @@ import ( "fmt" "os" "path/filepath" + "runtime" "slices" + "time" "github.com/bitrise-io/go-utils/command" "github.com/bitrise-io/go-utils/pathutil" @@ -12,7 +14,7 @@ import ( "github.com/bitrise-io/stepman/stepman" ) -var errStepNotAvailableOfflineMode error = fmt.Errorf("step not available in offline mode") +const precompiledStepsEnv = "BITRISE_EXPERIMENT_PRECOMPILED_STEPS" func ActivateStep(stepLibURI, id, version, destination, destinationStepYML string, log stepman.Logger, isOfflineMode bool) error { stepCollection, err := stepman.ReadStepSpec(stepLibURI) @@ -20,41 +22,34 @@ func ActivateStep(stepLibURI, id, version, destination, destinationStepYML strin return fmt.Errorf("failed to read %s steplib: %s", stepLibURI, err) } - step, version, err := queryStep(stepCollection, stepLibURI, id, version) + step, version, err := queryStepMetadata(stepCollection, stepLibURI, id, version) if err != nil { return fmt.Errorf("failed to find step: %s", err) } - srcFolder, err := activateStep(stepCollection, stepLibURI, id, version, step, log, isOfflineMode) - if err != nil { - if err == errStepNotAvailableOfflineMode { - availableVersions := ListCachedStepVersions(log, stepCollection, stepLibURI, id) - versionList := "Other versions available in the local cache:" - for _, version := range availableVersions { - versionList = versionList + fmt.Sprintf("\n- %s", version) + if os.Getenv(precompiledStepsEnv) == "true" { + platform := fmt.Sprintf("%s-%s", runtime.GOOS, runtime.GOARCH) + executableForPlatform, ok := step.Executables[platform] + if ok { + log.Debugf("Downloading executable for %s", platform) + downloadStart := time.Now() + err = activateStepExecutable(stepLibURI, id, version, executableForPlatform, destination, destinationStepYML) + if err != nil { + log.Warnf("Failed to download step executable, falling back to source build: %s", err) + return activateStepSource(stepCollection, stepLibURI, id, version, step, destination, destinationStepYML, log, isOfflineMode) } - - errMsg := fmt.Sprintf("version is not available in the local cache and $BITRISE_OFFLINE_MODE is set. %s", versionList) - return fmt.Errorf("failed to download step: %s", errMsg) + log.Debugf("Downloaded executable in %s", time.Since(downloadStart).Round(time.Millisecond)) + return nil + } else { + log.Infof("No prebuilt executable found for %s, falling back to source build", platform) + return activateStepSource(stepCollection, stepLibURI, id, version, step, destination, destinationStepYML, log, isOfflineMode) } - - return fmt.Errorf("failed to download step: %s", err) - } - - if err := copyStep(srcFolder, destination); err != nil { - return fmt.Errorf("copy step failed: %s", err) + } else { + return activateStepSource(stepCollection, stepLibURI, id, version, step, destination, destinationStepYML, log, isOfflineMode) } - - if destinationStepYML != "" { - if err := copyStepYML(stepLibURI, id, version, destinationStepYML); err != nil { - return fmt.Errorf("copy step.yml failed: %s", err) - } - } - - return nil } -func queryStep(stepLib models.StepCollectionModel, stepLibURI string, id, version string) (models.StepModel, string, error) { +func queryStepMetadata(stepLib models.StepCollectionModel, stepLibURI string, id, version string) (models.StepModel, string, error) { step, stepFound, versionFound := stepLib.GetStep(id, version) if !stepFound { @@ -75,46 +70,6 @@ func queryStep(stepLib models.StepCollectionModel, stepLibURI string, id, versio return step, version, nil } -func activateStep(stepLib models.StepCollectionModel, stepLibURI, id, version string, step models.StepModel, log stepman.Logger, isOfflineMode bool) (string, error) { - route, found := stepman.ReadRoute(stepLibURI) - if !found { - return "", fmt.Errorf("no route found for %s steplib", stepLibURI) - } - - stepCacheDir := stepman.GetStepCacheDirPath(route, id, version) - if exist, err := pathutil.IsPathExists(stepCacheDir); err != nil { - return "", fmt.Errorf("failed to check if %s path exist: %s", stepCacheDir, err) - } else if exist { - return stepCacheDir, nil - } - - // version specific source cache not exists - if isOfflineMode { - return "", errStepNotAvailableOfflineMode - } - - if err := stepman.DownloadStep(stepLibURI, stepLib, id, version, step.Source.Commit, log); err != nil { - return "", fmt.Errorf("download failed: %s", err) - } - - return stepCacheDir, nil -} - -func copyStep(src, dst string) error { - if exist, err := pathutil.IsPathExists(dst); err != nil { - return fmt.Errorf("failed to check if %s path exist: %s", dst, err) - } else if !exist { - if err := os.MkdirAll(dst, 0777); err != nil { - return fmt.Errorf("failed to create dir for %s path: %s", dst, err) - } - } - - if err := command.CopyDir(src+"/", dst, true); err != nil { - return fmt.Errorf("copy command failed: %s", err) - } - return nil -} - func copyStepYML(libraryURL, id, version, dest string) error { route, found := stepman.ReadRoute(libraryURL) if !found { @@ -138,8 +93,14 @@ func copyStepYML(libraryURL, id, version, dest string) error { func ListCachedStepVersions(log stepman.Logger, stepLib models.StepCollectionModel, stepLibURI, stepID string) []string { versions := []models.Semver{} - for version, step := range stepLib.Steps[stepID].Versions { - _, err := activateStep(stepLib, stepLibURI, stepID, version, step, log, true) + route, found := stepman.ReadRoute(stepLibURI) + if !found { + return nil + } + + for version := range stepLib.Steps[stepID].Versions { + stepCacheDir := stepman.GetStepCacheDirPath(route, stepID, version) + _, err := os.Stat(stepCacheDir) if err != nil { continue } diff --git a/vendor/github.com/bitrise-io/stepman/activator/steplib/activate_executable.go b/vendor/github.com/bitrise-io/stepman/activator/steplib/activate_executable.go new file mode 100644 index 000000000..7c0973b9f --- /dev/null +++ b/vendor/github.com/bitrise-io/stepman/activator/steplib/activate_executable.go @@ -0,0 +1,84 @@ +package steplib + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/bitrise-io/stepman/models" + "github.com/hashicorp/go-retryablehttp" +) + +func activateStepExecutable( + stepLibURI string, + stepID string, + version string, + executable models.Executable, + destination string, + destinationStepYML string, +) error { + resp, err := retryablehttp.Get(executable.Url) + if err != nil { + return fmt.Errorf("fetch from %s: %w", executable.Url, err) + } + defer resp.Body.Close() + + path := filepath.Join(destination, stepID) + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("create file %s: %w", path, err) + } + + _, err = io.Copy(file, resp.Body) + if err != nil { + return fmt.Errorf("download %s to %s: %w", executable.Url, path, err) + } + + err = validateHash(path, executable.Hash) + if err != nil { + return fmt.Errorf("validate hash: %s", err) + } + + err = os.Chmod(path, 0755) + if err != nil { + return fmt.Errorf("set executable permission on file: %s", err) + } + + if err := copyStepYML(stepLibURI, stepID, version, destinationStepYML); err != nil { + return fmt.Errorf("copy step.yml: %s", err) + } + + return nil +} + +func validateHash(filePath string, expectedHash string) error { + if expectedHash == "" { + return fmt.Errorf("hash is empty") + } + + if !strings.HasPrefix(expectedHash, "sha256-") { + return fmt.Errorf("only SHA256 hashes supported at this time, make sure to prefix the hash with `sha256-`. Found hash value: %s", expectedHash) + } + + expectedHash = strings.TrimPrefix(expectedHash, "sha256-") + + reader, err := os.Open(filePath) + if err != nil { + return err + } + + h := sha256.New() + _, err = io.Copy(h, reader) + if err != nil { + return fmt.Errorf("calculate hash: %w", err) + } + actualHash := hex.EncodeToString(h.Sum(nil)) + if actualHash != expectedHash { + return fmt.Errorf("hash mismatch: expected sha256-%s, got sha256-%s", expectedHash, actualHash) + } + return nil +} diff --git a/vendor/github.com/bitrise-io/stepman/activator/steplib/activate_source.go b/vendor/github.com/bitrise-io/stepman/activator/steplib/activate_source.go new file mode 100644 index 000000000..38975aea5 --- /dev/null +++ b/vendor/github.com/bitrise-io/stepman/activator/steplib/activate_source.go @@ -0,0 +1,77 @@ +package steplib + +import ( + "fmt" + "os" + + "github.com/bitrise-io/go-utils/command" + "github.com/bitrise-io/go-utils/pathutil" + "github.com/bitrise-io/stepman/models" + "github.com/bitrise-io/stepman/stepman" +) + +func activateStepSource( + stepLib models.StepCollectionModel, + stepLibURI, id, version string, + step models.StepModel, + destination string, + stepYMLDestination string, + log stepman.Logger, + isOfflineMode bool, +) error { + route, found := stepman.ReadRoute(stepLibURI) + if !found { + return fmt.Errorf("no route found for %s steplib", stepLibURI) + } + + stepCacheDir := stepman.GetStepCacheDirPath(route, id, version) + if exist, err := pathutil.IsPathExists(stepCacheDir); err != nil { + return fmt.Errorf("failed to check if %s path exist: %s", stepCacheDir, err) + } else if exist { + if err := copyStep(stepCacheDir, destination); err != nil { + return fmt.Errorf("copy step failed: %s", err) + } + return nil + } + + // version specific source cache not exists + if isOfflineMode { + availableVersions := ListCachedStepVersions(log, stepLib, stepLibURI, id) + versionList := "Other versions available in the local cache:" + for _, version := range availableVersions { + versionList = versionList + fmt.Sprintf("\n- %s", version) + } + + errMsg := fmt.Sprintf("version is not available in the local cache and $BITRISE_OFFLINE_MODE is set. %s", versionList) + return fmt.Errorf("download step: %s", errMsg) + } + + if err := stepman.DownloadStep(stepLibURI, stepLib, id, version, step.Source.Commit, log); err != nil { + return fmt.Errorf("download failed: %s", err) + } + + if err := copyStep(stepCacheDir, destination); err != nil { + return fmt.Errorf("copy step failed: %s", err) + } + + if err := copyStepYML(stepLibURI, id, version, stepYMLDestination); err != nil { + return fmt.Errorf("copy step.yml failed: %s", err) + } + + return nil +} + +func copyStep(src, dst string) error { + if exist, err := pathutil.IsPathExists(dst); err != nil { + return fmt.Errorf("failed to check if %s path exist: %s", dst, err) + } else if !exist { + if err := os.MkdirAll(dst, 0777); err != nil { + return fmt.Errorf("failed to create dir for %s path: %s", dst, err) + } + } + + if err := command.CopyDir(src+"/", dst, true); err != nil { + return fmt.Errorf("copy command failed: %s", err) + } + return nil +} diff --git a/vendor/github.com/bitrise-io/stepman/models/models.go b/vendor/github.com/bitrise-io/stepman/models/models.go index ec6a31c5d..98d81be81 100644 --- a/vendor/github.com/bitrise-io/stepman/models/models.go +++ b/vendor/github.com/bitrise-io/stepman/models/models.go @@ -56,12 +56,11 @@ type SwiftStepToolkitModel struct { } type StepToolkitModel struct { - Bash *BashStepToolkitModel `json:"bash,omitempty" yaml:"bash,omitempty"` - Go *GoStepToolkitModel `json:"go,omitempty" yaml:"go,omitempty"` - Swift *SwiftStepToolkitModel `json:"swift,omitempty" yaml:"swift,omitempty"` + Bash *BashStepToolkitModel `json:"bash,omitempty" yaml:"bash,omitempty"` + Go *GoStepToolkitModel `json:"go,omitempty" yaml:"go,omitempty"` + Swift *SwiftStepToolkitModel `json:"swift,omitempty" yaml:"swift,omitempty"` } -// StepModel ... type StepModel struct { Title *string `json:"title,omitempty" yaml:"title,omitempty"` Summary *string `json:"summary,omitempty" yaml:"summary,omitempty"` @@ -70,10 +69,13 @@ type StepModel struct { Website *string `json:"website,omitempty" yaml:"website,omitempty"` SourceCodeURL *string `json:"source_code_url,omitempty" yaml:"source_code_url,omitempty"` SupportURL *string `json:"support_url,omitempty" yaml:"support_url,omitempty"` + // auto-generated at share PublishedAt *time.Time `json:"published_at,omitempty" yaml:"published_at,omitempty"` Source *StepSourceModel `json:"source,omitempty" yaml:"source,omitempty"` + Executables Executables `json:"executables,omitempty" yaml:"executables,omitempty"` AssetURLs map[string]string `json:"asset_urls,omitempty" yaml:"asset_urls,omitempty"` + // HostOsTags []string `json:"host_os_tags,omitempty" yaml:"host_os_tags,omitempty"` ProjectTypeTags []string `json:"project_type_tags,omitempty" yaml:"project_type_tags,omitempty"` @@ -120,6 +122,15 @@ type StepGroupModel struct { Versions map[string]StepModel `json:"versions,omitempty" yaml:"versions,omitempty"` } +// Key: platform, as in runtime.GOOS + runtime.GOARCH +// Examples: darwin-arm64, linux-amd64 +type Executables map[string]Executable + +type Executable struct { + Url string `json:"url,omitempty" yaml:"url,omitempty"` + Hash string `json:"hash,omitempty" yaml:"hash,omitempty"` +} + func (stepGroup StepGroupModel) LatestVersion() (StepModel, bool) { step, found := stepGroup.Versions[stepGroup.LatestVersionNumber] if !found { @@ -174,4 +185,3 @@ type SteplibInfoModel struct { URI string `json:"uri,omitempty" yaml:"uri,omitempty"` SpecPath string `json:"spec_path,omitempty" yaml:"spec_path,omitempty"` } - diff --git a/vendor/modules.txt b/vendor/modules.txt index 889728fdc..d636fa2bd 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -72,7 +72,7 @@ github.com/bitrise-io/go-utils/v2/retryhttp # github.com/bitrise-io/goinp v0.0.0-20240103152431-054ed78518ef ## explicit; go 1.18 github.com/bitrise-io/goinp/goinp -# github.com/bitrise-io/stepman v0.0.0-20240828074035-6ae1a5f5efde +# github.com/bitrise-io/stepman v0.0.0-20240827073840-3b3418afd742 ## explicit; go 1.18 github.com/bitrise-io/stepman/activator github.com/bitrise-io/stepman/activator/steplib