diff --git a/cmd/sst/init.go b/cmd/sst/init.go index b9756e4bf5..044b8d14e4 100644 --- a/cmd/sst/init.go +++ b/cmd/sst/init.go @@ -190,7 +190,7 @@ func CmdInit(cli *cli.Cli) error { p = promptui.Select{ Label: "‏‏‎ ‎Where do you want to deploy your app? You can change this later", HideSelected: true, - Items: []string{"aws", "cloudflare"}, + Items: []string{"aws", "cloudflare", "gcp"}, HideHelp: true, } _, home, err = p.Run() diff --git a/go.mod b/go.mod index cb77d45953..79dc5af226 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/sst/sst/v3 go 1.23.1 require ( + cloud.google.com/go/secretmanager v1.11.5 + cloud.google.com/go/storage v1.39.1 github.com/BurntSushi/toml v1.2.1 github.com/Masterminds/semver/v3 v3.2.1 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 @@ -57,6 +59,7 @@ require ( golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 golang.org/x/sync v0.15.0 golang.org/x/term v0.32.0 + google.golang.org/api v0.169.0 google.golang.org/protobuf v1.36.0 ) @@ -65,7 +68,6 @@ require ( cloud.google.com/go/compute/metadata v0.5.0 // indirect cloud.google.com/go/iam v1.1.6 // indirect cloud.google.com/go/kms v1.15.7 // indirect - cloud.google.com/go/storage v1.39.1 // indirect dario.cat/mergo v1.0.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 // indirect @@ -309,7 +311,6 @@ require ( golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect - google.golang.org/api v0.169.0 // indirect google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect diff --git a/go.sum b/go.sum index bc53eb31bd..5936df6f16 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= cloud.google.com/go/kms v1.15.7 h1:7caV9K3yIxvlQPAcaFffhlT7d1qpxjB1wHBtjWa13SM= cloud.google.com/go/kms v1.15.7/go.mod h1:ub54lbsa6tDkUwnu4W7Yt1aAIFLnspgh0kPGToDukeI= +cloud.google.com/go/secretmanager v1.11.5 h1:82fpF5vBBvu9XW4qj0FU2C6qVMtj1RM/XHwKXUEAfYY= +cloud.google.com/go/secretmanager v1.11.5/go.mod h1:eAGv+DaCHkeVyQi0BeXgAHOU0RdrMeZIASKc+S7VqH4= cloud.google.com/go/storage v1.39.1 h1:MvraqHKhogCOTXTlct/9C3K3+Uy2jBmFYb3/Sp6dVtY= cloud.google.com/go/storage v1.39.1/go.mod h1:xK6xZmxZmo+fyP7+DEF6FhNc24/JAe95OLyOHCXFH1o= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= diff --git a/pkg/project/project.go b/pkg/project/project.go index 218705c773..8d6fff99e0 100644 --- a/pkg/project/project.go +++ b/pkg/project/project.go @@ -300,6 +300,8 @@ func (proj *Project) LoadHome() error { match = &provider.CloudflareProvider{} case "aws": match = provider.NewAwsProvider() + case "gcp": + match = &provider.GCPProvider{} } if match == nil { continue @@ -327,6 +329,8 @@ func (proj *Project) LoadHome() error { home = provider.NewAwsHome(loadedProviders["aws"].(*provider.AwsProvider)) case "cloudflare": home = provider.NewCloudflareHome(loadedProviders["cloudflare"].(*provider.CloudflareProvider)) + case "gcp": + home = provider.NewGCPHome(loadedProviders["gcp"].(*provider.GCPProvider)) default: return fmt.Errorf("Home provider %s is invalid", proj.app.Home) } diff --git a/pkg/project/provider/gcp.go b/pkg/project/provider/gcp.go new file mode 100644 index 0000000000..ea98c87a08 --- /dev/null +++ b/pkg/project/provider/gcp.go @@ -0,0 +1,507 @@ +package provider + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path" + "strings" + "sync" + + "cloud.google.com/go/compute/metadata" + secretmanager "cloud.google.com/go/secretmanager/apiv1" + secretmanagerpb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb" + "cloud.google.com/go/storage" + "github.com/sst/sst/v3/internal/util" + "golang.org/x/oauth2/google" + "google.golang.org/api/googleapi" + "google.golang.org/api/iterator" + "google.golang.org/api/option" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type GCPProvider struct { + project string + region string + zone string + credentials *google.Credentials +} + +func (g *GCPProvider) Env() (map[string]string, error) { + return map[string]string{ + "GOOGLE_PROJECT": g.project, + "GOOGLE_REGION": g.region, + "GOOGLE_ZONE": g.zone, + // fixme: pulumi and go sdk share the same underlying auth mechanism + // so we can just assume the pulumi uses works without any special auth vars + // but should confirm with dax + }, nil +} + +func (g *GCPProvider) Init(app, stage string, args map[string]any) error { + ctx := context.Background() + creds, err := google.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/cloud-platform") + if err != nil { + return util.NewReadableError(err, "Failed to find GCP credentials, pleasu use gcloud cli to login with application default credentials.") + } + + project := firstNonEmptyEnv( + "GOOGLE_PROJECT", + "GOOGLE_CLOUD_PROJECT", + "GCLOUD_PROJECT", + "CLOUDSDK_CORE_PROJECT", + ) + if args["project"] != nil { + project = args["project"].(string) + } + region := firstNonEmptyEnv( + "GOOGLE_REGION", + "GCLOUD_REGION", + "CLOUDSDK_COMPUTE_REGION", + "GOOGLE_CLOUD_REGION", + ) + if r, ok := args["region"].(string); ok && r != "" { + region = r + } + zone := firstNonEmptyEnv( + "GOOGLE_ZONE", + "GCLOUD_ZONE", + "CLOUDSDK_COMPUTE_ZONE", + "GOOGLE_CLOUD_ZONE", + ) + if z, ok := args["zone"].(string); ok && z != "" { + zone = z + } + + if metadata.OnGCE() { + client := metadata.NewClient(nil) + if project == "" { + if pid, err := client.ProjectIDWithContext(ctx); err == nil { + project = pid + } + } + + if region == "" || zone == "" { + if z, err := client.ZoneWithContext(ctx); err == nil { + if zone == "" { + zone = z + } + + // z :: us-central1-a + if region == "" { + if dash := strings.LastIndex(z, "-"); dash != -1 { + region = z[:dash] + } + } + } + } + } + + if project == "" { + return util.NewReadableError(nil, "GCP project not found. Please use GOOGLE_PROJECT environment variable or in the provider section of the project configuration file.") + } + + // fixme: if region is still empty, should we set it to a default value? + // some resources don't require region to be set, and pulumi/go sdk defaults to us + // should ask dax + + g.project = project + g.region = region + g.zone = zone + g.credentials = creds + slog.Info("gcp project selected", "project", project) + return nil +} + +type GCPHome struct { + provider *GCPProvider + bootstrapBucket string + gcsClient *storage.Client + secretClient *secretmanager.Client + once sync.Once + initErr error +} + +func NewGCPHome(provider *GCPProvider) *GCPHome { + return &GCPHome{ + provider: provider, + } +} + +func (g *GCPHome) initClients() { + g.once.Do(func() { + ctx := context.Background() + + gcsClient, err := storage.NewClient(ctx, option.WithCredentials(g.provider.credentials)) + if err != nil { + g.initErr = fmt.Errorf("failed to create GCS client: %w", err) + return + } + + secretClient, err := secretmanager.NewClient(ctx, option.WithCredentials(g.provider.credentials)) + if err != nil { + gcsClient.Close() + g.initErr = fmt.Errorf("failed to create Secret Manager client: %w", err) + return + } + + g.gcsClient = gcsClient + g.secretClient = secretClient + }) +} + +func (g *GCPHome) getGCSClient() (*storage.Client, error) { + g.initClients() + if g.initErr != nil { + return nil, g.initErr + } + if g.gcsClient == nil { + return nil, fmt.Errorf("GCS client not initialized") + } + return g.gcsClient, nil +} + +func (g *GCPHome) getSecretClient() (*secretmanager.Client, error) { + g.initClients() + if g.initErr != nil { + return nil, g.initErr + } + if g.secretClient == nil { + return nil, fmt.Errorf("Secret Manager client not initialized") + } + return g.secretClient, nil +} + +func (g *GCPHome) Bootstrap() error { + gcsClient, err := g.getGCSClient() + if err != nil { + return err + } + + ctx := context.Background() + + // fixme: projectId is unique, so sst-state-{projectId} should be unique + // should be fune but should ask dax anyway + bucketName := fmt.Sprintf("sst-state-%s", g.provider.project) + bucket := gcsClient.Bucket(bucketName) + + _, err = bucket.Attrs(ctx) + if errors.Is(err, storage.ErrBucketNotExist) { + slog.Info("creating new bucket", "bucket", bucketName) + if err := bucket.Create( + ctx, + g.provider.project, + &storage.BucketAttrs{ + Location: g.provider.region, + // just in case pulumi corrupts the state or something idk + VersioningEnabled: true, + }, + ); err != nil { + return err + } + } else if err != nil { + return err + } + slog.Info("found existing bucket", "bucket", bucketName) + g.bootstrapBucket = bucketName + + return nil +} + +func (g *GCPHome) getData(key, app, stage string) (io.Reader, error) { + gcsClient, err := g.getGCSClient() + if err != nil { + return nil, err + } + + ctx := context.Background() + bucket := gcsClient.Bucket(g.bootstrapBucket) + + obj := bucket.Object(g.pathForData(key, app, stage)) + r, err := obj.NewReader(ctx) + if err != nil { + if errors.Is(err, storage.ErrObjectNotExist) { + return nil, nil + } + return nil, err + } + return r, nil +} + +func (g *GCPHome) putData(key, app, stage string, data io.Reader) error { + gcsClient, err := g.getGCSClient() + if err != nil { + return err + } + + ctx := context.Background() + bucket := gcsClient.Bucket(g.bootstrapBucket) + + obj := bucket.Object(g.pathForData(key, app, stage)) + w := obj.NewWriter(ctx) + if _, err := io.Copy(w, data); err != nil { + w.Close() + return err + } + return w.Close() +} + +func (g *GCPHome) removeData(key, app, stage string) error { + gcsClient, err := g.getGCSClient() + if err != nil { + return err + } + + ctx := context.Background() + bucket := gcsClient.Bucket(g.bootstrapBucket) + + obj := bucket.Object(g.pathForData(key, app, stage)) + return obj.Delete(ctx) +} + +func (g *GCPHome) cleanup(key, app, stage string) error { + gcsClient, err := g.getGCSClient() + if err != nil { + return err + } + + ctx := context.Background() + bucket := gcsClient.Bucket(g.bootstrapBucket) + + prefix := path.Join(key, app, stage) + "/" + slog.Info("cleaning up folder", "bucket", g.bootstrapBucket, "prefix", prefix) + + it := bucket.Objects(ctx, &storage.Query{ + Prefix: prefix, + }) + + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return err + } + + obj := bucket.Object(attrs.Name) + if err := obj.Delete(ctx); err != nil { + return err + } + } + + slog.Info("folder cleanup complete", "prefix", prefix) + return nil +} + +func (g *GCPHome) getPassphrase(app string, stage string) (string, error) { + secretClient, err := g.getSecretClient() + if err != nil { + return "", err + } + + ctx := context.Background() + secretName := g.pathForPassphrase(app, stage) + name := fmt.Sprintf("projects/%s/secrets/%s/versions/latest", g.provider.project, secretName) + + req := &secretmanagerpb.AccessSecretVersionRequest{ + Name: name, + } + + result, err := secretClient.AccessSecretVersion(ctx, req) + if err != nil { + if isNotFoundError(err) { + return "", nil + } + return "", wrapError(err) + } + + return string(result.Payload.Data), nil +} + +func (g *GCPHome) setPassphrase(app, stage, passphrase string) error { + secretClient, err := g.getSecretClient() + if err != nil { + return err + } + + ctx := context.Background() + secretName := g.pathForPassphrase(app, stage) + parent := fmt.Sprintf("projects/%s", g.provider.project) + + getReq := &secretmanagerpb.GetSecretRequest{ + Name: fmt.Sprintf("projects/%s/secrets/%s", g.provider.project, secretName), + } + + _, err = secretClient.GetSecret(ctx, getReq) + if err != nil { + if isNotFoundError(err) { + return createSecretAndAddVersion(ctx, secretClient, parent, secretName, passphrase) + } + return wrapError(err) + } + + return addSecretVersion(ctx, secretClient, secretName, passphrase) +} + +func createSecretAndAddVersion(ctx context.Context, secretClient *secretmanager.Client, parent, secretName, passphrase string) error { + createReq := &secretmanagerpb.CreateSecretRequest{ + Parent: parent, + SecretId: secretName, + Secret: &secretmanagerpb.Secret{ + Replication: &secretmanagerpb.Replication{ + Replication: &secretmanagerpb.Replication_Automatic_{ + Automatic: &secretmanagerpb.Replication_Automatic{}, + }, + }, + }, + } + + secret, err := secretClient.CreateSecret(ctx, createReq) + if err != nil { + return wrapError(err) + } + + return addSecretVersion(ctx, secretClient, secret.Name, passphrase) +} + +func addSecretVersion(ctx context.Context, secretClient *secretmanager.Client, secretName, passphrase string) error { + addReq := &secretmanagerpb.AddSecretVersionRequest{ + Parent: secretName, + Payload: &secretmanagerpb.SecretPayload{ + Data: []byte(passphrase), + }, + } + + _, err := secretClient.AddSecretVersion(ctx, addReq) + if err != nil { + if st, ok := status.FromError(err); ok { + switch st.Code() { + case codes.PermissionDenied, codes.Unimplemented: + return wrapError(err) + } + } + + var gErr *googleapi.Error + if errors.As(err, &gErr) { + switch gErr.Code { + case 403, 501: + return util.NewReadableError(err, gErr.Message) + } + } + return err + } + return nil +} + +func (g *GCPHome) listStages(app string) ([]string, error) { + gcsClient, err := g.getGCSClient() + if err != nil { + return nil, err + } + + ctx := context.Background() + bucketName := fmt.Sprintf("sst-state-%s", g.provider.project) + bucket := gcsClient.Bucket(bucketName) + + prefix := path.Join("app", app) + "/" + it := bucket.Objects(ctx, &storage.Query{ + Prefix: prefix, + }) + + stages := []string{} + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, err + } + + filename := path.Base(attrs.Name) + if strings.HasSuffix(filename, ".json") { + stageName := strings.TrimSuffix(filename, ".json") + stages = append(stages, stageName) + } + } + + return stages, nil +} + +func (g *GCPHome) info() (util.KeyValuePairs[string], error) { + lines := util.KeyValuePairs[string]{ + {Key: "Provider", Value: "GCP"}, + {Key: "Project", Value: g.provider.project}, + } + if g.provider.region != "" { + lines = append(lines, util.KeyValuePair[string]{ + Key: "Region", Value: g.provider.region, + }) + } + if g.provider.zone != "" { + lines = append(lines, util.KeyValuePair[string]{ + Key: "Zone", Value: g.provider.zone, + }) + } + + return lines, nil +} + +func (g *GCPHome) pathForData(key, app, stage string) string { + return path.Join(key, app, fmt.Sprintf("%v.json", stage)) +} + +func (g *GCPHome) pathForPassphrase(app string, stage string) string { + return fmt.Sprintf("sst-passphrase-%s-%s", app, stage) +} + +func isNotFoundError(err error) bool { + if st, ok := status.FromError(err); ok { + return st.Code() == codes.NotFound + } + + var gErr *googleapi.Error + if errors.As(err, &gErr) { + return gErr.Code == 404 + } + + return strings.Contains(err.Error(), "NotFound") +} + +func wrapError(err error) error { + if st, ok := status.FromError(err); ok { + for _, detail := range st.Details() { + if detailWithMethods, ok := detail.(interface { + GetLocale() string + GetMessage() string + }); ok { + locale := detailWithMethods.GetLocale() + if locale == "en-US" || locale == "" { + return util.NewReadableError(err, detailWithMethods.GetMessage()) + } + } + } + return util.NewReadableError(err, st.Message()) + } + + var gErr *googleapi.Error + if errors.As(err, &gErr) { + return util.NewReadableError(err, gErr.Message) + } + + return err +} + +func firstNonEmptyEnv(envs ...string) string { + for _, env := range envs { + if value := os.Getenv(env); value != "" { + return value + } + } + return "" +} diff --git a/platform/src/config.ts b/platform/src/config.ts index a6f6602fec..d6ba6bc095 100644 --- a/platform/src/config.ts +++ b/platform/src/config.ts @@ -242,7 +242,7 @@ export interface App { * ``` * */ - home: "aws" | "cloudflare" | "local"; + home: "aws" | "cloudflare" | "gcp" | "local"; /** * If set to `true`, the `sst remove` CLI will not run and will error out.