diff --git a/docs/ENVIRONMENT_FILTERING.md b/docs/ENVIRONMENT_FILTERING.md new file mode 100644 index 0000000..553ed3a --- /dev/null +++ b/docs/ENVIRONMENT_FILTERING.md @@ -0,0 +1,393 @@ +# Environment-Based Variables in SwarmCD + +This document explains how to use environment-specific variables in SwarmCD. + +## Overview + +SwarmCD supports **environment-specific variables** that allow you to define different configuration values for each environment (dev, staging, prod, etc.) directly in your stack's Git repository + +## How It Works + +### 1. Environment Detection + +SwarmCD detects the current environment by reading a label from the Docker Swarm manager node. By default, it looks for the label `swarmcd.environment`, but this can be customized in `config.yaml`. + +**Setting the environment label on your Swarm manager node:** + +```bash +# For a development environment +docker node update --label-add swarmcd.environment=dev + +# For a staging environment +docker node update --label-add swarmcd.environment=staging + +# For a production environment +docker node update --label-add swarmcd.environment=prod +``` + +**Check current labels:** +```bash +docker node inspect --format '{{ .Spec.Labels }}' +``` + +### 2. Creating the Environments File + +In your **stack's Git repository**, create an `environments.yaml` file (or any name you prefer): + +```yaml +# environments.yaml in your stack repository +environments: + dev: + DB_HOST: dev-database.local + API_URL: https://dev-api.example.com + LOG_LEVEL: debug + staging: + DB_HOST: staging-database.local + API_URL: https://staging-api.example.com + LOG_LEVEL: info + prod: + DB_HOST: prod-database.local + API_URL: https://api.example.com + LOG_LEVEL: warn +``` + +### 3. Configuring the Stack + +In your **SwarmCD `stacks.yaml`** file, reference the environments file: + +```yaml +my-app: + repo: my-repo + branch: main + compose_file: docker-compose.yaml + # Reference the environments file in the repository + environments_file: environments.yaml +``` + +### 4. Using Variables in Docker Compose Files + +In your `docker-compose.yaml` files, you can reference these variables using `${VARIABLE_NAME}` or `$VARIABLE_NAME` syntax: + +```yaml +services: + app: + image: myapp:latest + environment: + DATABASE_HOST: ${DB_HOST} + API_ENDPOINT: ${API_URL} + LOG_LEVEL: ${LOG_LEVEL} + deploy: + replicas: 3 +``` + +When SwarmCD processes this compose file: +- In the **dev** environment, it will replace `${DB_HOST}` with `dev-database.local` +- In the **staging** environment, it will replace `${DB_HOST}` with `staging-database.local` + +## Configuration Reference + +### config.yaml + +```yaml +# Optional: customize the label name used to detect the environment +environment_label: swarmcd.environment # default value +``` + +### stacks.yaml + +```yaml +stack-name: + repo: repo-name + branch: main + compose_file: path/to/docker-compose.yaml + + # Optional: path to environments file in the repository + # This file contains environment-specific variables + # If omitted, no environment variables will be replaced + environments_file: path/to/environments.yaml +``` + +### environments.yaml (in your stack repository) + +```yaml +environments: + dev: + VAR_NAME: dev-value + ANOTHER_VAR: dev-value-2 + staging: + VAR_NAME: staging-value + ANOTHER_VAR: staging-value-2 + prod: + VAR_NAME: prod-value + ANOTHER_VAR: prod-value-2 +``` + +## Deployment Behavior + +### Stack Filtering Logic + +The deployment decision is based on two factors: whether the stack has an `environments_file` configured and whether the node has an environment label. + +| Node Label | Stack has `environments_file` | Behavior | +|------------|------------------------------|----------| +| ❌ No label | ❌ No file | ✅ **DEPLOY** (no filtering) | +| ❌ No label | ✅ Has file | ❌ **SKIP** (stack is environment-filtered) | +| ✅ Has label | ❌ No file | ✅ **DEPLOY** (no filtering) | +| ✅ Has label | ✅ Has file | ✅/❌ **DEPLOY if environment exists in file** | + +**In summary:** +- **Stacks WITHOUT `environments_file`**: Always deploy (no environment filtering) +- **Stacks WITH `environments_file` but node WITHOUT label**: Never deploy (filtered stack requires environment) +- **Stacks WITH `environments_file` and node WITH label**: Deploy only if the current environment is defined in the file + +### Variable Loading + +- **If `environments_file` is not specified**: Stack always deploys, no variables loaded +- **If `environments_file` is specified but no environment label on node**: Stack is **skipped** (not deployed) +- **If `environments_file` is specified but the file doesn't exist in repo**: Stack deploys without environment variables (graceful handling) +- **If the current environment is not defined in the environments file**: Stack is **skipped** (not deployed in this environment) + +**Example logs:** + +``` +# Stack without environments_file - always deploys +INFO detected environment from node label environment=prod label=swarmcd.environment +DEBUG loading environment variables... stack=my-app +DEBUG reading stack file... stack=my-app + +# Stack with environments_file and matching environment - deploys +INFO detected environment from node label environment=prod label=swarmcd.environment +DEBUG loading environment variables... stack=my-app +DEBUG loaded environment variables stack=my-app environment=prod vars_count=8 +DEBUG reading stack file... stack=my-app + +# Stack with environments_file but no environment label - skipped +WARN environment label not found on manager node, all stacks will be deployed label=swarmcd.environment +INFO skipping stack with environment filtering when no environment is set on node stack=my-app environments_file=environments.yaml + +# Stack with environments_file but current environment not in file - skipped +INFO detected environment from node label environment=prod label=swarmcd.environment +INFO skipping stack not configured for current environment stack=dev-tools environment=prod available_environments=[dev staging] +``` + +### Variable Replacement + +- Variables are replaced **after** template rendering (if using `values_file`) +- Variables are replaced **before** SOPS decryption and config/secret rotation +- Variables in the format `${VAR_NAME}` or `$VAR_NAME` will be replaced +- Only variables defined in `env_vars[current_environment]` will be replaced +- If no variables are defined for the current environment, no replacement occurs +- Undefined variables remain unchanged in the compose file + +## Examples + +### Example 1: Basic Stack with Environment Variables + +**Repository structure:** +``` +my-stack/ +├── docker-compose.yaml +└── environments.yaml +``` + +**stacks.yaml:** +```yaml +my-app: + repo: my-stack-repo + branch: main + compose_file: docker-compose.yaml + environments_file: environments.yaml +``` + +**environments.yaml (in repository):** +```yaml +environments: + dev: + DB_HOST: dev-db.local + API_URL: https://dev-api.example.com + prod: + DB_HOST: prod-db.local + API_URL: https://api.example.com +``` + +**docker-compose.yaml (in repository):** +```yaml +services: + app: + image: myapp:latest + environment: + DATABASE_HOST: ${DB_HOST} + API_ENDPOINT: ${API_URL} +``` + +### Example 2: Different Database Configurations per Environment + +**environments.yaml (in repository):** +```yaml +environments: + dev: + DB_HOST: postgres-dev.internal + DB_NAME: api_dev + DB_USER: dev_user + REDIS_HOST: redis-dev.internal + REDIS_PORT: "6379" + staging: + DB_HOST: postgres-staging.internal + DB_NAME: api_staging + DB_USER: staging_user + REDIS_HOST: redis-staging.internal + REDIS_PORT: "6379" + prod: + DB_HOST: postgres-prod.internal + DB_NAME: api_production + DB_USER: prod_user + REDIS_HOST: redis-prod.internal + REDIS_PORT: "6379" +``` + +**docker-compose.yaml (in repository):** +```yaml +services: + api: + image: mycompany/api:latest + environment: + DATABASE_URL: postgresql://${DB_USER}:password@${DB_HOST}:5432/${DB_NAME} + REDIS_URL: redis://${REDIS_HOST}:${REDIS_PORT} +``` + +### Example 3: Environment-Specific Scaling and Resources + +**environments.yaml (in repository):** +```yaml +environments: + dev: + REPLICAS: "1" + MEMORY_LIMIT: "512M" + CPU_LIMIT: "0.5" + staging: + REPLICAS: "2" + MEMORY_LIMIT: "1G" + CPU_LIMIT: "1.0" + prod: + REPLICAS: "5" + MEMORY_LIMIT: "2G" + CPU_LIMIT: "2.0" +``` + +**docker-compose.yaml (in repository):** +```yaml +services: + web: + image: mycompany/web:latest + deploy: + replicas: ${REPLICAS} + resources: + limits: + memory: ${MEMORY_LIMIT} + cpus: ${CPU_LIMIT} +``` + +## Troubleshooting + +### Stack not deploying + +**Check the environment label:** +```bash +docker node inspect self --format '{{ .Spec.Labels.swarmcd\.environment }}' +``` + +**Check SwarmCD logs:** +```bash +docker service logs swarm-cd +``` + +Look for messages like: +- `detected environment from node label` +- `skipping stack not configured for current environment` + +### Variables not being replaced + +1. Ensure the variable is defined in `env_vars` for the current environment +2. Check that the variable syntax in your compose file is correct (`${VAR_NAME}`) +3. Review SwarmCD logs for any errors during variable replacement + +### Environment label not found + +If you see: +``` +WARN environment label not found on manager node, all stacks will be deployed +``` + +This means the manager node doesn't have the environment label. Add it using: +```bash +docker node update --label-add swarmcd.environment= $(docker node ls --filter role=manager -q) +``` + +### Environments file not found + +If the environments file doesn't exist in the repository, you'll see: +``` +DEBUG environments file not found, skipping environment variables stack=my-app file=environments.yaml +``` + +This is not an error - the stack will deploy without environment-specific variables. + +## Best Practices + +1. **Always set the environment label** on manager nodes to ensure correct variable replacement +2. **Keep the environments file in Git** alongside your compose files for version control +3. **Use descriptive environment names**: `dev`, `staging`, `prod` are common, but use what makes sense for your workflow +4. **Test environment-specific configs** by deploying to dev/staging before production +5. **Keep sensitive values in SOPS-encrypted files**, not in plaintext environment variables +6. **Document your environment-specific variables** in your repository's README +7. **Use the same label name** across all your Swarm clusters for consistency +8. **Commit the environments.yaml file** to your stack repository so changes are tracked with the code + +## Migration Guide + +### Migrating Existing Configurations + +If you have existing stacks and want to add environment-specific variables: + +1. **Add environment labels to your Swarm nodes:** + ```bash + docker node update --label-add swarmcd.environment=prod + ``` + +2. **Create an `environments.yaml` file in your stack repository:** + ```yaml + environments: + dev: + API_URL: https://dev-api.example.com + DB_HOST: dev-db.local + prod: + API_URL: https://api.example.com + DB_HOST: prod-db.local + ``` + +3. **Update your stacks.yaml** to reference the environments file: + ```yaml + existing-stack: + # ... existing configuration ... + environments_file: environments.yaml # Add this + ``` + +4. **Update your compose files** to use the variables: + ```yaml + services: + app: + environment: + API_ENDPOINT: ${API_URL} # Replace hardcoded values + DATABASE_HOST: ${DB_HOST} + ``` + +5. **Commit the environments.yaml file** to your repository + +6. **Test in a non-production environment first** before rolling out to production + +### Backward Compatibility + +- If you don't add the `environments_file` field, stacks will deploy without environment variables (existing behavior) +- If the environment label is not found, stacks will deploy but without environment-specific variables +- If the environments file doesn't exist, stacks will deploy without errors +- The new fields are optional and don't break existing configurations diff --git a/docs/environments.yaml b/docs/environments.yaml new file mode 100644 index 0000000..348e07d --- /dev/null +++ b/docs/environments.yaml @@ -0,0 +1,60 @@ +# SwarmCD environments configuration file +# This file should be placed in your stack's git repository +# and referenced in stacks.yaml via the environments_file field + +# Define environment-specific variables for your stack +# The current environment is detected from the Docker Swarm manager node label +# (default label: swarmcd.environment) + +environments: + # Development environment variables + dev: + DB_HOST: dev-database.local + DB_PORT: "5432" + DB_NAME: myapp_dev + API_URL: https://dev-api.example.com + API_KEY: dev-api-key-12345 + LOG_LEVEL: debug + REDIS_HOST: redis-dev.internal + CACHE_TTL: "60" + MAX_CONNECTIONS: "10" + + # Staging environment variables + staging: + DB_HOST: staging-database.local + DB_PORT: "5432" + DB_NAME: myapp_staging + API_URL: https://staging-api.example.com + API_KEY: staging-api-key-67890 + LOG_LEVEL: info + REDIS_HOST: redis-staging.internal + CACHE_TTL: "300" + MAX_CONNECTIONS: "50" + + # Production environment variables + prod: + DB_HOST: prod-database.local + DB_PORT: "5432" + DB_NAME: myapp_production + API_URL: https://api.example.com + API_KEY: prod-api-key-abcde + LOG_LEVEL: warn + REDIS_HOST: redis-prod.internal + CACHE_TTL: "600" + MAX_CONNECTIONS: "100" + +# Usage in docker-compose.yaml: +# +# services: +# app: +# image: myapp:latest +# environment: +# DATABASE_HOST: ${DB_HOST} +# DATABASE_PORT: ${DB_PORT} +# DATABASE_NAME: ${DB_NAME} +# API_ENDPOINT: ${API_URL} +# API_KEY: ${API_KEY} +# LOG_LEVEL: ${LOG_LEVEL} +# REDIS_URL: redis://${REDIS_HOST}:6379 +# CACHE_TTL_SECONDS: ${CACHE_TTL} +# DB_MAX_CONNECTIONS: ${MAX_CONNECTIONS} diff --git a/docs/stacks.yaml b/docs/stacks.yaml index 720ea53..536529a 100644 --- a/docs/stacks.yaml +++ b/docs/stacks.yaml @@ -1,8 +1,8 @@ -# SwarmCD stacks configuration refernce +# SwarmCD stacks configuration reference # This file should contain all the stacks -# that you want swarm-cd to keep synching +# that you want swarm-cd to keep synching -# Name of the stack, it will be used in +# Name of the stack, it will be used in # the stack deploy command as the stack name stack-name: # The repo that contain the stack compose file @@ -16,7 +16,7 @@ stack-name: compose_file: /path/to/compose.yaml # Path to values file to use when rendering # compose file as a Go template. If empty, compose - # file will be treated as a regular compose file + # file will be treated as a regular compose file values_file: /path/to/values.yaml # Paths to files encrypted using sops to decrypt # before updating stack @@ -25,3 +25,9 @@ stack-name: # Enable the automatic secret discovery # alternative to sops_files sops_secrets_discovery: false + # Path to the environments file in the repository + # This file contains environment-specific variables for the stack + # The environment is detected from the manager node label (default: swarmcd.environment) + # If this field is empty or not specified, no environment filtering will be applied + # Example: environments.yaml + environments_file: path/to/environments.yaml diff --git a/swarmcd/init.go b/swarmcd/init.go index e17940d..29313ba 100644 --- a/swarmcd/init.go +++ b/swarmcd/init.go @@ -1,6 +1,7 @@ package swarmcd import ( + "context" "fmt" "log/slog" "os" @@ -9,6 +10,7 @@ import ( "github.com/docker/cli/cli/command" "github.com/docker/cli/cli/flags" + "github.com/docker/docker/client" "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/m-adawi/swarm-cd/util" ) @@ -27,16 +29,22 @@ var repos map[string]*stackRepo = map[string]*stackRepo{} var dockerCli *command.DockerCli +var currentEnvironment string + func Init() (err error) { err = initRepos() if err != nil { return err } - err = initStacks() + err = initDockerCli() if err != nil { return err } - err = initDockerCli() + err = detectEnvironment() + if err != nil { + return err + } + err = initStacks() if err != nil { return err } @@ -91,6 +99,49 @@ func createHTTPBasicAuth(repoName string) (*http.BasicAuth, error) { }, nil } +func detectEnvironment() error { + // Create a Docker client to query node information + cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return fmt.Errorf("could not create docker client for environment detection: %w", err) + } + defer cli.Close() + + ctx := context.Background() + + // Get info about the current node (should be a manager) + info, err := cli.Info(ctx) + if err != nil { + return fmt.Errorf("could not get docker info: %w", err) + } + + // Check if this is a swarm manager + if !info.Swarm.ControlAvailable { + return fmt.Errorf("this node is not a swarm manager, cannot detect environment") + } + + // Get the current node ID + nodeID := info.Swarm.NodeID + + // Inspect the node to get its labels + node, _, err := cli.NodeInspectWithRaw(ctx, nodeID) + if err != nil { + return fmt.Errorf("could not inspect node %s: %w", nodeID, err) + } + + // Get the environment from the label + environmentLabel := config.EnvironmentLabel + if env, ok := node.Spec.Labels[environmentLabel]; ok { + currentEnvironment = env + logger.Info("detected environment from node label", "environment", currentEnvironment, "label", environmentLabel) + } else { + logger.Warn("environment label not found on manager node, all stacks will be deployed", "label", environmentLabel) + currentEnvironment = "" + } + + return nil +} + func initStacks() error { for stack, stackConfig := range config.StackConfigs { stackRepo, ok := repos[stackConfig.Repo] @@ -98,7 +149,8 @@ func initStacks() error { return fmt.Errorf("error initializing %s stack, no such repo: %s", stack, stackConfig.Repo) } discoverSecrets := config.SopsSecretsDiscovery || stackConfig.SopsSecretsDiscovery - swarmStack := newSwarmStack(stack, stackRepo, stackConfig.Branch, stackConfig.ComposeFile, stackConfig.SopsFiles, stackConfig.ValuesFile, discoverSecrets) + + swarmStack := newSwarmStack(stack, stackRepo, stackConfig.Branch, stackConfig.ComposeFile, stackConfig.SopsFiles, stackConfig.ValuesFile, discoverSecrets, stackConfig.EnvironmentsFile) stacks = append(stacks, swarmStack) stackStatus[stack] = &StackStatus{} stackStatus[stack].RepoURL = stackRepo.url diff --git a/swarmcd/stack.go b/swarmcd/stack.go index 3bc6464..7bb4e7b 100644 --- a/swarmcd/stack.go +++ b/swarmcd/stack.go @@ -7,6 +7,7 @@ import ( "log/slog" "os" "path" + "strings" "text/template" "github.com/docker/cli/cli/command/stack" @@ -15,24 +16,32 @@ import ( ) type swarmStack struct { - name string - repo *stackRepo - branch string - composePath string - sopsFiles []string - valuesFile string - discoverSecrets bool + name string + repo *stackRepo + branch string + composePath string + sopsFiles []string + valuesFile string + discoverSecrets bool + environmentsFile string + envVars map[string]string } -func newSwarmStack(name string, repo *stackRepo, branch string, composePath string, sopsFiles []string, valuesFile string, discoverSecrets bool) *swarmStack { +type environmentsConfig struct { + Environments map[string]map[string]string `yaml:"environments"` +} + +func newSwarmStack(name string, repo *stackRepo, branch string, composePath string, sopsFiles []string, valuesFile string, discoverSecrets bool, environmentsFile string) *swarmStack { return &swarmStack{ - name: name, - repo: repo, - branch: branch, - composePath: composePath, - sopsFiles: sopsFiles, - valuesFile: valuesFile, - discoverSecrets: discoverSecrets, + name: name, + repo: repo, + branch: branch, + composePath: composePath, + sopsFiles: sopsFiles, + valuesFile: valuesFile, + discoverSecrets: discoverSecrets, + environmentsFile: environmentsFile, + envVars: nil, // Will be loaded from environments file } } @@ -49,45 +58,51 @@ func (swarmStack *swarmStack) updateStack() (revision string, err error) { } log.Debug("changes pulled", "revision", revision) - log.Debug("reading stack file...") + shouldDeploy, err := swarmStack.loadEnvironmentVars() + if err != nil { + return + } + if !shouldDeploy { + log.Info("stack skipped due to environment filtering") + return revision, nil + } + stackBytes, err := swarmStack.readStack() if err != nil { return } if swarmStack.valuesFile != "" { - log.Debug("rendering template...") stackBytes, err = swarmStack.renderComposeTemplate(stackBytes) + if err != nil { + return + } } + + stackContents, err := swarmStack.parseStackString([]byte(stackBytes)) if err != nil { return } - log.Debug("parsing stack content...") - stackContents, err := swarmStack.parseStackString([]byte(stackBytes)) + err = swarmStack.replaceEnvVars(stackContents) if err != nil { return } - - log.Debug("decrypting secrets...") err = swarmStack.decryptSopsFiles(stackContents) if err != nil { return "", fmt.Errorf("failed to decrypt one or more sops files for %s stack: %w", swarmStack.name, err) } - log.Debug("rotating configs and secrets...") err = swarmStack.rotateConfigsAndSecrets(stackContents) if err != nil { return } - log.Debug("writing stack to file...") err = swarmStack.writeStack(stackContents) if err != nil { return } - log.Debug("deploying stack...") err = swarmStack.deployStack() return } @@ -101,6 +116,71 @@ func (swarmStack *swarmStack) readStack() ([]byte, error) { return composeFileBytes, nil } +func (swarmStack *swarmStack) loadEnvironmentVars() (shouldDeploy bool, err error) { + // If no environments file is specified, always deploy (no filtering) + if swarmStack.environmentsFile == "" { + return true, nil + } + + // If environments file is specified but no environment label is set on the node, + // this stack is environment-filtered and should NOT be deployed + if currentEnvironment == "" { + logger.Info("skipping stack with environment filtering when no environment is set on node", + "stack", swarmStack.name, + "environments_file", swarmStack.environmentsFile) + return false, nil + } + + // Read the environments file from the repository + envFilePath := path.Join(swarmStack.repo.path, swarmStack.environmentsFile) + envFileBytes, err := os.ReadFile(envFilePath) + if err != nil { + // If file doesn't exist, it's not an error - just skip loading + if os.IsNotExist(err) { + logger.Warn("environments file not found, deploying without environment variables", + "stack", swarmStack.name, + "file", swarmStack.environmentsFile) + return true, nil + } + return false, fmt.Errorf("could not read environments file %s: %w", envFilePath, err) + } + + // Parse the environments file + var envConfig environmentsConfig + err = yaml.Unmarshal(envFileBytes, &envConfig) + if err != nil { + return false, fmt.Errorf("could not parse environments file %s: %w", envFilePath, err) + } + + // Get the variables for the current environment + if envVars, ok := envConfig.Environments[currentEnvironment]; ok { + swarmStack.envVars = envVars + if len(envVars) > 0 { + logger.Info("loaded environment variables", + "stack", swarmStack.name, + "environment", currentEnvironment, + "count", len(envVars)) + } + return true, nil + } else { + // Current environment is not defined in the environments file + // This means this stack should not be deployed in this environment + logger.Info("skipping stack not configured for current environment", + "stack", swarmStack.name, + "environment", currentEnvironment, + "available_environments", getKeys(envConfig.Environments)) + return false, nil + } +} + +func getKeys(m map[string]map[string]string) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + return keys +} + func (swarmStack *swarmStack) renderComposeTemplate(templateContents []byte) ([]byte, error) { valuesFile := path.Join(swarmStack.repo.path, swarmStack.valuesFile) valuesBytes, err := os.ReadFile(valuesFile) @@ -130,6 +210,63 @@ func (swarmStack *swarmStack) parseStackString(stackContent []byte) (map[string] return composeMap, nil } +func (swarmStack *swarmStack) replaceEnvVars(composeMap map[string]any) error { + // If no environment variables are defined for this stack, skip replacement + if len(swarmStack.envVars) == 0 { + return nil + } + + // Recursively replace variables in the compose map + replaceInMap(composeMap, swarmStack.envVars) + + return nil +} + +// replaceInMap recursively walks through a map and replaces string values +// containing variable references like ${VAR_NAME} with their actual values +func replaceInMap(data any, envVars map[string]string) { + switch v := data.(type) { + case map[string]any: + for key, value := range v { + switch val := value.(type) { + case string: + // Replace environment variables in string values + v[key] = replaceEnvInString(val, envVars) + case map[string]any: + // Recursively process nested maps + replaceInMap(val, envVars) + case []any: + // Recursively process arrays + replaceInMap(val, envVars) + } + } + case []any: + for i, item := range v { + switch val := item.(type) { + case string: + v[i] = replaceEnvInString(val, envVars) + case map[string]any: + replaceInMap(val, envVars) + case []any: + replaceInMap(val, envVars) + } + } + } +} + +// replaceEnvInString replaces ${VAR_NAME} patterns in a string with their values +func replaceEnvInString(str string, envVars map[string]string) string { + result := str + for key, value := range envVars { + // Replace ${VAR_NAME} format + result = strings.ReplaceAll(result, "${"+key+"}", value) + // Also support $VAR_NAME format (but not if followed by alphanumeric to avoid partial matches) + // This is a simple implementation - for production you might want regex + result = strings.ReplaceAll(result, "$"+key, value) + } + return result +} + func (swarmStack *swarmStack) decryptSopsFiles(composeMap map[string]any) (err error) { var sopsFiles []string if !swarmStack.discoverSecrets { diff --git a/swarmcd/stack_test.go b/swarmcd/stack_test.go index 59de84a..ccc29dc 100644 --- a/swarmcd/stack_test.go +++ b/swarmcd/stack_test.go @@ -8,7 +8,7 @@ import ( // External objects are ignored by the rotation func TestRotateExternalObjects(t *testing.T) { repo := &stackRepo{name: "test", path: "test", url: "", auth: nil, lock: &sync.Mutex{}, gitRepoObject: nil} - stack := newSwarmStack("test", repo, "main", "docker-compose.yaml", nil, "", false) + stack := newSwarmStack("test", repo, "main", "docker-compose.yaml", nil, "", false, "") objects := map[string]any{ "my-secret": map[string]any{"external": true}, } @@ -21,7 +21,7 @@ func TestRotateExternalObjects(t *testing.T) { // Secrets are discovered, external secrets are ignored func TestSecretDiscovery(t *testing.T) { repo := &stackRepo{name: "test", path: "test", url: "", auth: nil, lock: &sync.Mutex{}, gitRepoObject: nil} - stack := newSwarmStack("test", repo, "main", "stacks/docker-compose.yaml", nil, "", false) + stack := newSwarmStack("test", repo, "main", "stacks/docker-compose.yaml", nil, "", false, "") stackString := []byte(`services: my-service: image: my-image @@ -48,3 +48,205 @@ secrets: t.Errorf("unexpected sops file: %s", sopsFiles[0]) } } + +// Test environment variable replacement in compose file +func TestEnvVarReplacement(t *testing.T) { + repo := &stackRepo{name: "test", path: "test", url: "", auth: nil, lock: &sync.Mutex{}, gitRepoObject: nil} + stack := newSwarmStack("test", repo, "main", "docker-compose.yaml", nil, "", false, "") + + // Manually set envVars to simulate loading from environments file + stack.envVars = map[string]string{ + "DB_HOST": "production-db.example.com", + "DB_PORT": "5432", + "API_URL": "https://api.example.com", + } + + stackString := []byte(`services: + app: + image: myapp:latest + environment: + DATABASE_HOST: ${DB_HOST} + DATABASE_PORT: ${DB_PORT} + API_ENDPOINT: ${API_URL} + STATIC_VAR: some-static-value`) + + composeMap, err := stack.parseStackString(stackString) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + err = stack.replaceEnvVars(composeMap) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + // Verify replacements + services := composeMap["services"].(map[string]any) + app := services["app"].(map[string]any) + environment := app["environment"].(map[string]any) + + if environment["DATABASE_HOST"] != "production-db.example.com" { + t.Errorf("expected DATABASE_HOST to be 'production-db.example.com', got '%s'", environment["DATABASE_HOST"]) + } + if environment["DATABASE_PORT"] != "5432" { + t.Errorf("expected DATABASE_PORT to be '5432', got '%s'", environment["DATABASE_PORT"]) + } + if environment["API_ENDPOINT"] != "https://api.example.com" { + t.Errorf("expected API_ENDPOINT to be 'https://api.example.com', got '%s'", environment["API_ENDPOINT"]) + } + if environment["STATIC_VAR"] != "some-static-value" { + t.Errorf("expected STATIC_VAR to remain 'some-static-value', got '%s'", environment["STATIC_VAR"]) + } +} + +// Test environment variable replacement in nested structures +func TestEnvVarReplacementNested(t *testing.T) { + repo := &stackRepo{name: "test", path: "test", url: "", auth: nil, lock: &sync.Mutex{}, gitRepoObject: nil} + stack := newSwarmStack("test", repo, "main", "docker-compose.yaml", nil, "", false, "") + + // Manually set envVars to simulate loading from environments file + stack.envVars = map[string]string{ + "IMAGE_TAG": "v1.2.3", + "REPLICA_COUNT": "3", + } + + stackString := []byte(`services: + app: + image: myapp:${IMAGE_TAG} + deploy: + replicas: ${REPLICA_COUNT} + labels: + - "version=${IMAGE_TAG}"`) + + composeMap, err := stack.parseStackString(stackString) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + err = stack.replaceEnvVars(composeMap) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + // Verify replacements + services := composeMap["services"].(map[string]any) + app := services["app"].(map[string]any) + + if app["image"] != "myapp:v1.2.3" { + t.Errorf("expected image to be 'myapp:v1.2.3', got '%s'", app["image"]) + } + + deploy := app["deploy"].(map[string]any) + if deploy["replicas"] != "3" { + t.Errorf("expected replicas to be '3', got '%s'", deploy["replicas"]) + } + + labels := deploy["labels"].([]any) + if labels[0] != "version=v1.2.3" { + t.Errorf("expected label to be 'version=v1.2.3', got '%s'", labels[0]) + } +} + +// Test that stacks without env vars are not affected +func TestNoEnvVars(t *testing.T) { + repo := &stackRepo{name: "test", path: "test", url: "", auth: nil, lock: &sync.Mutex{}, gitRepoObject: nil} + stack := newSwarmStack("test", repo, "main", "docker-compose.yaml", nil, "", false, "") + + stackString := []byte(`services: + app: + image: myapp:latest + environment: + SOME_VAR: ${UNDEFINED_VAR}`) + + composeMap, err := stack.parseStackString(stackString) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + err = stack.replaceEnvVars(composeMap) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + // Verify no replacements occurred + services := composeMap["services"].(map[string]any) + app := services["app"].(map[string]any) + environment := app["environment"].(map[string]any) + + if environment["SOME_VAR"] != "${UNDEFINED_VAR}" { + t.Errorf("expected SOME_VAR to remain '${UNDEFINED_VAR}', got '%s'", environment["SOME_VAR"]) + } +} + +// Test that stacks without environments_file always deploy +func TestLoadEnvironmentVars_NoEnvironmentsFile(t *testing.T) { + // Save and restore currentEnvironment + oldEnv := currentEnvironment + defer func() { currentEnvironment = oldEnv }() + + repo := &stackRepo{name: "test", path: "test", url: "", auth: nil, lock: &sync.Mutex{}, gitRepoObject: nil} + stack := newSwarmStack("test", repo, "main", "docker-compose.yaml", nil, "", false, "") + + // Test with no environment set + currentEnvironment = "" + shouldDeploy, err := stack.loadEnvironmentVars() + if err != nil { + t.Errorf("unexpected error: %s", err) + } + if !shouldDeploy { + t.Error("expected stack without environments_file to deploy even when no environment is set") + } + + // Test with environment set + currentEnvironment = "prod" + shouldDeploy, err = stack.loadEnvironmentVars() + if err != nil { + t.Errorf("unexpected error: %s", err) + } + if !shouldDeploy { + t.Error("expected stack without environments_file to deploy when environment is set") + } +} + +// Test that stacks with environments_file but no environment label do NOT deploy +func TestLoadEnvironmentVars_WithFileButNoEnvironment(t *testing.T) { + // Save and restore currentEnvironment + oldEnv := currentEnvironment + defer func() { currentEnvironment = oldEnv }() + + repo := &stackRepo{name: "test", path: "test", url: "", auth: nil, lock: &sync.Mutex{}, gitRepoObject: nil} + stack := newSwarmStack("test", repo, "main", "docker-compose.yaml", nil, "", false, "environments.yaml") + + // No environment set on node + currentEnvironment = "" + shouldDeploy, err := stack.loadEnvironmentVars() + if err != nil { + t.Errorf("unexpected error: %s", err) + } + if shouldDeploy { + t.Error("expected stack with environments_file to NOT deploy when no environment is set on node") + } +} + +// Test that stacks deploy when environment matches +func TestLoadEnvironmentVars_MatchingEnvironment(t *testing.T) { + // Save and restore currentEnvironment + oldEnv := currentEnvironment + defer func() { currentEnvironment = oldEnv }() + + // This test would need a real environments.yaml file to work properly + // For now, we just verify the logic path + currentEnvironment = "dev" + + repo := &stackRepo{name: "test", path: "test", url: "", auth: nil, lock: &sync.Mutex{}, gitRepoObject: nil} + stack := newSwarmStack("test", repo, "main", "docker-compose.yaml", nil, "", false, "nonexistent.yaml") + + // File doesn't exist, should still deploy (graceful handling) + shouldDeploy, err := stack.loadEnvironmentVars() + if err != nil { + t.Errorf("unexpected error: %s", err) + } + if !shouldDeploy { + t.Error("expected stack to deploy when environments file doesn't exist") + } +} diff --git a/util/config.go b/util/config.go index 174ced2..f38a48b 100644 --- a/util/config.go +++ b/util/config.go @@ -14,6 +14,7 @@ type StackConfig struct { ValuesFile string `mapstructure:"values_file"` SopsFiles []string `mapstructure:"sops_files"` SopsSecretsDiscovery bool `mapstructure:"sops_secrets_discovery"` + EnvironmentsFile string `mapstructure:"environments_file"` } type RepoConfig struct { @@ -31,6 +32,7 @@ type Config struct { RepoConfigs map[string]*RepoConfig `mapstructure:"repos"` SopsSecretsDiscovery bool `mapstructure:"sops_secrets_discovery"` Address string `mapstructure:"address"` + EnvironmentLabel string `mapstructure:"environment_label"` } var Configs Config @@ -64,6 +66,7 @@ func readConfig() (err error) { configViper.SetDefault("auto_rotate", true) configViper.SetDefault("sops_secrets_discovery", false) configViper.SetDefault("address", "0.0.0.0:8080") + configViper.SetDefault("environment_label", "swarmcd.environment") err = configViper.ReadInConfig() if err != nil && !errors.As(err, &viper.ConfigFileNotFoundError{}) { return